APT_FammaMacbeth.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# APT model: Famma-Macbeth Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from rqdata_utils import *\n",
    "import pandas\n",
    "import numpy as np\n",
    "import scipy as sp\n",
    "import alphalens as al\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "price_df,instrument_df,equity_df = get_price_instrument_equity(\"cn_stock_price_2012_2018.csv\",\"cn_instrument_info_2012_2018.csv\",\"cn_equity_daily_2012_2018.csv\",\"sectorCode\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>return</th>\n",
       "      <th>close</th>\n",
       "      <th>total_turnover</th>\n",
       "      <th>volume</th>\n",
       "      <th>week</th>\n",
       "      <th>month</th>\n",
       "      <th>report_quarter</th>\n",
       "      <th>market_cap</th>\n",
       "      <th>a_share_market_val_2</th>\n",
       "      <th>cash_received_from_sales_of_goods</th>\n",
       "      <th>pb_ratio</th>\n",
       "      <th>net_profit</th>\n",
       "      <th>ps_ratio</th>\n",
       "      <th>sectorCode</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>date</th>\n",
       "      <th>order_book_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">2012-01-04</th>\n",
       "      <th>000001.XSHE</th>\n",
       "      <td>-0.027582</td>\n",
       "      <td>5.1224</td>\n",
       "      <td>2.275637e+08</td>\n",
       "      <td>40894428.0</td>\n",
       "      <td>0.5775</td>\n",
       "      <td>0.4331</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Financials</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>000002.XSHE</th>\n",
       "      <td>-0.018742</td>\n",
       "      <td>6.0525</td>\n",
       "      <td>3.559891e+08</td>\n",
       "      <td>47432958.0</td>\n",
       "      <td>0.3711</td>\n",
       "      <td>0.4030</td>\n",
       "      <td>2011q3</td>\n",
       "      <td>8.059489e+10</td>\n",
       "      <td>7.082120e+10</td>\n",
       "      <td>7.516785e+10</td>\n",
       "      <td>1.5216</td>\n",
       "      <td>4.106349e+09</td>\n",
       "      <td>0.8679</td>\n",
       "      <td>Financials</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>000004.XSHE</th>\n",
       "      <td>-0.022250</td>\n",
       "      <td>7.9100</td>\n",
       "      <td>3.763833e+06</td>\n",
       "      <td>465469.0</td>\n",
       "      <td>0.5720</td>\n",
       "      <td>0.7506</td>\n",
       "      <td>2011q3</td>\n",
       "      <td>6.642556e+08</td>\n",
       "      <td>6.634549e+08</td>\n",
       "      <td>5.949968e+07</td>\n",
       "      <td>8.8175</td>\n",
       "      <td>4.500363e+06</td>\n",
       "      <td>37.5796</td>\n",
       "      <td>HealthCare</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>000005.XSHE</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.8600</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>2011q3</td>\n",
       "      <td>3.529328e+09</td>\n",
       "      <td>3.527048e+09</td>\n",
       "      <td>2.565851e+07</td>\n",
       "      <td>5.3480</td>\n",
       "      <td>1.365665e+07</td>\n",
       "      <td>-347.2191</td>\n",
       "      <td>Industrials</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>000006.XSHE</th>\n",
       "      <td>-0.009756</td>\n",
       "      <td>2.6766</td>\n",
       "      <td>7.619286e+06</td>\n",
       "      <td>2513811.0</td>\n",
       "      <td>0.1416</td>\n",
       "      <td>0.1667</td>\n",
       "      <td>2011q3</td>\n",
       "      <td>4.015370e+09</td>\n",
       "      <td>3.929464e+09</td>\n",
       "      <td>2.531436e+09</td>\n",
       "      <td>1.4348</td>\n",
       "      <td>2.763917e+08</td>\n",
       "      <td>1.4139</td>\n",
       "      <td>Financials</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            return   close  total_turnover      volume  \\\n",
       "date       order_book_id                                                 \n",
       "2012-01-04 000001.XSHE   -0.027582  5.1224    2.275637e+08  40894428.0   \n",
       "           000002.XSHE   -0.018742  6.0525    3.559891e+08  47432958.0   \n",
       "           000004.XSHE   -0.022250  7.9100    3.763833e+06    465469.0   \n",
       "           000005.XSHE    0.000000  3.8600    0.000000e+00         0.0   \n",
       "           000006.XSHE   -0.009756  2.6766    7.619286e+06   2513811.0   \n",
       "\n",
       "                            week   month report_quarter    market_cap  \\\n",
       "date       order_book_id                                                \n",
       "2012-01-04 000001.XSHE    0.5775  0.4331            NaN           NaN   \n",
       "           000002.XSHE    0.3711  0.4030         2011q3  8.059489e+10   \n",
       "           000004.XSHE    0.5720  0.7506         2011q3  6.642556e+08   \n",
       "           000005.XSHE    0.0000  0.0000         2011q3  3.529328e+09   \n",
       "           000006.XSHE    0.1416  0.1667         2011q3  4.015370e+09   \n",
       "\n",
       "                          a_share_market_val_2  \\\n",
       "date       order_book_id                         \n",
       "2012-01-04 000001.XSHE                     NaN   \n",
       "           000002.XSHE            7.082120e+10   \n",
       "           000004.XSHE            6.634549e+08   \n",
       "           000005.XSHE            3.527048e+09   \n",
       "           000006.XSHE            3.929464e+09   \n",
       "\n",
       "                          cash_received_from_sales_of_goods  pb_ratio  \\\n",
       "date       order_book_id                                                \n",
       "2012-01-04 000001.XSHE                                  NaN       NaN   \n",
       "           000002.XSHE                         7.516785e+10    1.5216   \n",
       "           000004.XSHE                         5.949968e+07    8.8175   \n",
       "           000005.XSHE                         2.565851e+07    5.3480   \n",
       "           000006.XSHE                         2.531436e+09    1.4348   \n",
       "\n",
       "                            net_profit  ps_ratio   sectorCode  \n",
       "date       order_book_id                                       \n",
       "2012-01-04 000001.XSHE             NaN       NaN   Financials  \n",
       "           000002.XSHE    4.106349e+09    0.8679   Financials  \n",
       "           000004.XSHE    4.500363e+06   37.5796   HealthCare  \n",
       "           000005.XSHE    1.365665e+07 -347.2191  Industrials  \n",
       "           000006.XSHE    2.763917e+08    1.4139   Financials  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "equity_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "164"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "healthcareUniverse = instrument_df.index[instrument_df.sectorCode=='HealthCare'].values\n",
    "len(healthcareUniverse)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def equity_universe_filtering(equity_df, universe):\n",
    "    universeFilter = [book_id in set(universe) for book_id in equity_df.index.get_level_values(level=1).values]\n",
    "    return equity_df[universeFilter]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>return</th>\n",
       "      <th>close</th>\n",
       "      <th>total_turnover</th>\n",
       "      <th>volume</th>\n",
       "      <th>week</th>\n",
       "      <th>month</th>\n",
       "      <th>report_quarter</th>\n",
       "      <th>market_cap</th>\n",
       "      <th>a_share_market_val_2</th>\n",
       "      <th>cash_received_from_sales_of_goods</th>\n",
       "      <th>pb_ratio</th>\n",
       "      <th>net_profit</th>\n",
       "      <th>ps_ratio</th>\n",
       "      <th>sectorCode</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>date</th>\n",
       "      <th>order_book_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">2012-01-04</th>\n",
       "      <th>000004.XSHE</th>\n",
       "      <td>-0.022250</td>\n",
       "      <td>7.9100</td>\n",
       "      <td>3763832.88</td>\n",
       "      <td>465469.0</td>\n",
       "      <td>0.5720</td>\n",
       "      <td>0.7506</td>\n",
       "      <td>2011q3</td>\n",
       "      <td>6.642556e+08</td>\n",
       "      <td>6.634549e+08</td>\n",
       "      <td>5.949968e+07</td>\n",
       "      <td>8.8175</td>\n",
       "      <td>4.500363e+06</td>\n",
       "      <td>37.5796</td>\n",
       "      <td>HealthCare</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>000028.XSHE</th>\n",
       "      <td>-0.045433</td>\n",
       "      <td>19.8422</td>\n",
       "      <td>9326924.28</td>\n",
       "      <td>450553.0</td>\n",
       "      <td>0.4201</td>\n",
       "      <td>0.2722</td>\n",
       "      <td>2011q3</td>\n",
       "      <td>5.872485e+09</td>\n",
       "      <td>4.753820e+09</td>\n",
       "      <td>1.053298e+10</td>\n",
       "      <td>4.3493</td>\n",
       "      <td>2.481834e+08</td>\n",
       "      <td>0.3414</td>\n",
       "      <td>HealthCare</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>000150.XSHE</th>\n",
       "      <td>-0.030295</td>\n",
       "      <td>3.1737</td>\n",
       "      <td>3109304.50</td>\n",
       "      <td>952600.0</td>\n",
       "      <td>0.3460</td>\n",
       "      <td>0.3610</td>\n",
       "      <td>2011q3</td>\n",
       "      <td>1.036800e+09</td>\n",
       "      <td>1.036800e+09</td>\n",
       "      <td>4.913279e+07</td>\n",
       "      <td>1.4763</td>\n",
       "      <td>3.657858e+06</td>\n",
       "      <td>7.8956</td>\n",
       "      <td>HealthCare</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>000153.XSHE</th>\n",
       "      <td>-0.028053</td>\n",
       "      <td>5.7700</td>\n",
       "      <td>9673054.49</td>\n",
       "      <td>1596020.0</td>\n",
       "      <td>0.6830</td>\n",
       "      <td>2.4594</td>\n",
       "      <td>2011q3</td>\n",
       "      <td>1.531454e+09</td>\n",
       "      <td>1.360856e+09</td>\n",
       "      <td>1.329425e+09</td>\n",
       "      <td>2.1169</td>\n",
       "      <td>1.560397e+07</td>\n",
       "      <td>0.7818</td>\n",
       "      <td>HealthCare</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>000403.XSHE</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.1625</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HealthCare</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            return    close  total_turnover     volume  \\\n",
       "date       order_book_id                                                 \n",
       "2012-01-04 000004.XSHE   -0.022250   7.9100      3763832.88   465469.0   \n",
       "           000028.XSHE   -0.045433  19.8422      9326924.28   450553.0   \n",
       "           000150.XSHE   -0.030295   3.1737      3109304.50   952600.0   \n",
       "           000153.XSHE   -0.028053   5.7700      9673054.49  1596020.0   \n",
       "           000403.XSHE    0.000000   3.1625            0.00        0.0   \n",
       "\n",
       "                            week   month report_quarter    market_cap  \\\n",
       "date       order_book_id                                                \n",
       "2012-01-04 000004.XSHE    0.5720  0.7506         2011q3  6.642556e+08   \n",
       "           000028.XSHE    0.4201  0.2722         2011q3  5.872485e+09   \n",
       "           000150.XSHE    0.3460  0.3610         2011q3  1.036800e+09   \n",
       "           000153.XSHE    0.6830  2.4594         2011q3  1.531454e+09   \n",
       "           000403.XSHE    0.0000  0.0000            NaN           NaN   \n",
       "\n",
       "                          a_share_market_val_2  \\\n",
       "date       order_book_id                         \n",
       "2012-01-04 000004.XSHE            6.634549e+08   \n",
       "           000028.XSHE            4.753820e+09   \n",
       "           000150.XSHE            1.036800e+09   \n",
       "           000153.XSHE            1.360856e+09   \n",
       "           000403.XSHE                     NaN   \n",
       "\n",
       "                          cash_received_from_sales_of_goods  pb_ratio  \\\n",
       "date       order_book_id                                                \n",
       "2012-01-04 000004.XSHE                         5.949968e+07    8.8175   \n",
       "           000028.XSHE                         1.053298e+10    4.3493   \n",
       "           000150.XSHE                         4.913279e+07    1.4763   \n",
       "           000153.XSHE                         1.329425e+09    2.1169   \n",
       "           000403.XSHE                                  NaN       NaN   \n",
       "\n",
       "                            net_profit  ps_ratio  sectorCode  \n",
       "date       order_book_id                                      \n",
       "2012-01-04 000004.XSHE    4.500363e+06   37.5796  HealthCare  \n",
       "           000028.XSHE    2.481834e+08    0.3414  HealthCare  \n",
       "           000150.XSHE    3.657858e+06    7.8956  HealthCare  \n",
       "           000153.XSHE    1.560397e+07    0.7818  HealthCare  \n",
       "           000403.XSHE             NaN       NaN  HealthCare  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "healthcare_equity_df = equity_universe_filtering(equity_df, healthcareUniverse)\n",
    "healthcare_equity_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "universe ratio: 6.210331877919959%\n"
     ]
    }
   ],
   "source": [
    "print(\"universe ratio: {}%\".format(len(healthcare_equity_df)/len(equity_df)*100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### benchmark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "benchmark_df = pd.read_csv(\"cn_SH_healthcare_index_2012_2018.csv\",names=['date','value'])\n",
    "benchmark_df = benchmark_df.set_index('date',drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>value</th>\n",
       "      <th>return</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>date</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2012-01-04</th>\n",
       "      <td>2891.462</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2012-01-05</th>\n",
       "      <td>2766.955</td>\n",
       "      <td>0.044015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2012-01-06</th>\n",
       "      <td>2744.793</td>\n",
       "      <td>0.008042</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2012-01-09</th>\n",
       "      <td>2833.219</td>\n",
       "      <td>-0.031708</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2012-01-10</th>\n",
       "      <td>2929.594</td>\n",
       "      <td>-0.033450</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               value    return\n",
       "date                          \n",
       "2012-01-04  2891.462  0.000000\n",
       "2012-01-05  2766.955  0.044015\n",
       "2012-01-06  2744.793  0.008042\n",
       "2012-01-09  2833.219 -0.031708\n",
       "2012-01-10  2929.594 -0.033450"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "benchmark_df['return'] = np.log(benchmark_df.shift(1)/benchmark_df).fillna(0)\n",
    "benchmark_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Factor Returns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def equity_factor_return(equity_df, factorColumn, nAllocations, longTop=True):\n",
    "    equity_copy = equity_df.copy()\n",
    "#     equity_copy[\"{}_rank\".format(factorColumn)] = equity_copy.groupby(level='date')[factorColumn].rank()\n",
    "#     equity_copy[equity_copy.groupby(level='date')[factorColumn].nlargest(nAllocations).index][\"biggest_{}_{}\".format(nAllocations,factorColumn)]=True\n",
    "    largest = equity_copy[factorColumn].groupby(level='date').nlargest(nAllocations).reset_index(level=0,drop=True)\n",
    "    smallest = equity_copy[factorColumn].groupby(level='date').nsmallest(nAllocations).reset_index(level=0,drop=True)\n",
    "    r_largest = equity_copy.loc[largest.index,'return'].groupby(level='date').mean()\n",
    "    r_smallest = equity_copy.loc[smallest.index,'return'].groupby(level='date').mean()\n",
    "    LMS = r_largest - r_smallest\n",
    "    if(longTop):\n",
    "        return LMS\n",
    "    else:\n",
    "        return -LMS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "date\n",
       "2012-01-04    0.005983\n",
       "2012-01-05   -0.009098\n",
       "2012-01-06   -0.004155\n",
       "2012-01-09    0.014615\n",
       "2012-01-10    0.006728\n",
       "Name: return, dtype: float64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SMB = equity_factor_return(healthcare_equity_df, 'market_cap', 20,longTop=False)\n",
    "SMB.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "date\n",
       "2012-01-04    0.005302\n",
       "2012-01-05   -0.007223\n",
       "2012-01-06    0.006031\n",
       "2012-01-09   -0.002597\n",
       "2012-01-10   -0.010780\n",
       "Name: return, dtype: float64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "HML = equity_factor_return(healthcare_equity_df, 'pb_ratio', 20,longTop=True)\n",
    "HML.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import itertools\n",
    "import statsmodels.api as sm\n",
    "from statsmodels import regression,stats\n",
    "import scipy\n",
    "\n",
    "data = healthcare_equity_df[['return']] # dataframe\n",
    "data = data.set_index(healthcare_equity_df.index) # elimilate redundant index (whole universe)\n",
    "asset_list_sizes = [group[1].size for group in data.groupby(level=0)]\n",
    "\n",
    "# Spreading the factor portfolio data across all assets for each day\n",
    "SMB_column = [[SMB.loc[group[0]]] * size for group, size \\\n",
    "              in zip(data.groupby(level=0), asset_list_sizes)]\n",
    "data['SMB'] = list(itertools.chain(*SMB_column))\n",
    "\n",
    "HML_column = [[HML.loc[group[0]]] * size for group, size \\\n",
    "              in zip(data.groupby(level=0), asset_list_sizes)]\n",
    "data['HML'] = list(itertools.chain(*HML_column))\n",
    "data = sm.add_constant(data.dropna())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>const</th>\n",
       "      <th>return</th>\n",
       "      <th>SMB</th>\n",
       "      <th>HML</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>date</th>\n",
       "      <th>order_book_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">2012-01-04</th>\n",
       "      <th>000004.XSHE</th>\n",
       "      <td>1.0</td>\n",
       "      <td>-0.022250</td>\n",
       "      <td>0.005983</td>\n",
       "      <td>0.005302</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>000028.XSHE</th>\n",
       "      <td>1.0</td>\n",
       "      <td>-0.045433</td>\n",
       "      <td>0.005983</td>\n",
       "      <td>0.005302</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>000150.XSHE</th>\n",
       "      <td>1.0</td>\n",
       "      <td>-0.030295</td>\n",
       "      <td>0.005983</td>\n",
       "      <td>0.005302</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>000153.XSHE</th>\n",
       "      <td>1.0</td>\n",
       "      <td>-0.028053</td>\n",
       "      <td>0.005983</td>\n",
       "      <td>0.005302</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>000403.XSHE</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.005983</td>\n",
       "      <td>0.005302</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          const    return       SMB       HML\n",
       "date       order_book_id                                     \n",
       "2012-01-04 000004.XSHE      1.0 -0.022250  0.005983  0.005302\n",
       "           000028.XSHE      1.0 -0.045433  0.005983  0.005302\n",
       "           000150.XSHE      1.0 -0.030295  0.005983  0.005302\n",
       "           000153.XSHE      1.0 -0.028053  0.005983  0.005302\n",
       "           000403.XSHE      1.0  0.000000  0.005983  0.005302"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Factor Exposures ($\\beta$)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "assets = data.index.levels[1].unique()\n",
    "Y = [data.xs(asset,level=1)['return'] for asset in assets]\n",
    "X = [data.xs(asset,level=1)[['SMB','HML','const']] for asset in assets]\n",
    "reg_results = [regression.linear_model.OLS(y,x).fit().params for y,x in zip(Y,X) if not(x.empty or y.empty)]\n",
    "indices = [asset for y, x, asset in zip(Y, X, assets) if not(x.empty or y.empty)]\n",
    "betas = pd.DataFrame(reg_results, index=indices)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SMB</th>\n",
       "      <th>HML</th>\n",
       "      <th>const</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>000004.XSHE</th>\n",
       "      <td>0.883906</td>\n",
       "      <td>0.048757</td>\n",
       "      <td>0.002002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>000028.XSHE</th>\n",
       "      <td>-0.003029</td>\n",
       "      <td>-0.064295</td>\n",
       "      <td>0.001073</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>000150.XSHE</th>\n",
       "      <td>0.354122</td>\n",
       "      <td>0.066071</td>\n",
       "      <td>0.002031</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>000153.XSHE</th>\n",
       "      <td>0.620706</td>\n",
       "      <td>-0.082229</td>\n",
       "      <td>0.001405</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>000403.XSHE</th>\n",
       "      <td>2.032192</td>\n",
       "      <td>11.457418</td>\n",
       "      <td>-0.017412</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  SMB        HML     const\n",
       "000004.XSHE  0.883906   0.048757  0.002002\n",
       "000028.XSHE -0.003029  -0.064295  0.001073\n",
       "000150.XSHE  0.354122   0.066071  0.002031\n",
       "000153.XSHE  0.620706  -0.082229  0.001405\n",
       "000403.XSHE  2.032192  11.457418 -0.017412"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "betas.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Factor Premium"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"simpletable\">\n",
       "<caption>OLS Regression Results</caption>\n",
       "<tr>\n",
       "  <th>Dep. Variable:</th>         <td>return</td>      <th>  R-squared:         </th> <td>   0.398</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Model:</th>                   <td>OLS</td>       <th>  Adj. R-squared:    </th> <td>   0.391</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Method:</th>             <td>Least Squares</td>  <th>  F-statistic:       </th> <td>   53.26</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Date:</th>             <td>Sat, 05 May 2018</td> <th>  Prob (F-statistic):</th> <td>1.77e-18</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Time:</th>                 <td>21:03:25</td>     <th>  Log-Likelihood:    </th> <td>  1012.1</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>No. Observations:</th>      <td>   164</td>      <th>  AIC:               </th> <td>  -2018.</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Df Residuals:</th>          <td>   161</td>      <th>  BIC:               </th> <td>  -2009.</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Df Model:</th>              <td>     2</td>      <th>                     </th>     <td> </td>   \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Covariance Type:</th>      <td>nonrobust</td>    <th>                     </th>     <td> </td>   \n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "    <td></td>       <th>coef</th>     <th>std err</th>      <th>t</th>      <th>P>|t|</th>  <th>[0.025</th>    <th>0.975]</th>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>const</th> <td>    0.0017</td> <td> 6.72e-05</td> <td>   24.956</td> <td> 0.000</td> <td>    0.002</td> <td>    0.002</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>SMB</th>   <td>-7.597e-05</td> <td>    0.000</td> <td>   -0.599</td> <td> 0.550</td> <td>   -0.000</td> <td>    0.000</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>HML</th>   <td>    0.0005</td> <td> 4.81e-05</td> <td>    9.695</td> <td> 0.000</td> <td>    0.000</td> <td>    0.001</td>\n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "  <th>Omnibus:</th>       <td>39.154</td> <th>  Durbin-Watson:     </th> <td>   1.906</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Prob(Omnibus):</th> <td> 0.000</td> <th>  Jarque-Bera (JB):  </th> <td>  78.545</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Skew:</th>          <td> 1.087</td> <th>  Prob(JB):          </th> <td>8.80e-18</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Kurtosis:</th>      <td> 5.601</td> <th>  Cond. No.          </th> <td>    3.92</td>\n",
       "</tr>\n",
       "</table>"
      ],
      "text/plain": [
       "<class 'statsmodels.iolib.summary.Summary'>\n",
       "\"\"\"\n",
       "                            OLS Regression Results                            \n",
       "==============================================================================\n",
       "Dep. Variable:                 return   R-squared:                       0.398\n",
       "Model:                            OLS   Adj. R-squared:                  0.391\n",
       "Method:                 Least Squares   F-statistic:                     53.26\n",
       "Date:                Sat, 05 May 2018   Prob (F-statistic):           1.77e-18\n",
       "Time:                        21:03:25   Log-Likelihood:                 1012.1\n",
       "No. Observations:                 164   AIC:                            -2018.\n",
       "Df Residuals:                     161   BIC:                            -2009.\n",
       "Df Model:                           2                                         \n",
       "Covariance Type:            nonrobust                                         \n",
       "==============================================================================\n",
       "                 coef    std err          t      P>|t|      [0.025      0.975]\n",
       "------------------------------------------------------------------------------\n",
       "const          0.0017   6.72e-05     24.956      0.000       0.002       0.002\n",
       "SMB        -7.597e-05      0.000     -0.599      0.550      -0.000       0.000\n",
       "HML            0.0005   4.81e-05      9.695      0.000       0.000       0.001\n",
       "==============================================================================\n",
       "Omnibus:                       39.154   Durbin-Watson:                   1.906\n",
       "Prob(Omnibus):                  0.000   Jarque-Bera (JB):               78.545\n",
       "Skew:                           1.087   Prob(JB):                     8.80e-18\n",
       "Kurtosis:                       5.601   Cond. No.                         3.92\n",
       "==============================================================================\n",
       "\n",
       "Warnings:\n",
       "[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.\n",
       "\"\"\""
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "betas = sm.add_constant(betas.drop('const', axis=1))\n",
    "\n",
    "R = data['return'].mean(axis=0, level=1)\n",
    "\n",
    "# Second regression step: estimating the risk premia\n",
    "risk_free_rate = benchmark_df['return'].mean()\n",
    "\n",
    "final_results = regression.linear_model.OLS(R - risk_free_rate, betas).fit()\n",
    "\n",
    "final_results.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fama-Macbeth Test Conclusion: \n",
    "although our individual factors are significant, we have a very low  $R^2$ . What this may suggest is that there is a real link between our factors and the returns of our assets, but that there still remains a lot of unexplained noise!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}