BasicRNN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import pandas as pd\n",
    "import urllib\n",
    "import tarfile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "url = 'https://github.com/le-scientifique/torchDatasets/raw/master/dbpedia_csv.tar.gz'\n",
    "\n",
    "def getunzipped(theurl):\n",
    "    name = './dbpedia_csv.tar.gz'\n",
    "    try:\n",
    "        name, hdrs = urllib.request.urlretrieve(theurl, name)\n",
    "    except IOError as e:\n",
    "        print(\"Can't retrieve %r to %r: %s\" % (theurl, thedir, e))\n",
    "        return\n",
    "    try:\n",
    "        z = tarfile.open(name, \"r:gz\")\n",
    "        z.extractall()\n",
    "        z.close()\n",
    "    except tarfile.error as e:\n",
    "        print(\"Bad zipfile (from %r): %s\" % (theurl, e))\n",
    "        return\n",
    "\n",
    "    print(\"Data Downloaded and unzipped!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data Downloaded and unzipped!\n"
     ]
    }
   ],
   "source": [
    "getunzipped(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>class</th>\n",
       "      <th>title</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>E. D. Abbott Ltd</td>\n",
       "      <td>Abbott of Farnham E D Abbott Limited was a Br...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Schwan-Stabilo</td>\n",
       "      <td>Schwan-STABILO is a German maker of pens for ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>Q-workshop</td>\n",
       "      <td>Q-workshop is a Polish company located in Poz...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>Marvell Software Solutions Israel</td>\n",
       "      <td>Marvell Software Solutions Israel known as RA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>Bergan Mercy Medical Center</td>\n",
       "      <td>Bergan Mercy Medical Center is a hospital loc...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   class                              title  \\\n",
       "0      1                   E. D. Abbott Ltd   \n",
       "1      1                     Schwan-Stabilo   \n",
       "2      1                         Q-workshop   \n",
       "3      1  Marvell Software Solutions Israel   \n",
       "4      1        Bergan Mercy Medical Center   \n",
       "\n",
       "                                                text  \n",
       "0   Abbott of Farnham E D Abbott Limited was a Br...  \n",
       "1   Schwan-STABILO is a German maker of pens for ...  \n",
       "2   Q-workshop is a Polish company located in Poz...  \n",
       "3   Marvell Software Solutions Israel known as RA...  \n",
       "4   Bergan Mercy Medical Center is a hospital loc...  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv('dbpedia_csv/train.csv',header=None,names=['class','title','text'])\n",
    "train.sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>class</th>\n",
       "      <th>title</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>TY KU</td>\n",
       "      <td>TY KU /taɪkuː/ is an American alcoholic bever...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Odd Lot Entertainment</td>\n",
       "      <td>OddLot Entertainment founded in 2001 by longt...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>Henkel</td>\n",
       "      <td>Henkel AG &amp; Company KGaA operates worldwide w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>GOAT Store</td>\n",
       "      <td>The GOAT Store (Games Of All Type Store) LLC ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>RagWing Aircraft Designs</td>\n",
       "      <td>RagWing Aircraft Designs (also called the Rag...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   class                     title  \\\n",
       "0      1                     TY KU   \n",
       "1      1     Odd Lot Entertainment   \n",
       "2      1                    Henkel   \n",
       "3      1                GOAT Store   \n",
       "4      1  RagWing Aircraft Designs   \n",
       "\n",
       "                                                text  \n",
       "0   TY KU /taɪkuː/ is an American alcoholic bever...  \n",
       "1   OddLot Entertainment founded in 2001 by longt...  \n",
       "2   Henkel AG & Company KGaA operates worldwide w...  \n",
       "3   The GOAT Store (Games Of All Type Store) LLC ...  \n",
       "4   RagWing Aircraft Designs (also called the Rag...  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test = pd.read_csv('dbpedia_csv/test.csv',header=None,names=['class','title','text'])\n",
    "test.sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X_train = train['text'].get_values()\n",
    "X_test = test['text'].get_values()\n",
    "y_train = train['class'].get_values()\n",
    "y_test= test['class'].get_values()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "max_doc_length = 50\n",
    "\n",
    "train_lengths = train['text'].apply(lambda x: min(max_doc_length, len(x.split(' '))))\n",
    "test_lengths = test['text'].apply(lambda x: min(max_doc_length, len(x.split(' '))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train_lengths = train_lengths.values\n",
    "test_lengths = test_lengths.values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### VocabularyProcessor \n",
    "Has an sklearn type interface - instantiate, call .fit() and .transform()\n",
    "\n",
    "Creates a mapping between words and ID's.\n",
    "\n",
    "Fit creates the mapping word-> ID, and transform replaces the words with relevant IDS.\n",
    "\n",
    "We can then use tf.nn.embedding_lookup(...) to look up the vector representations that correspond to each ID."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "preprocessor = tf.contrib.learn.preprocessing.VocabularyProcessor(max_doc_length,min_frequency=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X_train_transformed = np.array(list(preprocessor.fit_transform(X_train)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As you can see, we have replaced each word with a numerical ID corresponding to a word in our vocabulary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The word 'the' corresponds to ID: 1\n",
      "ID 25 corresponds to: known\n"
     ]
    }
   ],
   "source": [
    "print(\"The word 'the' corresponds to ID:\",preprocessor.vocabulary_.get('the')) #Get the ID for a word\n",
    "print(\"ID 25 corresponds to:\",preprocessor.vocabulary_.reverse(25)) #Get the word that corresponds to an ID"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vocab_length = len(preprocessor.vocabulary_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X_test_transformed = np.array(list(preprocessor.transform(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[     0,  33095,      0,      4,     15,     34,  13638,  12026,\n",
       "            58,     32,   2092,      2,  34220,      6,    154,  19432,\n",
       "             8,      0,     58,      7,    108,      2,    254,      6,\n",
       "             4,    672,      2,     46,    111,    133,     46,    111,\n",
       "          1789,     69,      2,     46,    111,      0,  33095,   1012,\n",
       "         12873,     40,    218,      2,    342,    137,      5,   2554,\n",
       "          2603,     17],\n",
       "       [  7874,  13438,   1130,    108,      2,    346,      9,   5924,\n",
       "          3640,  26563,  53628,      6,   9446,   4389, 115134,      8,\n",
       "         10402, 102537,      4,      5,     22,    454,      6,  11213,\n",
       "            58,     69,      2,  14655,    133,    144,   7874,  13438,\n",
       "           138,      1,     22,    579,      3,   9822,   1198,  38006,\n",
       "         13657,     82,  27928,   2842,     56,     22,    579,      3,\n",
       "           155,     82]], dtype=int64)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test_transformed[:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Abbott of Farnham E D Abbott Limited was a British coachbuilding business based in Farnham Surrey trading under that name from 1929 A major part of their output was under sub-contract to motor vehicle manufacturers Their business closed in 1972 <UNK> <UNK> <UNK> <UNK> <UNK> <UNK> <UNK> <UNK> <UNK> <UNK>'"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "next(preprocessor.reverse(X_train_transformed[0,None].tolist()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Now our data is prepared we can start modelling..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The basics have been implemented, but you need to implement the RNN functionality, and define the loss function.\n",
    "\n",
    "As a starting point, you several comments have been provided in the relevant name scopes."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is a stylised architecture that we want to use. In reality we will do the lookups all in one go when we feed IDs to the graph, rather than as we feed each step. Each node in the output step corresponds to a particular class, so for 14 classes we have 14 nodes etc."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"src/rnnclassifier.png\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For this exercise we will use the BasicRNNCell in tensorflow. This is shown below."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"src/basicrnncell.png\", height=\"400\", width=\"400\">"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "graph = tf.Graph()\n",
    "embedding_size=100\n",
    "rnn_cell_size = 100\n",
    "n_classes=15\n",
    "learning_rate = 0.005\n",
    "\n",
    "with graph.as_default(), tf.device('/cpu:0'):\n",
    "    with tf.name_scope('placeholders'):\n",
    "        tf_X = tf.placeholder(shape=[None, max_doc_length], dtype=tf.int32) #This is a matrix of word IDs\n",
    "        tf_y = tf.placeholder(shape=[None], dtype=tf.int64) #This is a list of class labels e.g. 1,5,3 etc.\n",
    "        tf_lengths = tf.placeholder(shape=[None], dtype=tf.int32) #This is the lengths \n",
    "                                                                    #of each document in sample\n",
    "        \n",
    "    with tf.name_scope('variables'):\n",
    "        embedding_matrix = tf.Variable(tf.random_uniform([vocab_length,embedding_size],-1.0,1.0)) \n",
    "        \n",
    "        W = tf.Variable(tf.truncated_normal(shape=[rnn_cell_size, n_classes ], dtype=tf.float32))\n",
    "        b = tf.Variable(tf.random_uniform([n_classes], -1.0,1.0))\n",
    "        tf_y_one_hot = tf.one_hot(tf_y,n_classes, dtype=tf.int64)\n",
    "    \n",
    "    with tf.name_scope('embbeding_lookup'):\n",
    "        #You need to look up the word ID and get replace with the associated word vector.\n",
    "        #Verify the size is (?, max_doc_length, embedding_size)\n",
    "        \n",
    "        \n",
    "    with tf.name_scope('run_rnn'):\n",
    "        #You need to define a cell type - use BasicRNNCell\n",
    "        #Also need to feed data into the RNN and get the final_state - use dynamic_rnn\n",
    "        \n",
    "        \n",
    "    with tf.name_scope('output_layer'):\n",
    "        #Create a variable logits which is final_state * W + b\n",
    "        #Pass this through a softmax to create prediction probabilities.\n",
    "        \n",
    "        \n",
    "    with tf.name_scope('loss'):\n",
    "        #Add a loss and an optimizer\n",
    "        \n",
    "        \n",
    "    with tf.name_scope('validation'):\n",
    "        correct_prediction = tf.equal(tf_y, predictions)\n",
    "        accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "        \n",
    "    with tf.name_scope('init'):\n",
    "        init_op = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|                                                                                          | 0/251 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "average loss: 0.322727813721\n",
      "accuracy: 0.0782813\n",
      "---------------------------------------- step 0 ----------------------------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 10%|████████                                                                         | 25/251 [00:12<01:19,  2.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "average loss: 5.31750082016\n",
      "accuracy: 0.174023\n",
      "---------------------------------------- step 25 ----------------------------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 20%|████████████████▏                                                                | 50/251 [00:24<01:08,  2.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "average loss: 3.42656703949\n",
      "accuracy: 0.233594\n",
      "---------------------------------------- step 50 ----------------------------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 30%|████████████████████████▏                                                        | 75/251 [00:36<00:59,  2.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "average loss: 2.96837429047\n",
      "accuracy: 0.242344\n",
      "---------------------------------------- step 75 ----------------------------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 40%|███████████████████████████████▊                                                | 100/251 [00:49<00:51,  2.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "average loss: 2.66550458908\n",
      "accuracy: 0.274883\n",
      "---------------------------------------- step 100 ----------------------------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50%|███████████████████████████████████████▊                                        | 125/251 [01:01<00:42,  2.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "average loss: 2.60701519966\n",
      "accuracy: 0.281016\n",
      "---------------------------------------- step 125 ----------------------------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 60%|███████████████████████████████████████████████▊                                | 150/251 [01:14<00:34,  2.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "average loss: 2.6524164772\n",
      "accuracy: 0.284297\n",
      "---------------------------------------- step 150 ----------------------------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 70%|███████████████████████████████████████████████████████▊                        | 175/251 [01:26<00:25,  2.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "average loss: 2.48520344734\n",
      "accuracy: 0.300273\n",
      "---------------------------------------- step 175 ----------------------------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|███████████████████████████████████████████████████████████████▋                | 200/251 [01:39<00:17,  2.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "average loss: 2.47928232193\n",
      "accuracy: 0.312773\n",
      "---------------------------------------- step 200 ----------------------------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 90%|███████████████████████████████████████████████████████████████████████▋        | 225/251 [01:52<00:09,  2.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "average loss: 2.33688541889\n",
      "accuracy: 0.291953\n",
      "---------------------------------------- step 225 ----------------------------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████▋| 250/251 [02:04<00:00,  3.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "average loss: 2.36772595406\n",
      "accuracy: 0.341836\n",
      "---------------------------------------- step 250 ----------------------------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 251/251 [02:08<00:00,  1.49s/it]\n"
     ]
    }
   ],
   "source": [
    "batch_size = 128\n",
    "print_step = 50\n",
    "from tqdm import tqdm\n",
    "\n",
    "with tf.Session(graph=graph) as session:\n",
    "    number_of_steps = 251\n",
    "    train_indicies = np.arange(X_train_transformed.shape[0])\n",
    "    test_indicies = np.arange(X_test_transformed.shape[0])\n",
    "    \n",
    "    session.run(init_op)\n",
    "    average_loss = 0\n",
    "    \n",
    "    for step in tqdm(range(number_of_steps)):\n",
    "        ind =  np.random.choice(np.arange(X_train_transformed.shape[0]), batch_size)\n",
    "        feed_dict = {tf_X:X_train_transformed[ind], \n",
    "                     tf_y:y_train[ind], \n",
    "                     tf_lengths: train_lengths[ind] }\n",
    "        l,_ = session.run([loss,optimizer ], feed_dict=feed_dict)\n",
    "        average_loss+=l\n",
    "        \n",
    "        if step%print_step == 0:\n",
    "            print (\"average loss:\",average_loss/print_step)\n",
    "            ind2 =  np.random.choice(np.arange(X_test_transformed.shape[0]), batch_size*200)\n",
    "            test_dict = {tf_X:X_test_transformed[ind2], \n",
    "                     tf_y:y_test[ind2], \n",
    "                     tf_lengths: test_lengths[ind2] }\n",
    "            print(\"accuracy:\",accuracy.eval(feed_dict=test_dict))\n",
    "            #print(predictions.eval(feed_dict=test_dict))\n",
    "            #print(y_test[ind2])\n",
    "            print('-'*40, \"step \"+str(step), '-'*40)\n",
    "            \n",
    "            average_loss=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda env:p3]",
   "language": "python",
   "name": "conda-env-p3-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}