PredictReadAndCategory.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gzip\n",
    "from collections import defaultdict\n",
    "import math\n",
    "import scipy.optimize\n",
    "from sklearn import svm\n",
    "import numpy\n",
    "import string\n",
    "import random\n",
    "import string\n",
    "from sklearn import linear_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def readGz(path):\n",
    "    for l in gzip.open(path, 'rt'):\n",
    "        yield eval(l)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def readCSV(path):\n",
    "    f = gzip.open(path, 'rt')\n",
    "    f.readline()\n",
    "    for l in f:\n",
    "        u,b,r = l.strip().split(',')\n",
    "        r = int(r)\n",
    "        yield u,b,r"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3 style=\"font-size: 30px; font-weight: bold;\"> Read Prediction </h3>\n",
    "<p style=\"font-size: 15px;\"> The following code uses Jaccard similarity and optimized thresholds to predict for each (user,book) pair in pairs_Read.csv if the user read that book (1) or not (0). The fraction of correct predictions was used to determine the accuracy of the model.</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "allRatings = []\n",
    "for l in readCSV(\"train_Interactions.csv.gz\"):\n",
    "    allRatings.append(l)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "ratingsTrain = allRatings[:190000]\n",
    "ratingsValid = allRatings[190000:]\n",
    "ratingsPerUser = defaultdict(list)\n",
    "ratingsPerBook = defaultdict(list)\n",
    "for u,b,r in ratingsTrain:\n",
    "    ratingsPerUser[u].append((b,r))\n",
    "    ratingsPerBook[b].append((u,r))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create new validation set that also contains negative ratings (one randomly chosed per positive rating)\n",
    "allBooks = []\n",
    "for user,book,rating in allRatings:\n",
    "    if book not in allBooks:\n",
    "        allBooks.append(book)\n",
    "\n",
    "negRatings = []\n",
    "for user,book,rating in ratingsValid:\n",
    "    userRatings = []\n",
    "    for b,r in ratingsPerUser[user]:\n",
    "        userRatings.append(b)\n",
    "    randomBook = random.choice(allBooks)\n",
    "    while randomBook in userRatings:\n",
    "        randomBook = random.choice(allBooks)\n",
    "    negRatings.append((user, randomBook, 0))\n",
    "\n",
    "newValid = ratingsValid + negRatings  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Jaccard(s1, s2):\n",
    "    numer = len(s1 and s2)\n",
    "    denom = len(s1 or s2)\n",
    "    if denom > 0:\n",
    "        return numer/denom\n",
    "    return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "return2 = set()\n",
    "\n",
    "for u,b,r in ratingsValid:\n",
    "    similarities = []\n",
    "    users = []\n",
    "    for user,rating in ratingsPerBook[b]:\n",
    "        users.append(user)\n",
    "    books = []\n",
    "    for book,rating in ratingsPerUser[u]:\n",
    "        books.append(book)\n",
    "    for b1 in books:\n",
    "        users1 = []\n",
    "        for user1,rating1 in ratingsPerBook[b1]:\n",
    "            users1.append(user1)\n",
    "        similarities.append(Jaccard(users1,users))\n",
    "    if len(similarities) > 0:\n",
    "        maxSim = max(similarities)\n",
    "        if maxSim > 5 or len(ratingsPerBook[b]) > 35:\n",
    "            return2.add(b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "c = 0\n",
    "for user,book,rating in newValid:\n",
    "    if book in return2:\n",
    "        c += 1\n",
    "\n",
    "acc = c/len(newValid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# write predictions to file\n",
    "predictions = open(\"predictions_Read.csv\", 'w')\n",
    "for l in open(\"pairs_Read.csv\"):\n",
    "    if l.startswith(\"userID\"):\n",
    "        predictions.write(l)\n",
    "        continue\n",
    "    u,b = l.strip().split(',')\n",
    "    if b in return2:\n",
    "        predictions.write(u + ',' + b + \",1\\n\")\n",
    "    else:\n",
    "        predictions.write(u + ',' + b + \",0\\n\")\n",
    "\n",
    "predictions.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3 style=\"font-size: 30px; font-weight: bold;\"> Category Prediction </h3>\n",
    "<p style=\"font-size: 15px;\"> The following code uses Bag-of-Words feature matrix for a Linear Regression model to predict the genre of books from user reviews.</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = []\n",
    "\n",
    "for d in readGz(\"train_Category.json.gz\"):\n",
    "    data.append(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'user_id': 'u75242413',\n",
       " 'review_id': 'r45843137',\n",
       " 'rating': 4,\n",
       " 'review_text': \"a clever book with a deeply troubling premise and an intriguing protagonist. Thompson's clean, sparse prose style kept each page feeling light even as some rather heavy existential questions dropped upon them. I enjoyed it. \\n and that cover design is boom-pow gorgeous.\",\n",
       " 'n_votes': 1,\n",
       " 'genre': 'mystery_thriller_crime',\n",
       " 'genreID': 3}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "wordCount = defaultdict(int)\n",
    "punctuation = set(string.punctuation)\n",
    "for d in data:\n",
    "    r = ''.join([c for c in d['review_text'].lower() if not c in punctuation])\n",
    "    for w in r.split():\n",
    "        wordCount[w] += 1\n",
    "\n",
    "counts = [(wordCount[w],w) for w in wordCount]\n",
    "counts.sort()\n",
    "counts.reverse()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "commonWords = [x[1] for x in counts[:4000]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "wordId = dict(zip(commonWords, range(len(commonWords))))\n",
    "wordSet = set(commonWords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def feature(d):\n",
    "    feat = [0]*len(commonWords)\n",
    "    r = ''.join([c for c in d['review_text'].lower() if not c in punctuation])\n",
    "    for w in r.split():\n",
    "        if w in commonWords:\n",
    "            feat[wordId[w]] += 1\n",
    "    feat.append(1)\n",
    "    return feat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = [feature(d) for d in data]\n",
    "y = [d['genreID'] for d in data]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "Xtrain = X[:9*len(X)//10]\n",
    "ytrain = y[:9*len(y)//10]\n",
    "Xvalid = X[9*len(X)//10:]\n",
    "yvalid = y[9*len(y)//10:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def accuracy(predictions, y):\n",
    "    TP = sum(numpy.logical_and(predictions, y))\n",
    "    FP = sum(numpy.logical_and(predictions, numpy.logical_not(y)))\n",
    "    TN = sum(numpy.logical_and(numpy.logical_not(predictions), numpy.logical_not(y)))\n",
    "    FN = sum(numpy.logical_and(numpy.logical_not(predictions), y))\n",
    "    return (TP + TN) / (TP + FP + TN + FN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>LogisticRegression(C=0.01)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">LogisticRegression</label><div class=\"sk-toggleable__content\"><pre>LogisticRegression(C=0.01)</pre></div></div></div></div></div>"
      ],
      "text/plain": [
       "LogisticRegression(C=0.01)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mod = linear_model.LogisticRegression(C=0.01)\n",
    "mod.fit(Xtrain,ytrain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataTest = []\n",
    "for d in readGz(\"test_Category.json.gz\"):\n",
    "    dataTest.append(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "Xtest = [feature(d) for d in dataTest] \n",
    "preds = mod.predict(Xtest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = open(\"predictions_Category.csv\", 'w')\n",
    "pos = 0\n",
    "\n",
    "for l in open(\"pairs_Category.csv\"):\n",
    "    if l.startswith(\"userID\"):\n",
    "        predictions.write(l)\n",
    "        continue\n",
    "    u,b = l.strip().split(',')\n",
    "    predictions.write(u + ',' + b + \",\" + str(preds[pos]) + \"\\n\")\n",
    "    pos += 1\n",
    "            \n",
    "predictions.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = open(\"README.md\", 'w')\n",
    "f.write(\"Read Prediction: To predict whether or not a book was read, I iterated through the validation set to collect the following information. For the book in the validation set, I found the users that had read that book in the training set and for the user, I found the books that user read in the training set. Now, for each of these books from the training set that the user had read, I determined the users that had read them. Then I found the Jaccard similarities between the users in the training set that had read the book in the validation set and the users that read each book read by the user in the validation set. If the maximum similarity was greater than 5 or the number of times the book had been read was greater than 35 (I tried many thresholds, and this was the most optimal), I added the book to a set called return2. To make the predictions, I checked if the book was in the set return2. If the book was in the return2 set, I predicted 1 (meaning that the book is predicted to have been read) and if the book was not in the set, I predicted 0 (meaning that the book is predicted to be unread).\")\n",
    "f.write(\"\\n\")\n",
    "f.write(\"\\n\")\n",
    "f.write(\"Category Prediction: To predict the category of books, I created a feature function that creates a feature vector for a data point in which every time of the 4000 most common words is seen in the review's text, 1 is added to the location in the feature vector that corresponds to that common word. Also, a 1 is added to the end of the feature vector for the constant term. I created a matrix of all the feature vectors for all data points in the dataset and did a linear regression model on the matrix and the genreIDs for each data point. I was able to then create a feature matrix for the test set and use the model to predict the genres of the test set. For the model, I tried many different values for C and found that 0.01 had the best accuracy.\")\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}