process_dataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_path = '/home/ayush/Downloads/train-v1.1.json'\n",
    "data = pd.read_json(data_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(442, 2)\n"
     ]
    }
   ],
   "source": [
    "print data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def preprocess_dataset(data):\n",
    "    processed_data = []\n",
    "    for i in data['data']:\n",
    "        data_i = []\n",
    "        for d in i['paragraphs']:\n",
    "            qas = []\n",
    "            for q in d['qas']:\n",
    "                for text in q['answers']:\n",
    "                    qas.append([q['question'], text['text']])\n",
    "            data_i.append([d['context'], qas])\n",
    "        processed_data.append(data_i)\n",
    "    return processed_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "442\n"
     ]
    }
   ],
   "source": [
    "processed_data = preprocess_dataset(data)\n",
    "print len(processed_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[u'The Continental Army was created on 14 June 1775 by the Continental Congress as a unified army for the colonies to fight Great Britain, with George Washington appointed as its commander. The army was initially led by men who had served in the British Army or colonial militias and who brought much of British military heritage with them. As the Revolutionary War progressed, French aid, resources, and military thinking influenced the new army. A number of European soldiers came on their own to help, such as Friedrich Wilhelm von Steuben, who taught the army Prussian tactics and organizational skills.', [[u'When was the Continental Army created?', u'14 June 1775'], [u'Who was the commander of the Continental Army?', u'George Washington'], [u'From what country did Friedrich Wilhelm von Steuben come from?', u'Europe'], [u'What did Friedrich Wilhelm von Steuben teach the Continental Army?', u'Prussian tactics and organizational skills']]]\n"
     ]
    }
   ],
   "source": [
    "print (processed_data[162][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}