preprocess.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import nltk\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import spacy\n",
    "import regex\n",
    "model_path = '/home/ayush/spaCy/models/en_core_web_sm-2.0.0/en_core_web_sm/en_core_web_sm-2.0.0'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "nlpSpacy = spacy.load(model_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "1. Tokenization\n",
    "2. Pos Tagging\n",
    "3. NER (Named Entity Recognition) : \n",
    "    a. Split the sentences.\n",
    "    b. For the sentence use the ne_chunk using its tags\n",
    "    c. return the pos_tags, named_entities\n",
    "\"\"\"\n",
    "def preprocess(S):\n",
    "    sentences = S.split('.')\n",
    "    pos_tags = []\n",
    "    named_entity = []\n",
    "    for sentence in sentences:\n",
    "        pos_tags.append(nltk.pos_tag(nltk.word_tokenize(sentence)))\n",
    "        parse_tree = nltk.ne_chunk(nltk.tag.pos_tag(sentence.split()), binary=True)\n",
    "        for tree in parse_tree.subtrees():\n",
    "            if tree.label()=='NE':\n",
    "                entity = \"\"\n",
    "                for t in tree:\n",
    "                    entity+=t[0]+\" \"\n",
    "                named_entity.append(entity[:-1])\n",
    "        \n",
    "    return (pos_tags, named_entity)\n",
    "\n",
    "def to_nltk_tree(node):\n",
    "    if node.n_lefts + node.n_rights > 0:\n",
    "        return nltk.Tree(node.orth_, [to_nltk_tree(child) for child in node.children])\n",
    "    else:\n",
    "        return node.orth_\n",
    "\n",
    "\n",
    "def dependency_tree(sentence):\n",
    "    sentence = unicode(sentence, 'utf-8')\n",
    "    doc = nlpSpacy(sentence)\n",
    "\n",
    "    for ent in doc.ents:\n",
    "        print(ent.text, ent.start_char, ent.end_char, ent.label_)\n",
    "    \n",
    "    [to_nltk_tree(sent.root).pretty_print() for sent in doc.sents]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sen =  \"Architecturally, the school has a Catholic character. Atop the Main Building gold dome is a golden statue of the Virgin Mary. Immediately in front of the Main Building and facing it, is a copper statue of Christ with arms upraised with the legend Venite Ad Me Omnes. Next to the Main Building is the Basilica of the Sacred Heart. Immediately behind the basilica is the Grotto, a Marian place of prayer and reflection. It is a replica of the grotto at Lourdes, France where the Virgin Mary reputedly appeared to Saint Bernadette Soubirous in 1858. At the end of the main drive (and in a direct line that connects through 3 statues and the Gold Dome), is a simple, modern stone statue of Mary.\"\n",
    "# sen = sen.split('.')\n",
    "# del sen[-1]\n",
    "# for s in sen:\n",
    "#     print s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "set(['Catholic', 'Basilica', 'Christ', 'Marian', 'Saint Bernadette Soubirous', 'Virgin', 'Venite Ad Me Omnes', 'Gold', 'Sacred Heart', 'Main Building'])\n"
     ]
    }
   ],
   "source": [
    "print set(preprocess(sen)[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Architecturally, school has Catholic character\n",
      "Atop Main Building gold dome is golden statue of Virgin Mary\n",
      "Immediately in front of Main Building and facing it, is copper statue of Christ with arms upraised with legend Venite Ad Me Omnes\n",
      "Next to Main Building is Basilica of Sacred Heart\n",
      "Immediately behind basilica is Grotto, Marian place of prayer and reflection\n",
      "It is replica of grotto at Lourdes, France where Virgin Mary reputedly appeared to Saint Bernadette Soubirous in 1858\n",
      "At end of main drive, is simple, modern stone statue of Mary\n"
     ]
    }
   ],
   "source": [
    "import regex\n",
    "sen =  \"Architecturally, the school has a Catholic character. Atop the Main Building\\'s gold dome is a golden statue of the Virgin Mary. Immediately in front of the Main Building and facing it, is a copper statue of Christ with arms upraised with the legend \\\"Venite Ad Me Omnes\\\". Next to the Main Building is the Basilica of the Sacred Heart. Immediately behind the basilica is the Grotto, a Marian place of prayer and reflection. It is a replica of the grotto at Lourdes, France where the Virgin Mary reputedly appeared to Saint Bernadette Soubirous in 1858. At the end of the main drive (and in a direct line that connects through 3 statues and the Gold Dome), is a simple, modern stone statue of Mary.\"\n",
    "sen = sen.split('.')\n",
    "del sen[-1]\n",
    "\n",
    "stop_words = ['a', 'an', 'the']\n",
    "replace = [\"\\\"\", \"\\'s\", \"\\'\"]\n",
    "for i in range(len(sen)):\n",
    "    sen[i] = regex.sub(r' \\(.*\\)', '', sen[i])\n",
    "    sen[i] = regex.sub(r'\\(.*\\)', '', sen[i])\n",
    "    for r in replace:\n",
    "        sen[i] = sen[i].replace(r, \"\")\n",
    "    sen[i] = \" \".join([s for s in sen[i].split(\" \") if s not in stop_words])\n",
    "    sen[i] = \" \".join(sen[i].split())\n",
    "for s in sen:\n",
    "    print s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}