nyt_comments.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# UASB03 - NY Times Comments"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Description des données"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Articles\n",
    "Nous chargeons tout d'abord les articles rédigés dans le New York Times entre janvier et mai 2017.\n",
    "\n",
    "#### Descriptions des variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import glob\n",
    "\n",
    "data_dir = 'C:\\\\Users\\\\julien.piquot\\\\workspace\\\\nyt-comments\\\\data\\\\'\n",
    "dateparse = lambda x: pd.datetime.strptime(x, '%Y-%m-%d %H:%M:%S')\n",
    "\n",
    "def read_all_csv(pattern):\n",
    "    allFiles = glob.glob(data_dir + pattern)\n",
    "    frame = pd.DataFrame()\n",
    "    list_ = []\n",
    "    for file_ in allFiles:\n",
    "        df = pd.read_csv(file_,index_col=None, header=0, parse_dates=['pubDate'], date_parser=dateparse)\n",
    "        list_.append(df)\n",
    "    frame = pd.concat(list_, sort=True)\n",
    "    return frame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>abstract</th>\n",
       "      <th>articleID</th>\n",
       "      <th>articleWordCount</th>\n",
       "      <th>byline</th>\n",
       "      <th>documentType</th>\n",
       "      <th>headline</th>\n",
       "      <th>keywords</th>\n",
       "      <th>multimedia</th>\n",
       "      <th>newDesk</th>\n",
       "      <th>printPage</th>\n",
       "      <th>pubDate</th>\n",
       "      <th>sectionName</th>\n",
       "      <th>snippet</th>\n",
       "      <th>source</th>\n",
       "      <th>typeOfMaterial</th>\n",
       "      <th>webURL</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>288</th>\n",
       "      <td>NaN</td>\n",
       "      <td>58ed0c267c459f24986d9a37</td>\n",
       "      <td>1180</td>\n",
       "      <td>By PETE WELLS</td>\n",
       "      <td>article</td>\n",
       "      <td>Where a Chicken’s Parts Are Greater Than the Sum</td>\n",
       "      <td>['Restaurants', 'Torishin (Manhattan, NY, Rest...</td>\n",
       "      <td>3</td>\n",
       "      <td>Dining</td>\n",
       "      <td>5</td>\n",
       "      <td>2017-04-11 17:02:23</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>At a yakitori restaurant in Hell’s Kitchen, ea...</td>\n",
       "      <td>The New York Times</td>\n",
       "      <td>Review</td>\n",
       "      <td>https://www.nytimes.com/2017/04/11/dining/tori...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>472</th>\n",
       "      <td>NaN</td>\n",
       "      <td>58a6d87495d0e024746371b7</td>\n",
       "      <td>1214</td>\n",
       "      <td>By ALEXANDRA S. LEVINE and AMY ZERBA</td>\n",
       "      <td>article</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>['New York City']</td>\n",
       "      <td>1</td>\n",
       "      <td>Metro</td>\n",
       "      <td>0</td>\n",
       "      <td>2017-02-17 11:03:09</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>Friday: How children can celebrate the holiday...</td>\n",
       "      <td>The New York Times</td>\n",
       "      <td>briefing</td>\n",
       "      <td>https://www.nytimes.com/2017/02/17/nyregion/ne...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    abstract                 articleID  articleWordCount  \\\n",
       "288      NaN  58ed0c267c459f24986d9a37              1180   \n",
       "472      NaN  58a6d87495d0e024746371b7              1214   \n",
       "\n",
       "                                   byline documentType  \\\n",
       "288                         By PETE WELLS      article   \n",
       "472  By ALEXANDRA S. LEVINE and AMY ZERBA      article   \n",
       "\n",
       "                                             headline  \\\n",
       "288  Where a Chicken’s Parts Are Greater Than the Sum   \n",
       "472                                           Unknown   \n",
       "\n",
       "                                              keywords  multimedia newDesk  \\\n",
       "288  ['Restaurants', 'Torishin (Manhattan, NY, Rest...           3  Dining   \n",
       "472                                  ['New York City']           1   Metro   \n",
       "\n",
       "     printPage             pubDate sectionName  \\\n",
       "288          5 2017-04-11 17:02:23     Unknown   \n",
       "472          0 2017-02-17 11:03:09     Unknown   \n",
       "\n",
       "                                               snippet              source  \\\n",
       "288  At a yakitori restaurant in Hell’s Kitchen, ea...  The New York Times   \n",
       "472  Friday: How children can celebrate the holiday...  The New York Times   \n",
       "\n",
       "    typeOfMaterial                                             webURL  \n",
       "288         Review  https://www.nytimes.com/2017/04/11/dining/tori...  \n",
       "472       briefing  https://www.nytimes.com/2017/02/17/nyregion/ne...  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "articles = read_all_csv('Articles*2017.csv')\n",
    "articles.sample(2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4566, 16)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "articles.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>unique</th>\n",
       "      <th>top</th>\n",
       "      <th>freq</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>abstract</th>\n",
       "      <td>156</td>\n",
       "      <td>151</td>\n",
       "      <td>Photos by The New York Times and by photograph...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>articleID</th>\n",
       "      <td>4566</td>\n",
       "      <td>4548</td>\n",
       "      <td>5906e1f17c459f24986dd044</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>byline</th>\n",
       "      <td>4566</td>\n",
       "      <td>1520</td>\n",
       "      <td>By DEB AMLEN</td>\n",
       "      <td>160</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>documentType</th>\n",
       "      <td>4566</td>\n",
       "      <td>2</td>\n",
       "      <td>article</td>\n",
       "      <td>4410</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>headline</th>\n",
       "      <td>4566</td>\n",
       "      <td>4084</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>314</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>keywords</th>\n",
       "      <td>4566</td>\n",
       "      <td>3752</td>\n",
       "      <td>[]</td>\n",
       "      <td>295</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>newDesk</th>\n",
       "      <td>4566</td>\n",
       "      <td>35</td>\n",
       "      <td>OpEd</td>\n",
       "      <td>942</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sectionName</th>\n",
       "      <td>4566</td>\n",
       "      <td>47</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>3209</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>snippet</th>\n",
       "      <td>4566</td>\n",
       "      <td>4379</td>\n",
       "      <td>Look closely at this image, stripped of its ca...</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>source</th>\n",
       "      <td>4566</td>\n",
       "      <td>2</td>\n",
       "      <td>The New York Times</td>\n",
       "      <td>4537</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>typeOfMaterial</th>\n",
       "      <td>4566</td>\n",
       "      <td>11</td>\n",
       "      <td>News</td>\n",
       "      <td>2811</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>webURL</th>\n",
       "      <td>4566</td>\n",
       "      <td>4416</td>\n",
       "      <td>https://www.nytimes.com/2017/03/02/magazine/ho...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               count unique  \\\n",
       "abstract         156    151   \n",
       "articleID       4566   4548   \n",
       "byline          4566   1520   \n",
       "documentType    4566      2   \n",
       "headline        4566   4084   \n",
       "keywords        4566   3752   \n",
       "newDesk         4566     35   \n",
       "sectionName     4566     47   \n",
       "snippet         4566   4379   \n",
       "source          4566      2   \n",
       "typeOfMaterial  4566     11   \n",
       "webURL          4566   4416   \n",
       "\n",
       "                                                              top  freq  \n",
       "abstract        Photos by The New York Times and by photograph...     4  \n",
       "articleID                                5906e1f17c459f24986dd044     2  \n",
       "byline                                               By DEB AMLEN   160  \n",
       "documentType                                              article  4410  \n",
       "headline                                                  Unknown   314  \n",
       "keywords                                                       []   295  \n",
       "newDesk                                                      OpEd   942  \n",
       "sectionName                                               Unknown  3209  \n",
       "snippet         Look closely at this image, stripped of its ca...    18  \n",
       "source                                         The New York Times  4537  \n",
       "typeOfMaterial                                               News  2811  \n",
       "webURL          https://www.nytimes.com/2017/03/02/magazine/ho...     2  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "articles.describe(include = ['O']).transpose()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>min</th>\n",
       "      <th>25%</th>\n",
       "      <th>50%</th>\n",
       "      <th>75%</th>\n",
       "      <th>max</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>articleWordCount</th>\n",
       "      <td>4566.0</td>\n",
       "      <td>1056.813403</td>\n",
       "      <td>817.325807</td>\n",
       "      <td>11.0</td>\n",
       "      <td>663.0</td>\n",
       "      <td>933.5</td>\n",
       "      <td>1264.0</td>\n",
       "      <td>12049.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>multimedia</th>\n",
       "      <td>4566.0</td>\n",
       "      <td>2.942838</td>\n",
       "      <td>7.316654</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>68.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>printPage</th>\n",
       "      <td>4566.0</td>\n",
       "      <td>7.440429</td>\n",
       "      <td>10.874987</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>94.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   count         mean         std   min    25%    50%     75%  \\\n",
       "articleWordCount  4566.0  1056.813403  817.325807  11.0  663.0  933.5  1264.0   \n",
       "multimedia        4566.0     2.942838    7.316654   0.0    1.0    3.0     3.0   \n",
       "printPage         4566.0     7.440429   10.874987   0.0    0.0    1.0    12.0   \n",
       "\n",
       "                      max  \n",
       "articleWordCount  12049.0  \n",
       "multimedia           68.0  \n",
       "printPage            94.0  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "articles.describe().transpose()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Le tableau de données formé par les données provenant des articles contient 4566 individus et 16 variables. Certaines de ces variables sont des informations techniques qui présentent peu d'intérêt pour une étude. Les variables que nous jugeons \"intéressantes\" sont donc les suivantes :\n",
    "\n",
    "Variables qualitatives nominales :\n",
    "* documentType : 'article' ou 'blogpost'\n",
    "* newDesk : la rubrique dans laquelle l'article a été publié (Sports, Culture, Editorial, etc.)\n",
    "* sectionName : la section dans laquelle l'article a été publié (Tennis, Hockey, Media, Politics, Canada, etc.)\n",
    "* source : 'The New York Times', 'International New York Times'\n",
    "* typeOfMaterial : la nature de l'article (News, Blog, Editorial, Question, etc.)\n",
    "\n",
    "Variables qualitatives ordinales :\n",
    "* pubDate : la date de publication de l'article\n",
    "* printPage : la page ou l'article a été imprimé\n",
    "\n",
    "Variables quantitatives :\n",
    "* articleWordCount : le nombre de terme de l'article complet\n",
    "\n",
    "Autre type de variables (texte, JSON, liste)\n",
    "* byline : autheur(s) de l'article\n",
    "* headline : le titre de l'article\n",
    "* keywords : la liste de mots clés associés à l'article au format JSON\n",
    "* snippet : un court apperçu de l'article"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Etude de la taille des articles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAD8CAYAAACcjGjIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAGsdJREFUeJzt3X901fWd5/HniySaFhUExhwl2NgptVC6tW3WOi07TWCqYt2BP9pRxh1pxaW761C7U2th2XPctYMHwepYO2VkDTM4x4lVZ2a1FdphJdG1/qhQWy1mPLLVShCxFaLlpyS894/7Cb3he0Pg3gs3V16Pc3Lu/b6/n+/3vm8bfOX7WxGBmZlZvhGVbsDMzIYfh4OZmWU4HMzMLMPhYGZmGQ4HMzPLcDiYmVmGw8HMzDIcDmZmluFwMDOzjNpKN1CscePGRVNTU6XbMMvYtWsXI0eOrHQbZgVt2LDhNxHxe0ONq9pwaGpqYv369ZVuwyyjs7OTlpaWSrdhVpCkXx3JOO9WMjOzDIeDmZllOBzMzCzD4WBmZhkOBzMzyxgyHCStlPSGpF/k1ZZJ+ldJz0n6Z0mj8+YtlLRJ0ouSLsqrX5xqmyQtyKufI+lpSS9J+p6kk8r5Bc2Ol/b2dqZMmcL06dOZMmUK7e3tlW7JrGhHcirr3wHfAe7Oq60FFkZEr6SbgYXANyRNBi4HPgycBfwfSR9My/w18FmgG3hG0kMR8QJwM3BbRNwr6W+AucDy0r+a2fHT3t7OokWLaGtro6+vj5qaGubOnQvA7NmzK9yd2dEbcsshIh4Dth9S+5eI6E2TTwGN6f1M4N6I2BcRLwObgPPTz6aI+GVEvAPcC8yUJGAa8EBafhUwq8TvZHbcLV68mLa2NlpbW6mtraW1tZW2tjYWL15c6dbMilKOYw5XAWvS+/HA5rx53ak2WH0s0JMXNP11s6rS1dXF1KlTB9SmTp1KV1dXhToyK01JV0hLWgT0Avf0lwoMCwqHUBxm/GCfNw+YB9DQ0EBnZ+fRtGt2zJx99tl85zvf4WMf+xg7d+6ks7OTZ599lrPPPtu/p1aVig4HSXOAS4HpEdH/H/RuYELesEbgtfS+UP03wGhJtWnrIX98RkSsAFYANDc3h29RYMPFTTfddPCYQ319PRHBHXfcwU033eRbaVhVKiocJF0MfAP4TETszpv1EPAPkm4ld0B6IvATclsIEyWdA2whd9D6TyMiJHUAnyd3HGIO8GCxX8asUvoPOs+fP5+uri4mTZrE4sWLfTDaqpZ+90f/IAOkdqAFGAdsA24gd3bSycCbadhTEfGf0vhF5I5D9AJfjYg1qX4J8FdADbAyIhan+vvJBcMY4FngP0TEvqEab25uDt94z4Yj33jPhjNJGyKieahxQ245REShP33aDjN+MZA5RSMiVgOrC9R/Se5sJjMzGyZ8hbSZmWU4HMzMLMPhYGZmGQ4HMzPLcDiYmVmGw8HMzDIcDmZmluFwMDOzDIeDmZllOBzMzCzD4WBmZhkOBzMzy3A4mJlZhsPBzMwyHA5mZpbhcDAzswyHg5mZZTgczMwsw+FgZmYZDgczM8twOJiZWYbDwczMMhwOZmaW4XAwM7OMIcNB0kpJb0j6RV5tjKS1kl5Kr6enuiR9W9ImSc9J+njeMnPS+Jckzcmrf0LS82mZb0tSub+kmZkdnSPZcvg74OJDaguARyJiIvBImgaYAUxMP/OA5ZALE+AG4JPA+cAN/YGSxszLW+7QzzIzs+NsyHCIiMeA7YeUZwKr0vtVwKy8+t2R8xQwWtKZwEXA2ojYHhE7gLXAxWneaRHxZEQEcHfeuszMrEKKPebQEBFbAdLrGak+HticN6471Q5X7y5QNzOzCqot8/oKHS+IIuqFVy7NI7cLioaGBjo7O4to0ezY2rlzp383reoVGw7bJJ0ZEVvTrqE3Ur0bmJA3rhF4LdVbDql3pnpjgfEFRcQKYAVAc3NztLS0DDbUrGI6Ozvx76ZVu2J3Kz0E9J9xNAd4MK9+ZTpr6QLgrbTb6UfAhZJOTweiLwR+lOb9VtIF6SylK/PWZWZmFTLkloOkdnJ/9Y+T1E3urKMlwH2S5gKvAl9Iw1cDlwCbgN3AlwAiYrukbwLPpHE3RkT/Qe7/TO6MqPcAa9KPmZlV0JDhEBGzB5k1vcDYAK4ZZD0rgZUF6uuBKUP1YWZmx4+vkDYzswyHg5mZZTgczMwsw+FgZmYZDgczM8twOJiZWYbDwczMMhwOZmaW4XAwM7MMh4OZmWU4HMzMLMPhYGZmGQ4HszJpb29nypQpTJ8+nSlTptDe3l7plsyKVu4nwZmdkNrb21m0aBFtbW309fVRU1PD3LlzAZg9e7AbG5sNX95yMCuDxYsX09bWRmtrK7W1tbS2ttLW1sbixYsr3ZpZURwOZmXQ1dXF1KlTB9SmTp1KV1dXhToyK43DwawMJk2axOOPPz6g9vjjjzNp0qQKdWRWGh9zMCuDRYsWcdlllzFy5Eh+9atf8b73vY9du3Zx++23V7o1s6I4HMzKZO/evfT09BARbNmyhfr6+kq3ZFY05R77XH2am5tj/fr1lW7DDIAJEybQ19fHPffcc/BspSuuuIKamho2b95c6fbMDpK0ISKahxrnYw5mZdDd3c2qVasGnK20atUquru7K92aWVEcDmZmluFwMCuDxsZGrrzySjo6Oujt7aWjo4Mrr7ySxsbGSrdmVhQfkDYrg6VLl3Lttddy1VVXHTxbqa+vj1tvvbXSrZkVpaQtB0n/VdJGSb+Q1C6pXtI5kp6W9JKk70k6KY09OU1vSvOb8tazMNVflHRRaV/J7PibPXs2l112GVu3biUi2Lp1K5dddplvnWFVq+hwkDQe+ArQHBFTgBrgcuBm4LaImAjsAOamReYCOyLiA8BtaRySJqflPgxcDHxXUk2xfZlVQnt7Ow8//DBr1qxh7dq1rFmzhocfftg337OqVeoxh1rgPZJqgfcCW4FpwANp/ipgVno/M02T5k+XpFS/NyL2RcTLwCbg/BL7MjuufG8le7cpOhwiYgtwC/AquVB4C9gA9EREbxrWDYxP78cDm9OyvWn82Px6gWXMqkJXVxf3338/9fX1tLa2Ul9fz/333+97K1nVKvqAtKTTyf3Vfw7QA9wPzCgwtP8qOw0yb7B6oc+cB8wDaGhooLOz8+iaNjtGRo4cyZ133smXv/xlpk2bxrp167jzzjsZOXKkf0+tKpVyttIfAS9HxK8BJP0T8ClgtKTatHXQCLyWxncDE4DutBtqFLA9r94vf5kBImIFsAJyV0i3tLSU0L5Z+ezZs4e6ujruuusuli9fTl1dHXV1dezZswf/nlo1KuWYw6vABZLem44dTAdeADqAz6cxc4AH0/uH0jRp/rrI3bvjIeDydDbTOcBE4Ccl9GV23PX29rJ//37GjBkDwJgxY9i/fz+9vb1DLGk2PJVyzOFpcgeWfwo8n9a1AvgG8BeSNpE7ptCWFmkDxqb6XwAL0no2AveRC5YfAtdERF+xfZlVyqWXXsrrr79OR0cHr7/+OpdeemmlWzIrmm+8Z1YGkpDEGWecwRtvvHHwNSKo1n9j9u50pDfe8xXSZmVQW1vLiBEj2L59OxHB9u3bqaur48CBA5VuzawovreSWRmcdtpp9PX1sWTJEtasWcOSJUvo6+vjtNNOq3RrZkXxloNZGfT09DBt2jSuu+46IgJJTJ8+nXXr1lW6NbOiOBzMyuCss85i48aNPPLIIwMe9nPWWWdVujWzojgczMpk9+7dA+7Kunv3bk499dRKt2VWFB9zMCuDLVu20NfXx5YtWw4+Q7p/2qwaORzMyqCmpoaIYPz48YwYMYLx48cTEdTU+AbDVp28W8msDHp7e+nr62PPnj1EBHv27GH37t2+xsGqlrcczMrkpJNOGnCdw0knnVTplsyK5nAwK5P9+/cPuM5h//79lW7JrGjerWRWJvX19dxxxx0Hz1aqr69n9+7dlW7LrCjecjArk9ra2sNOm1UTh4NZGTQ2NtLb2zvgVNbe3l4aGxsr3ZpZURwOZmUwa9Ys9u7dy5gxY5DEmDFj2Lt3L7NmzRp6YbNhyOFgVgYdHR0sXLiQcePGIYlx48axcOFCOjo6Kt2aWVEcDmZl0NXVxbnnnjugdu6559LV1VWhjsxK44f9mJXBhAkTePPNNw8+LrSuro7a2lrGjh3L5s2bK92e2UFH+rAfbzmYlcGOHTvYs2cPV199Nd///ve5+uqr2bNnDzt27Kh0a2ZFcTiYlcGuXbuYPXs2jz32GDNnzuSxxx5j9uzZ7Nq1q9KtmRXF4WBWJoeeturTWK2a+SodszKoqanhlltu4ZZbbmHy5Mm88MILXHfddb4rq1Uth4NZGYwaNYqenh6WLl3Ktm3baGhoQBKjRo2qdGtmRfFuJbMy6OnpYfLkyWzbtg2Abdu2MXnyZHp6eircmVlxvOVgVgajR4+mq6uLb33rWwd3K11//fWMHj260q2ZFaWkcJA0GrgLmAIEcBXwIvA9oAl4BfiTiNghScDtwCXAbuCLEfHTtJ45wH9Pq/3LiFhVSl9mx9vbb79NTU0NCxYsOHidQ01NDW+//XalWzMrSqm7lW4HfhgRHwI+CnQBC4BHImIi8EiaBpgBTEw/84DlAJLGADcAnwTOB26QdHqJfZkdV729vbzzzjscOHAAgAMHDvDOO+/Q29tb4c7MilN0OEg6DfhDoA0gIt6JiB5gJtD/l/8qoP/OYzOBuyPnKWC0pDOBi4C1EbE9InYAa4GLi+3LrFLq6uqYMGECI0aMYMKECdTV1VW6JbOilbLl8H7g18DfSnpW0l2SRgINEbEVIL2ekcaPB/LvI9CdaoPVzarK/v37mTFjBg8++CAzZszwk+CsqpVyzKEW+DgwPyKelnQ7v9uFVIgK1OIw9ewKpHnkdknR0NBAZ2fnUTVsdiw1NTVx1113sXz5curq6mhqauKVV17x76lVpVLCoRvojoin0/QD5MJhm6QzI2Jr2m30Rt74CXnLNwKvpXrLIfXOQh8YESuAFZC78V5LS0uhYWYV8eqrr7Js2bKDZyt9/etfB8C/p1aNig6HiHhd0mZJ50bEi8B04IX0MwdYkl4fTIs8BPy5pHvJHXx+KwXIj4Cb8g5CXwgsLLYvs0pobGzkzTffHHC20sknn8zYsWMr3ZpZUUo9W2k+cI+k54DzgJvIhcJnJb0EfDZNA6wGfglsAv4X8F8AImI78E3gmfRzY6qZVY2lS5cCHDzO0P/aXzerNiVd5xARPwMK3Rd8eoGxAVwzyHpWAitL6cWskp544gn27t1LbW0tvb291NbWsnfvXp544glmz55d6fbMjpof9mNWBv2nreZf11Bbm/vby2ct2XBypA/78e0zzMqgPxRGjBjBgQMHGDFihC+As6rmG++ZldGyZctYs2YNy5Ytq3QrZiVxOJiZWYZ3K5mVyYgRI/ja1742YLr/Xktm1cZbDmZlcuDAAU455RQkccoppzgYrKo5HMzKYMyYMQDs3LmTiGDnzp0D6mbVxuFgVgbbtxe+bnOwutlw53AwK5P6+nqampqQRFNTE/X19ZVuyaxoDgezMtm3bx/z589n9erVzJ8/n3379lW6JbOi+WwlszJpbW1l5cqVdHV1MWnSJFpbW1m3bl2l2zIrisPBrEweffRRli5devCW3ddff32lWzIrmsPBrAwaGxvp7u4ecJ1Df92sGvmYg1kZ7N69+6jqZsOdw8GsDHwqq73bOBzMzCzD4WBmZhkOBzMzy3A4mJlZhsPBzMwyHA5mZpbhcDAzswyHg5mZZTgczMwso+RwkFQj6VlJP0jT50h6WtJLkr4n6aRUPzlNb0rzm/LWsTDVX5R0Uak9mVWKpAGvZtWqHFsO1wJdedM3A7dFxERgBzA31ecCOyLiA8BtaRySJgOXAx8GLga+K6mmDH2ZHXcRMeDVrFqVFA6SGoHPAXelaQHTgAfSkFXArPR+ZpomzZ+exs8E7o2IfRHxMrAJOL+Uvswqpf/pb34KnFW7Um/Z/VfA9cCpaXos0BMRvWm6Gxif3o8HNgNERK+kt9L48cBTeevMX2YASfOAeQANDQ10dnaW2L5Zee3du3fAK+DfU6tKRYeDpEuBNyJig6SW/nKBoTHEvMMtM7AYsQJYAdDc3BwtLS2FhpkNK/49tWpUypbDp4E/lnQJUA+cRm5LYrSk2rT10Ai8lsZ3AxOAbkm1wChge169X/4yZlWhpqaGvr6+gnWzalT0MYeIWBgRjRHRRO6A8rqIuALoAD6fhs0BHkzvH0rTpPnrInfU7iHg8nQ20znAROAnxfZlVgmFguFwdbPh7lg8JvQbwL2S/hJ4FmhL9Tbg7yVtIrfFcDlARGyUdB/wAtALXBMR/hdlZlZBqtZT7pqbm2P9+vWVbsMMOPx1DdX6b8zenSRtiIjmocb5CmmzMqqrqxvwalatHA5mZbR///4Br2bVyuFgZmYZDgczM8twOJiZWYbDwczMMhwOZmaW4XAwM7MMh4OZmWU4HMzMLMPhYGZmGQ4HMzPLcDiYmVmGw8HMzDIcDmZmluFwMDOzDIeDmZllOBzMzCzD4WBmZhkOBzMzy3A4mJlZhsPBzMwyHA5mZpbhcDAzs4yiw0HSBEkdkrokbZR0baqPkbRW0kvp9fRUl6RvS9ok6TlJH89b15w0/iVJc0r/WmZmVopSthx6ga9FxCTgAuAaSZOBBcAjETEReCRNA8wAJqafecByyIUJcAPwSeB84Ib+QDEbDiQN+VPq8kOtw+x4KzocImJrRPw0vf8t0AWMB2YCq9KwVcCs9H4mcHfkPAWMlnQmcBGwNiK2R8QOYC1wcbF9mZVbRAz5U+ryQ63D7HirLcdKJDUBHwOeBhoiYivkAkTSGWnYeGBz3mLdqTZYvdDnzCO31UFDQwOdnZ3laN+sZB0dHbS2thas+/fUqlHJ4SDpFOAfga9GxNuH2TwuNCMOU88WI1YAKwCam5ujpaXlqPs1O1b6//pvWvAwryz5XIW7MStNSWcrSaojFwz3RMQ/pfK2tLuI9PpGqncDE/IWbwReO0zdzMwqpJSzlQS0AV0RcWverIeA/jOO5gAP5tWvTGctXQC8lXY//Qi4UNLp6UD0halmZmYVUspupU8DfwY8L+lnqfbfgCXAfZLmAq8CX0jzVgOXAJuA3cCXACJiu6RvAs+kcTdGxPYS+jIzsxIVHQ4R8TiFjxcATC8wPoBrBlnXSmBlsb2YmVl5+QppMzPLcDiYmVmGw8HMzDIcDmZmluFwMDOzDIeDmZlllOXeSmbV4qP/8194a8/+Y/45TQsePqbrH/WeOn5+w4XH9DPsxOZwsBPKW3v2H/P7HnV2dnKs7/t1rMPHzLuVzMwsw+FgZmYZDgczM8twOJiZWYYPSNsJ5dRJC/jIqgVDDyzVqqGHlOLUSQB+oJAdOw4HO6H8tmuJz1YyOwLerWRmZhkOBzMzy/BuJTvhHJddMj889ldImx1LDgc7oRzr4w2QC5/j8Tlmx5J3K5mZWYbDwczMMhwOZmaW4XAwM7MMh4OZmWUMm3CQdLGkFyVtknQc7m9gZmaDGRbhIKkG+GtgBjAZmC1pcmW7MjM7cQ2LcADOBzZFxC8j4h3gXmBmhXsyMzthDZdwGA9szpvuTjUzM6uA4XKFtArUIjNImgfMA2hoaKCzs/MYt2UGra2tR72Mbj76z+no6Dj6hcyOkeESDt3AhLzpRuC1QwdFxApgBUBzc3Mc69simwFEZP5OOazjcctus2NtuOxWegaYKOkcSScBlwMPVbgnM7MT1rDYcoiIXkl/DvwIqAFWRsTGCrdlZnbCGhbhABARq4HVle7DzMyGz24lMzMbRhwOZmaW4XAwM7MMh4OZmWU4HMzMLENHe4HPcCHp18CvKt2HWQHjgN9UugmzQbwvIn5vqEFVGw5mw5Wk9RHRXOk+zErh3UpmZpbhcDAzswyHg1n5rah0A2al8jEHMzPL8JaDmZllOBzsXUXSVyW9N296taTRhxn/PyRdd5j5oyW9KUlp+g8khaTGND1K0nZJRf9bkrQz7/0HU8+bJHVJuk9SQ7HrHuTzZvkZ7TYUh4O9a0iqAb4KHAyHiLgkInqKXWda9nVgUip9Cng2vQJcADwdEQeOsMdB74QsqR54GFgeER+IiEnAcmDIc9KP0izA4WCH5XCwqiHpf0vaIGljemQsknZKulHS08Ai4CygQ1JHmv+KpHHp/ZWSnpP0c0l/X2D9vy/ph+kz/q+kD6VZP+Z3YfAp4LZDpp9Iy58n6an0Gf8s6fRU75R0k6RHgWvTQ62elPSMpG/mtfCnwJMR8f3+QkR0RMQvJNVL+ltJz0t6VlJrWvcXJX0n7zv8QFJL3v82i9P3fUpSg6RPAX8MLJP0M0m/X9z/G/Zu53CwanJVRHwCaAa+ImksMBL4RUR8MiJuJPd42daIGPDgZ0kfJhce0yLio8C1Bda/ApifPuM64Lup/gS/C4P3A/enHkj1H6f3dwPfiIh/AzwP3JC37tER8ZmI+BZwO7mtg39Lbquk3xRgwyDf/RqAiPgIMBtYlbY0Dmck8FT6vo8B/zEiniD3lMWvR8R5EfH/hliHnaAcDlZNviLp58BT5J45PhHoA/7xCJadBjwQEb8BiIjt+TMlnULuP/T3S/oZcCdwZpr9Y+BTks4BXomIvblFdArwCeAnkkaRC4BH0zKrgD/M+4jv5b3/NNCe3me2YAYxtX9sRPwruVvHfHCIZd4BfpDebwCajvCzzIbPk+DMDiftKvkj4A8iYrekTqAe2BsRfUeyCuBw522PAHoi4rxDZ0TES2kX0b8HnkzlDcCXgJcjYmcKh8PZdehqC4zZCHxmkOU1SL2XgX/k5W9N7I/fnaveh/+921HwloNVi1HAjhQMHyJ3ILiQ3wKnFqg/AvxJ2hWFpDH5MyPibeBlSV9I8yXpo3lDniS3K+rJvOmvko43RMRbwA5J/y7N/zPgUQr7MXB5en9FXv0fyG2hfK6/IOliSR8ht1voilT7IHA28CLwCnCepBGSJgDnD/KZ+Qb738jsIIeDVYsfArWSngO+SW7XUiErgDX9B6T7RcRGYDHwaNo1dWuBZa8A5qb5G4GZefN+TG5X1vo0/SS54w9P5I2ZQ+5A73PAecCNg/R4LXCNpGfIhV5/j3uAS4H5kl6S9ALwReANcsc/aiQ9T24X1RcjYl/q62VyxzhuAX46yGfmuxf4ejqw7QPSVpCvkDYzswxvOZiZWYbDwczMMhwOZmaW4XAwM7MMh4OZmWU4HMzMLMPhYGZmGQ4HMzPL+P80dY02p9UirgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "axes = articles.boxplot(column='articleWordCount')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "La taille moyenne d'un article est d'environ 1057 mots tandis que la médianne se situe a environ 934 termes, nous constatons qu'au delà de 2000 termes, il existe un certain nombre d'article long mais qu'ils constituent une exception."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Etude de la numérotation des articles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEXCAYAAAC6baP3AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3XmcHFW99/HPF8KaACEs80BAAwhI2CGy+ngHRXZvQAHBgKzGBQQ1XEWvSwS5xtc1oqJyn6iBgIGAgk/CohCBEfEKhCAQQkAiRAgJYQ8M2yXyu3/UGVIM0z1VPdMzPdPf9+vVr+46farqV2dq+td1TnWVIgIzM7OiVunvAMzMbGBx4jAzs1KcOMzMrBQnDjMzK8WJw8zMSnHiMDOzUpw4BhlJ/yXpG720rHdJape0appuk3Rqbyw7Le93kk7oreVVWc+Jkm6rcd5e3eZ6k7SRpHsk7d7Lyx0n6cbeXGaVdYWk9/TFuqw2ThwDiKRFkl6V9JKkFyT9t6TPSHrr7xgRn4mIcwsua/9qdSLisYgYFhH/7IXYJ0r6VaflHxwR03q6bMtIWg2YBnwuIub2YDmj0of3kI6yiJgeEQf0Rpw28A3pvoo1mI9ExB8krQf8C/AjYE/gpN5ciaQhEbGiN5dp9RURbwCHdFdP0qq98WXAmpePOAaoiFgeEbOAjwMnSNoBQNLFkr6TXm8o6dp0dPKcpD9JWkXSpcC7gGtSV9SXc98yT5H0GHBzV988ga0k3SlpuaSZkkakdbVKWpyPseOoRtJBwNeAj6f13Zvef6sbKMX1dUn/kPSUpEtScsx/Az5B0mOSnpH075XaRtIGkmZJelHSncBWnd5/r6TZqU0eknR0kTaXtJWkmyU9m2KYLml4lfqRjggflvS8pJ9KUnrvbUdgnds6tc130lFlu6Rr0nZNT9s1R9KoItuU9okLJV0v6WVgP0mHSvprWtbjkibmQr81Pb+Q1r135+4+SfukGJan531y77VJOlfSn9PR8Y2SNqzSTv8maamkJZJO7vTeGpK+n/7uy5R1xa5VYTknpnVekOJ6UNKHcu+fJGlBiukRSZ/uNP+Xc3GcqlyXWZk4mkJE+DFAHsAiYP8uyh8DPpteXwx8J73+LvBfwGrp8X8BdbUsYBQQwCXAUGCtXNmQVKcNeALYIdW5CvhVeq8VWFwpXmBiR93c+23Aqen1ycBCYEtgGHA1cGmn2H6e4toZeB3YrkI7zQCuTDHukGK+Lb03FHic7AhtCLAb8AywfYVl5WN8D/BhYA1gI7IP2B9W+XsFcC0wnCxRPw0c1FV7VGjrhWRJbz3gAeBvwP4p7kuAi4psU9onlgP7kn1ZXDP9vXZM0zsBy4DDu4ollZ2Ya8MRwPPA8Wl9x6bpDXKx/x3YJv292oBJFdrooLTujn3qsrTu96T3fwjMSutcB7gG+G6FZZ0IrAC+SLa/fzxt94j0/qGpPUV2tP4KsFsujieB7YG1gUtrjaMZHj7iGByWkO3Qnb0BbAK8OyLeiIg/RfovqGJiRLwcEa9WeP/SiLg/Il4GvgEcrTR43kPjgB9ExCMR0Q58FThGbz/a+XZEvBoR9wL3kiWQt0mxfAz4ZtqO+8n6/TscBiyKiIsiYkVE3E2WAI/sLsCIWBgRsyPi9Yh4GvgB2QdQNZMi4oWIeAy4Bdilu/XkXBQRf4+I5cDvgL9HxB8i60L8NbBriW2aGRF/jog3I+K1iGiLiHlp+j7g8gLb0uFQ4OGIuDSt73LgQeAjnWL/W9qPrqyy3Uenuh371MSON9LR2aeAL0bEcxHxEvAfwDFVYnuKLJm/ERFXAA+leImI61J7RkT8EbiR7MtUPo75EfEK8O0exjGoeYxjcBgJPNdF+X+S/SPemHpIpkTEpG6W9XiJ9/9B9s2uYjdECZum5eWXPQRoyZU9mXv9CtmRSWcbpfk6x9nh3cCekl7IlQ0h+4ZZlaSNgR+TfdisQ/Zt/fluZisScyXLcq9f7WK6Y1lFtultf1dJewKTyL7pr052FPXrgnF1/luRpkfmpotu96ZAfiA/v9yNyL79z037L2RHC9W+qDzR6cvRP9I6kHQw8C2yI6FV0rLn5eK4Kzdfvr1qiWNQ8xHHACfpfWT/sO843TQiXoqICRGxJdm3wS/l+nwrHXl0d0Syee71u8iOap4BXib75+qIa1Wyf7iiy11C9gGYX/YK3v5hWcTTab7OcXZ4HPhjRAzPPYZFxGcLLPu7ZNuxU0SsCxxH9gFSi7e1F/B/alwOFNumzu1/GVnXy+YRsR5Zl6Yq1O2s898KsjZ+oobYl1L5b/UMWYLcPrdd60VEteQ7UrlP97S8JZLWIDsK+z7QEhHDgetZuc1Lgc1y8+VjqiWOQc2JY4CStK6kw8j6838VEfO6qHOYpPekf6QXgX+mB2QfyFvWsOrjJI2WtDZwDvCbyM7Q+RuwZhp0XQ34Otm32A7LgFHKnTrcyeXAFyVtIWkYWVfAFVHyzK4Uy9XARElrSxoN5H8rci2wjaTjJa2WHu+TtF2Bxa8DtJMNGo8E/q1MbJ3cA3xA2W9l1iPrmqtVLdu0DvBcRLwmaQ/gE7n3ngbepPL+cX1a3yckDZH0cWB0iqOsK4ETc/vUtzreiIg3yca1zk9He0gaKenAKsvbGDgjtcFRwHYp3o6jqqeBFenoI3968ZXASZK2S3F8s4dxDGpOHAPPNZJeIvuW+e9k/eyVTsXdGvgD2YfdX4CfRURbeu+7wNeVnXF1Von1X0o22Pok2SDrGZCd5QV8DvgF2TfPl4H8WVYd3SDPSrq7i+VOTcu+FXgUeA34fIm48k4n6xp5MsV6UccbqX/6ALL+6SWpzvd4e5Kr5NtkA8/LgevIElRNImI2cAVwH1lXTS0fuh3LqmWbPgeck/alb5J9cHYs7xXgPODPaf/Yq9P6niUbV5kAPAt8GTgsIp6pIfbfkQ0830x2MsDNnap8JZXfLulFsv152yqLvINsv38mbcOREfFsaqMz0nY+T5YoZ3WK48dk41ALyf5fIDsJo5Y4BrWOM2zMzAY0SSeSnQH3/l5Y1nbA/cAaZY96m4GPOMzMAElHSFpd0vpkR2zXOGl0zYnDzCzzabIxkL+TjQUWOWGiKbmryszMSvERh5mZleLEYWZmpQzKX45vuOGGMWrUqJrnf/nllxk6dGjvBTTANPv2g9sA3AbQfG0wd+7cZyJio+7qDcrEMWrUKO66667uK1bQ1tZGa2tr7wU0wDT79oPbANwG0HxtIKnzpWS65K4qMzMrxYnDzMxKceIwM7NSnDjMzKwUJw4zMyvFicPMzEpx4jAzs1KcOMzMrJRB+QPAnpr3xHJOPPu6muZdNOnQXo7GzKyx+IjDzMxKceIwM7NSnDjMzKwUJw4zMyvFicPMzEpx4jAzs1KcOMzMrBQnDjMzK8WJw8zMSnHiMDOzUpw4zMysFCcOMzMrxYnDzMxKceIwM7NSnDjMzKwUJw4zMyvFicPMzEqpW+KQtLmkWyQtkDRf0pmpfKKkJyTdkx6H5Ob5qqSFkh6SdGCu/KBUtlDS2fWK2czMulfPW8euACZExN2S1gHmSpqd3js/Ir6fryxpNHAMsD2wKfAHSdukt38KfBhYDMyRNCsiHqhj7GZmVkHdEkdELAWWptcvSVoAjKwyy1hgRkS8DjwqaSGwR3pvYUQ8AiBpRqrrxGFm1g/6ZIxD0ihgV+COVHS6pPskTZW0fiobCTyem21xKqtUbmZm/aCeXVUASBoGXAV8ISJelHQhcC4Q6XkycDKgLmYPuk5u0cV6xgPjAVpaWmhra6s55pa1YMKOK2qatyfrbRTt7e2DYjt6wm3gNgC3QSV1TRySViNLGtMj4mqAiFiWe//nwLVpcjGweW72zYAl6XWl8rdExBRgCsCYMWOitbW15rgvmD6TyfNqa5pF42pfb6Noa2ujJ+03GLgN3AbgNqiknmdVCfglsCAifpAr3yRX7Qjg/vR6FnCMpDUkbQFsDdwJzAG2lrSFpNXJBtBn1StuMzOrrp5HHPsCxwPzJN2Tyr4GHCtpF7LupkXApwEiYr6kK8kGvVcAp0XEPwEknQ7cAKwKTI2I+XWM28zMqqjnWVW30fW4xfVV5jkPOK+L8uurzWdmZn3Hvxw3M7NSnDjMzKwUJw4zMyvFicPMzEpx4jAzs1KcOMzMrBQnDjMzK8WJw8zMSnHiMDOzUpw4zMysFCcOMzMrxYnDzMxKceIwM7NSnDjMzKwUJw4zMyvFicPMzEpx4jAzs1KcOMzMrBQnDjMzK8WJw8zMSnHiMDOzUpw4zMysFCcOMzMrxYnDzMxKceIwM7NSnDjMzKwUJw4zMyvFicPMzEoplTgkrS9pp4J1N5d0i6QFkuZLOjOVj5A0W9LD6Xn9VC5JP5a0UNJ9knbLLeuEVP9hSSeUidnMzHpXt4lDUpukdSWNAO4FLpL0gwLLXgFMiIjtgL2A0ySNBs4GboqIrYGb0jTAwcDW6TEeuDCtfwTwLWBPYA/gWx3JxszM+l6RI471IuJF4KPARRGxO7B/dzNFxNKIuDu9fglYAIwExgLTUrVpwOHp9VjgksjcDgyXtAlwIDA7Ip6LiOeB2cBBhbfQzMx6VZHEMSR9gB8NXFvLSiSNAnYF7gBaImIpZMkF2DhVGwk8npttcSqrVG5mZv1gSIE65wA3AH+OiDmStgQeLroCScOAq4AvRMSLkipW7aIsqpR3Xs94si4uWlpaaGtrKxriO7SsBRN2XFHTvD1Zb6Nob28fFNvRE24DtwG4DSrpNnFExK+BX+emHwE+VmThklYjSxrTI+LqVLxM0iYRsTQdyTyVyhcDm+dm3wxYkspbO5W3dRHnFGAKwJgxY6K1tbVzlcIumD6TyfOK5NR3WjSu9vU2ira2NnrSfoOB28BtAG6DSooMjm8j6SZJ96fpnSR9vcB8An4JLIiI/GD6LKDjzKgTgJm58k+ms6v2ApanrqwbgAPSGV3rAwekMjMz6wdFxjh+DnwVeAMgIu4Djikw377A8cAHJd2THocAk4APS3oY+HCaBrgeeARYmNb5ubS+54BzgTnpcU4qMzOzflCkP2btiLiz09hEtwMAEXEbXY9PAHyoi/oBnFZhWVOBqd2HamZm9VbkiOMZSVuRBqQlHQksrWtUZmbWsIoccZxGNuj8XklPAI8Cx9U1KjMza1hFzqp6BNhf0lBglfRjPjMza1IVE4ekL1UoB6DTmVJmZtYkqh1xrNNnUZiZ2YBRMXFExLf7MhAzMxsYivwAcJqk4bnp9SX51FgzsyZV5HTcnSLihY6JdIXaXesXkpmZNbIiiWOV/P0v0v0xaruQk5mZDXhFEsBk4L8l/SZNHwWcV7+QzMyskRX5Hcclku4CPkh2CZGPRsQDdY/MzMwaUrXfcayb7p8xAngSuCz33ghfaNDMrDlVO+K4DDgMmMvbb5ykNL1lHeMyM7MGVe13HIel5y36LhwzM2t0RX7HcVORMjMzaw7VxjjWBNYGNkyn43bcW2NdYNM+iM3MzBpQtTGOTwNfIEsSc1mZOF4EflrnuMzMrEFVG+P4kaSfAF+LiHP7MCYzM2tgVcc4IuKfwCF9FIuZmQ0ARS45cqOkj6nTTcfNzKw5FbnkyJeAocAKSa+RfscREevWNTIzM2tIRS454hs6mZnZWwpd5Tadjrs1sGZHWUTcWq+gzMyscXWbOCSdCpwJbAbcA+wF/IXsoodmZtZkigyOnwm8D/hHROxHdhOnp+salZmZNawiieO1iHgNQNIaEfEgsG19wzIzs0ZVZIxjcbrn+P8HZkt6HlhS37DMzKxRFTmr6oj0cqKkW4D1gN/XNSozM2tYRbqq3hIRf4yIWRHxP93VlTRV0lOS7s+VTZT0hKR70uOQ3HtflbRQ0kOSDsyVH5TKFko6u0y8ZmbW+yomDkmPSnpE0h01Lvti4KAuys+PiF3S4/q0rtHAMcD2aZ6fSVpV0qpkF1Q8GBgNHJvqmplZP6l2kcMe3cApIm6VNKpg9bHAjIh4HXhU0kJgj/Tewoh4BEDSjFTX9zw3M+sn1e7HsVu1GSPi7hrXebqkTwJ3ARMi4nlgJHB7rs7iVAbweKfyPWtcr5mZ9YJqg+OT0/OawBjgXrLrVO0E3AG8v4b1XQicS3bP8nPTOk5m5b0+8oKuu9KiizIkjQfGA7S0tNDW1lZDeJmWtWDCjitqmrcn620U7e3tg2I7esJt4DYAt0El1bqq9oO3uofGR8S8NL0DcFYtK4uIZR2vJf0cuDZNLgY2z1XdjJWn/FYq77zsKcAUgDFjxkRra2stIQJwwfSZTJ5X6Gos77BoXO3rbRRtbW30pP0GA7eB2wDcBpUUOavqvR1JAyAi7gd2qWVlkjbJTR4BdJxxNQs4RtIakrYguy7WncAcYGtJW0hanWwAfVYt6zYzs95R5Gv1Akm/AH5F1k10HLCgu5kkXQ60kt2zfDHwLaBV0i5pOYvIbk9LRMyXdCXZoPcK4LR0EykknQ7cAKwKTI2I+WU20MzMeleRxHES8Fmya1YB3Eo2VlFVRBzbRfEvq9Q/Dzivi/LrgesLxGlmZn2gyC/HXwPOTw8zM2typX45bmZm5sRhZmalFE4ckobWMxAzMxsYuk0ckvaR9ADpTCpJO0v6Wd0jMzOzhlTkiON84EDgWYCIuBf4QD2DMjOzxlWoqyoiHu9U9M86xGJmZgNAkd9xPC5pHyDSr7fPoMAPAM3MbHAqcsTxGeA0sqvVLia73Mhp9QzKzMwaV9UjjnQjpeMjYlwfxWNmZg2u6hFHul7U2D6KxczMBoAiYxx/lvQT4Arg5Y7CHtzIyczMBrAiiWOf9HxOriyAD/Z+OGZm1uiKXORwv74IxMzMBoYivxzfQNKPJd0taa6kH0naoC+CMzOzxlPkdNwZwNPAx4Aj0+sr6hmUmZk1riJjHCMi4tzc9HckHV6vgMzMrLEVOeK4RdIxklZJj6OB6+odmJmZNaaKRxySXiI7e0rAl8juOQ5Zsmknu4e4mZk1mYqJIyLW6ctAzMxsYCgyxoGknYBR+foRcXWdYjIzswbWbeKQNBXYCZgPvJmKA3DiMDNrQkWOOPaKiNF1j8TMzAaEImdV/UWSE4eZmQHFjjimkSWPJ4HXyc6yiojYqa6RmZlZQyqSOKYCxwPzWDnGYWZmTapI4ngsImbVPRIzMxsQiiSOByVdBlxD1lUF+HRcM7NmVWRwfC2yhHEA8JH0OKy7mSRNlfSUpPtzZSMkzZb0cHpeP5UrXYF3oaT7JO2Wm+eEVP9hSSeU3UAzM+tdRe7HcVKNy74Y+AlwSa7sbOCmiJgk6ew0/RXgYGDr9NgTuBDYU9IIskubjCH77chcSbMi4vkaYzIzsx4qcj+OzST9Nh09LJN0laTNupsvIm4FnutUPJbsLC3S8+G58ksiczswXNImwIHA7Ih4LiWL2cBBxTbNzMzqoUhX1UXALGBTYCTZWMdFNa6vJSKWAqTnjVP5SODxXL3FqaxSuZmZ9ZMig+MbRUQ+UVws6Qu9HIe6KIsq5e9cgDQeGA/Q0tJCW1tbzcG0rAUTdlxR07w9WW+jaG9vHxTb0RNuA7cBuA0qKZI4npF0HHB5mj4WeLbG9S2TtElELE1dUU+l8sXA5rl6mwFLUnlrp/K2rhYcEVOAKQBjxoyJ1tbWrqoVcsH0mUyeV+j6j++waFzt620UbW1t9KT9BgO3gdsA3AaVFOmqOhk4GngSWEp2+9iTa1zfLKDjzKgTgJm58k+ms6v2ApanrqwbgAMkrZ/OwDoglZmZWT8pclbVY8C/ll2wpMvJjhY2lLSY7OyoScCVkk4BHgOOStWvBw4BFgKvACeldT8n6VxgTqp3TkR0HnA3M7M+VO0OgN+sMl90ug95VxWOrfDWh7paGHBaheVMJbvsiZmZNYBqRxwvd1E2FDgF2AComjjMzGxwqnbr2MkdryWtA5xJ1oU0A5hcaT4zMxvcqo5xpF9ufwkYR/aDvd38q20zs+ZWbYzjP4GPkp3iumNEtPdZVGZm1rCqnY47gezX4l8Hlkh6MT1ekvRi34RnZmaNptoYR5HfeJiZWZNxcjAzs1KcOMzMrBQnDjMzK8WJw8zMSnHiMDOzUpw4zMysFCcOMzMrxYnDzMxKceIwM7NSnDjMzKwUJw4zMyvFicPMzEpx4jAzs1KcOMzMrBQnDjMzK8WJw8zMSnHiMDOzUpw4zMysFCcOMzMrxYnDzMxKceIwM7NSnDjMzKwUJw4zMyulXxKHpEWS5km6R9JdqWyEpNmSHk7P66dySfqxpIWS7pO0W3/EbGZmmf484tgvInaJiDFp+mzgpojYGrgpTQMcDGydHuOBC/s8UjMze0sjdVWNBaal19OAw3Pll0TmdmC4pE36I0AzMwNFRN+vVHoUeB4I4P9FxBRJL0TE8Fyd5yNifUnXApMi4rZUfhPwlYi4q9Myx5MdkdDS0rL7jBkzao7vqeeWs+zV2ubdceR6Na+3UbS3tzNs2LD+DqNfuQ3cBtB8bbDffvvNzfUCVTSkL4Lpwr4RsUTSxsBsSQ9Wqasuyt6R7SJiCjAFYMyYMdHa2lpzcBdMn8nkebU1zaJxta+3UbS1tdGT9hsM3AZuA3AbVNIvXVURsSQ9PwX8FtgDWNbRBZWen0rVFwOb52bfDFjSd9GamVlenx9xSBoKrBIRL6XXBwDnALOAE4BJ6XlmmmUWcLqkGcCewPKIWNrXcRc16uzrap530aRDezESM7P66I+uqhbgt5I61n9ZRPxe0hzgSkmnAI8BR6X61wOHAAuBV4CT+j5kMzPr0OeJIyIeAXbuovxZ4ENdlAdwWh+EZmZmBfTX4Lg1mHwX24QdV3BiiS43d7GZNZdG+h2HmZkNAE4cZmZWihOHmZmV4sRhZmalOHGYmVkpThxmZlaKE4eZmZXixGFmZqU4cZiZWSn+5XgD8QUSzWwg8BGHmZmV4sRhZmalOHGYmVkpHuOwHvPYjFlz8RGHmZmV4iOOQaIn3/rNzMrwEYeZmZXixGFmZqU4cZiZWSlOHGZmVooTh5mZleLEYWZmpThxmJlZKU4cZmZWihOHmZmV4l+OW7/yda7MBh4fcZiZWSkDJnFIOkjSQ5IWSjq7v+MxM2tWAyJxSFoV+ClwMDAaOFbS6P6NysysOQ2UMY49gIUR8QiApBnAWOCBfo3K+lU9rwg8YccVnFhl+T0ZX/G4jg10ioj+jqFbko4EDoqIU9P08cCeEXF6rs54YHya3BZ4qAer3BB4pgfzD3TNvv3gNgC3ATRfG7w7IjbqrtJAOeJQF2Vvy3gRMQWY0isrk+6KiDG9sayBqNm3H9wG4DYAt0ElA2KMA1gMbJ6b3gxY0k+xmJk1tYGSOOYAW0vaQtLqwDHArH6OycysKQ2IrqqIWCHpdOAGYFVgakTMr+Mqe6XLawBr9u0HtwG4DcBt0KUBMThuZmaNY6B0VZmZWYNw4jAzs1KcOHKa8bImkjaXdIukBZLmSzozlY+QNFvSw+l5/f6OtZ4krSrpr5KuTdNbSLojbf8V6aSMQUvScEm/kfRg2hf2bsJ94Ivpf+B+SZdLWrPZ9oOinDiSJr6syQpgQkRsB+wFnJa2+2zgpojYGrgpTQ9mZwILctPfA85P2/88cEq/RNV3fgT8PiLeC+xM1hZNsw9IGgmcAYyJiB3ITsI5hubbDwpx4ljprcuaRMT/AB2XNRnUImJpRNydXr9E9oExkmzbp6Vq04DD+yfC+pO0GXAo8Is0LeCDwG9SlcG+/esCHwB+CRAR/xMRL9BE+0AyBFhL0hBgbWApTbQflOHEsdJI4PHc9OJU1jQkjQJ2Be4AWiJiKWTJBdi4/yKrux8CXwbeTNMbAC9ExIo0Pdj3hS2Bp4GLUnfdLyQNpYn2gYh4Avg+8BhZwlgOzKW59oPCnDhW6vayJoOZpGHAVcAXIuLF/o6nr0g6DHgqIubmi7uoOpj3hSHAbsCFEbEr8DKDuFuqK2n8ZiywBbApMJSs27qzwbwfFObEsVLTXtZE0mpkSWN6RFydipdJ2iS9vwnwVH/FV2f7Av8qaRFZ9+QHyY5AhqcuCxj8+8JiYHFE3JGmf0OWSJplHwDYH3g0Ip6OiDeAq4F9aK79oDAnjpWa8rImqT//l8CCiPhB7q1ZwAnp9QnAzL6OrS9ExFcjYrOIGEX2N785IsYBtwBHpmqDdvsBIuJJ4HFJ26aiD5HdsqAp9oHkMWAvSWun/4mONmia/aAM/3I8R9IhZN82Oy5rcl4/h1R3kt4P/AmYx8o+/q+RjXNcCbyL7J/qqIh4rl+C7COSWoGzIuIwSVuSHYGMAP4KHBcRr/dnfPUkaReykwNWBx4BTiL7Ytk0+4CkbwMfJzvT8K/AqWRjGk2zHxTlxGFmZqW4q8rMzEpx4jAzs1KcOMzMrBQnDjMzK8WJw6wCSUMknS5pjf6OxayROHHYgCQpJE3OTZ8laWIvLl9kp2bf15unX0q6WNKR3dd8q/5ESWf11vrNeoMThw1UrwMflbRhPRYemdMj4tau3s/9mtis6Thx2EC1gux+0F/s/Ebnb/WS2tNzq6Q/SrpS0t8kTZI0TtKdkuZJ2irV20jSVZLmpMe+qXyipCmSbgQuSfdruCjN+1dJ+3URiyT9RNIDkq4jd6FASbuneOZKuqHj8h6VSPpUiufeFN/aXdSZKOlSSTene0h8KpUPk3STpLtTvGNz83wj3YdjdroPxVmpfCtJv0/x/UnSe6v/SaxZOHHYQPZTYJyk9UrMszPZvTd2BI4HtomIPch+Nf35VOdHZPdgeB/wsfReh92BsRHxCeA0gIjYETgWmCZpzU7rOwLYNq3vU2TXP+q4PtgFwJERsTswFejuSgVXR8T7IqLjfhmV7g2xE9ll4vcGvilpU+A14IiI2A3YD5icktqYtI27Ah8FxuSWMwX4fIrvLOBn3cRnTcKH2zZgRcSLki4huwHPqwVnm9NxqXBJfwduTOXzyD5QIbvg3ehsmAOAdSWtk17PioiOdb2f7MOfiHhQ0j+AbYD7cuv7AHB5RPwTWCLp5lS+LbADMDutZ1Wyy3lXs4Ok7wDDgWHADRXqzUwxvirpFrJ7zVwH/IekD5BdWmYk0JJVios2AAAB1UlEQVS2oaM+kq5Jz8PIktyvc+3gkwQMcOKwge+HwN3ARbmyFaSj6TTInb/dZ36g+83c9Jus/H9YBdg7lyBIy4LskuNvFRWMsavr+giYHxF7F1wGwMXA4RFxr6QTgdaC6wtgHLARsHtEvJGuBrwmlbdhFbJ7UexSIj5rEu6qsgEtXXTvSt7ebbOIrEsJsnssrFZysTcCp3dMpAsAduVWsg9kJG1DdjHAh7qoc4yye5pvwsqjmoeAjSTtneZfTdL23cS1DrA0dXONq1JvbBp/2YAsucwB1iO778gbaSzm3anubcBHUv1hZF1cpHuyPCrpqBSfJO3cTXzWJJw4bDCYDOTPrvo58C+S7gT25O1HCUWcAYyRdJ+kB4DPVKj3M2BVSfOAK4ATuzh197fAw2RdYRcCf4Ts9qxkl+v+nqR7gXtI4x9VfIPsqsWzgQer1LuTrGvqduDciFgCTE/bdBdZ0nkwxTGH7PLp95Ldg+IusrvfkeqdkuKbTxPcStmK8dVxzQaR9FuW9oj4fol5hkVEezpL61ZgfMd96M264jEOM5siaTTZmMc0Jw3rjo84zMysFI9xmJlZKU4cZmZWihOHmZmV4sRhZmalOHGYmVkpThxmZlbK/wJE5b2xQ3/RygAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "axes = articles.printPage.hist(bins=20)\n",
    "plt.title('Distribution de la numération de page')\n",
    "plt.ylabel(\"Nombre d'articles\")\n",
    "plt.xlabel(\"Numéro de la page\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Nous constatons que la majorité des articles que nous traitons apparaissent en première page du New York Times. Cela peut reveler un biais lors de la selection des articles dans le jeu de données car les premiers articles génère probablement plus de commentaires et sont donc un sujet d'étude plus intéressant."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Etude du rubricage des articles"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Etude des aperçus d'articles\n",
    "\n",
    "La difficulté ici consiste a traiter une colonne dont les valeurs correspondent à du texte. La construction de colonnes artificelles pour chaque terme du vocabulaire est malaisé car ce dernier est de grande taille et la tableau ainsi obtenu est très creux. L'analyse sémantique latente permet de résoudre ce problème en regroupant conjointement les articles possédant les même termes et les termes se trouvant dans les même articles.\n",
    "\n",
    "Afin d'appliquer l'analyse sémantique latente"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['one', 'largest', 'photo', 'display', 'time', 'histori', 'peopl', 'fled', 'boko', 'haram', 'requir', 'effort', 'immens']\n"
     ]
    }
   ],
   "source": [
    "from nltk.tokenize import RegexpTokenizer\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem.porter import PorterStemmer\n",
    "\n",
    "def preprocess_text(texts):\n",
    "    tokenizer = RegexpTokenizer(r'\\w+')\n",
    "    en_stop = set(stopwords.words('english'))\n",
    "    p_stemmer = PorterStemmer()\n",
    "    preprocessed_texts = []\n",
    "    for text in texts:\n",
    "        tokens = tokenizer.tokenize(text)\n",
    "        tokens = [token for token in tokens if token not in en_stop]\n",
    "        tokens = [p_stemmer.stem(token) for token in tokens]\n",
    "        preprocessed_texts.append(tokens)\n",
    "    return preprocessed_texts\n",
    "\n",
    "def prepare_corpus\n",
    "\n",
    "preprocessed_texts = preprocess_text(articles.snippet)\n",
    "print(preprocessed_texts[0])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "One of the largest photo displays in Times history, of people who have fled Boko Haram, required an effort just as immense. 120$\n",
      "['One', 'of', 'the', 'largest', 'photo', 'displays', 'in', 'Times', 'history', 'of', 'people', 'who', 'have', 'fled', 'Boko', 'Haram', 'required', 'an', 'effort', 'just', 'as', 'immense', '120']\n",
      "['one', 'largest', 'photo', 'displays', 'times', 'history', 'people', 'fled', 'boko', 'haram', 'required', 'effort', 'immense', '120']\n"
     ]
    }
   ],
   "source": [
    "sentence = sentence + ' 120$'\n",
    "print(sentence)\n",
    "tokens = tokenizer.tokenize(sentence)\n",
    "print(tokens)\n",
    "tokens = [token.lower() for token in tokens]\n",
    "tokens = [token for token in tokens if token not in stopwords.words('english')]\n",
    "print(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}