CrossDomain_Emotion_Sentiment_POS.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Generic_model.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "ga5ReBiY1yjo",
        "AbYi7hg711eB",
        "ttAjHtO45AI6",
        "SzTT1MLc00GR",
        "-5I83ZoK02ly",
        "lFPxzLii080O"
      ]
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ga5ReBiY1yjo"
      },
      "source": [
        "#Libraries "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jzF1sGQ-1ya3",
        "outputId": "bb19ff0f-5f4a-4f11-984e-fa5b4227b079"
      },
      "source": [
        "# !pip install wordcloud\n",
        "# !pip install nltk\n",
        "#! pip install text2emotion\n",
        "import nltk\n",
        "nltk.download('stopwords')\n",
        "nltk.download('punkt')\n",
        "nltk.download('averaged_perceptron_tagger')\n",
        "nltk.download('vader_lexicon')\n",
        "\n",
        "import pandas as pd \n",
        "import numpy as np \n",
        "import time\n",
        "\n",
        "#For Text Cleaning\n",
        "import nltk\n",
        "from nltk.tokenize import word_tokenize\n",
        "from nltk.stem import PorterStemmer\n",
        "import re\n",
        "from nltk.corpus import stopwords\n",
        "stop = stopwords.words('english')\n",
        "\n",
        "#For EDA\n",
        "from PIL import Image\n",
        "from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "#For text vectorizing\n",
        "from nltk.tokenize import word_tokenize\n",
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "from nltk.tokenize import RegexpTokenizer\n",
        "from gensim import corpora\n",
        "from gensim import models\n",
        "import random\n",
        "\n",
        "#For naives bayes DF\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.naive_bayes import MultinomialNB\n",
        "from sklearn import metrics\n",
        "from sklearn.metrics import classification_report\n",
        "from sklearn.metrics import confusion_matrix   \n",
        "\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "from sklearn.feature_extraction.text import TfidfTransformer\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "\n",
        "from sklearn import model_selection, naive_bayes, svm\n",
        "from sklearn.metrics import accuracy_score\n",
        "from sklearn.preprocessing import LabelEncoder\n",
        "from sklearn.linear_model import SGDClassifier\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "from xgboost import XGBClassifier\n",
        "from sklearn.tree import DecisionTreeClassifier\n",
        "from sklearn.ensemble import AdaBoostClassifier\n",
        "from sklearn.neighbors import KNeighborsClassifier\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "\n",
        "\n",
        "# Emotion Analysis \n",
        "import text2emotion as te\n",
        "\n",
        "# pos tagging \n",
        "from collections import Counter\n",
        "\n",
        "# ploting \n",
        "import seaborn as sn\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "# VADER generation \n",
        "#nltk.download('vader_lexicon')\n",
        "from nltk.sentiment.vader import SentimentIntensityAnalyzer\n"
      ],
      "execution_count": 154,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n",
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n",
            "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
            "[nltk_data]     /root/nltk_data...\n",
            "[nltk_data]   Package averaged_perceptron_tagger is already up-to-\n",
            "[nltk_data]       date!\n",
            "[nltk_data] Downloading package vader_lexicon to /root/nltk_data...\n",
            "[nltk_data]   Package vader_lexicon is already up-to-date!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AbYi7hg711eB"
      },
      "source": [
        "##Required Functions "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nt5aNP-P11IC"
      },
      "source": [
        "# for preprocessing\n",
        "\n",
        "# Function to clean the compiled dataframe\n",
        "def preprocess(review):\n",
        "    review = \" \".join([stemmer.stem(w.lower()) for w in word_tokenize(review) if not w in stop_words])\n",
        "    return review\n",
        "\n",
        "def emotion_detection(sents):\n",
        "    \"\"\"Main algo for convertion for the 5 emotions \"\"\"\n",
        "    sent_emotion = te.get_emotion(sents)\n",
        "    return sent_emotion\n",
        "    \n",
        "\n",
        "# Generating the Emotions \n",
        "def generate_emotions(news_dup):\n",
        "    \n",
        "    \"\"\"Use to generate the dataframe that appends the orginal text and the emotion label vector\"\"\"\n",
        "    \n",
        "    emotion_list = []\n",
        "    for i, row in news_dup.iterrows():\n",
        "        emotion_dict = emotion_detection(row[2])\n",
        "        emotion_dict['text'] = row[2]\n",
        "        emotion_dict['label'] = row[3]\n",
        "        emotion_list.append(emotion_dict)\n",
        "        \n",
        "        \n",
        "    emotion_df = pd.DataFrame(emotion_list)\n",
        "    horizontal_stack = news_dup.merge(emotion_df, how='left', on='text')\n",
        "    horizontal_stack.drop(['index'], inplace=True, axis=1)       \n",
        "            \n",
        "    return horizontal_stack\n",
        "        \n",
        "\n",
        "def generate_pos_tag_dist(df):\n",
        "    \n",
        "    \"\"\"Takes in a df : this dataframe contains the emotions,\n",
        "        and outputs the dataframe with additional pos tags above that have no more than 30% missing \"\"\"\n",
        "        \n",
        "    emotion_text = list(df['text'])\n",
        "    counts = []\n",
        "    for sentences in emotion_text:\n",
        "        tokens = nltk.word_tokenize(sentences)    \n",
        "        tags = nltk.pos_tag(tokens)\n",
        "        counts.append(Counter( tag for word,  tag in tags))\n",
        "\n",
        "    # creating the pos count feature and setting non rare pos tag features ,\n",
        "    df_post_dist = pd.DataFrame.from_records(counts)\n",
        "    df_post_dist_non_null = df_post_dist.loc[:,df_post_dist.columns[df_post_dist.isnull().mean() < 0.7]].reset_index()\n",
        "    df_post_dist_non_null.fillna(0,inplace=True)\n",
        "    \n",
        "    # combining the dataframe \n",
        "    return df_post_dist_non_null\n",
        "\n",
        "# generating Vader https://towardsdatascience.com/sentimental-analysis-using-vader-a3415fef7664\n",
        "def vader_score_generation(emo_pos_df):\n",
        "    \n",
        "    \"\"\"Generates the vader scores for the dataframe to create neu pos neg tags base on the text \"\"\"\n",
        "    sid = SentimentIntensityAnalyzer()\n",
        "    polarity_Score = emo_pos_df['text'].apply(lambda review: sid.polarity_scores(review))\n",
        "    df_p_scores = polarity_Score.apply(pd.Series).reset_index() # generating the scores and appending it the dataframe end \n",
        "    emo_pos_dist_pscore =  emo_pos_df.merge(df_p_scores, left_on='index', right_on='index')\n",
        "    return emo_pos_dist_pscore\n",
        "\n",
        "def zerolistmaker(n):\n",
        "    listofzeros = [0] * n\n",
        "    return listofzeros\n",
        "\n",
        "def group_pos_features(pos_tagged_df):\n",
        "\n",
        "    \"\"\"This function takes in the POS feature dataframe and group POS tags together.\n",
        "      Comments means that the POS tags in this data set is not prevelant as they either are not present or contain too much missing values\n",
        "\n",
        "      Unpresent POS tags  for now it is known to be insignificant as mentioned in the research paper :\n",
        "          pos_tagged_df['group_e'] = pos_tagged_df['EX']\n",
        "          pos_tagged_df['NR'] + pos_tagged_df['NPS']\n",
        "          pos_tagged_df['group_p'] = pos_tagged_df['PDT'] + pos_tagged_df['POS'] + pos_tagged_df['PP']\n",
        "          pos_tagged_df['group_t'] = pos_tagged_df['TO']\n",
        "          pos_tagged_df['group_u'] = pos_tagged_df['UH'] \n",
        "          pos_tagged_df['group_w'] = pos_tagged_df['WDT'] + pos_tagged_df['WP'] + pos_tagged_df['WP$'] + pos_tagged_df['WRB']\n",
        "          # Could be a future work to include using different POS tagging dictionaries\n",
        "     \"\"\" \n",
        "    #pos_group = ['group_c','group_d','group_f','group_i','group_j','group_m','group_n','group_r','group_v']\n",
        "    pos_list = ['CC','CD','DT','FW','IN','JJ','JJR','JJS','MD','NN', 'NNS', 'RBR','RB','VB','VBD', 'VBN','VBP','VBZ']\n",
        "    delete_pos_list = []\n",
        "    pos_dict = {}\n",
        "    for pos_l in pos_list:\n",
        "      \n",
        "      if pos_l in pos_tagged_df.columns:\n",
        "        pos_dict[pos_l] = np.array(list(pos_tagged_df[pos_l]))\n",
        "        delete_pos_list.append(pos_l)\n",
        "      else:\n",
        "        pos_dict[pos_l] = list(zerolistmaker(len(pos_tagged_df)))\n",
        "      \n",
        "    #print(np.sum([pos_dict['CC'],pos_dict['CD']] , axis=0))\n",
        "    pos_tagged_df['group_c'] = np.sum([pos_dict['CC'],pos_dict['CD']], axis=0) \n",
        "    pos_tagged_df['group_d'] = pos_dict['DT'] \n",
        "    pos_tagged_df['group_f'] = pos_dict['FW'] \n",
        "    pos_tagged_df['group_i'] = pos_dict['IN']\n",
        "    pos_tagged_df['group_j'] = np.sum([pos_dict['JJ'],pos_dict['JJR'], pos_dict['JJS']], axis=0)  \n",
        "    \n",
        "    pos_tagged_df['group_m'] = pos_dict['MD']\n",
        "    pos_tagged_df['group_n'] = np.sum([pos_dict['NN'],pos_dict['NNS']], axis=0) \n",
        "    \n",
        "    # changed RBP to RBR \n",
        "    pos_tagged_df['group_r'] = np.sum([pos_dict['RBR'],pos_dict['RB']], axis=0) \n",
        "    pos_tagged_df['group_v'] =  np.sum([pos_dict['VB'],pos_dict['VBD'], pos_dict['VBN'],  pos_dict['VBP'],  pos_dict['VBZ']], axis=0)  \n",
        "\n",
        "    return pos_tagged_df, delete_pos_list\n",
        "\n",
        "\n",
        "  "
      ],
      "execution_count": 156,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lazbmb6e1x2E"
      },
      "source": [
        "Data Preprocessing and Cleaning "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n_J15CNcz-b7"
      },
      "source": [
        "# Data Reading and Cleaning \n",
        "maindf = pd.read_excel('sample_data/other_domain_text.xlsx')\n",
        "stop_words = set(stopwords.words('english'))\n",
        "stemmer = PorterStemmer()\n",
        "\n",
        "#The bottom two lines will remove all stop words and stem the column text, takes very long do not run\n",
        "maindf['text'] = maindf.apply(lambda x: preprocess(x['text']), axis=1)\n",
        "\n",
        "#Export the new cleaned data into excel file\n",
        "maindf.to_excel(r'sample_data/next_100.xlsx', index = False, header=True)\n",
        "\n"
      ],
      "execution_count": 176,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "B8Og7CFH0sTh"
      },
      "source": [
        "#Feature Engineering "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ttAjHtO45AI6"
      },
      "source": [
        "##Emotion Analysis "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4YOpzg0M5B2l"
      },
      "source": [
        "news  = pd.read_excel('sample_data/next_100.xlsx')\n",
        "news_dup = news.copy()\n",
        "emp_df = generate_emotions(news_dup)\n",
        "emp_df.drop_duplicates(inplace=True)\n",
        "emp_df.to_excel('sample_data/emotion_analysis_final.xlsx')"
      ],
      "execution_count": 177,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SzTT1MLc00GR"
      },
      "source": [
        "##Pos Tagging and Vader Text "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AcaN5bqn00ny"
      },
      "source": [
        "emp_df = pd.read_excel('sample_data/emotion_analysis_final.xlsx')\n",
        "\n",
        "# Generating the POS tag dist \n",
        "emo_pos_df = generate_pos_tag_dist(emp_df)\n",
        "emp_df = emp_df.reset_index()\n",
        "emp_df = emp_df[emp_df.columns.drop(list(emp_df.filter(regex='index')))]\n",
        "emp_df = emp_df.rename(columns={emp_df.columns[0]: 'index'})\n",
        "comb_emo_pos_dist =  emp_df.merge(emo_pos_df, left_on='index', right_on='index')\n",
        "\n",
        "# genrating vader text \n",
        "emo_pos_dist_pscore = vader_score_generation(comb_emo_pos_dist)\n",
        "emo_pos_dist_pscore.to_excel('sample_data/emo_pos_dist_pscore.xlsx')\n"
      ],
      "execution_count": 178,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-5I83ZoK02ly"
      },
      "source": [
        "##POS Grouping analysis "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "efy493rA0uHe"
      },
      "source": [
        "# Generating of Pos Grouping Features \n",
        "emotional_df_stack = pd.read_excel('sample_data/emo_pos_dist_pscore.xlsx')\n",
        "del emotional_df_stack['compound']\n",
        "\n",
        "emotional_df_stack_groupped_pos, pos_tags = group_pos_features(emotional_df_stack)\n",
        "emotional_df_stack_groupped_pos.drop(columns=pos_tags, inplace=True)\n",
        "emotional_df_stack_groupped_pos = emotional_df_stack_groupped_pos.loc[:, (emotional_df_stack_groupped_pos != 0).any(axis=0)]\n",
        "emotional_df_stack_groupped_pos.to_excel('sample_data/pos_groupings_vader_emotion.xlsx')\n",
        "\n",
        "\n"
      ],
      "execution_count": 179,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lFPxzLii080O"
      },
      "source": [
        "## Entity Regconition Features "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3hBb0WeT1F3L",
        "outputId": "fd547a0a-6869-49d1-b9df-1b5b57b63717"
      },
      "source": [
        "import sys\n",
        "import json\n",
        "import ast\n",
        "nltk.download('punkt')\n",
        "nltk.download('averaged_perceptron_tagger')\n",
        "nltk.download('maxent_ne_chunker')\n",
        "nltk.download('words')\n",
        "emo_posgroup_vader_er = emotional_df_stack_groupped_pos.copy()"
      ],
      "execution_count": 180,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n",
            "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
            "[nltk_data]     /root/nltk_data...\n",
            "[nltk_data]   Package averaged_perceptron_tagger is already up-to-\n",
            "[nltk_data]       date!\n",
            "[nltk_data] Downloading package maxent_ne_chunker to\n",
            "[nltk_data]     /root/nltk_data...\n",
            "[nltk_data]   Package maxent_ne_chunker is already up-to-date!\n",
            "[nltk_data] Downloading package words to /root/nltk_data...\n",
            "[nltk_data]   Package words is already up-to-date!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xAOQ8h1dvIPx"
      },
      "source": [
        "def getEntityDic(df):\n",
        "  entity_dic = {}\n",
        "  counter = 1\n",
        "  total_records = df.shape[0]\n",
        "  for index, row in df.iterrows():\n",
        "    text = row['text'] + \" \"+ row['title']\n",
        "    sys.stdout.write('\\rCompletion progress: ' + str(counter) + ' of ' + str(total_records) + \" articles\")\n",
        "    sys.stdout.flush()\n",
        "\n",
        "    ## Put full stop behind each sentence\n",
        "    new = []\n",
        "    if \"\\n\" in text:\n",
        "        text = text.replace(\" \\n\", \". \")\n",
        "    text_list = text.split(\". \")\n",
        "    for x in text_list:\n",
        "        new.append(x + \".\")\n",
        "    \n",
        "    ## Cycle through each sentence\n",
        "    for x in new:\n",
        "        sent_tokens = word_tokenize(x)\n",
        "        tagged_sent = nltk.pos_tag(sent_tokens)\n",
        "        ne_tree = nltk.ne_chunk(tagged_sent)\n",
        "        ne_list = extract_ne_from_tree(ne_tree)\n",
        "        \n",
        "        ## Insert into dictionary\n",
        "        for y in ne_list:\n",
        "            if y[0] not in entity_dic:\n",
        "                entity_dic[y[0]] = {}\n",
        "            if y[0] in entity_dic:\n",
        "                string = y[1].lower()\n",
        "                if string not in entity_dic[y[0]]:\n",
        "                    entity_dic[y[0]][string] = 0\n",
        "                if string in entity_dic[y[0]]:\n",
        "                    entity_dic[y[0]][string] += 1\n",
        "    counter += 1\n",
        "  return entity_dic"
      ],
      "execution_count": 181,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DB0BvvCCvISI"
      },
      "source": [
        "def extract_ne_from_tree ( tree ):\n",
        "    result = []\n",
        "    for s in tree.subtrees():\n",
        "        label = s.label()\n",
        "        if (label == 'PERSON' or label == 'ORGANIZATION' or label == 'GPE'):\n",
        "            leaves = s.leaves()\n",
        "            ne = ''\n",
        "            for l in leaves:\n",
        "                ne = ne + ' ' + l[0]\n",
        "            result.append((label, ne[1:]))\n",
        "    return result"
      ],
      "execution_count": 182,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kiW1LzcHvIUO"
      },
      "source": [
        "def addEntitiesToDataframe2(df):\n",
        "\n",
        "    entity_dic = {}\n",
        "    counter = 1\n",
        "    total_records = df.shape[0]\n",
        "    i = 0\n",
        "    for index, row in df.iterrows():\n",
        "        entity_dic['PERSON'] = {}\n",
        "        entity_dic['ORGANIZATION'] = {}\n",
        "        entity_dic['GPE'] = {}\n",
        "        text = row['text'] + \" \"+ row['title']\n",
        "        sys.stdout.write('\\rCompletion progress: ' + str(counter) + ' of ' + str(total_records) + \" articles\")\n",
        "        sys.stdout.flush()\n",
        "\n",
        "        ## Put full stop behind each sentence\n",
        "        new = []\n",
        "        if \"\\n\" in text:\n",
        "          text = text.replace(\" \\n\", \". \")\n",
        "        text_list = text.split(\". \")\n",
        "        for x in text_list:\n",
        "            new.append(x + \".\")\n",
        "\n",
        "        ## Cycle through each sentence\n",
        "        for x in new:\n",
        "            sent_tokens = word_tokenize(x)\n",
        "            tagged_sent = nltk.pos_tag(sent_tokens)\n",
        "            ne_tree = nltk.ne_chunk(tagged_sent)\n",
        "            ne_list = extract_ne_from_tree(ne_tree)\n",
        "\n",
        "            ## Insert into dictionary\n",
        "            for y in ne_list:\n",
        "                if y[0] not in entity_dic:\n",
        "                    entity_dic[y[0]] = {}\n",
        "                if y[0] in entity_dic:\n",
        "                    string = y[1].lower()\n",
        "                    if string not in entity_dic[y[0]]:\n",
        "                        entity_dic[y[0]][string] = 0\n",
        "                    if string in entity_dic[y[0]]:\n",
        "                        entity_dic[y[0]][string] += 1\n",
        "        counter += 1\n",
        "\n",
        "        # Get df index\n",
        "        ind = index_list[i]\n",
        "\n",
        "        # get summation values\n",
        "        total_p = 0\n",
        "        for x in entity_dic['PERSON']:\n",
        "          total_p += entity_dic['PERSON'][x]\n",
        "\n",
        "        total_o = 0\n",
        "        for x in entity_dic['ORGANIZATION']:\n",
        "          total_o += entity_dic['ORGANIZATION'][x]\n",
        "\n",
        "        total_g = 0\n",
        "        for x in entity_dic['GPE']:\n",
        "          total_g += entity_dic['GPE'][x]\n",
        "        \n",
        "        # insert into DF\n",
        "        df.loc[ind:ind+1, 'person entities'] = str(entity_dic['PERSON'])\n",
        "        df.loc[ind:ind+1, 'organisation entities'] = str(entity_dic['ORGANIZATION'])\n",
        "        df.loc[ind:ind+1, 'location entities'] = str(entity_dic['GPE'])\n",
        "        df.loc[ind:ind+1, 'person entity count'] = total_p\n",
        "        df.loc[ind:ind+1, 'organisation entity count'] = total_o\n",
        "        df.loc[ind:ind+1, 'location entity count'] = total_g\n",
        "        entity_dic = {}\n",
        "        i+=1\n",
        "        \n",
        "    return df"
      ],
      "execution_count": 183,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MjOHHqW1vIW7"
      },
      "source": [
        "def attachTopEntities(df, fakeTop, realTop):\n",
        "  store = []\n",
        "  for x in trueTop:\n",
        "    for y in trueTop[x]:\n",
        "      store.append((\"(\" + str(x) + \")\" + y))\n",
        "\n",
        "  for x in fakeTop:\n",
        "    for y in fakeTop[x]:\n",
        "      item = (\"(\" + str(x) + \")\" + y)\n",
        "      if item not in store:\n",
        "        store.append(item)\n",
        "\n",
        "  for x in store:\n",
        "    df[x] = 0\n",
        "\n",
        "  return df, store"
      ],
      "execution_count": 184,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RZUxv8OdvIY1"
      },
      "source": [
        "def getTopEntityDic(entity_dic, n):\n",
        "  print()\n",
        "  bigstore = {}\n",
        "  top10entities = {}\n",
        "  for x in entity_dic:\n",
        "    first = str(x)[0] + \"_\"\n",
        "    bigstore[x] = []\n",
        "    print(x)\n",
        "    print(\"----------------\")\n",
        "    sorted_dic = sorted(entity_dic[x], key=entity_dic[x].get, reverse=True)\n",
        "    count = 0\n",
        "    for y in sorted_dic:\n",
        "      top10entities[y] = entity_dic[x][y]\n",
        "      print(y.ljust(40), \":\" + str(entity_dic[x][y]))\n",
        "      count += 1\n",
        "      if count == n:\n",
        "          break\n",
        "    top10 = dict(sorted(top10entities.items(), key=lambda item: item[1], reverse = True))\n",
        "    count = 0\n",
        "    result = []\n",
        "    for z in top10:\n",
        "      count += 1\n",
        "      result.append(str(z))\n",
        "      if count == n:\n",
        "        bigstore[x] = result \n",
        "        break\n",
        "    print()\n",
        "\n",
        "  return bigstore"
      ],
      "execution_count": 185,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1ba9A2nmvIa3"
      },
      "source": [
        "def getFinalEntityDF(finalEntitydf, store):\n",
        "  total_records = finalEntitydf.shape[0]\n",
        "  for index, row in finalEntitydf.iterrows():\n",
        "    sys.stdout.write('\\rCompletion progress: ' + str(index+1) + ' of ' + str(total_records) + \" articles\")\n",
        "    sys.stdout.flush()\n",
        "    # Person Entities\n",
        "    json_string = row['person entities']\n",
        "    dicp = ast.literal_eval(json_string)\n",
        "    for x in dicp:\n",
        "      item = \"(PERSON)\" + str(x)\n",
        "      if item in store:\n",
        "        finalEntitydf.loc[index, item] = dicp[x]\n",
        "\n",
        "    # Location Entities\n",
        "    json_string = row['location entities']\n",
        "    dicp = ast.literal_eval(json_string)\n",
        "    for x in dicp:\n",
        "      item = \"(GPE)\" + str(x)\n",
        "      if item in store:\n",
        "        finalEntitydf.loc[index, item] = dicp[x]\n",
        "    \n",
        "    # GPE Entities\n",
        "    json_string = row['organisation entities']\n",
        "    dicp = ast.literal_eval(json_string)\n",
        "    for x in dicp:\n",
        "      item = \"(ORGANISATION)\" + str(x)\n",
        "      if item in store:\n",
        "        finalEntitydf.loc[index, item] = dicp[x]\n",
        "\n",
        "  return finalEntitydf"
      ],
      "execution_count": 186,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LEGrxSBJ9viA"
      },
      "source": [
        "er_main_df= pd.read_excel('sample_data/covid_real_fake.xlsx')"
      ],
      "execution_count": 187,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "C8qsWd8svIdK",
        "outputId": "dd3e9b67-3f05-43a7-c11f-cf2480cd6fcb"
      },
      "source": [
        "# Fake News\n",
        "df1fake = er_main_df[er_main_df['label'] == 0]                                 # 1) Get fake news article dataframe\n",
        "index_list = df1fake.index.tolist()                                 # 2) Get index of fake dataframe\n",
        "# fakeEntityDic = getEntityDic(df1fake)                               # 3) Get sorted dict of {fake entities : count}\n",
        "# fakeTop = getTopEntityDic(fakeEntityDic, 10)                        # 4) Get dic of {entitiy type : [sorted list of top 10 entities]}\n",
        "df1_fakeEntities = addEntitiesToDataframe2(df1fake)                 # 5) Get DF with all entities, entity type count per article"
      ],
      "execution_count": 188,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Completion progress: 13 of 129 articles"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/pandas/core/indexing.py:1596: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  self.obj[key] = _infer_fill_value(value)\n",
            "/usr/local/lib/python3.7/dist-packages/pandas/core/indexing.py:1763: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  isetter(loc, value)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Completion progress: 129 of 129 articles"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GxFCnaUzvZR9",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f64713de-cf9f-4658-d7f2-11ba9b0734e8"
      },
      "source": [
        "# Real News\n",
        "df1true = er_main_df[er_main_df['label'] == 1]                                 # 1) Get real news article dataframe\n",
        "index_list = df1true.index.tolist()                                 # 2) Get index of real dataframe\n",
        "# trueEntityDic = getEntityDic(df1true)                               # 3) Get sorted dict of {real entities : count}\n",
        "# trueTop = getTopEntityDic(trueEntityDic, 10)                        # 4) Get dic of {entitiy type : [sorted list of top 10 entities]}\n",
        "df1_trueEntities = addEntitiesToDataframe2(df1true)                 # 5) Get DF of entities, entity count per article"
      ],
      "execution_count": 189,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Completion progress: 14 of 70 articles"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/pandas/core/indexing.py:1596: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  self.obj[key] = _infer_fill_value(value)\n",
            "/usr/local/lib/python3.7/dist-packages/pandas/core/indexing.py:1763: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  isetter(loc, value)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Completion progress: 70 of 70 articles"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_AM6wwFTvZUF",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 193
        },
        "outputId": "b2891964-9e4b-4024-dcd6-d89c1da444a2"
      },
      "source": [
        "df1_trueEntities.head()"
      ],
      "execution_count": 190,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>index</th>\n",
              "      <th>title</th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "      <th>person entities</th>\n",
              "      <th>organisation entities</th>\n",
              "      <th>location entities</th>\n",
              "      <th>person entity count</th>\n",
              "      <th>organisation entity count</th>\n",
              "      <th>location entity count</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>covid</td>\n",
              "      <td>The supposed phrase of the daughter of the pr...</td>\n",
              "      <td>1</td>\n",
              "      <td>{'banco santander': 1}</td>\n",
              "      <td>{}</td>\n",
              "      <td>{'portugal': 1}</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>covid</td>\n",
              "      <td>Smokers are less likely to be hospitalized fo...</td>\n",
              "      <td>1</td>\n",
              "      <td>{}</td>\n",
              "      <td>{}</td>\n",
              "      <td>{}</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>covid</td>\n",
              "      <td>A photo of the room of a hotel in AlmerÃ­a, S...</td>\n",
              "      <td>1</td>\n",
              "      <td>{}</td>\n",
              "      <td>{}</td>\n",
              "      <td>{'spain': 1}</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>covid</td>\n",
              "      <td>Drugstores in France are forbidden to buy and...</td>\n",
              "      <td>1</td>\n",
              "      <td>{}</td>\n",
              "      <td>{}</td>\n",
              "      <td>{'france': 1}</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>covid</td>\n",
              "      <td>It is recommended to disinfect disposable mas...</td>\n",
              "      <td>1</td>\n",
              "      <td>{}</td>\n",
              "      <td>{}</td>\n",
              "      <td>{}</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   index  title  ... organisation entity count  location entity count\n",
              "0      0  covid  ...                       0.0                    1.0\n",
              "1      1  covid  ...                       0.0                    0.0\n",
              "2      2  covid  ...                       0.0                    1.0\n",
              "3      3  covid  ...                       0.0                    1.0\n",
              "4      4  covid  ...                       0.0                    0.0\n",
              "\n",
              "[5 rows x 10 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 190
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FTLdIT5WvZWV",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 152
        },
        "outputId": "12e05795-9a2b-40f3-d5d5-66dcdd95d9ab"
      },
      "source": [
        "# Combining Real and Fake News Dataframes\n",
        "finalEntitydf = pd.concat([df1_trueEntities, df1_fakeEntities])     # Union fake & real dataframes\n",
        "finalEntitydf.sort_index(ascending=True, inplace=True)              # Re-index them to original state\n",
        "\n",
        "# Attaching Top Entities as Column Headers to Main Dataframe                   \n",
        "finalEntitydf, store = attachTopEntities(finalEntitydf, fakeTop, trueTop)       \n",
        "\n",
        "# Get Final DF with Top Entity counts\n",
        "mainEntitydf = getFinalEntityDF(finalEntitydf, store)\n",
        "mainEntitydf.head(3)"
      ],
      "execution_count": 191,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Completion progress: 199 of 199 articles"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>index</th>\n",
              "      <th>title</th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "      <th>person entities</th>\n",
              "      <th>organisation entities</th>\n",
              "      <th>location entities</th>\n",
              "      <th>person entity count</th>\n",
              "      <th>organisation entity count</th>\n",
              "      <th>location entity count</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>covid</td>\n",
              "      <td>The supposed phrase of the daughter of the pr...</td>\n",
              "      <td>1</td>\n",
              "      <td>{'banco santander': 1}</td>\n",
              "      <td>{}</td>\n",
              "      <td>{'portugal': 1}</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>covid</td>\n",
              "      <td>Smokers are less likely to be hospitalized fo...</td>\n",
              "      <td>1</td>\n",
              "      <td>{}</td>\n",
              "      <td>{}</td>\n",
              "      <td>{}</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>covid</td>\n",
              "      <td>A photo of the room of a hotel in AlmerÃ­a, S...</td>\n",
              "      <td>1</td>\n",
              "      <td>{}</td>\n",
              "      <td>{}</td>\n",
              "      <td>{'spain': 1}</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   index  title  ... organisation entity count  location entity count\n",
              "0      0  covid  ...                       0.0                    1.0\n",
              "1      1  covid  ...                       0.0                    0.0\n",
              "2      2  covid  ...                       0.0                    1.0\n",
              "\n",
              "[3 rows x 10 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 191
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pqUEmTc20yfW"
      },
      "source": [
        "del mainEntitydf['person entities']\n",
        "del mainEntitydf['organisation entities']\n",
        "del mainEntitydf['location entities']\n",
        "del emotional_df_stack_groupped_pos['Unnamed: 0']"
      ],
      "execution_count": 192,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uThNDObr-vSh"
      },
      "source": [
        "horizontal_stack = emotional_df_stack_groupped_pos.merge(mainEntitydf, how='left', on='index')\n",
        "horizontal_stack = horizontal_stack.loc[:,~horizontal_stack.columns.duplicated()]\n",
        "del horizontal_stack['label']\n",
        "del horizontal_stack['label_x']\n",
        "del horizontal_stack['title_y']\n",
        "del horizontal_stack['text_y']\n",
        "del horizontal_stack[',']\n",
        "del horizontal_stack['.']"
      ],
      "execution_count": 193,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cn9xrJPwARgG"
      },
      "source": [
        "horizontal_stack.to_excel('final_feature_engineered.xlsx')"
      ],
      "execution_count": 194,
      "outputs": []
    }
  ]
}