flipkart review  (1).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "9fd17f26-864f-4a8a-8215-656660e8dbc8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "b73363e3-9eb2-4b5e-bed2-97af91db0e44",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "for dirname, _, filenames in os.walk('/kaggle/input'):\n",
    "    for filename in filenames:\n",
    "        print(os.path.join(dirname, filename))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "fe79ef3a-cc8f-4f9c-80f0-9af64bfbcf0a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "%matplotlib inline\n",
    "from matplotlib import style\n",
    "style.use('ggplot')\n",
    "import csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "aa879304-2d2a-4528-af67-a512b73c2b28",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>product_name</th>\n",
       "      <th>product_price</th>\n",
       "      <th>Rate</th>\n",
       "      <th>Review</th>\n",
       "      <th>Summary</th>\n",
       "      <th>Sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Candes 12 L Room/Personal Air Cooler??????(Whi...</td>\n",
       "      <td>3999</td>\n",
       "      <td>5</td>\n",
       "      <td>super!</td>\n",
       "      <td>great cooler excellent air flow and for this p...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Candes 12 L Room/Personal Air Cooler??????(Whi...</td>\n",
       "      <td>3999</td>\n",
       "      <td>5</td>\n",
       "      <td>awesome</td>\n",
       "      <td>best budget 2 fit cooler nice cooling</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Candes 12 L Room/Personal Air Cooler??????(Whi...</td>\n",
       "      <td>3999</td>\n",
       "      <td>3</td>\n",
       "      <td>fair</td>\n",
       "      <td>the quality is good but the power of air is de...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Candes 12 L Room/Personal Air Cooler??????(Whi...</td>\n",
       "      <td>3999</td>\n",
       "      <td>1</td>\n",
       "      <td>useless product</td>\n",
       "      <td>very bad product its a only a fan</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Candes 12 L Room/Personal Air Cooler??????(Whi...</td>\n",
       "      <td>3999</td>\n",
       "      <td>3</td>\n",
       "      <td>fair</td>\n",
       "      <td>ok ok product</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        product_name product_price Rate  \\\n",
       "0  Candes 12 L Room/Personal Air Cooler??????(Whi...          3999    5   \n",
       "1  Candes 12 L Room/Personal Air Cooler??????(Whi...          3999    5   \n",
       "2  Candes 12 L Room/Personal Air Cooler??????(Whi...          3999    3   \n",
       "3  Candes 12 L Room/Personal Air Cooler??????(Whi...          3999    1   \n",
       "4  Candes 12 L Room/Personal Air Cooler??????(Whi...          3999    3   \n",
       "\n",
       "            Review                                            Summary  \\\n",
       "0           super!  great cooler excellent air flow and for this p...   \n",
       "1          awesome              best budget 2 fit cooler nice cooling   \n",
       "2             fair  the quality is good but the power of air is de...   \n",
       "3  useless product                  very bad product its a only a fan   \n",
       "4             fair                                      ok ok product   \n",
       "\n",
       "  Sentiment  \n",
       "0  positive  \n",
       "1  positive  \n",
       "2  positive  \n",
       "3  negative  \n",
       "4   neutral  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df=pd.read_csv(\"Dataset-SA.csv\",encoding=\"unicode_escape\")\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "6efbed0f-38cc-4e8f-8150-2acc25b4020e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The dataset has 205052 number of rows and 6 number of columns.\n"
     ]
    }
   ],
   "source": [
    "print(f'The dataset has {df.shape[0]} number of rows and {df.shape[1]} number of columns.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "a5667281-4315-44ce-b375-e98f7aba8b47",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['product_name', 'product_price', 'Rate', 'Review', 'Summary',\n",
       "       'Sentiment'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "cb76bc44-c91f-491a-82cb-e24a27d8d13f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "958"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['product_name'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "02f52ac1-b261-4784-a141-5fd0f9e63108",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 205052 entries, 0 to 205051\n",
      "Data columns (total 6 columns):\n",
      " #   Column         Non-Null Count   Dtype \n",
      "---  ------         --------------   ----- \n",
      " 0   product_name   205052 non-null  object\n",
      " 1   product_price  205052 non-null  object\n",
      " 2   Rate           205052 non-null  object\n",
      " 3   Review         180388 non-null  object\n",
      " 4   Summary        205041 non-null  object\n",
      " 5   Sentiment      205052 non-null  object\n",
      "dtypes: object(6)\n",
      "memory usage: 9.4+ MB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "27500160-ccfb-4d67-9e38-f0fbfeff2185",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "product_name     0\n",
       "product_price    0\n",
       "Rate             0\n",
       "Review           0\n",
       "Summary          0\n",
       "Sentiment        0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "e48cb35d-d0aa-4bf7-b4af-4956a1583bc0",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "bafbc83c-653f-4892-baaf-da2b7dc6ac8e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "super! \n",
      "\n",
      "awesome \n",
      "\n",
      "fair \n",
      "\n"
     ]
    }
   ],
   "source": [
    "for review in df['Review'][0:3]:\n",
    "    print(review,'\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "d51753c9-bfaa-4927-a5b5-fd7f8f14b63e",
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "import nltk\n",
    "import re\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem.snowball import SnowballStemmer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "d5174dc3-1afe-438c-a61f-3f93476aa96c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Collecting bs4\n",
      "  Downloading bs4-0.0.1.tar.gz (1.1 kB)\n",
      "  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hCollecting beautifulsoup4 (from bs4)\n",
      "  Downloading beautifulsoup4-4.12.2-py3-none-any.whl (142 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m143.0/143.0 kB\u001b[0m \u001b[31m3.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hCollecting soupsieve>1.2 (from beautifulsoup4->bs4)\n",
      "  Downloading soupsieve-2.5-py3-none-any.whl (36 kB)\n",
      "Building wheels for collected packages: bs4\n",
      "  Building wheel for bs4 (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for bs4: filename=bs4-0.0.1-py3-none-any.whl size=1257 sha256=6632fc6fdca1b90725f5292ce6e5a0163fc9ee9a75e4fbd7994da45da9ac0f40\n",
      "  Stored in directory: /home/u203083/.cache/pip/wheels/73/2b/cb/099980278a0c9a3e57ff1a89875ec07bfa0b6fcbebb9a8cad3\n",
      "Successfully built bs4\n",
      "Installing collected packages: soupsieve, beautifulsoup4, bs4\n",
      "Successfully installed beautifulsoup4-4.12.2 bs4-0.0.1 soupsieve-2.5\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install bs4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "eada29d9-bdb7-47c0-8845-f891208ee719",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Collecting nltk\n",
      "  Downloading nltk-3.8.1-py3-none-any.whl (1.5 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.5/1.5 MB\u001b[0m \u001b[31m3.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hCollecting click (from nltk)\n",
      "  Downloading click-8.1.7-py3-none-any.whl (97 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m97.9/97.9 kB\u001b[0m \u001b[31m2.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: joblib in /glob/development-tools/versions/oneapi/2023.2.0.1/oneapi/intelpython/python3.9/lib/python3.9/site-packages (from nltk) (1.2.0)\n",
      "Collecting regex>=2021.8.3 (from nltk)\n",
      "  Downloading regex-2023.8.8-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (771 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m771.4/771.4 kB\u001b[0m \u001b[31m3.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: tqdm in /glob/development-tools/versions/oneapi/2023.2.0.1/oneapi/intelpython/python3.9/lib/python3.9/site-packages (from nltk) (4.65.0)\n",
      "Installing collected packages: regex, click, nltk\n",
      "Successfully installed click-8.1.7 nltk-3.8.1 regex-2023.8.8\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "8008b66a-17f7-4a2d-8cd8-56ef79e8ceec",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Collecting scikit-learn-intelex\n",
      "  Downloading scikit_learn_intelex-2023.2.1-py39-none-manylinux1_x86_64.whl (128 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m128.7/128.7 kB\u001b[0m \u001b[31m2.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hCollecting daal4py==2023.2.1 (from scikit-learn-intelex)\n",
      "  Downloading daal4py-2023.2.1-py39-none-manylinux1_x86_64.whl (14.1 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m14.1/14.1 MB\u001b[0m \u001b[31m3.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: scikit-learn>=0.22 in /glob/development-tools/versions/oneapi/2023.2.0.1/oneapi/intelpython/python3.9/lib/python3.9/site-packages (from scikit-learn-intelex) (1.2.2)\n",
      "Collecting daal==2023.2.1 (from daal4py==2023.2.1->scikit-learn-intelex)\n",
      "  Downloading daal-2023.2.1-py2.py3-none-manylinux1_x86_64.whl (75.3 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.3/75.3 MB\u001b[0m \u001b[31m2.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: numpy>=1.19 in /glob/development-tools/versions/oneapi/2023.2.0.1/oneapi/intelpython/python3.9/lib/python3.9/site-packages (from daal4py==2023.2.1->scikit-learn-intelex) (1.24.3)\n",
      "Collecting tbb==2021.* (from daal==2023.2.1->daal4py==2023.2.1->scikit-learn-intelex)\n",
      "  Downloading tbb-2021.10.0-py2.py3-none-manylinux1_x86_64.whl (4.4 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.4/4.4 MB\u001b[0m \u001b[31m3.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: scipy>=1.3.2 in /glob/development-tools/versions/oneapi/2023.2.0.1/oneapi/intelpython/python3.9/lib/python3.9/site-packages (from scikit-learn>=0.22->scikit-learn-intelex) (1.10.1)\n",
      "Requirement already satisfied: joblib>=1.1.1 in /glob/development-tools/versions/oneapi/2023.2.0.1/oneapi/intelpython/python3.9/lib/python3.9/site-packages (from scikit-learn>=0.22->scikit-learn-intelex) (1.2.0)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /glob/development-tools/versions/oneapi/2023.2.0.1/oneapi/intelpython/python3.9/lib/python3.9/site-packages (from scikit-learn>=0.22->scikit-learn-intelex) (3.1.0)\n",
      "Installing collected packages: tbb, daal, daal4py, scikit-learn-intelex\n",
      "Successfully installed daal-2023.2.1 daal4py-2023.2.1 scikit-learn-intelex-2023.2.1 tbb-2021.10.0\n"
     ]
    }
   ],
   "source": [
    "!pip install scikit-learn-intelex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "c3d98b37-7727-4f1f-8d9a-1672cad0c0d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "import nltk\n",
    "import re\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem.snowball import SnowballStemmer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "c58cf04c-828c-40a1-821d-2506688dd28d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     /home/u203083/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "nltk.download('stopwords')\n",
    "\n",
    "stops = set(stopwords.words('english'))\n",
    "\n",
    "stemmer = SnowballStemmer('english') \n",
    "def review_to_words(raw_review):\n",
    "    # 1. Delete HTML \n",
    "    review_text = BeautifulSoup(raw_review, 'html.parser').get_text()\n",
    "    # 2. Make a space\n",
    "    letters_only = re.sub('[^a-zA-Z]', ' ', review_text)\n",
    "    # 3. lower letters\n",
    "    words = letters_only.lower().split()\n",
    "    # 5. Stopwords \n",
    "    meaningful_words = [w for w in words if not w in stops]\n",
    "    # 6. Stemming\n",
    "    stemming_words = [stemmer.stem(w) for w in meaningful_words]\n",
    "    # 7. space join words\n",
    "    return( ' '.join(stemming_words))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "1fedc4d1-9bec-48c5-8fbc-eeb3bc2279b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def review_to_words(raw_review):\n",
    "    # 1. Delete HTML \n",
    "    review_text = BeautifulSoup(raw_review, 'html.parser').get_text()\n",
    "    # 2. Make a space\n",
    "    letters_only = re.sub('[^a-zA-Z]', ' ', review_text)\n",
    "    # 3. lower letters\n",
    "    words = letters_only.lower().split()\n",
    "    # 5. Stopwords \n",
    "    meaningful_words = [w for w in words if not w in stops]\n",
    "    # 6. Stemming\n",
    "    stemming_words = [stemmer.stem(w) for w in meaningful_words]\n",
    "    # 7. space join words\n",
    "    return( ' '.join(stemming_words))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "b8025b5b-86a0-4c1c-afff-430848814cd3",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/u203083/tmp/ipykernel_2838637/2607154216.py:3: MarkupResemblesLocatorWarning: The input looks more like a filename than markup. You may want to open this file and pass the filehandle into Beautiful Soup.\n",
      "  review_text = BeautifulSoup(raw_review, 'html.parser').get_text()\n"
     ]
    }
   ],
   "source": [
    "df['Review'] = df['Review'].apply(review_to_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "5cfd8c6c-aa1a-4b85-ac20-748c575caa33",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/intel/inteloneapi/intelpython/latest/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/intel/inteloneapi/intelpython/latest/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n"
     ]
    }
   ],
   "source": [
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "# apply label encoding to the sentiment column\n",
    "encoder = LabelEncoder()\n",
    "df['sentiment_encoded'] = encoder.fit_transform(df['Sentiment'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "a8d74560-7cb6-46a5-99a3-f0e99ce071b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split #import train_test_split\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer #import TfidfVectorizer \n",
    "from sklearn.metrics import confusion_matrix #import confusion_matrix\n",
    "from sklearn.naive_bayes import MultinomialNB #import MultinomialNB\n",
    "from sklearn.ensemble import RandomForestClassifier  #import RandomForestClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "bee7939e-b64d-4830-bf41-edd568226df1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(180379, 1018)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer = TfidfVectorizer()\n",
    "reviews_corpus = vectorizer.fit_transform(df.Review)\n",
    "reviews_corpus.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "36e1c6ab-c7b1-4d1b-b669-a0e3c3682e17",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(180379,)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentiment = df['sentiment_encoded']\n",
    "sentiment.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "f7aa8574-e444-4a0d-b511-66781b2b8cb8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train data shape  (120853, 1018) (120853,)\n",
      "Test data shape  (59526, 1018) (59526,)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train,X_test,Y_train,Y_test = train_test_split(reviews_corpus,sentiment,test_size=0.33,random_state=42)\n",
    "print('Train data shape ',X_train.shape,Y_train.shape)\n",
    "print('Test data shape ',X_test.shape,Y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "59604739-c695-4066-be70-14a00c78bed3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9029163726774855\n",
      "Confusion Matrix\n",
      "[[ 5804   511   493]\n",
      " [   18     7    93]\n",
      " [ 2273  2391 47936]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/intel/inteloneapi/intelpython/latest/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/intel/inteloneapi/intelpython/latest/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/intel/inteloneapi/intelpython/latest/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/intel/inteloneapi/intelpython/latest/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/intel/inteloneapi/intelpython/latest/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/intel/inteloneapi/intelpython/latest/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/intel/inteloneapi/intelpython/latest/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/intel/inteloneapi/intelpython/latest/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/intel/inteloneapi/intelpython/latest/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/intel/inteloneapi/intelpython/latest/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/intel/inteloneapi/intelpython/latest/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/intel/inteloneapi/intelpython/latest/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/intel/inteloneapi/intelpython/latest/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/intel/inteloneapi/intelpython/latest/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n"
     ]
    }
   ],
   "source": [
    "clf = MultinomialNB().fit(X_train, Y_train) #fit the training data\n",
    "\n",
    "pred = clf.predict(X_test) #predict the sentiment for test data\n",
    "\n",
    "print(\"Accuracy: %s\" % str(clf.score(X_test, Y_test))) #check accuracy\n",
    "print(\"Confusion Matrix\") \n",
    "print(confusion_matrix(pred, Y_test)) #print confusion matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "c8b1959e-08bf-4dde-bc16-d861fc94b086",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/intel/inteloneapi/intelpython/latest/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/intel/inteloneapi/intelpython/latest/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9096361253905856\n",
      "Confusion Matrix\n",
      "[[ 6118   524   492]\n",
      " [    0     0     1]\n",
      " [ 1977  2385 48029]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/intel/inteloneapi/intelpython/latest/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/intel/inteloneapi/intelpython/latest/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/intel/inteloneapi/intelpython/latest/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/intel/inteloneapi/intelpython/latest/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/intel/inteloneapi/intelpython/latest/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/intel/inteloneapi/intelpython/latest/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/intel/inteloneapi/intelpython/latest/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/intel/inteloneapi/intelpython/latest/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/intel/inteloneapi/intelpython/latest/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/intel/inteloneapi/intelpython/latest/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n",
      "/opt/intel/inteloneapi/intelpython/latest/lib/python3.9/site-packages/sklearn/utils/validation.py:605: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype):\n",
      "/opt/intel/inteloneapi/intelpython/latest/lib/python3.9/site-packages/sklearn/utils/validation.py:614: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(pd_dtype) or not is_extension_array_dtype(pd_dtype):\n"
     ]
    }
   ],
   "source": [
    "clf = RandomForestClassifier().fit(X_train, Y_train)\n",
    "\n",
    "pred = clf.predict(X_test)\n",
    "\n",
    "print(\"Accuracy: %s\" % str(clf.score(X_test, Y_test)))\n",
    "print(\"Confusion Matrix\")\n",
    "print(confusion_matrix(pred, Y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "33b0beb3-c260-4597-8fee-3148ee1e1579",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (Intel® oneAPI 2023.2)",
   "language": "python",
   "name": "c009-intel_distribution_of_python_3_oneapi-beta05-python"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}