MLP (1).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import pandas as pd\n",
    "df = pd.read_csv(\"F:/Project/application_train.csv\")\n",
    "dum_df = pd.get_dummies(df)\n",
    "dum_df1 = dum_df.drop('TARGET', axis=1)\n",
    "df2=dum_df1.dropna(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import warnings filter\n",
    "from warnings import simplefilter\n",
    "# ignore all future warnings\n",
    "simplefilter(action='ignore', category=FutureWarning)\n",
    "#ignore all Depricated Warning\n",
    "simplefilter(action='ignore', category=DeprecationWarning)\n",
    "simplefilter(action='ignore', category=UserWarning)\n",
    "simplefilter(action='ignore', category=RuntimeWarning)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split \n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import classification_report, accuracy_score\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df2.iloc[:,0:186]\n",
    "y = df.iloc[:,1]\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y,test_size = 0.3, \n",
    "                                                    random_state=2018,\n",
    "                                                    stratify=y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "mlp = MLPClassifier(hidden_layer_sizes=(3,3,2),activation='tanh',\n",
    "                    random_state=2018)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MLPClassifier(activation='tanh', alpha=0.0001, batch_size='auto', beta_1=0.9,\n",
       "       beta_2=0.999, early_stopping=False, epsilon=1e-08,\n",
       "       hidden_layer_sizes=(3, 3, 2), learning_rate='constant',\n",
       "       learning_rate_init=0.001, max_iter=200, momentum=0.9,\n",
       "       nesterovs_momentum=True, power_t=0.5, random_state=2018,\n",
       "       shuffle=True, solver='adam', tol=0.0001, validation_fraction=0.1,\n",
       "       verbose=False, warm_start=False)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mlp.fit( X_train , y_train )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, ..., 0, 0, 0], dtype=int64)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = mlp.predict(X_test)\n",
    "y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[84806     0]\n",
      " [ 7448     0]]\n"
     ]
    }
   ],
   "source": [
    "print(confusion_matrix(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.92      1.00      0.96     84806\n",
      "          1       0.00      0.00      0.00      7448\n",
      "\n",
      "avg / total       0.85      0.92      0.88     92254\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9192663732737876\n"
     ]
    }
   ],
   "source": [
    "print(accuracy_score(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_curve, roc_auc_score\n",
    "\n",
    "y_pred_prob = mlp.predict_proba(X_test)[:,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "fpr, tpr, thresholds = roc_curve(y_test, y_pred_prob)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.plot([0, 1], [0, 1], 'k--')\n",
    "plt.plot(fpr, tpr)\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.title('ROC Curve')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5025749179089304"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "roc_auc_score(y_test, y_pred_prob)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'n_neighbors': array([1, 3, 5, 7, 9])}\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "import numpy as np\n",
    "parameters = {'n_neighbors': np.array([1,3,5,7,9])}\n",
    "print(parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "knn = KNeighborsClassifier()\n",
    "cv = GridSearchCV(knn, param_grid=parameters,\n",
    "                  cv=5,scoring='roc_auc')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise',\n",
       "       estimator=KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
       "           metric_params=None, n_jobs=1, n_neighbors=5, p=2,\n",
       "           weights='uniform'),\n",
       "       fit_params=None, iid=True, n_jobs=1,\n",
       "       param_grid={'n_neighbors': array([1, 3, 5, 7, 9])},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='roc_auc', verbose=0)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv.fit( X , y )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'mean_fit_time': array([8.66123023, 8.74172797, 8.39301481, 8.38385477, 8.90432854]), 'std_fit_time': array([0.23375318, 0.10616414, 0.06455418, 0.0873752 , 0.35902544]), 'mean_score_time': array([19.12513604, 22.31692271, 23.67248173, 25.22252522, 28.55605321]), 'std_score_time': array([3.87520959, 3.75222146, 3.27388535, 3.18161056, 4.59686796]), 'param_n_neighbors': masked_array(data=[1, 3, 5, 7, 9],\n",
      "             mask=[False, False, False, False, False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'params': [{'n_neighbors': 1}, {'n_neighbors': 3}, {'n_neighbors': 5}, {'n_neighbors': 7}, {'n_neighbors': 9}], 'split0_test_score': array([0.50216416, 0.51036267, 0.5175471 , 0.52260944, 0.52347507]), 'split1_test_score': array([0.51301469, 0.52710028, 0.53128023, 0.53451477, 0.53520065]), 'split2_test_score': array([0.50934936, 0.51339678, 0.51849668, 0.52181479, 0.52428387]), 'split3_test_score': array([0.50359576, 0.5099115 , 0.51453947, 0.51913153, 0.51970278]), 'split4_test_score': array([0.50025877, 0.50243066, 0.5074608 , 0.51081159, 0.51142172]), 'mean_test_score': array([0.50567654, 0.51264037, 0.51786485, 0.52177643, 0.52281682]), 'std_test_score': array([0.00476144, 0.00808125, 0.00774305, 0.00761818, 0.00768752]), 'rank_test_score': array([5, 4, 3, 2, 1]), 'split0_train_score': array([1.        , 0.9334673 , 0.88565917, 0.84998142, 0.82246872]), 'split1_train_score': array([1.        , 0.93343661, 0.88600992, 0.85051245, 0.82272365]), 'split2_train_score': array([1.        , 0.93319091, 0.88561327, 0.85064766, 0.82252736]), 'split3_train_score': array([1.        , 0.93337288, 0.88611517, 0.85121874, 0.82363213]), 'split4_train_score': array([1.        , 0.93361151, 0.885957  , 0.85077296, 0.8227567 ]), 'mean_train_score': array([1.        , 0.93341584, 0.8858709 , 0.85062665, 0.82282171]), 'std_train_score': array([0.        , 0.00013697, 0.0001988 , 0.00040045, 0.00041997])}\n"
     ]
    }
   ],
   "source": [
    "print(cv.cv_results_  )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'n_neighbors': 9}\n"
     ]
    }
   ],
   "source": [
    "print(cv.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5228168215594952\n"
     ]
    }
   ],
   "source": [
    "print(cv.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
      "           metric_params=None, n_jobs=1, n_neighbors=9, p=2,\n",
      "           weights='uniform')\n"
     ]
    }
   ],
   "source": [
    "print(cv.best_estimator_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Using imputer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test = pd.read_csv('F:/Project/application_test.csv') \n",
    "\n",
    "df_test = pd.get_dummies(df_test, drop_first=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = pd.read_csv(\"F:/Project/application_train.csv\")\n",
    "\n",
    "df_train = pd.get_dummies(df_train, drop_first=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_labels = df_train['TARGET']\n",
    "train_sk_id_curr = df_train['SK_ID_CURR']\n",
    "test_sk_id_curr =df_test['SK_ID_CURR']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train.drop('SK_ID_CURR', inplace=True, axis=1)\n",
    "df_test.drop('SK_ID_CURR', inplace=True, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Features shape:  (307511, 225)\n",
      "Testing Features shape:  (48744, 225)\n"
     ]
    }
   ],
   "source": [
    "df_train, df_test = df_train.align(df_test, join = 'inner', axis = 1)\n",
    "print('Training Features shape: ', df_train.shape)\n",
    "print('Testing Features shape: ', df_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import Imputer\n",
    "imputer = Imputer(strategy = 'median')\n",
    "imputer.fit(df_train)\n",
    "df_train = imputer.transform(df_train)\n",
    "df_test = imputer.transform(df_test)\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(df_train, train_labels,test_size = 0.3, \n",
    "                                                    random_state=2018,\n",
    "                                                    stratify=train_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "mlp = MLPClassifier(hidden_layer_sizes=(3,3,2),activation='tanh',\n",
    "                    random_state=2018)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MLPClassifier(activation='tanh', alpha=0.0001, batch_size='auto', beta_1=0.9,\n",
       "       beta_2=0.999, early_stopping=False, epsilon=1e-08,\n",
       "       hidden_layer_sizes=(3, 3, 2), learning_rate='constant',\n",
       "       learning_rate_init=0.001, max_iter=200, momentum=0.9,\n",
       "       nesterovs_momentum=True, power_t=0.5, random_state=2018,\n",
       "       shuffle=True, solver='adam', tol=0.0001, validation_fraction=0.1,\n",
       "       verbose=False, warm_start=False)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mlp.fit( X_train , y_train )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = mlp.predict(df_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "submit = pd.DataFrame({'SK_ID_CURR': test_sk_id_curr, 'TARGET':pred})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "submit.to_csv(\"F:/Project/submit_mlp2.csv\",\n",
    "              index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}