XGBoost_Trial_Only_LabelEncoded.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ran\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import io\n",
    "import numpy as np\n",
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import pickle\n",
    "print ('Ran')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of training data: (307511, 122)\n",
      "Shape of test data: (48744, 121)\n"
     ]
    }
   ],
   "source": [
    "app_train = pd.read_csv('D:/Project_Data/application_train.csv')\n",
    "app_test = pd.read_csv('D:/Project_Data/application_test.csv')\n",
    "\n",
    "print(\"Shape of training data: {}\".format(app_train.shape))\n",
    "print(\"Shape of test data: {}\".format(app_test.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SK_ID_CURR</th>\n",
       "      <th>TARGET</th>\n",
       "      <th>NAME_CONTRACT_TYPE</th>\n",
       "      <th>CODE_GENDER</th>\n",
       "      <th>FLAG_OWN_CAR</th>\n",
       "      <th>FLAG_OWN_REALTY</th>\n",
       "      <th>CNT_CHILDREN</th>\n",
       "      <th>AMT_INCOME_TOTAL</th>\n",
       "      <th>AMT_CREDIT</th>\n",
       "      <th>AMT_ANNUITY</th>\n",
       "      <th>...</th>\n",
       "      <th>FLAG_DOCUMENT_18</th>\n",
       "      <th>FLAG_DOCUMENT_19</th>\n",
       "      <th>FLAG_DOCUMENT_20</th>\n",
       "      <th>FLAG_DOCUMENT_21</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_HOUR</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_DAY</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_WEEK</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_MON</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_QRT</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_YEAR</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>100002</td>\n",
       "      <td>1</td>\n",
       "      <td>Cash loans</td>\n",
       "      <td>M</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "      <td>0</td>\n",
       "      <td>202500.0</td>\n",
       "      <td>406597.5</td>\n",
       "      <td>24700.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>100003</td>\n",
       "      <td>0</td>\n",
       "      <td>Cash loans</td>\n",
       "      <td>F</td>\n",
       "      <td>N</td>\n",
       "      <td>N</td>\n",
       "      <td>0</td>\n",
       "      <td>270000.0</td>\n",
       "      <td>1293502.5</td>\n",
       "      <td>35698.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>100004</td>\n",
       "      <td>0</td>\n",
       "      <td>Revolving loans</td>\n",
       "      <td>M</td>\n",
       "      <td>Y</td>\n",
       "      <td>Y</td>\n",
       "      <td>0</td>\n",
       "      <td>67500.0</td>\n",
       "      <td>135000.0</td>\n",
       "      <td>6750.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>100006</td>\n",
       "      <td>0</td>\n",
       "      <td>Cash loans</td>\n",
       "      <td>F</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "      <td>0</td>\n",
       "      <td>135000.0</td>\n",
       "      <td>312682.5</td>\n",
       "      <td>29686.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>100007</td>\n",
       "      <td>0</td>\n",
       "      <td>Cash loans</td>\n",
       "      <td>M</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "      <td>0</td>\n",
       "      <td>121500.0</td>\n",
       "      <td>513000.0</td>\n",
       "      <td>21865.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 122 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   SK_ID_CURR  TARGET NAME_CONTRACT_TYPE CODE_GENDER FLAG_OWN_CAR  \\\n",
       "0      100002       1         Cash loans           M            N   \n",
       "1      100003       0         Cash loans           F            N   \n",
       "2      100004       0    Revolving loans           M            Y   \n",
       "3      100006       0         Cash loans           F            N   \n",
       "4      100007       0         Cash loans           M            N   \n",
       "\n",
       "  FLAG_OWN_REALTY  CNT_CHILDREN  AMT_INCOME_TOTAL  AMT_CREDIT  AMT_ANNUITY  \\\n",
       "0               Y             0          202500.0    406597.5      24700.5   \n",
       "1               N             0          270000.0   1293502.5      35698.5   \n",
       "2               Y             0           67500.0    135000.0       6750.0   \n",
       "3               Y             0          135000.0    312682.5      29686.5   \n",
       "4               Y             0          121500.0    513000.0      21865.5   \n",
       "\n",
       "   ...  FLAG_DOCUMENT_18 FLAG_DOCUMENT_19 FLAG_DOCUMENT_20 FLAG_DOCUMENT_21  \\\n",
       "0  ...                 0                0                0                0   \n",
       "1  ...                 0                0                0                0   \n",
       "2  ...                 0                0                0                0   \n",
       "3  ...                 0                0                0                0   \n",
       "4  ...                 0                0                0                0   \n",
       "\n",
       "  AMT_REQ_CREDIT_BUREAU_HOUR AMT_REQ_CREDIT_BUREAU_DAY  \\\n",
       "0                        0.0                       0.0   \n",
       "1                        0.0                       0.0   \n",
       "2                        0.0                       0.0   \n",
       "3                        NaN                       NaN   \n",
       "4                        0.0                       0.0   \n",
       "\n",
       "   AMT_REQ_CREDIT_BUREAU_WEEK  AMT_REQ_CREDIT_BUREAU_MON  \\\n",
       "0                         0.0                        0.0   \n",
       "1                         0.0                        0.0   \n",
       "2                         0.0                        0.0   \n",
       "3                         NaN                        NaN   \n",
       "4                         0.0                        0.0   \n",
       "\n",
       "   AMT_REQ_CREDIT_BUREAU_QRT  AMT_REQ_CREDIT_BUREAU_YEAR  \n",
       "0                        0.0                         1.0  \n",
       "1                        0.0                         0.0  \n",
       "2                        0.0                         0.0  \n",
       "3                        NaN                         NaN  \n",
       "4                        0.0                         0.0  \n",
       "\n",
       "[5 rows x 122 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "app_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Features shape:  (307511, 120)\n",
      "Testing Features shape:  (48744, 120)\n"
     ]
    }
   ],
   "source": [
    "train_labels = app_train['TARGET']\n",
    "train_sk_id_curr = app_train['SK_ID_CURR']\n",
    "test_sk_id_curr = app_test['SK_ID_CURR']\n",
    "\n",
    "app_train.drop('SK_ID_CURR', inplace=True, axis=1)\n",
    "app_test.drop('SK_ID_CURR', inplace=True, axis=1)\n",
    "\n",
    "app_train, app_test = app_train.align(app_test, join = 'inner', axis = 1)\n",
    "print('Training Features shape: ', app_train.shape)\n",
    "print('Testing Features shape: ', app_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat_features = [f for f in app_train.columns if app_train[f].dtype == 'object']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def column_index(df, query_cols):\n",
    "    cols = df.columns.values\n",
    "    sidx = np.argsort(cols)\n",
    "    return sidx[np.searchsorted(cols, query_cols, sorter=sidx)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cat features are: ['NAME_CONTRACT_TYPE', 'CODE_GENDER', 'FLAG_OWN_CAR', 'FLAG_OWN_REALTY', 'NAME_TYPE_SUITE', 'NAME_INCOME_TYPE', 'NAME_EDUCATION_TYPE', 'NAME_FAMILY_STATUS', 'NAME_HOUSING_TYPE', 'OCCUPATION_TYPE', 'WEEKDAY_APPR_PROCESS_START', 'ORGANIZATION_TYPE', 'FONDKAPREMONT_MODE', 'HOUSETYPE_MODE', 'WALLSMATERIAL_MODE', 'EMERGENCYSTATE_MODE']\n",
      "[ 0  1  2  3  9 10 11 12 13 26 30 38 84 85 87 88]\n"
     ]
    }
   ],
   "source": [
    "cat_features_inds = column_index(app_train, cat_features)    \n",
    "print(\"Cat features are: %s\" % [f for f in cat_features])\n",
    "print(cat_features_inds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import LabelEncoder\n",
    "le=LabelEncoder()\n",
    "for col in cat_features:\n",
    "    app_train[col] = le.fit_transform(app_train[col].astype(str))\n",
    "    app_test[col] = le.fit_transform(app_test[col].astype(str))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "from sklearn.preprocessing import LabelEncoder\n",
    "le = LabelEncoder()\n",
    "le_count = 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Iterate through the columns\n",
    "for col in app_train:\n",
    "    if app_train[col].dtype == 'object':\n",
    "        # If 2 or fewer unique categories\n",
    "        if len(list(app_train[col].unique())) <= 2:\n",
    "            # Train on the training data\n",
    "            le.fit(app_train[col])\n",
    "            # Transform both training and testing data\n",
    "            app_train[col] = le.transform(app_train[col])\n",
    "            app_test[col] = le.transform(app_test[col])\n",
    "            \n",
    "            # Keep track of how many columns were label encoded\n",
    "            le_count += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "print('%d columns were label encoded.' % le_count)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "app_train = pd.get_dummies(app_train)\n",
    "app_test = pd.get_dummies(app_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Features shape:  (307511, 120)\n",
      "Testing Features shape:  (48744, 120)\n"
     ]
    }
   ],
   "source": [
    "print('Training Features shape: ', app_train.shape)\n",
    "print('Testing Features shape: ', app_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.impute import SimpleImputer\n",
    "imputer = SimpleImputer(strategy = 'median')\n",
    "imputer.fit(app_train)\n",
    "app_train = imputer.transform(app_train)\n",
    "app_test = imputer.transform(app_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split, cross_val_score\n",
    "import xgboost as xgb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "11.387150050352467"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratio = (train_labels == 0).sum()/ (train_labels == 1).sum()\n",
    "ratio"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "0 - Non-defaulter and 1- Defaulter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier\n",
    "from sklearn.model_selection import cross_val_score, StratifiedKFold\n",
    "from sklearn.metrics import roc_auc_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Postive examples in train set: 226148\n",
      "Negative examples in train set: 19860\n",
      "Postive examples in test set: 56538\n",
      "Negative examples in test set: 4965\n"
     ]
    }
   ],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(app_train, train_labels, test_size=0.2, stratify=train_labels, random_state=1)\n",
    "print(\"Postive examples in train set: {}\".format(np.sum(y_train==0)))\n",
    "print(\"Negative examples in train set: {}\".format(np.sum(y_train==1)))\n",
    "\n",
    "print(\"Postive examples in test set: {}\".format(np.sum(y_test==0)))\n",
    "print(\"Negative examples in test set: {}\".format(np.sum(y_test==1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-auc:0.684694\n",
      "Will train until validation_0-auc hasn't improved in 10 rounds.\n",
      "[1]\tvalidation_0-auc:0.69767\n",
      "[2]\tvalidation_0-auc:0.701373\n",
      "[3]\tvalidation_0-auc:0.705643\n",
      "[4]\tvalidation_0-auc:0.70868\n",
      "[5]\tvalidation_0-auc:0.710959\n",
      "[6]\tvalidation_0-auc:0.712839\n",
      "[7]\tvalidation_0-auc:0.714636\n",
      "[8]\tvalidation_0-auc:0.715176\n",
      "[9]\tvalidation_0-auc:0.717006\n",
      "[10]\tvalidation_0-auc:0.718398\n",
      "[11]\tvalidation_0-auc:0.720092\n",
      "[12]\tvalidation_0-auc:0.721173\n",
      "[13]\tvalidation_0-auc:0.724314\n",
      "[14]\tvalidation_0-auc:0.72539\n",
      "[15]\tvalidation_0-auc:0.726071\n",
      "[16]\tvalidation_0-auc:0.728094\n",
      "[17]\tvalidation_0-auc:0.728817\n",
      "[18]\tvalidation_0-auc:0.730411\n",
      "[19]\tvalidation_0-auc:0.730801\n",
      "[20]\tvalidation_0-auc:0.732192\n",
      "[21]\tvalidation_0-auc:0.733783\n",
      "[22]\tvalidation_0-auc:0.734385\n",
      "[23]\tvalidation_0-auc:0.734824\n",
      "[24]\tvalidation_0-auc:0.735173\n",
      "[25]\tvalidation_0-auc:0.735963\n",
      "[26]\tvalidation_0-auc:0.737605\n",
      "[27]\tvalidation_0-auc:0.738684\n",
      "[28]\tvalidation_0-auc:0.738828\n",
      "[29]\tvalidation_0-auc:0.739251\n",
      "[30]\tvalidation_0-auc:0.740075\n",
      "[31]\tvalidation_0-auc:0.740237\n",
      "[32]\tvalidation_0-auc:0.741283\n",
      "[33]\tvalidation_0-auc:0.741857\n",
      "[34]\tvalidation_0-auc:0.742791\n",
      "[35]\tvalidation_0-auc:0.743281\n",
      "[36]\tvalidation_0-auc:0.743684\n",
      "[37]\tvalidation_0-auc:0.744116\n",
      "[38]\tvalidation_0-auc:0.744421\n",
      "[39]\tvalidation_0-auc:0.745159\n",
      "[40]\tvalidation_0-auc:0.745255\n",
      "[41]\tvalidation_0-auc:0.745389\n",
      "[42]\tvalidation_0-auc:0.74595\n",
      "[43]\tvalidation_0-auc:0.74603\n",
      "[44]\tvalidation_0-auc:0.746428\n",
      "[45]\tvalidation_0-auc:0.746789\n",
      "[46]\tvalidation_0-auc:0.74693\n",
      "[47]\tvalidation_0-auc:0.747547\n",
      "[48]\tvalidation_0-auc:0.747876\n",
      "[49]\tvalidation_0-auc:0.748326\n",
      "[50]\tvalidation_0-auc:0.748497\n",
      "[51]\tvalidation_0-auc:0.748607\n",
      "[52]\tvalidation_0-auc:0.748837\n",
      "[53]\tvalidation_0-auc:0.748774\n",
      "[54]\tvalidation_0-auc:0.74934\n",
      "[55]\tvalidation_0-auc:0.749513\n",
      "[56]\tvalidation_0-auc:0.749747\n",
      "[57]\tvalidation_0-auc:0.749831\n",
      "[58]\tvalidation_0-auc:0.750139\n",
      "[59]\tvalidation_0-auc:0.750354\n",
      "[60]\tvalidation_0-auc:0.750487\n",
      "[61]\tvalidation_0-auc:0.750509\n",
      "[62]\tvalidation_0-auc:0.750866\n",
      "[63]\tvalidation_0-auc:0.751147\n",
      "[64]\tvalidation_0-auc:0.751515\n",
      "[65]\tvalidation_0-auc:0.751596\n",
      "[66]\tvalidation_0-auc:0.751811\n",
      "[67]\tvalidation_0-auc:0.752163\n",
      "[68]\tvalidation_0-auc:0.752574\n",
      "[69]\tvalidation_0-auc:0.752625\n",
      "[70]\tvalidation_0-auc:0.752753\n",
      "[71]\tvalidation_0-auc:0.753035\n",
      "[72]\tvalidation_0-auc:0.75323\n",
      "[73]\tvalidation_0-auc:0.75341\n",
      "[74]\tvalidation_0-auc:0.753464\n",
      "[75]\tvalidation_0-auc:0.753544\n",
      "[76]\tvalidation_0-auc:0.753663\n",
      "[77]\tvalidation_0-auc:0.753812\n",
      "[78]\tvalidation_0-auc:0.753837\n",
      "[79]\tvalidation_0-auc:0.754214\n",
      "[80]\tvalidation_0-auc:0.754287\n",
      "[81]\tvalidation_0-auc:0.754376\n",
      "[82]\tvalidation_0-auc:0.754472\n",
      "[83]\tvalidation_0-auc:0.75445\n",
      "[84]\tvalidation_0-auc:0.754535\n",
      "[85]\tvalidation_0-auc:0.754609\n",
      "[86]\tvalidation_0-auc:0.754703\n",
      "[87]\tvalidation_0-auc:0.754865\n",
      "[88]\tvalidation_0-auc:0.754963\n",
      "[89]\tvalidation_0-auc:0.755085\n",
      "[90]\tvalidation_0-auc:0.755226\n",
      "[91]\tvalidation_0-auc:0.755322\n",
      "[92]\tvalidation_0-auc:0.755369\n",
      "[93]\tvalidation_0-auc:0.755411\n",
      "[94]\tvalidation_0-auc:0.755578\n",
      "[95]\tvalidation_0-auc:0.755605\n",
      "[96]\tvalidation_0-auc:0.755808\n",
      "[97]\tvalidation_0-auc:0.755966\n",
      "[98]\tvalidation_0-auc:0.756105\n",
      "[99]\tvalidation_0-auc:0.756175\n",
      "[100]\tvalidation_0-auc:0.756254\n",
      "[101]\tvalidation_0-auc:0.756335\n",
      "[102]\tvalidation_0-auc:0.756383\n",
      "[103]\tvalidation_0-auc:0.75658\n",
      "[104]\tvalidation_0-auc:0.756682\n",
      "[105]\tvalidation_0-auc:0.756711\n",
      "[106]\tvalidation_0-auc:0.756803\n",
      "[107]\tvalidation_0-auc:0.75703\n",
      "[108]\tvalidation_0-auc:0.757089\n",
      "[109]\tvalidation_0-auc:0.757087\n",
      "[110]\tvalidation_0-auc:0.757102\n",
      "[111]\tvalidation_0-auc:0.757149\n",
      "[112]\tvalidation_0-auc:0.757154\n",
      "[113]\tvalidation_0-auc:0.757148\n",
      "[114]\tvalidation_0-auc:0.757381\n",
      "[115]\tvalidation_0-auc:0.757503\n",
      "[116]\tvalidation_0-auc:0.757497\n",
      "[117]\tvalidation_0-auc:0.757513\n",
      "[118]\tvalidation_0-auc:0.757595\n",
      "[119]\tvalidation_0-auc:0.757706\n",
      "[120]\tvalidation_0-auc:0.757761\n",
      "[121]\tvalidation_0-auc:0.757722\n",
      "[122]\tvalidation_0-auc:0.757796\n",
      "[123]\tvalidation_0-auc:0.757831\n",
      "[124]\tvalidation_0-auc:0.757852\n",
      "[125]\tvalidation_0-auc:0.757868\n",
      "[126]\tvalidation_0-auc:0.757908\n",
      "[127]\tvalidation_0-auc:0.757934\n",
      "[128]\tvalidation_0-auc:0.757998\n",
      "[129]\tvalidation_0-auc:0.758001\n",
      "[130]\tvalidation_0-auc:0.757977\n",
      "[131]\tvalidation_0-auc:0.758\n",
      "[132]\tvalidation_0-auc:0.757965\n",
      "[133]\tvalidation_0-auc:0.757966\n",
      "[134]\tvalidation_0-auc:0.757978\n",
      "[135]\tvalidation_0-auc:0.758067\n",
      "[136]\tvalidation_0-auc:0.758053\n",
      "[137]\tvalidation_0-auc:0.758057\n",
      "[138]\tvalidation_0-auc:0.758044\n",
      "[139]\tvalidation_0-auc:0.758103\n",
      "[140]\tvalidation_0-auc:0.758431\n",
      "[141]\tvalidation_0-auc:0.75845\n",
      "[142]\tvalidation_0-auc:0.758418\n",
      "[143]\tvalidation_0-auc:0.758437\n",
      "[144]\tvalidation_0-auc:0.758433\n",
      "[145]\tvalidation_0-auc:0.758449\n",
      "[146]\tvalidation_0-auc:0.75852\n",
      "[147]\tvalidation_0-auc:0.758537\n",
      "[148]\tvalidation_0-auc:0.758641\n",
      "[149]\tvalidation_0-auc:0.758793\n",
      "[150]\tvalidation_0-auc:0.75888\n",
      "[151]\tvalidation_0-auc:0.758822\n",
      "[152]\tvalidation_0-auc:0.758781\n",
      "[153]\tvalidation_0-auc:0.758779\n",
      "[154]\tvalidation_0-auc:0.758715\n",
      "[155]\tvalidation_0-auc:0.758856\n",
      "[156]\tvalidation_0-auc:0.758859\n",
      "[157]\tvalidation_0-auc:0.758811\n",
      "[158]\tvalidation_0-auc:0.758939\n",
      "[159]\tvalidation_0-auc:0.758988\n",
      "[160]\tvalidation_0-auc:0.758976\n",
      "[161]\tvalidation_0-auc:0.759016\n",
      "[162]\tvalidation_0-auc:0.758971\n",
      "[163]\tvalidation_0-auc:0.758944\n",
      "[164]\tvalidation_0-auc:0.759157\n",
      "[165]\tvalidation_0-auc:0.759163\n",
      "[166]\tvalidation_0-auc:0.759188\n",
      "[167]\tvalidation_0-auc:0.75924\n",
      "[168]\tvalidation_0-auc:0.759282\n",
      "[169]\tvalidation_0-auc:0.759228\n",
      "[170]\tvalidation_0-auc:0.759226\n",
      "[171]\tvalidation_0-auc:0.759246\n",
      "[172]\tvalidation_0-auc:0.759317\n",
      "[173]\tvalidation_0-auc:0.759316\n",
      "[174]\tvalidation_0-auc:0.759398\n",
      "[175]\tvalidation_0-auc:0.759473\n",
      "[176]\tvalidation_0-auc:0.759498\n",
      "[177]\tvalidation_0-auc:0.759552\n",
      "[178]\tvalidation_0-auc:0.759606\n",
      "[179]\tvalidation_0-auc:0.759616\n",
      "[180]\tvalidation_0-auc:0.759592\n",
      "[181]\tvalidation_0-auc:0.759648\n",
      "[182]\tvalidation_0-auc:0.759668\n",
      "[183]\tvalidation_0-auc:0.759676\n",
      "[184]\tvalidation_0-auc:0.759634\n",
      "[185]\tvalidation_0-auc:0.75964\n",
      "[186]\tvalidation_0-auc:0.759666\n",
      "[187]\tvalidation_0-auc:0.759668\n",
      "[188]\tvalidation_0-auc:0.759652\n",
      "[189]\tvalidation_0-auc:0.759659\n",
      "[190]\tvalidation_0-auc:0.759654\n",
      "[191]\tvalidation_0-auc:0.759768\n",
      "[192]\tvalidation_0-auc:0.759802\n",
      "[193]\tvalidation_0-auc:0.759809\n",
      "[194]\tvalidation_0-auc:0.759912\n",
      "[195]\tvalidation_0-auc:0.759925\n",
      "[196]\tvalidation_0-auc:0.760031\n",
      "[197]\tvalidation_0-auc:0.760126\n",
      "[198]\tvalidation_0-auc:0.760262\n",
      "[199]\tvalidation_0-auc:0.760336\n",
      "[200]\tvalidation_0-auc:0.760315\n",
      "[201]\tvalidation_0-auc:0.760534\n",
      "[202]\tvalidation_0-auc:0.760445\n",
      "[203]\tvalidation_0-auc:0.760534\n",
      "[204]\tvalidation_0-auc:0.760582\n",
      "[205]\tvalidation_0-auc:0.760545\n",
      "[206]\tvalidation_0-auc:0.76063\n",
      "[207]\tvalidation_0-auc:0.760602\n",
      "[208]\tvalidation_0-auc:0.760574\n",
      "[209]\tvalidation_0-auc:0.760681\n",
      "[210]\tvalidation_0-auc:0.760762\n",
      "[211]\tvalidation_0-auc:0.760734\n",
      "[212]\tvalidation_0-auc:0.760743\n",
      "[213]\tvalidation_0-auc:0.76082\n",
      "[214]\tvalidation_0-auc:0.760885\n",
      "[215]\tvalidation_0-auc:0.76085\n",
      "[216]\tvalidation_0-auc:0.760951\n",
      "[217]\tvalidation_0-auc:0.760935\n",
      "[218]\tvalidation_0-auc:0.760898\n",
      "[219]\tvalidation_0-auc:0.761007\n",
      "[220]\tvalidation_0-auc:0.760996\n",
      "[221]\tvalidation_0-auc:0.76102\n",
      "[222]\tvalidation_0-auc:0.761025\n",
      "[223]\tvalidation_0-auc:0.76109\n",
      "[224]\tvalidation_0-auc:0.761066\n",
      "[225]\tvalidation_0-auc:0.761042\n",
      "[226]\tvalidation_0-auc:0.76109\n",
      "[227]\tvalidation_0-auc:0.761093\n",
      "[228]\tvalidation_0-auc:0.761108\n",
      "[229]\tvalidation_0-auc:0.76115\n",
      "[230]\tvalidation_0-auc:0.761216\n",
      "[231]\tvalidation_0-auc:0.761336\n",
      "[232]\tvalidation_0-auc:0.761339\n",
      "[233]\tvalidation_0-auc:0.761307\n",
      "[234]\tvalidation_0-auc:0.761282\n",
      "[235]\tvalidation_0-auc:0.761351\n",
      "[236]\tvalidation_0-auc:0.761393\n",
      "[237]\tvalidation_0-auc:0.761411\n",
      "[238]\tvalidation_0-auc:0.761457\n",
      "[239]\tvalidation_0-auc:0.761467\n",
      "[240]\tvalidation_0-auc:0.7615\n",
      "[241]\tvalidation_0-auc:0.761449\n",
      "[242]\tvalidation_0-auc:0.761446\n",
      "[243]\tvalidation_0-auc:0.761421\n",
      "[244]\tvalidation_0-auc:0.761364\n",
      "[245]\tvalidation_0-auc:0.76135\n",
      "[246]\tvalidation_0-auc:0.761326\n",
      "[247]\tvalidation_0-auc:0.761295\n",
      "[248]\tvalidation_0-auc:0.761294\n",
      "[249]\tvalidation_0-auc:0.761319\n",
      "[250]\tvalidation_0-auc:0.761277\n",
      "Stopping. Best iteration:\n",
      "[240]\tvalidation_0-auc:0.7615\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "       colsample_bynode=1, colsample_bytree=1, gamma=0.1,\n",
       "       learning_rate=0.1, max_delta_step=0, max_depth=3,\n",
       "       min_child_weight=1, missing=None, n_estimators=1000, n_jobs=1,\n",
       "       nthread=None, objective='binary:logistic', random_state=0,\n",
       "       reg_alpha=0, reg_lambda=1, scale_pos_weight=11.387150050352467,\n",
       "       seed=None, silent=None, subsample=0.5, verbosity=1)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf = XGBClassifier(n_estimators=1000, objective='binary:logistic', gamma=0.1, subsample=0.5, scale_pos_weight=ratio )\n",
    "clf.fit(X_train, y_train, eval_set=[(X_test, y_test)], eval_metric='auc', early_stopping_rounds=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy score  0.7004698957774418\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "y_pred = clf.predict(X_test)\n",
    "print('Accuracy score ',accuracy_score(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-auc:0.686937\n",
      "[1]\tvalidation_0-auc:0.691054\n",
      "[2]\tvalidation_0-auc:0.706216\n",
      "[3]\tvalidation_0-auc:0.708388\n",
      "[4]\tvalidation_0-auc:0.708699\n",
      "[5]\tvalidation_0-auc:0.713207\n",
      "[6]\tvalidation_0-auc:0.714295\n",
      "[7]\tvalidation_0-auc:0.716832\n",
      "[8]\tvalidation_0-auc:0.717939\n",
      "[9]\tvalidation_0-auc:0.719787\n",
      "[10]\tvalidation_0-auc:0.720925\n",
      "[11]\tvalidation_0-auc:0.72301\n",
      "[12]\tvalidation_0-auc:0.723734\n",
      "[13]\tvalidation_0-auc:0.726398\n",
      "[14]\tvalidation_0-auc:0.727451\n",
      "[15]\tvalidation_0-auc:0.728416\n",
      "[16]\tvalidation_0-auc:0.728998\n",
      "[17]\tvalidation_0-auc:0.729529\n",
      "[18]\tvalidation_0-auc:0.73024\n",
      "[19]\tvalidation_0-auc:0.731471\n",
      "[20]\tvalidation_0-auc:0.73262\n",
      "[21]\tvalidation_0-auc:0.733088\n",
      "[22]\tvalidation_0-auc:0.734061\n",
      "[23]\tvalidation_0-auc:0.735106\n",
      "[24]\tvalidation_0-auc:0.73597\n",
      "[25]\tvalidation_0-auc:0.736875\n",
      "[26]\tvalidation_0-auc:0.738188\n",
      "[27]\tvalidation_0-auc:0.738648\n",
      "[28]\tvalidation_0-auc:0.739556\n",
      "[29]\tvalidation_0-auc:0.740707\n",
      "[30]\tvalidation_0-auc:0.741544\n",
      "[31]\tvalidation_0-auc:0.741986\n",
      "[32]\tvalidation_0-auc:0.742549\n",
      "[33]\tvalidation_0-auc:0.743101\n",
      "[34]\tvalidation_0-auc:0.743609\n",
      "[35]\tvalidation_0-auc:0.744061\n",
      "[36]\tvalidation_0-auc:0.744285\n",
      "[37]\tvalidation_0-auc:0.744748\n",
      "[38]\tvalidation_0-auc:0.745227\n",
      "[39]\tvalidation_0-auc:0.74556\n",
      "[40]\tvalidation_0-auc:0.746166\n",
      "[41]\tvalidation_0-auc:0.746605\n",
      "[42]\tvalidation_0-auc:0.746748\n",
      "[43]\tvalidation_0-auc:0.746984\n",
      "[44]\tvalidation_0-auc:0.747182\n",
      "[45]\tvalidation_0-auc:0.747565\n",
      "[46]\tvalidation_0-auc:0.747769\n",
      "[47]\tvalidation_0-auc:0.748489\n",
      "[48]\tvalidation_0-auc:0.748876\n",
      "[49]\tvalidation_0-auc:0.749223\n",
      "[50]\tvalidation_0-auc:0.749851\n",
      "[51]\tvalidation_0-auc:0.750178\n",
      "[52]\tvalidation_0-auc:0.750338\n",
      "[53]\tvalidation_0-auc:0.750356\n",
      "[54]\tvalidation_0-auc:0.75059\n",
      "[55]\tvalidation_0-auc:0.750852\n",
      "[56]\tvalidation_0-auc:0.751081\n",
      "[57]\tvalidation_0-auc:0.751498\n",
      "[58]\tvalidation_0-auc:0.751792\n",
      "[59]\tvalidation_0-auc:0.752075\n",
      "[60]\tvalidation_0-auc:0.752289\n",
      "[61]\tvalidation_0-auc:0.752519\n",
      "[62]\tvalidation_0-auc:0.7526\n",
      "[63]\tvalidation_0-auc:0.752733\n",
      "[64]\tvalidation_0-auc:0.752971\n",
      "[65]\tvalidation_0-auc:0.753182\n",
      "[66]\tvalidation_0-auc:0.753429\n",
      "[67]\tvalidation_0-auc:0.75358\n",
      "[68]\tvalidation_0-auc:0.753799\n",
      "[69]\tvalidation_0-auc:0.754033\n",
      "[70]\tvalidation_0-auc:0.754218\n",
      "[71]\tvalidation_0-auc:0.754381\n",
      "[72]\tvalidation_0-auc:0.754566\n",
      "[73]\tvalidation_0-auc:0.754889\n",
      "[74]\tvalidation_0-auc:0.755042\n",
      "[75]\tvalidation_0-auc:0.755215\n",
      "[76]\tvalidation_0-auc:0.755583\n",
      "[77]\tvalidation_0-auc:0.755772\n",
      "[78]\tvalidation_0-auc:0.755952\n",
      "[79]\tvalidation_0-auc:0.756135\n",
      "[80]\tvalidation_0-auc:0.756236\n",
      "[81]\tvalidation_0-auc:0.756371\n",
      "[82]\tvalidation_0-auc:0.756649\n",
      "[83]\tvalidation_0-auc:0.756923\n",
      "[84]\tvalidation_0-auc:0.757102\n",
      "[85]\tvalidation_0-auc:0.75723\n",
      "[86]\tvalidation_0-auc:0.757327\n",
      "[87]\tvalidation_0-auc:0.757422\n",
      "[88]\tvalidation_0-auc:0.757579\n",
      "[89]\tvalidation_0-auc:0.757709\n",
      "[90]\tvalidation_0-auc:0.757799\n",
      "[91]\tvalidation_0-auc:0.758008\n",
      "[92]\tvalidation_0-auc:0.758126\n",
      "[93]\tvalidation_0-auc:0.758244\n",
      "[94]\tvalidation_0-auc:0.758451\n",
      "[95]\tvalidation_0-auc:0.758616\n",
      "[96]\tvalidation_0-auc:0.75878\n",
      "[97]\tvalidation_0-auc:0.758933\n",
      "[98]\tvalidation_0-auc:0.759029\n",
      "[99]\tvalidation_0-auc:0.759213\n",
      "[100]\tvalidation_0-auc:0.759301\n",
      "[101]\tvalidation_0-auc:0.759545\n",
      "[102]\tvalidation_0-auc:0.759605\n",
      "[103]\tvalidation_0-auc:0.7597\n",
      "[104]\tvalidation_0-auc:0.759764\n",
      "[105]\tvalidation_0-auc:0.759877\n",
      "[106]\tvalidation_0-auc:0.759993\n",
      "[107]\tvalidation_0-auc:0.760037\n",
      "[108]\tvalidation_0-auc:0.760143\n",
      "[109]\tvalidation_0-auc:0.760268\n",
      "[110]\tvalidation_0-auc:0.760363\n",
      "[111]\tvalidation_0-auc:0.760454\n",
      "[112]\tvalidation_0-auc:0.760594\n",
      "[113]\tvalidation_0-auc:0.760687\n",
      "[114]\tvalidation_0-auc:0.760761\n",
      "[115]\tvalidation_0-auc:0.76085\n",
      "[116]\tvalidation_0-auc:0.760917\n",
      "[117]\tvalidation_0-auc:0.761053\n",
      "[118]\tvalidation_0-auc:0.761234\n",
      "[119]\tvalidation_0-auc:0.761425\n",
      "[120]\tvalidation_0-auc:0.761546\n",
      "[121]\tvalidation_0-auc:0.761657\n",
      "[122]\tvalidation_0-auc:0.761825\n",
      "[123]\tvalidation_0-auc:0.761926\n",
      "[124]\tvalidation_0-auc:0.762003\n",
      "[125]\tvalidation_0-auc:0.762119\n",
      "[126]\tvalidation_0-auc:0.762206\n",
      "[127]\tvalidation_0-auc:0.762413\n",
      "[128]\tvalidation_0-auc:0.762537\n",
      "[129]\tvalidation_0-auc:0.762649\n",
      "[130]\tvalidation_0-auc:0.762751\n",
      "[131]\tvalidation_0-auc:0.762803\n",
      "[132]\tvalidation_0-auc:0.762937\n",
      "[133]\tvalidation_0-auc:0.763003\n",
      "[134]\tvalidation_0-auc:0.763063\n",
      "[135]\tvalidation_0-auc:0.763149\n",
      "[136]\tvalidation_0-auc:0.763222\n",
      "[137]\tvalidation_0-auc:0.763455\n",
      "[138]\tvalidation_0-auc:0.763561\n",
      "[139]\tvalidation_0-auc:0.763648\n",
      "[140]\tvalidation_0-auc:0.763739\n",
      "[141]\tvalidation_0-auc:0.763821\n",
      "[142]\tvalidation_0-auc:0.763945\n",
      "[143]\tvalidation_0-auc:0.764082\n",
      "[144]\tvalidation_0-auc:0.764133\n",
      "[145]\tvalidation_0-auc:0.764227\n",
      "[146]\tvalidation_0-auc:0.764285\n",
      "[147]\tvalidation_0-auc:0.764342\n",
      "[148]\tvalidation_0-auc:0.764371\n",
      "[149]\tvalidation_0-auc:0.764478\n",
      "[150]\tvalidation_0-auc:0.764678\n",
      "[151]\tvalidation_0-auc:0.764815\n",
      "[152]\tvalidation_0-auc:0.764876\n",
      "[153]\tvalidation_0-auc:0.764948\n",
      "[154]\tvalidation_0-auc:0.765063\n",
      "[155]\tvalidation_0-auc:0.765149\n",
      "[156]\tvalidation_0-auc:0.765231\n",
      "[157]\tvalidation_0-auc:0.765279\n",
      "[158]\tvalidation_0-auc:0.765361\n",
      "[159]\tvalidation_0-auc:0.765443\n",
      "[160]\tvalidation_0-auc:0.765528\n",
      "[161]\tvalidation_0-auc:0.765557\n",
      "[162]\tvalidation_0-auc:0.765633\n",
      "[163]\tvalidation_0-auc:0.765699\n",
      "[164]\tvalidation_0-auc:0.765761\n",
      "[165]\tvalidation_0-auc:0.765804\n",
      "[166]\tvalidation_0-auc:0.765927\n",
      "[167]\tvalidation_0-auc:0.766193\n",
      "[168]\tvalidation_0-auc:0.766283\n",
      "[169]\tvalidation_0-auc:0.766346\n",
      "[170]\tvalidation_0-auc:0.766419\n",
      "[171]\tvalidation_0-auc:0.766512\n",
      "[172]\tvalidation_0-auc:0.766547\n",
      "[173]\tvalidation_0-auc:0.766647\n",
      "[174]\tvalidation_0-auc:0.766728\n",
      "[175]\tvalidation_0-auc:0.766777\n",
      "[176]\tvalidation_0-auc:0.766831\n",
      "[177]\tvalidation_0-auc:0.766859\n",
      "[178]\tvalidation_0-auc:0.76696\n",
      "[179]\tvalidation_0-auc:0.767\n",
      "[180]\tvalidation_0-auc:0.767055\n",
      "[181]\tvalidation_0-auc:0.767087\n",
      "[182]\tvalidation_0-auc:0.767138\n",
      "[183]\tvalidation_0-auc:0.767183\n",
      "[184]\tvalidation_0-auc:0.767214\n",
      "[185]\tvalidation_0-auc:0.76739\n",
      "[186]\tvalidation_0-auc:0.767506\n",
      "[187]\tvalidation_0-auc:0.76756\n",
      "[188]\tvalidation_0-auc:0.76762\n",
      "[189]\tvalidation_0-auc:0.767705\n",
      "[190]\tvalidation_0-auc:0.767751\n",
      "[191]\tvalidation_0-auc:0.767782\n",
      "[192]\tvalidation_0-auc:0.767921\n",
      "[193]\tvalidation_0-auc:0.768011\n",
      "[194]\tvalidation_0-auc:0.768109\n",
      "[195]\tvalidation_0-auc:0.768191\n",
      "[196]\tvalidation_0-auc:0.768217\n",
      "[197]\tvalidation_0-auc:0.76829\n",
      "[198]\tvalidation_0-auc:0.768338\n",
      "[199]\tvalidation_0-auc:0.768353\n",
      "[200]\tvalidation_0-auc:0.768494\n",
      "[201]\tvalidation_0-auc:0.768598\n",
      "[202]\tvalidation_0-auc:0.768661\n",
      "[203]\tvalidation_0-auc:0.768747\n",
      "[204]\tvalidation_0-auc:0.768816\n",
      "[205]\tvalidation_0-auc:0.768947\n",
      "[206]\tvalidation_0-auc:0.769012\n",
      "[207]\tvalidation_0-auc:0.769071\n",
      "[208]\tvalidation_0-auc:0.769095\n",
      "[209]\tvalidation_0-auc:0.769156\n",
      "[210]\tvalidation_0-auc:0.769212\n",
      "[211]\tvalidation_0-auc:0.769322\n",
      "[212]\tvalidation_0-auc:0.769403\n",
      "[213]\tvalidation_0-auc:0.769476\n",
      "[214]\tvalidation_0-auc:0.76967\n",
      "[215]\tvalidation_0-auc:0.769783\n",
      "[216]\tvalidation_0-auc:0.769864\n",
      "[217]\tvalidation_0-auc:0.769926\n",
      "[218]\tvalidation_0-auc:0.770041\n",
      "[219]\tvalidation_0-auc:0.770054\n",
      "[220]\tvalidation_0-auc:0.7701\n",
      "[221]\tvalidation_0-auc:0.770173\n",
      "[222]\tvalidation_0-auc:0.770247\n",
      "[223]\tvalidation_0-auc:0.770299\n",
      "[224]\tvalidation_0-auc:0.770413\n",
      "[225]\tvalidation_0-auc:0.770458\n",
      "[226]\tvalidation_0-auc:0.770503\n",
      "[227]\tvalidation_0-auc:0.770541\n",
      "[228]\tvalidation_0-auc:0.770595\n",
      "[229]\tvalidation_0-auc:0.770661\n",
      "[230]\tvalidation_0-auc:0.770729\n",
      "[231]\tvalidation_0-auc:0.770801\n",
      "[232]\tvalidation_0-auc:0.770874\n",
      "[233]\tvalidation_0-auc:0.770956\n",
      "[234]\tvalidation_0-auc:0.771061\n",
      "[235]\tvalidation_0-auc:0.771126\n",
      "[236]\tvalidation_0-auc:0.771193\n",
      "[237]\tvalidation_0-auc:0.771255\n",
      "[238]\tvalidation_0-auc:0.771295\n",
      "[239]\tvalidation_0-auc:0.771298\n",
      "[240]\tvalidation_0-auc:0.771343\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "       colsample_bynode=1, colsample_bytree=1, gamma=0.1,\n",
       "       learning_rate=0.1, max_delta_step=0, max_depth=3,\n",
       "       min_child_weight=1, missing=None, n_estimators=241, n_jobs=1,\n",
       "       nthread=None, objective='binary:logistic', random_state=0,\n",
       "       reg_alpha=0, reg_lambda=1, scale_pos_weight=11.387150050352467,\n",
       "       seed=None, silent=None, subsample=0.5, verbosity=1)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n_estimators = clf.best_ntree_limit\n",
    "clf = XGBClassifier(n_estimators=n_estimators, objective='binary:logistic', gamma=0.1, subsample=0.5, scale_pos_weight=ratio )\n",
    "clf.fit(app_train, train_labels, eval_set=[(app_train, train_labels)], eval_metric='auc')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC: 0.7753978796069996\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import roc_auc_score\n",
    "print('AUC:', roc_auc_score(y_test, clf.predict_proba(X_test)[:,1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = clf.predict(app_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission = pd.DataFrame({'SK_ID_CURR': test_sk_id_curr, 'TARGET': predictions})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SK_ID_CURR</th>\n",
       "      <th>TARGET</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>100001</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>100005</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>100013</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>100028</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>100038</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   SK_ID_CURR  TARGET\n",
       "0      100001       0\n",
       "1      100005       1\n",
       "2      100013       0\n",
       "3      100028       0\n",
       "4      100038       1"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submission.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission.to_csv('baseline_xgboost_trial_only_labelencoded.csv', index = False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}