Yellow messenger predict-lgbm-lstm-distillbert.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_kg_hide-output": true,
    "execution": {
     "iopub.execute_input": "2020-10-20T12:38:27.921220Z",
     "iopub.status.busy": "2020-10-20T12:38:27.920412Z",
     "iopub.status.idle": "2020-10-20T12:38:42.689980Z",
     "shell.execute_reply": "2020-10-20T12:38:42.688954Z"
    },
    "papermill": {
     "duration": 14.786036,
     "end_time": "2020-10-20T12:38:42.690135",
     "exception": false,
     "start_time": "2020-10-20T12:38:27.904099",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting sentence-transformers\r\n",
      "  Downloading sentence-transformers-0.3.8.tar.gz (66 kB)\r\n",
      "\u001b[K     |████████████████████████████████| 66 kB 1.7 MB/s \r\n",
      "\u001b[?25hCollecting transformers<3.4.0,>=3.1.0\r\n",
      "  Downloading transformers-3.3.1-py3-none-any.whl (1.1 MB)\r\n",
      "\u001b[K     |████████████████████████████████| 1.1 MB 7.7 MB/s \r\n",
      "\u001b[?25hRequirement already satisfied, skipping upgrade: tqdm in /opt/conda/lib/python3.7/site-packages (from sentence-transformers) (4.45.0)\r\n",
      "Requirement already satisfied, skipping upgrade: torch>=1.2.0 in /opt/conda/lib/python3.7/site-packages (from sentence-transformers) (1.6.0)\r\n",
      "Requirement already satisfied, skipping upgrade: numpy in /opt/conda/lib/python3.7/site-packages (from sentence-transformers) (1.18.5)\r\n",
      "Requirement already satisfied, skipping upgrade: scikit-learn in /opt/conda/lib/python3.7/site-packages (from sentence-transformers) (0.23.2)\r\n",
      "Requirement already satisfied, skipping upgrade: scipy in /opt/conda/lib/python3.7/site-packages (from sentence-transformers) (1.4.1)\r\n",
      "Requirement already satisfied, skipping upgrade: nltk in /opt/conda/lib/python3.7/site-packages (from sentence-transformers) (3.2.4)\r\n",
      "Requirement already satisfied, skipping upgrade: requests in /opt/conda/lib/python3.7/site-packages (from transformers<3.4.0,>=3.1.0->sentence-transformers) (2.23.0)\r\n",
      "Requirement already satisfied, skipping upgrade: regex!=2019.12.17 in /opt/conda/lib/python3.7/site-packages (from transformers<3.4.0,>=3.1.0->sentence-transformers) (2020.4.4)\r\n",
      "Collecting tokenizers==0.8.1.rc2\r\n",
      "  Downloading tokenizers-0.8.1rc2-cp37-cp37m-manylinux1_x86_64.whl (3.0 MB)\r\n",
      "\u001b[K     |████████████████████████████████| 3.0 MB 54.1 MB/s \r\n",
      "\u001b[?25hRequirement already satisfied, skipping upgrade: sacremoses in /opt/conda/lib/python3.7/site-packages (from transformers<3.4.0,>=3.1.0->sentence-transformers) (0.0.43)\r\n",
      "Requirement already satisfied, skipping upgrade: packaging in /opt/conda/lib/python3.7/site-packages (from transformers<3.4.0,>=3.1.0->sentence-transformers) (20.1)\r\n",
      "Requirement already satisfied, skipping upgrade: sentencepiece!=0.1.92 in /opt/conda/lib/python3.7/site-packages (from transformers<3.4.0,>=3.1.0->sentence-transformers) (0.1.91)\r\n",
      "Requirement already satisfied, skipping upgrade: filelock in /opt/conda/lib/python3.7/site-packages (from transformers<3.4.0,>=3.1.0->sentence-transformers) (3.0.10)\r\n",
      "Requirement already satisfied, skipping upgrade: future in /opt/conda/lib/python3.7/site-packages (from torch>=1.2.0->sentence-transformers) (0.18.2)\r\n",
      "Requirement already satisfied, skipping upgrade: threadpoolctl>=2.0.0 in /opt/conda/lib/python3.7/site-packages (from scikit-learn->sentence-transformers) (2.1.0)\r\n",
      "Requirement already satisfied, skipping upgrade: joblib>=0.11 in /opt/conda/lib/python3.7/site-packages (from scikit-learn->sentence-transformers) (0.14.1)\r\n",
      "Requirement already satisfied, skipping upgrade: six in /opt/conda/lib/python3.7/site-packages (from nltk->sentence-transformers) (1.14.0)\r\n",
      "Requirement already satisfied, skipping upgrade: idna<3,>=2.5 in /opt/conda/lib/python3.7/site-packages (from requests->transformers<3.4.0,>=3.1.0->sentence-transformers) (2.9)\r\n",
      "Requirement already satisfied, skipping upgrade: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /opt/conda/lib/python3.7/site-packages (from requests->transformers<3.4.0,>=3.1.0->sentence-transformers) (1.24.3)\r\n",
      "Requirement already satisfied, skipping upgrade: certifi>=2017.4.17 in /opt/conda/lib/python3.7/site-packages (from requests->transformers<3.4.0,>=3.1.0->sentence-transformers) (2020.6.20)\r\n",
      "Requirement already satisfied, skipping upgrade: chardet<4,>=3.0.2 in /opt/conda/lib/python3.7/site-packages (from requests->transformers<3.4.0,>=3.1.0->sentence-transformers) (3.0.4)\r\n",
      "Requirement already satisfied, skipping upgrade: click in /opt/conda/lib/python3.7/site-packages (from sacremoses->transformers<3.4.0,>=3.1.0->sentence-transformers) (7.1.1)\r\n",
      "Requirement already satisfied, skipping upgrade: pyparsing>=2.0.2 in /opt/conda/lib/python3.7/site-packages (from packaging->transformers<3.4.0,>=3.1.0->sentence-transformers) (2.4.7)\r\n",
      "Building wheels for collected packages: sentence-transformers\r\n",
      "  Building wheel for sentence-transformers (setup.py) ... \u001b[?25l-\b \b\\\b \b|\b \bdone\r\n",
      "\u001b[?25h  Created wheel for sentence-transformers: filename=sentence_transformers-0.3.8-py3-none-any.whl size=101994 sha256=d0c0fc21a4ee49e3a67f91302bac0daa6eaeee70228836e825a195b81db9833a\r\n",
      "  Stored in directory: /root/.cache/pip/wheels/1c/43/65/fe0f3ea9327623e749a79eb5dfad85a809c84064b1cc4682c1\r\n",
      "Successfully built sentence-transformers\r\n",
      "Installing collected packages: tokenizers, transformers, sentence-transformers\r\n",
      "  Attempting uninstall: tokenizers\r\n",
      "    Found existing installation: tokenizers 0.8.1rc1\r\n",
      "    Uninstalling tokenizers-0.8.1rc1:\r\n",
      "      Successfully uninstalled tokenizers-0.8.1rc1\r\n",
      "  Attempting uninstall: transformers\r\n",
      "    Found existing installation: transformers 3.0.2\r\n",
      "    Uninstalling transformers-3.0.2:\r\n",
      "      Successfully uninstalled transformers-3.0.2\r\n",
      "\u001b[31mERROR: After October 2020 you may experience errors when installing or updating packages. This is because pip will change the way that it resolves dependency conflicts.\r\n",
      "\r\n",
      "We recommend you use --use-feature=2020-resolver to test your packages with the new resolver before it becomes the default.\r\n",
      "\r\n",
      "allennlp 1.1.0 requires transformers<3.1,>=3.0, but you'll have transformers 3.3.1 which is incompatible.\u001b[0m\r\n",
      "Successfully installed sentence-transformers-0.3.8 tokenizers-0.8.1rc2 transformers-3.3.1\r\n",
      "\u001b[33mWARNING: You are using pip version 20.2.3; however, version 20.2.4 is available.\r\n",
      "You should consider upgrading via the '/opt/conda/bin/python3.7 -m pip install --upgrade pip' command.\u001b[0m\r\n"
     ]
    }
   ],
   "source": [
    "!pip install -U sentence-transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2020-10-20T12:38:42.759622Z",
     "iopub.status.busy": "2020-10-20T12:38:42.758783Z",
     "iopub.status.idle": "2020-10-20T12:38:52.348097Z",
     "shell.execute_reply": "2020-10-20T12:38:52.347460Z"
    },
    "papermill": {
     "duration": 9.628368,
     "end_time": "2020-10-20T12:38:52.348227",
     "exception": false,
     "start_time": "2020-10-20T12:38:42.719859",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m W&B installed but not logged in.  Run `wandb login` or set the WANDB_API_KEY env variable.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np \n",
    "import pandas as pd \n",
    "import os\n",
    "from tqdm import tqdm\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.layers import Dense, Input\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.callbacks import ModelCheckpoint\n",
    "from kaggle_datasets import KaggleDatasets\n",
    "import transformers\n",
    "from tokenizers import BertWordPieceTokenizer\n",
    "from sentence_transformers import SentenceTransformer\n",
    "import matplotlib.pyplot as plt\n",
    "import lightgbm as lgb\n",
    "import keras.layers as L\n",
    "from keras.models import Model as M\n",
    "from keras.optimizers import Adam as A"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-10-20T12:38:52.409571Z",
     "iopub.status.busy": "2020-10-20T12:38:52.408748Z",
     "iopub.status.idle": "2020-10-20T12:38:52.415173Z",
     "shell.execute_reply": "2020-10-20T12:38:52.414457Z"
    },
    "papermill": {
     "duration": 0.043743,
     "end_time": "2020-10-20T12:38:52.415294",
     "exception": false,
     "start_time": "2020-10-20T12:38:52.371551",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.utils import to_categorical\n",
    "from sklearn import preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-10-20T12:38:52.473118Z",
     "iopub.status.busy": "2020-10-20T12:38:52.471963Z",
     "iopub.status.idle": "2020-10-20T12:38:52.499624Z",
     "shell.execute_reply": "2020-10-20T12:38:52.498824Z"
    },
    "papermill": {
     "duration": 0.059243,
     "end_time": "2020-10-20T12:38:52.499758",
     "exception": false,
     "start_time": "2020-10-20T12:38:52.440515",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "TEXT_COL = 'Utterance'\n",
    "EMB_PATH = '../input/fasttext-crawl-300d-2m/crawl-300d-2M.vec'\n",
    "train=pd.read_csv('../input/data-set-augment-intent/data_file.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a",
    "execution": {
     "iopub.execute_input": "2020-10-20T12:38:52.563503Z",
     "iopub.status.busy": "2020-10-20T12:38:52.562479Z",
     "iopub.status.idle": "2020-10-20T12:38:52.565149Z",
     "shell.execute_reply": "2020-10-20T12:38:52.565750Z"
    },
    "papermill": {
     "duration": 0.042677,
     "end_time": "2020-10-20T12:38:52.565918",
     "exception": false,
     "start_time": "2020-10-20T12:38:52.523241",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n",
    "\n",
    "def load_embeddings(embed_dir=EMB_PATH):\n",
    "    \n",
    "    embedding_index = dict(get_coefs(*o.strip().split(\" \")) for o in tqdm(open(embed_dir)))\n",
    "    return embedding_index\n",
    "\n",
    "def build_embedding_matrix(word_index, embeddings_index, max_features, lower = True, verbose = True):\n",
    "    \n",
    "    embedding_matrix = np.zeros((max_features, 300))\n",
    "    for word, i in tqdm(word_index.items(),disable = not verbose):\n",
    "        if lower:\n",
    "            word = word.lower()\n",
    "        if i >= max_features: continue\n",
    "        try:\n",
    "            embedding_vector = embeddings_index[word]\n",
    "        except:\n",
    "            embedding_vector = embeddings_index[\"unknown\"]\n",
    "        if embedding_vector is not None:\n",
    "            # words not found in embedding index will be all-zeros.\n",
    "            embedding_matrix[i] = embedding_vector\n",
    "    return embedding_matrix\n",
    "\n",
    "def build_matrix(word_index, embeddings_index):\n",
    "    \n",
    "    embedding_matrix = np.zeros((len(word_index) + 1,300))\n",
    "    for word, i in word_index.items():\n",
    "        try:\n",
    "            embedding_matrix[i] = embeddings_index[word]\n",
    "        except:\n",
    "            embedding_matrix[i] = embeddings_index[\"unknown\"]\n",
    "    return embedding_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-10-20T12:38:52.628607Z",
     "iopub.status.busy": "2020-10-20T12:38:52.627550Z",
     "iopub.status.idle": "2020-10-20T12:38:52.631042Z",
     "shell.execute_reply": "2020-10-20T12:38:52.630290Z"
    },
    "papermill": {
     "duration": 0.041821,
     "end_time": "2020-10-20T12:38:52.631164",
     "exception": false,
     "start_time": "2020-10-20T12:38:52.589343",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sklearn import preprocessing\n",
    "le = preprocessing.LabelEncoder()\n",
    "labels = le.fit_transform(train['Intent'].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-10-20T12:38:52.711871Z",
     "iopub.status.busy": "2020-10-20T12:38:52.710994Z",
     "iopub.status.idle": "2020-10-20T12:39:22.947713Z",
     "shell.execute_reply": "2020-10-20T12:39:22.946787Z"
    },
    "papermill": {
     "duration": 30.293019,
     "end_time": "2020-10-20T12:39:22.947889",
     "exception": false,
     "start_time": "2020-10-20T12:38:52.654870",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a09a83247fbf4ee0b765942e6f9f8929",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=231508.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "391e4b80669e4e22baf17cdbc2bd70cb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=442.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "eb1c628bf16d4ece9a0e5a51b9224329",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=363423424.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing TFDistilBertModel: ['vocab_transform', 'activation_13', 'vocab_projector', 'vocab_layer_norm']\n",
      "- This IS expected if you are initializing TFDistilBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPretraining model).\n",
      "- This IS NOT expected if you are initializing TFDistilBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "All the weights of TFDistilBertModel were initialized from the model checkpoint at distilbert-base-uncased.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFDistilBertModel for predictions without further training.\n"
     ]
    }
   ],
   "source": [
    "def build_model_lstm(embedding_matrix , word_index , verbose = False, compile = True):\n",
    "    \n",
    "    sequence_input = L.Input(shape=(maxlen,), dtype='int32')\n",
    "    embedding_layer = L.Embedding(len(word_index) + 1,\n",
    "                                300,\n",
    "                                weights=[embedding_matrix],\n",
    "                                input_length=maxlen,\n",
    "                                trainable=False)\n",
    "    x = embedding_layer(sequence_input)\n",
    "    x = L.SpatialDropout1D(0.2)(x)\n",
    "    x = L.Bidirectional(L.GRU(128, return_sequences=True))(x)\n",
    "    avg_pool1 = L.GlobalAveragePooling1D()(x)\n",
    "    preds = L.Dense(144, activation='softmax')(avg_pool1)\n",
    "\n",
    "\n",
    "    model = M(sequence_input, preds)\n",
    "    if verbose:\n",
    "        model.summary()\n",
    "    if compile:\n",
    "        model.compile(loss='categorical_crossentropy',optimizer=A(0.0001),metrics=['acc'])\n",
    "    return model\n",
    "\n",
    "\n",
    "def fast_encode(texts, tokenizer, chunk_size=256, maxlen=128):\n",
    "    tokenizer.enable_truncation(max_length=maxlen)\n",
    "    tokenizer.enable_padding(length=maxlen)\n",
    "    all_ids = []\n",
    "    \n",
    "    for i in tqdm(range(0, len(texts), chunk_size)):\n",
    "        text_chunk = texts[i:i+chunk_size].tolist()\n",
    "        encs = tokenizer.encode_batch(text_chunk)\n",
    "        all_ids.extend([enc.ids for enc in encs])\n",
    "    \n",
    "    return np.array(all_ids)\n",
    "\n",
    "def build_model_bert(transformer, max_len=512):\n",
    "    \n",
    "    input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name=\"input_word_ids\")\n",
    "    sequence_output = transformer(input_word_ids)[0]\n",
    "    cls_token = sequence_output[:, 0, :]\n",
    "    out = Dense(144, activation='softmax')(cls_token)\n",
    "    \n",
    "    model = Model(inputs=input_word_ids, outputs=out)\n",
    "    model.compile(Adam(lr=1e-5), loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "    \n",
    "    return model\n",
    "\n",
    "LGB_BO = {'feature_fraction': 0.05,\n",
    " 'lambda_l1': 0.0,\n",
    " 'lambda_l2': 5.0,\n",
    " 'learning_rate': 0.01,\n",
    " 'max_depth': 4.882082219992168,\n",
    " 'min_data_in_leaf': 5.07479887170428,\n",
    " 'min_gain_to_split': 0.0,\n",
    " 'min_sum_hessian_in_leaf': 0.01,\n",
    " 'num_leaves': 5.0}\n",
    "\n",
    "param_lgb = {\n",
    "        'num_leaves': int(LGB_BO['num_leaves']), # remember to int here\n",
    "        'max_bin': 63,\n",
    "        'min_data_in_leaf': int(LGB_BO['min_data_in_leaf']), # remember to int here\n",
    "        'learning_rate': LGB_BO['learning_rate'],\n",
    "        'min_sum_hessian_in_leaf': LGB_BO['min_sum_hessian_in_leaf'],\n",
    "        'bagging_fraction': 1.0, \n",
    "        'bagging_freq': 5, \n",
    "        'num_class':144,\n",
    "        'feature_fraction': LGB_BO['feature_fraction'],\n",
    "        'lambda_l1': LGB_BO['lambda_l1'],\n",
    "        'lambda_l2': LGB_BO['lambda_l2'],\n",
    "        'min_gain_to_split': LGB_BO['min_gain_to_split'],\n",
    "        'max_depth': int(LGB_BO['max_depth']), # remember to int here\n",
    "        'save_binary': True,\n",
    "        'seed': 1337,\n",
    "        'feature_fraction_seed': 1337,\n",
    "        'bagging_seed': 1337,\n",
    "        'drop_seed': 1337,\n",
    "        'data_random_seed': 1337,\n",
    "        'objective': 'multiclass',\n",
    "        'boosting_type': 'gbdt',\n",
    "        'verbose': 1,\n",
    "        'metric': 'multi_error',\n",
    "        'is_unbalance': True,\n",
    "        'boost_from_average': False}\n",
    "\n",
    "\n",
    "# Configuration\n",
    "EPOCHS = 20\n",
    "BATCH_SIZE = 16\n",
    "MAX_LEN = 64\n",
    "maxlen = 64\n",
    "max_features = 100000\n",
    "embed_size = 300\n",
    "\n",
    "\n",
    "\n",
    "tokenizer = transformers.DistilBertTokenizer.from_pretrained('distilbert-base-uncased')\n",
    "tokenizer.save_pretrained('.')\n",
    "fast_tokenizer = BertWordPieceTokenizer('vocab.txt', lowercase=False)\n",
    "transformer_layer = transformers.TFDistilBertModel.from_pretrained('distilbert-base-uncased')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-10-20T12:39:23.028887Z",
     "iopub.status.busy": "2020-10-20T12:39:23.025156Z",
     "iopub.status.idle": "2020-10-20T12:39:23.032637Z",
     "shell.execute_reply": "2020-10-20T12:39:23.031871Z"
    },
    "papermill": {
     "duration": 0.055612,
     "end_time": "2020-10-20T12:39:23.032765",
     "exception": false,
     "start_time": "2020-10-20T12:39:22.977153",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def predict_func(input_data,bert_weight,lgbm_weight):\n",
    "    \"\"\" \n",
    "    Okay so this is the most important part of the code\n",
    "    \n",
    "    this function predicts the data with the models trained\n",
    "    with the augmented data on the models\n",
    "    \n",
    "    1. DistillBERT base uncase\n",
    "    2. LGBM trained and hypertuned on the \n",
    "        token embeddings a pre-trained bert model\n",
    "    3. naive LSTM model with fasttext pre trained embeddings\n",
    "    \n",
    "    parameters:-\n",
    "    input_data : this where you put data to be predict an intent\n",
    "                 so how not to break a code \n",
    "                 you can pass a series of data like \n",
    "                 train['Utterance'] shown in the short example\n",
    "    \n",
    "    bert_weight: weight of the bert in the ensemble\n",
    "    lgbm_weight: weight of lgbm model in the ensemble\n",
    "    lstm_weight: 1-bert_weight-lgbm_weight \n",
    "    \n",
    "    returns: A list of outputs of the instances as labels\n",
    "    \n",
    "    \"\"\"\n",
    "    \n",
    "    \n",
    "    lstm_weight = 1 - bert_weight - lgbm_weight\n",
    "    \n",
    "    \"\"\" BERT Inference \"\"\"\n",
    "    x_test = fast_encode(input_data.astype(str), fast_tokenizer, maxlen=MAX_LEN)\n",
    "    model_bert = build_model_bert(transformer_layer, max_len=MAX_LEN)\n",
    "    bert_5_model_preds = []\n",
    "    for i in os.listdir('../input/classfy-intent-distillbert/'):\n",
    "        try:\n",
    "            model_bert.load_weights('../input/classfy-intent-distillbert/'+i)\n",
    "            preds_bert = model_bert.predict(x_test)\n",
    "            bert_5_model_preds.append(preds_bert)\n",
    "        except:\n",
    "            pass\n",
    "    print('bert_inference_completed')\n",
    "    \n",
    "    \"\"\" LGBM Inference \"\"\"\n",
    "    model_lgbm = SentenceTransformer('distilbert-base-nli-mean-tokens')\n",
    "    embeddings = model_lgbm.encode(input_data.astype(str), show_progress_bar=True)\n",
    "    x_test = pd.DataFrame(embeddings)\n",
    "    lgbm_5_model_preds = []\n",
    "    for i in os.listdir('../input/cluster-bert-lgbm/'):\n",
    "        try:\n",
    "            model_lgbm = lgb.Booster(model_file='../input/cluster-bert-lgbm/'+i)\n",
    "            preds_lgbm = model_lgbm.predict(x_test)\n",
    "            lgbm_5_model_preds.append(preds_lgbm)\n",
    "        except:\n",
    "            pass\n",
    "    print('lgbm_inference_completed')\n",
    "    \n",
    "    \"\"\" LSTM Inference \"\"\"\n",
    "    tokenizer = Tokenizer(num_words=max_features, lower=True)\n",
    "    train=pd.read_csv('../input/data-set-augment-intent/data_file.csv')\n",
    "    tokenizer.fit_on_texts(list(train['Utterance']))\n",
    "    word_index = tokenizer.word_index\n",
    "    X_train = tokenizer.texts_to_sequences(list(input_data))\n",
    "    X_train = pad_sequences(X_train, maxlen=maxlen)    \n",
    "    embedding_matrix = build_matrix(tokenizer.word_index, load_embeddings())\n",
    "    model_lstm = build_model_lstm(embedding_matrix=embedding_matrix,word_index=word_index)\n",
    "    lstm_5_model_preds = []\n",
    "    \n",
    "    for i in os.listdir('../input/intent-classification-lstm/'):\n",
    "        model_lstm.load_weights('../input/intent-classification-lstm/'+i)\n",
    "        preds_lstm = model_lstm.predict(X_train)\n",
    "        lstm_5_model_preds.append(preds_lstm)\n",
    "    print('lstm_inference_completed')\n",
    "    \n",
    "    bert_avg_preds = np.average(bert_5_model_preds,axis=0)\n",
    "    lgbm_avg_preds = np.average(lgbm_5_model_preds,axis=0)\n",
    "    lstm_avg_preds = np.average(lstm_5_model_preds,axis=0)\n",
    "    \n",
    "    final_average = bert_avg_preds*bert_weight + lgbm_avg_preds*lgbm_weight + lstm_avg_preds*lstm_weight\n",
    "    return np.argmax(final_average,axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-10-20T12:39:23.097331Z",
     "iopub.status.busy": "2020-10-20T12:39:23.096250Z",
     "iopub.status.idle": "2020-10-20T12:45:45.887495Z",
     "shell.execute_reply": "2020-10-20T12:45:45.886680Z"
    },
    "papermill": {
     "duration": 382.826997,
     "end_time": "2020-10-20T12:45:45.887734",
     "exception": false,
     "start_time": "2020-10-20T12:39:23.060737",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1/1 [00:00<00:00, 500.81it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bert_inference_completed\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245M/245M [00:14<00:00, 16.9MB/s]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bba615e925a348afb219a7e486c7e471",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Batches', max=1.0, style=ProgressStyle(description_width=…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "lgbm_inference_completed\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2000001it [05:15, 6329.80it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "lstm_inference_completed\n"
     ]
    }
   ],
   "source": [
    "final_average = predict_func(train['Utterance'][:20],0.5,0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 1.32611,
     "end_time": "2020-10-20T12:45:48.565555",
     "exception": false,
     "start_time": "2020-10-20T12:45:47.239445",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "papermill": {
   "duration": 448.306559,
   "end_time": "2020-10-20T12:45:50.906659",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2020-10-20T12:38:22.600100",
   "version": "2.1.0"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "07f1bbc2a83d4595a6f6de081f2e36b0": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "09fb0ecf496544299cca9b10ea7aa373": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "Downloading: 100%",
       "description_tooltip": null,
       "layout": "IPY_MODEL_42e124c9920448b78c08be723e18984c",
       "max": 363423424.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_68be87a0301d4ee0b11bc50fb6559c3a",
       "value": 363423424.0
      }
     },
     "0dd225ccc75d46f8ac38bc5a2d2655dc": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "3079c9578d1b4f1db5d522292f00c1a2": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": "initial"
      }
     },
     "31e8389f099b4a76b0eab05700189f8c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_cc8c8e31fdf84e59a2c25d7a2539a611",
       "placeholder": "​",
       "style": "IPY_MODEL_d99d18c41bb54e09b5389660def918c5",
       "value": " 363M/363M [00:26&lt;00:00, 13.6MB/s]"
      }
     },
     "391e4b80669e4e22baf17cdbc2bd70cb": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_b5f130562c3141a9a609fb13d78f8e45",
        "IPY_MODEL_cec7d60f69f34d4281ef07b600e668bd"
       ],
       "layout": "IPY_MODEL_07f1bbc2a83d4595a6f6de081f2e36b0"
      }
     },
     "3c2dce12bc3e4e7eac33f0138eb43b67": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_7c4e2d1b8ec34322acacdddbe4b64661",
       "placeholder": "​",
       "style": "IPY_MODEL_75b9c0818fae401592a6e83c15e579a5",
       "value": " 1/1 [00:29&lt;00:00, 29.88s/it]"
      }
     },
     "42e124c9920448b78c08be723e18984c": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "4665fde75e844e71b2d167371a3abd82": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "Downloading: 100%",
       "description_tooltip": null,
       "layout": "IPY_MODEL_dede450911cf459e90f62135e444fab2",
       "max": 231508.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_eb45e6946de54245a899fe600a28ad35",
       "value": 231508.0
      }
     },
     "4be45eaa08c444be8737fcee502af5cf": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "55f0c8776a864b339e4fff8d803aa914": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_d37cce93e3ee4ce49a4b1e583c8a3620",
       "placeholder": "​",
       "style": "IPY_MODEL_ed49db7c50a24d54aa091eeb9fd69838",
       "value": " 232k/232k [00:00&lt;00:00, 1.86MB/s]"
      }
     },
     "68be87a0301d4ee0b11bc50fb6559c3a": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": "initial"
      }
     },
     "6bbb44a16476479daf01b942eee69c61": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "6eb3e3cda4e4409eaeda43465423f5e4": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "7144ae251c4f4331b7f36edbb54686a4": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "75b9c0818fae401592a6e83c15e579a5": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "7c4e2d1b8ec34322acacdddbe4b64661": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "81b0355297764c30914650b39c81d101": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "9eb9971f60b8476f811bfe8d2a357e9c": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "a09a83247fbf4ee0b765942e6f9f8929": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_4665fde75e844e71b2d167371a3abd82",
        "IPY_MODEL_55f0c8776a864b339e4fff8d803aa914"
       ],
       "layout": "IPY_MODEL_4be45eaa08c444be8737fcee502af5cf"
      }
     },
     "ad40ffe5d75a49b09fd28134b8fc0cd2": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "Batches: 100%",
       "description_tooltip": null,
       "layout": "IPY_MODEL_0dd225ccc75d46f8ac38bc5a2d2655dc",
       "max": 1.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_d881435e69064fafb55c4ff65b9346ad",
       "value": 1.0
      }
     },
     "b5f130562c3141a9a609fb13d78f8e45": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "Downloading: 100%",
       "description_tooltip": null,
       "layout": "IPY_MODEL_7144ae251c4f4331b7f36edbb54686a4",
       "max": 442.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_3079c9578d1b4f1db5d522292f00c1a2",
       "value": 442.0
      }
     },
     "bba615e925a348afb219a7e486c7e471": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_ad40ffe5d75a49b09fd28134b8fc0cd2",
        "IPY_MODEL_3c2dce12bc3e4e7eac33f0138eb43b67"
       ],
       "layout": "IPY_MODEL_81b0355297764c30914650b39c81d101"
      }
     },
     "cc8c8e31fdf84e59a2c25d7a2539a611": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "cec7d60f69f34d4281ef07b600e668bd": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_6eb3e3cda4e4409eaeda43465423f5e4",
       "placeholder": "​",
       "style": "IPY_MODEL_6bbb44a16476479daf01b942eee69c61",
       "value": " 442/442 [00:00&lt;00:00, 629B/s]"
      }
     },
     "d37cce93e3ee4ce49a4b1e583c8a3620": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "d881435e69064fafb55c4ff65b9346ad": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": "initial"
      }
     },
     "d99d18c41bb54e09b5389660def918c5": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "dede450911cf459e90f62135e444fab2": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "eb1c628bf16d4ece9a0e5a51b9224329": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_09fb0ecf496544299cca9b10ea7aa373",
        "IPY_MODEL_31e8389f099b4a76b0eab05700189f8c"
       ],
       "layout": "IPY_MODEL_9eb9971f60b8476f811bfe8d2a357e9c"
      }
     },
     "eb45e6946de54245a899fe600a28ad35": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": "initial"
      }
     },
     "ed49db7c50a24d54aa091eeb9fd69838": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}