Yellow messenger distilbert-model-2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [],
   "source": [
    "\"\"\" \n",
    "    A simple stratified 5 fold implementation of Distill bert \n",
    "    with augment data\n",
    "    \n",
    "\"\"\"\n",
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.layers import Dense, Input\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.callbacks import ModelCheckpoint\n",
    "from kaggle_datasets import KaggleDatasets\n",
    "import transformers\n",
    "from tqdm.notebook import tqdm\n",
    "from tokenizers import BertWordPieceTokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2.7.0'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "transformers.__version__"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Helper Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [],
   "source": [
    "def fast_encode(texts, tokenizer, chunk_size=256, maxlen=128):\n",
    "    tokenizer.enable_truncation(max_length=maxlen)\n",
    "    tokenizer.enable_padding(max_length=maxlen)\n",
    "    all_ids = []\n",
    "    \n",
    "    for i in tqdm(range(0, len(texts), chunk_size)):\n",
    "        text_chunk = texts[i:i+chunk_size].tolist()\n",
    "        encs = tokenizer.encode_batch(text_chunk)\n",
    "        all_ids.extend([enc.ids for enc in encs])\n",
    "    \n",
    "    return np.array(all_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_model(transformer, max_len=512):\n",
    "    \n",
    "    input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name=\"input_word_ids\")\n",
    "    sequence_output = transformer(input_word_ids)[0]\n",
    "    cls_token = sequence_output[:, 0, :]\n",
    "    out = Dense(144, activation='softmax')(cls_token)\n",
    "    \n",
    "    model = Model(inputs=input_word_ids, outputs=out)\n",
    "    model.compile(Adam(lr=1e-5), loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TPU Configs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "REPLICAS:  1\n"
     ]
    }
   ],
   "source": [
    "# Detect hardware, return appropriate distribution strategy\n",
    "try:\n",
    "    # TPU detection. No parameters necessary if TPU_NAME environment variable is\n",
    "    # set: this is always the case on Kaggle.\n",
    "    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n",
    "    print('Running on TPU ', tpu.master())\n",
    "except ValueError:\n",
    "    tpu = None\n",
    "\n",
    "if tpu:\n",
    "    tf.config.experimental_connect_to_cluster(tpu)\n",
    "    tf.tpu.experimental.initialize_tpu_system(tpu)\n",
    "    strategy = tf.distribute.experimental.TPUStrategy(tpu)\n",
    "else:\n",
    "    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n",
    "    strategy = tf.distribute.get_strategy()\n",
    "\n",
    "print(\"REPLICAS: \", strategy.num_replicas_in_sync)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "AUTO = tf.data.experimental.AUTOTUNE\n",
    "\n",
    "# Data access\n",
    "GCS_DS_PATH = KaggleDatasets().get_gcs_path()\n",
    "\n",
    "# Configuration\n",
    "EPOCHS = 20\n",
    "BATCH_SIZE = 16 * strategy.num_replicas_in_sync\n",
    "MAX_LEN = 64"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create fast tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d85f61c6faa244ac9311edb628a86f24",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=231508.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Tokenizer(vocabulary_size=30522, model=BertWordPiece, add_special_tokens=True, unk_token=[UNK], sep_token=[SEP], cls_token=[CLS], clean_text=True, handle_chinese_chars=True, strip_accents=True, lowercase=False, wordpieces_prefix=##)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# First load the real tokenizer\n",
    "tokenizer = transformers.DistilBertTokenizer.from_pretrained('distilbert-base-uncased')\n",
    "# Save the loaded tokenizer locally\n",
    "tokenizer.save_pretrained('.')\n",
    "# Reload it with the huggingface tokenizers library\n",
    "fast_tokenizer = BertWordPieceTokenizer('vocab.txt', lowercase=False)\n",
    "fast_tokenizer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load text data into memory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv('../input/data-set-augment-intent/data_file.csv')\n",
    "from sklearn import preprocessing\n",
    "le = preprocessing.LabelEncoder()\n",
    "labels = le.fit_transform(train['Intent'].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "144"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(set(labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4c26051874bd457ab604d8ca9fc5725c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=38.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.utils import shuffle\n",
    "x_train = fast_encode(train.Utterance.astype(str), fast_tokenizer, maxlen=MAX_LEN)\n",
    "y_train = tf.keras.utils.to_categorical(labels, num_classes=len(set(labels)), dtype='float32')\n",
    "x_train,y_train = shuffle(x_train,y_train, random_state=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build datasets objects"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = (\n",
    "    tf.data.Dataset\n",
    "    .from_tensor_slices((x_train, y_train))\n",
    "    .repeat()\n",
    "    .shuffle(2048)\n",
    "    .batch(BATCH_SIZE)\n",
    "    .prefetch(AUTO)\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load model into the TPU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5c9ff7ac1cd14aecb176e507f9ca6e83",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=442.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6c5c3628912f4eb985e158d5107ac69c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=363423424.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Model: \"model\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_word_ids (InputLayer)  [(None, 64)]              0         \n",
      "_________________________________________________________________\n",
      "tf_distil_bert_model (TFDist ((None, 64, 768),)        66362880  \n",
      "_________________________________________________________________\n",
      "tf_op_layer_strided_slice (T [(None, 768)]             0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 144)               110736    \n",
      "=================================================================\n",
      "Total params: 66,473,616\n",
      "Trainable params: 66,473,616\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "CPU times: user 14.3 s, sys: 2.51 s, total: 16.8 s\n",
      "Wall time: 17.9 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "transformer_layer = transformers.TFDistilBertModel.from_pretrained('distilbert-base-uncased')\n",
    "model = build_model(transformer_layer, max_len=MAX_LEN)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First, we train on the subset of the training set, which is completely in English."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 7696 samples, validate on 1925 samples\n",
      "Epoch 1/15\n",
      "7680/7696 [============================>.] - ETA: 0s - loss: 4.0117 - accuracy: 0.2398\n",
      "Epoch 00001: saving model to distill_bert_0.hdf5\n",
      "7696/7696 [==============================] - 47s 6ms/sample - loss: 4.0078 - accuracy: 0.2406 - val_loss: 1.9966 - val_accuracy: 0.6904\n",
      "Epoch 2/15\n",
      "7680/7696 [============================>.] - ETA: 0s - loss: 1.3624 - accuracy: 0.8025\n",
      "Epoch 00002: saving model to distill_bert_0.hdf5\n",
      "7696/7696 [==============================] - 40s 5ms/sample - loss: 1.3613 - accuracy: 0.8028 - val_loss: 0.7960 - val_accuracy: 0.8655\n",
      "Epoch 3/15\n",
      "7680/7696 [============================>.] - ETA: 0s - loss: 0.6158 - accuracy: 0.9098\n",
      "Epoch 00003: saving model to distill_bert_0.hdf5\n",
      "7696/7696 [==============================] - 40s 5ms/sample - loss: 0.6152 - accuracy: 0.9100 - val_loss: 0.5234 - val_accuracy: 0.8914\n",
      "Epoch 4/15\n",
      "7680/7696 [============================>.] - ETA: 0s - loss: 0.3607 - accuracy: 0.9409\n",
      "Epoch 00004: saving model to distill_bert_0.hdf5\n",
      "7696/7696 [==============================] - 39s 5ms/sample - loss: 0.3612 - accuracy: 0.9407 - val_loss: 0.4076 - val_accuracy: 0.9096\n",
      "Epoch 5/15\n",
      "7680/7696 [============================>.] - ETA: 0s - loss: 0.2375 - accuracy: 0.9630\n",
      "Epoch 00005: saving model to distill_bert_0.hdf5\n",
      "7696/7696 [==============================] - 39s 5ms/sample - loss: 0.2373 - accuracy: 0.9631 - val_loss: 0.3709 - val_accuracy: 0.9153\n",
      "Epoch 6/15\n",
      "7680/7696 [============================>.] - ETA: 0s - loss: 0.1655 - accuracy: 0.9750\n",
      "Epoch 00006: saving model to distill_bert_0.hdf5\n",
      "7696/7696 [==============================] - 40s 5ms/sample - loss: 0.1652 - accuracy: 0.9751 - val_loss: 0.3422 - val_accuracy: 0.9210\n",
      "Epoch 7/15\n",
      "7680/7696 [============================>.] - ETA: 0s - loss: 0.1228 - accuracy: 0.9811\n",
      "Epoch 00007: saving model to distill_bert_0.hdf5\n",
      "7696/7696 [==============================] - 40s 5ms/sample - loss: 0.1226 - accuracy: 0.9812 - val_loss: 0.3134 - val_accuracy: 0.9252\n",
      "Epoch 8/15\n",
      "7680/7696 [============================>.] - ETA: 0s - loss: 0.0941 - accuracy: 0.9837\n",
      "Epoch 00008: saving model to distill_bert_0.hdf5\n",
      "7696/7696 [==============================] - 40s 5ms/sample - loss: 0.0944 - accuracy: 0.9836 - val_loss: 0.3255 - val_accuracy: 0.9221\n",
      "Epoch 9/15\n",
      "7680/7696 [============================>.] - ETA: 0s - loss: 0.0778 - accuracy: 0.9858\n",
      "Epoch 00009: saving model to distill_bert_0.hdf5\n",
      "7696/7696 [==============================] - 39s 5ms/sample - loss: 0.0777 - accuracy: 0.9858 - val_loss: 0.3109 - val_accuracy: 0.9299\n",
      "Epoch 10/15\n",
      "7680/7696 [============================>.] - ETA: 0s - loss: 0.0634 - accuracy: 0.9878\n",
      "Epoch 00010: saving model to distill_bert_0.hdf5\n",
      "7696/7696 [==============================] - 39s 5ms/sample - loss: 0.0634 - accuracy: 0.9878 - val_loss: 0.3262 - val_accuracy: 0.9252\n",
      "Epoch 00010: early stopping\n",
      "Train on 7697 samples, validate on 1924 samples\n",
      "Epoch 1/15\n",
      "3600/7697 [=============>................] - ETA: 33s - loss: 3.4711 - accuracy: 0.4039\n",
      "Epoch 00003: saving model to distill_bert_1.hdf5\n",
      "7697/7697 [==============================] - 45s 6ms/sample - loss: 0.2285 - accuracy: 0.9689 - val_loss: 0.1615 - val_accuracy: 0.9699\n",
      "Epoch 4/15\n",
      "7696/7697 [============================>.] - ETA: 0s - loss: 0.1433 - accuracy: 0.9775\n",
      "Epoch 00004: saving model to distill_bert_1.hdf5\n",
      "7697/7697 [==============================] - 45s 6ms/sample - loss: 0.1433 - accuracy: 0.9775 - val_loss: 0.1359 - val_accuracy: 0.9725\n",
      "Epoch 5/15\n",
      "7696/7697 [============================>.] - ETA: 0s - loss: 0.0993 - accuracy: 0.9839\n",
      "Epoch 00005: saving model to distill_bert_1.hdf5\n",
      "7697/7697 [==============================] - 45s 6ms/sample - loss: 0.0993 - accuracy: 0.9839 - val_loss: 0.1340 - val_accuracy: 0.9725\n",
      "Epoch 6/15\n",
      "7696/7697 [============================>.] - ETA: 0s - loss: 0.0834 - accuracy: 0.9839\n",
      "Epoch 00006: saving model to distill_bert_1.hdf5\n",
      "7697/7697 [==============================] - 45s 6ms/sample - loss: 0.0834 - accuracy: 0.9839 - val_loss: 0.1227 - val_accuracy: 0.9735\n",
      "Epoch 7/15\n",
      "7696/7697 [============================>.] - ETA: 0s - loss: 0.0605 - accuracy: 0.9886\n",
      "Epoch 00007: saving model to distill_bert_1.hdf5\n",
      "7697/7697 [==============================] - 45s 6ms/sample - loss: 0.0605 - accuracy: 0.9886 - val_loss: 0.1276 - val_accuracy: 0.9719\n",
      "Epoch 8/15\n",
      "7696/7697 [============================>.] - ETA: 0s - loss: 0.0568 - accuracy: 0.9890\n",
      "Epoch 00008: saving model to distill_bert_1.hdf5\n",
      "7697/7697 [==============================] - 45s 6ms/sample - loss: 0.0568 - accuracy: 0.9890 - val_loss: 0.1269 - val_accuracy: 0.9740\n",
      "Epoch 9/15\n",
      "7696/7697 [============================>.] - ETA: 0s - loss: 0.0504 - accuracy: 0.9884\n",
      "Epoch 00009: saving model to distill_bert_1.hdf5\n",
      "7697/7697 [==============================] - 44s 6ms/sample - loss: 0.0504 - accuracy: 0.9884 - val_loss: 0.1423 - val_accuracy: 0.9683\n",
      "Epoch 00009: early stopping\n",
      "Train on 7697 samples, validate on 1924 samples\n",
      "Epoch 1/15\n",
      "7696/7697 [============================>.] - ETA: 0s - loss: 1.7964 - accuracy: 0.7406\n",
      "Epoch 00001: saving model to distill_bert_2.hdf5\n",
      "7697/7697 [==============================] - 57s 7ms/sample - loss: 1.7962 - accuracy: 0.7407 - val_loss: 0.2890 - val_accuracy: 0.9771\n",
      "Epoch 2/15\n",
      "7696/7697 [============================>.] - ETA: 0s - loss: 0.2758 - accuracy: 0.9727\n",
      "Epoch 00002: saving model to distill_bert_2.hdf5\n",
      "7697/7697 [==============================] - 45s 6ms/sample - loss: 0.2757 - accuracy: 0.9727 - val_loss: 0.1239 - val_accuracy: 0.9818\n",
      "Epoch 3/15\n",
      "7696/7697 [============================>.] - ETA: 0s - loss: 0.1339 - accuracy: 0.9822\n",
      "Epoch 00003: saving model to distill_bert_2.hdf5\n",
      "7697/7697 [==============================] - 45s 6ms/sample - loss: 0.1339 - accuracy: 0.9822 - val_loss: 0.0957 - val_accuracy: 0.9818\n",
      "Epoch 4/15\n",
      "7696/7697 [============================>.] - ETA: 0s - loss: 0.0897 - accuracy: 0.9861\n",
      "Epoch 00004: saving model to distill_bert_2.hdf5\n",
      "7697/7697 [==============================] - 46s 6ms/sample - loss: 0.0897 - accuracy: 0.9861 - val_loss: 0.0932 - val_accuracy: 0.9808\n",
      "Epoch 5/15\n",
      "7696/7697 [============================>.] - ETA: 0s - loss: 0.0719 - accuracy: 0.9871\n",
      "Epoch 00005: saving model to distill_bert_2.hdf5\n",
      "7697/7697 [==============================] - 45s 6ms/sample - loss: 0.0719 - accuracy: 0.9871 - val_loss: 0.0899 - val_accuracy: 0.9818\n",
      "Epoch 6/15\n",
      "7696/7697 [============================>.] - ETA: 0s - loss: 0.0563 - accuracy: 0.9890\n",
      "Epoch 00006: saving model to distill_bert_2.hdf5\n",
      "7697/7697 [==============================] - 45s 6ms/sample - loss: 0.0563 - accuracy: 0.9890 - val_loss: 0.0846 - val_accuracy: 0.9828\n",
      "Epoch 7/15\n",
      "7696/7697 [============================>.] - ETA: 0s - loss: 0.0489 - accuracy: 0.9888\n",
      "Epoch 00007: saving model to distill_bert_2.hdf5\n",
      "7697/7697 [==============================] - 46s 6ms/sample - loss: 0.0489 - accuracy: 0.9888 - val_loss: 0.0762 - val_accuracy: 0.9854\n",
      "Epoch 8/15\n",
      "7696/7697 [============================>.] - ETA: 0s - loss: 0.0443 - accuracy: 0.9897\n",
      "Epoch 00008: saving model to distill_bert_2.hdf5\n",
      "7697/7697 [==============================] - 45s 6ms/sample - loss: 0.0443 - accuracy: 0.9897 - val_loss: 0.0799 - val_accuracy: 0.9839\n",
      "Epoch 9/15\n",
      "7696/7697 [============================>.] - ETA: 0s - loss: 0.0418 - accuracy: 0.9896\n",
      "Epoch 00009: saving model to distill_bert_2.hdf5\n",
      "7697/7697 [==============================] - 45s 6ms/sample - loss: 0.0418 - accuracy: 0.9896 - val_loss: 0.0799 - val_accuracy: 0.9808\n",
      "Epoch 10/15\n",
      "7696/7697 [============================>.] - ETA: 0s - loss: 0.0397 - accuracy: 0.9904\n",
      "Epoch 00010: saving model to distill_bert_2.hdf5\n",
      "7697/7697 [==============================] - 45s 6ms/sample - loss: 0.0397 - accuracy: 0.9904 - val_loss: 0.0879 - val_accuracy: 0.9782\n",
      "Epoch 00010: early stopping\n",
      "Train on 7697 samples, validate on 1924 samples\n",
      "Epoch 1/15\n",
      "7696/7697 [============================>.] - ETA: 0s - loss: 1.5131 - accuracy: 0.8012\n",
      "Epoch 00001: saving model to distill_bert_3.hdf5\n",
      "7697/7697 [==============================] - 58s 8ms/sample - loss: 1.5129 - accuracy: 0.8012 - val_loss: 0.1892 - val_accuracy: 0.9891\n",
      "Epoch 2/15\n",
      "7696/7697 [============================>.] - ETA: 0s - loss: 0.1824 - accuracy: 0.9834\n",
      "Epoch 00002: saving model to distill_bert_3.hdf5\n",
      "7697/7697 [==============================] - 45s 6ms/sample - loss: 0.1824 - accuracy: 0.9834 - val_loss: 0.0837 - val_accuracy: 0.9896\n",
      "Epoch 3/15\n",
      "7696/7697 [============================>.] - ETA: 0s - loss: 0.0942 - accuracy: 0.9873\n",
      "Epoch 00003: saving model to distill_bert_3.hdf5\n",
      "7697/7697 [==============================] - 45s 6ms/sample - loss: 0.0941 - accuracy: 0.9873 - val_loss: 0.0672 - val_accuracy: 0.9886\n",
      "Epoch 4/15\n",
      "7696/7697 [============================>.] - ETA: 0s - loss: 0.0671 - accuracy: 0.9879\n",
      "Epoch 00004: saving model to distill_bert_3.hdf5\n",
      "7697/7697 [==============================] - 46s 6ms/sample - loss: 0.0671 - accuracy: 0.9879 - val_loss: 0.0645 - val_accuracy: 0.9880\n",
      "Epoch 5/15\n",
      "4992/7697 [==================>...........] - ETA: 14s - loss: 0.0517 - accuracy: 0.9882"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "splits = list(StratifiedKFold(n_splits=5).split(x_train,y_train.argmax(1)))\n",
    "\n",
    "from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint\n",
    "import keras.backend as K\n",
    "import numpy as np\n",
    "BATCH_SIZE = 16\n",
    "NUM_EPOCHS = 15\n",
    "\n",
    "oof_preds = np.zeros((x_train.shape[0],len(set(labels))))\n",
    "for fold in [0,1,2,3,4]:\n",
    "    K.clear_session()\n",
    "    tr_ind, val_ind = splits[fold]\n",
    "    ckpt = ModelCheckpoint(f'distill_bert_{fold}.hdf5',save_weights_only=True, verbose=1)\n",
    "    es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=3,min_delta=0.008)\n",
    "    model = build_model(transformer_layer, max_len=MAX_LEN)\n",
    "    model.fit(x_train[tr_ind],\n",
    "        y_train[tr_ind],\n",
    "        batch_size=BATCH_SIZE,\n",
    "        epochs=NUM_EPOCHS,\n",
    "        validation_data=(x_train[val_ind], y_train[val_ind]),\n",
    "        callbacks = [es,ckpt])\n",
    "\n",
    "    oof_preds[val_ind] += model.predict(x_train[val_ind])\n",
    "    #test_preds += model.predict(X_test)[:,0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Submission"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "07fa6a849956492792deaf1e35e9c92b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "Downloading: 100%",
       "description_tooltip": null,
       "layout": "IPY_MODEL_d9764a4adc444cd0b8d8729452b17487",
       "max": 231508.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_26b4c0f0621f42ac9847ffe0a326092c",
       "value": 231508.0
      }
     },
     "150bad3393db4acba84449ce00c28443": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "1f212bc852ee4a55872cd2ef579e7cc8": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_ec6d306ea6de48a28857e8c1fc8e8e5a",
       "placeholder": "​",
       "style": "IPY_MODEL_aeb763a4f25b4dd187099692b0e176f8",
       "value": " 38/38 [00:23&lt;00:00,  1.61it/s]"
      }
     },
     "223d0fd940744e3ba70f810f67d0b292": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "Downloading: 100%",
       "description_tooltip": null,
       "layout": "IPY_MODEL_c97fa1b69903404eae6cf86fdc494f80",
       "max": 442.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_7cfd02648d45493298d03d3b453d62c1",
       "value": 442.0
      }
     },
     "2682f3f277c54f0e99442beb4917bd23": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "26b4c0f0621f42ac9847ffe0a326092c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": "initial"
      }
     },
     "2d12ec90aaac46efbcbc2827ba0fd997": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "311abda8ced744138653d9236086f36b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_2682f3f277c54f0e99442beb4917bd23",
       "placeholder": "​",
       "style": "IPY_MODEL_39dd4bdc6cdd469999a810a6831414e8",
       "value": " 232k/232k [00:00&lt;00:00, 2.12MB/s]"
      }
     },
     "31390a1fa0d248baadc4b446149e195c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": "initial"
      }
     },
     "352bbaac7c6e49d5b1c0d630898fe6f1": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "39dd4bdc6cdd469999a810a6831414e8": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "4561a0a369a14e338529d711698095f3": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "100%",
       "description_tooltip": null,
       "layout": "IPY_MODEL_fa88fa0babfd44daa274a6733201d9d7",
       "max": 38.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_ec57335a82574ae4bbb3ea144676aa62",
       "value": 38.0
      }
     },
     "4a4d464a6323482a9f3fc8474d36b133": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "4c26051874bd457ab604d8ca9fc5725c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_4561a0a369a14e338529d711698095f3",
        "IPY_MODEL_1f212bc852ee4a55872cd2ef579e7cc8"
       ],
       "layout": "IPY_MODEL_150bad3393db4acba84449ce00c28443"
      }
     },
     "59109d3e9ffb4dd1b4a3f0261e60ef9d": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_65c18c169e254af3842dea83b183c117",
       "placeholder": "​",
       "style": "IPY_MODEL_352bbaac7c6e49d5b1c0d630898fe6f1",
       "value": " 363M/363M [00:11&lt;00:00, 31.9MB/s]"
      }
     },
     "5c9ff7ac1cd14aecb176e507f9ca6e83": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_223d0fd940744e3ba70f810f67d0b292",
        "IPY_MODEL_c33a2b606c234a23b3bb908a7ca111e7"
       ],
       "layout": "IPY_MODEL_2d12ec90aaac46efbcbc2827ba0fd997"
      }
     },
     "65c18c169e254af3842dea83b183c117": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "6c5c3628912f4eb985e158d5107ac69c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_b8483f1c1d0e4b0fbcec3dfb68409c97",
        "IPY_MODEL_59109d3e9ffb4dd1b4a3f0261e60ef9d"
       ],
       "layout": "IPY_MODEL_bad2295f11af415ebb443afbb4b34610"
      }
     },
     "7cfd02648d45493298d03d3b453d62c1": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": "initial"
      }
     },
     "a25e1acc41574bb599124e563c75092a": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "aeb763a4f25b4dd187099692b0e176f8": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "b8483f1c1d0e4b0fbcec3dfb68409c97": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "Downloading: 100%",
       "description_tooltip": null,
       "layout": "IPY_MODEL_4a4d464a6323482a9f3fc8474d36b133",
       "max": 363423424.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_31390a1fa0d248baadc4b446149e195c",
       "value": 363423424.0
      }
     },
     "ba20f34f7621499c81b3e44f9626310a": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "bad2295f11af415ebb443afbb4b34610": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "c33a2b606c234a23b3bb908a7ca111e7": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_ba20f34f7621499c81b3e44f9626310a",
       "placeholder": "​",
       "style": "IPY_MODEL_fdd9a7dd28384b08b73acf3f9e364566",
       "value": " 442/442 [00:00&lt;00:00, 2.65kB/s]"
      }
     },
     "c97fa1b69903404eae6cf86fdc494f80": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "d85f61c6faa244ac9311edb628a86f24": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_07fa6a849956492792deaf1e35e9c92b",
        "IPY_MODEL_311abda8ced744138653d9236086f36b"
       ],
       "layout": "IPY_MODEL_a25e1acc41574bb599124e563c75092a"
      }
     },
     "d9764a4adc444cd0b8d8729452b17487": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "ec57335a82574ae4bbb3ea144676aa62": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": "initial"
      }
     },
     "ec6d306ea6de48a28857e8c1fc8e8e5a": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "fa88fa0babfd44daa274a6733201d9d7": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "fdd9a7dd28384b08b73acf3f9e364566": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}