t5/t5_tf_huggingface.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Text Summary with T5 from Huggingface Tensorflow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "DckiD07HU32-"
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from transformers import T5Tokenizer, TFT5Model, TFT5ForConditionalGeneration\n",
    "import tensorflow_datasets as tfds\n",
    "import time"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_x49QLWNT5XK"
   },
   "outputs": [],
   "source": [
    "BATCH_SIZE = 16\n",
    "\n",
    "SHUFFEL_SIZE = 1024\n",
    "\n",
    "learning_rate = 3e-5"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define Pretrained Model and Tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 66,
     "referenced_widgets": [
      "a1df937387d74c85bdf401d513643587",
      "0b17614a42b249ffaf5b6dee2cbaf2e3",
      "d8ee845e5f8c47b98ca3bfdd9d36f57a",
      "8dc3ae85b5674482827a9e79799f16e6",
      "0ffe8b9161f04645b1efe34ef7eae8fa",
      "62bf0ecaccd14d48b8b38b7c3b1e7da1",
      "4bad82a79e474cb2ad5545e2182238ef",
      "59a2bf0c1b0d4512afe698d135537534"
     ]
    },
    "colab_type": "code",
    "id": "BVinRzgfIdkr",
    "outputId": "7f176e00-a553-45e6-ef29-b3300268fbc6"
   },
   "outputs": [],
   "source": [
    "tokenizer = T5Tokenizer.from_pretrained('t5-small')\n",
    "model = TFT5ForConditionalGeneration.from_pretrained('t5-small')\n",
    "\n",
    "task_specific_params = model.config.task_specific_params\n",
    "if task_specific_params is not None:\n",
    "    model.config.update(task_specific_params.get(\"summarization\", {}))\n",
    "    \n",
    "pad_token_id = tokenizer.pad_token_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 266
    },
    "colab_type": "code",
    "id": "aI7FIzmmnA5l",
    "outputId": "7ccc442a-300f-474b-bb9f-8526c56a3bc0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"tf_t5for_conditional_generation\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "shared (TFSharedEmbeddings)  multiple                  16449536  \n",
      "_________________________________________________________________\n",
      "encoder (TFT5MainLayer)      multiple                  18881280  \n",
      "_________________________________________________________________\n",
      "decoder (TFT5MainLayer)      multiple                  25176064  \n",
      "=================================================================\n",
      "Total params: 60,506,880\n",
      "Trainable params: 60,506,880\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate, epsilon=1e-08, clipnorm=1.0)\n",
    "loss_object = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n",
    "\n",
    "train_loss = tf.keras.metrics.Mean(name='train_loss')\n",
    "train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='train_accuracy')\n",
    "\n",
    "val_loss = tf.keras.metrics.Mean(name='val_loss')\n",
    "val_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='val_accuracy')\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000,
     "referenced_widgets": [
      "5d77a35f61fd47a087c67776ad6ed41f",
      "42bf617e559746fbb14ded624e992cac",
      "0374645206be4041bb7446fa8e5ff399",
      "224cdc484e4840afb58817c86a6ee205",
      "b2c0a44a936c4713bffbc635e1eaf26e",
      "7ec12b01c04b4f01bdcfe54c232c3f3f",
      "785065aa28264ab78f374aace63247c1",
      "6232d0ebbbfa44ecb95f00e5d8ead87f",
      "7f6cdc48bbcb4fa38faac6e7e12e6d4f",
      "90b113917f1f47da8ff6e8c3b6396a48",
      "7ba6854529a64443b9761f26e75f60cd",
      "2adfcf3adc484714a9be55ed23c793d6",
      "5595f863787d48a1ad37456868c412a0",
      "516554f0a5dd439abed88e1ef3b1df4e",
      "87606d2b0fda450687d66c226c8877d1",
      "8b55482761a446daa6ccfb2cd2e9bc4a",
      "37d07d9e8bdc488e99395ffc56b8dba2",
      "ff651d29638e4d55b491eb50ac9abaf0",
      "e301d715fe9043299869ac8807b6b733",
      "9166f632078044cda8d4589042860180",
      "32aff92a4044485fae3ee1f79c2a7c29",
      "9f7e7c4e542047baa0ab3b5243b70902",
      "1e6b2553ef864a19a59bf9d5e936e898",
      "8737fdc83ec346a8a757335a5143ff4d",
      "9fb029de2c8c4ac1a99256e580d58938",
      "586a5811447c4180943013ee1d255066",
      "b792afc29f2d4b4ba6df23bc688adb3e",
      "bcb4fa71770d40f694244a6551c8208f",
      "c8ba21378f21430dae451410a722413d",
      "f79bdbbc8123453b8cd49bec3de46e5f",
      "016bc870a66b4b9b8a6a75962bef9d87",
      "422ef5f2294b46f7ab4c9badc6d89480",
      "58c2ee2b115045a49d0d297354456e84",
      "6942cfd7f0964f2db9c97bf95775926f",
      "62d4332d052c4ad8a1f4e9e983944f95",
      "2d6af80aa66043de8f383a33836a9980",
      "c75f830ef1804e1c87e0f50c97687be3",
      "f44cdbcc689c4fbaa3848fcbfc097faa",
      "5263262f561149b981ffebb3c341cbb1",
      "f8f1b594fb344a34a4be3c0f35dd4d12",
      "f24137e06248452c8d80a8e066d6bd7c",
      "bf2bf4f333a043aeb87c5e6cb957d825",
      "2397d41cf49b4598a38cebb786783536",
      "b777595dc2554bdca40193880a51c9aa",
      "9786bef8c4f44fd4a67f36dce0921ab4",
      "a0160ed7e1074046b912be35937478f0",
      "a0ccd8ef5139495c95d8ecb55dacaf0c",
      "abc0758352014fb4a58e64480ab64bba",
      "50dd294dbcc6448f9cc0b7899b58b6b6",
      "6c1e205cd773412483caa305b2e66129",
      "8fdb82b31dd4406c86c8ed2ed5873a57",
      "7cdb9b751dff473ca5e70a919cdfe150",
      "134aa7d0201e4bb4b89ab1c233ab7e05",
      "8f2c8a6bfca04fb3972f0f81c9e6424b",
      "c9f2d8e2e176482aa307f45fa8e344c3",
      "419429f21a0e4166a2c1c89af4e2badc",
      "ddada92688e74e079b791b3b6b253a07",
      "6fde331cda954c588707c46eca4e9bc9",
      "e82ed2feedad4d4687314bf88bfe7e1d",
      "d5ab4da3f61448ba91fa4a7cbc2c5dec",
      "b222ab334ad1411abdf989aa3f4efeef",
      "74b176ec2fc944389fc19f0d85ee4ba4",
      "e70beb853fa348f8bf2e9019a1b4ae5d",
      "a0fb3a8862d2482cb718e05a55c95e75",
      "9b98165875e14228b84b159690bd11f8",
      "3ba8fe9b8aee4ccb90d9dfe64a777b0e",
      "7848ae7c618c4387ad7815b8c965d5d6",
      "ea5fc83661d14aca880b64a25cbf5384",
      "76c2cbf2fee74bec9314b92ed8652cc5",
      "307b983aca0049308008999ac3adb30d",
      "60013e94ef89481ea6e8ad78689446fb",
      "ad77770d1ebb4d73a0b3038cc65dc0c1"
     ]
    },
    "colab_type": "code",
    "id": "KCUpDIKS78T7",
    "outputId": "6d8110c5-6215-4ae8-89aa-1b84675ec99b"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:absl:No config specified, defaulting to first: cnn_dailymail/plain_text\n",
      "INFO:absl:Load dataset info from /home/yannik/tensorflow_datasets/cnn_dailymail/plain_text/3.0.0\n",
      "INFO:absl:Field info.description from disk and from code do not match. Keeping the one from code.\n",
      "INFO:absl:Field info.citation from disk and from code do not match. Keeping the one from code.\n",
      "INFO:absl:Reusing dataset cnn_dailymail (/home/yannik/tensorflow_datasets/cnn_dailymail/plain_text/3.0.0)\n",
      "INFO:absl:Constructing tf.data.Dataset for split None, from /home/yannik/tensorflow_datasets/cnn_dailymail/plain_text/3.0.0\n"
     ]
    }
   ],
   "source": [
    "cnn_dailymail = tfds.load(name=\"cnn_dailymail\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "8kMfGUlK7_KF"
   },
   "outputs": [],
   "source": [
    "train_tfds = cnn_dailymail['train']\n",
    "test_tfds = cnn_dailymail['test']\n",
    "val_tfds = cnn_dailymail['validation']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "len_train = len(list(train_tfds))\n",
    "len_test = len(list(test_tfds))\n",
    "len_val = len(list(val_tfds))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Bs4aDeST8TcI"
   },
   "outputs": [],
   "source": [
    "def normalize_text(text):\n",
    "    \"\"\"Lowercase and remove quotes from a TensorFlow string.\"\"\"\n",
    "    text = tf.strings.lower(text)\n",
    "    text = tf.strings.regex_replace(text,\"'(.*)'\", r\"\\1\")\n",
    "    return text.numpy().decode('UTF-8')\n",
    "\n",
    "def tokenize_articles(text):\n",
    "    text = normalize_text(text)\n",
    "    ids = tokenizer.encode_plus((model.config.prefix + text), return_tensors=\"tf\", max_length=512) \n",
    "\n",
    "    return tf.squeeze(ids['input_ids']), tf.squeeze(ids['attention_mask'])\n",
    "        \n",
    "def tokenize_highlights(text):\n",
    "    text = normalize_text(text)\n",
    "    ids = tokenizer.encode(text, return_tensors=\"tf\", max_length=150)\n",
    "    return tf.squeeze(ids)\n",
    "\n",
    "\n",
    "\n",
    "def map_func(features):\n",
    "\n",
    "    article_ids, attention_mask = tf.py_function(tokenize_articles, inp=[features[\"article\"]], Tout=(tf.int32, tf.int32))\n",
    "    highlights_ids = tf.py_function(tokenize_highlights, inp=[features[\"highlights\"]], Tout=tf.int32)\n",
    "\n",
    "    return article_ids, attention_mask, highlights_ids\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Z3gEDMH9Gm7n"
   },
   "outputs": [],
   "source": [
    "train_ds = train_tfds.map(map_func)\\\n",
    "    .shuffle(SHUFFEL_SIZE)\\\n",
    "    .padded_batch(BATCH_SIZE, padded_shapes=([512],[512],[150]))\\\n",
    "    .prefetch(tf.data.experimental.AUTOTUNE)\n",
    "\n",
    "val_ds = val_tfds.map(map_func)\\\n",
    "    .shuffle(SHUFFEL_SIZE)\\\n",
    "    .padded_batch(BATCH_SIZE, padded_shapes=([512],[512],[150]))\\\n",
    "    .prefetch(tf.data.experimental.AUTOTUNE)\n",
    "\n",
    "test_ds = test_tfds.map(map_func)\\\n",
    "    .shuffle(SHUFFEL_SIZE)\\\n",
    "    .padded_batch(BATCH_SIZE, padded_shapes=([512],[512],[150]))\\\n",
    "    .prefetch(tf.data.experimental.AUTOTUNE)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define Train and Validation Step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "@tf.function\n",
    "def train_step(input_ids, input_mask, y):\n",
    "    # https://github.com/huggingface/transformers/blob/master/examples/summarization/bart/finetune.py\n",
    "    y_ids = y[:, :-1]\n",
    "    lm_labels = tf.identity(y[:, 1:])\n",
    "    lm_labels = tf.where(tf.equal(y[:, 1:],pad_token_id), -100, lm_labels)\n",
    "\n",
    "    with tf.GradientTape() as tape:\n",
    "        # prediction_scores: (bs, 150, 32128)\n",
    "        # decoder_past_key_value_states: (bs, 512, 512), (bs, 8, 150, 64)\n",
    "        # z: (bs, 512, 512)\n",
    "        predictions, _, _ = model(input_ids, attention_mask=input_mask, decoder_input_ids=y_ids, lm_labels=lm_labels, training=True)\n",
    "        loss = loss_object(y[:, 1:], predictions)\n",
    "    gradients = tape.gradient(loss, model.trainable_variables)\n",
    "    optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n",
    "    \n",
    "    train_loss(loss)\n",
    "    train_accuracy(y[:, 1:], predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "@tf.function\n",
    "def val_step(input_ids, input_mask, y):\n",
    "    # https://github.com/huggingface/transformers/blob/master/examples/summarization/bart/finetune.py\n",
    "    y_ids = y[:, :-1]\n",
    "    lm_labels = tf.identity(y[:, 1:])\n",
    "    lm_labels = tf.where(tf.equal(y[:, 1:],pad_token_id), -100, lm_labels)\n",
    "    \n",
    "    predictions, _, _ = model(input_ids, attention_mask=input_mask, decoder_input_ids=y_ids, lm_labels=lm_labels, training=False)\n",
    "    v_loss = loss_object(y[:, 1:], predictions)\n",
    "\n",
    "    val_loss(v_loss)\n",
    "    val_accuracy(y[:, 1:], predictions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "qQZFQvNxpVJw",
    "outputId": "ab3098cc-2008-4247-8ad4-df5e997ca5a8"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/yannik/Dokumente/text_summarization/venv/lib/python3.8/site-packages/tensorflow/python/framework/indexed_slices.py:433: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   0 | [    0/287113] | ms/batch 81.02 | train acc 28.57 | train acc 34.94 |loss  9.09 | val loss  8.66\n",
      "| epoch   0 | [  200/287113] | ms/batch 315.95 | train acc 62.01 | train acc 51.91 |loss  2.78 | val loss  5.17\n",
      "| epoch   0 | [  400/287113] | ms/batch 319.16 | train acc 68.77 | train acc 58.54 |loss  2.05 | val loss  3.94\n",
      "| epoch   0 | [  600/287113] | ms/batch 320.77 | train acc 71.55 | train acc 63.86 |loss  1.77 | val loss  3.20\n",
      "| epoch   0 | [  800/287113] | ms/batch 321.61 | train acc 73.08 | train acc 66.10 |loss  1.61 | val loss  2.82\n",
      "| epoch   0 | [ 1000/287113] | ms/batch 324.29 | train acc 74.08 | train acc 67.99 |loss  1.52 | val loss  2.53\n",
      "| epoch   0 | [ 1200/287113] | ms/batch 322.67 | train acc 74.76 | train acc 69.61 |loss  1.45 | val loss  2.32\n",
      "| epoch   0 | [ 1400/287113] | ms/batch 322.85 | train acc 75.32 | train acc 70.84 |loss  1.40 | val loss  2.17\n",
      "| epoch   0 | [ 1600/287113] | ms/batch 322.79 | train acc 75.75 | train acc 72.05 |loss  1.36 | val loss  2.02\n",
      "| epoch   0 | [ 1800/287113] | ms/batch 322.96 | train acc 76.07 | train acc 72.90 |loss  1.33 | val loss  1.92\n",
      "| epoch   0 | [ 2000/287113] | ms/batch 322.98 | train acc 76.39 | train acc 73.75 |loss  1.30 | val loss  1.83\n",
      "| epoch   0 | [ 2200/287113] | ms/batch 323.15 | train acc 76.57 | train acc 74.07 |loss  1.28 | val loss  1.77\n",
      "| epoch   0 | [ 2400/287113] | ms/batch 323.06 | train acc 76.76 | train acc 74.34 |loss  1.27 | val loss  1.72\n",
      "| epoch   0 | [ 2600/287113] | ms/batch 323.29 | train acc 76.92 | train acc 74.91 |loss  1.25 | val loss  1.65\n",
      "| epoch   0 | [ 2800/287113] | ms/batch 323.21 | train acc 77.08 | train acc 75.47 |loss  1.24 | val loss  1.60\n",
      "| epoch   0 | [ 3000/287113] | ms/batch 323.38 | train acc 77.24 | train acc 75.34 |loss  1.22 | val loss  1.58\n",
      "| epoch   0 | [ 3200/287113] | ms/batch 323.25 | train acc 77.34 | train acc 75.72 |loss  1.21 | val loss  1.54\n",
      "| epoch   0 | [ 3400/287113] | ms/batch 323.40 | train acc 77.43 | train acc 75.78 |loss  1.20 | val loss  1.52\n",
      "| epoch   0 | [ 3600/287113] | ms/batch 323.05 | train acc 77.53 | train acc 75.95 |loss  1.20 | val loss  1.50\n",
      "| epoch   0 | [ 3800/287113] | ms/batch 323.33 | train acc 77.63 | train acc 76.15 |loss  1.19 | val loss  1.47\n",
      "| epoch   0 | [ 4000/287113] | ms/batch 323.22 | train acc 77.69 | train acc 76.22 |loss  1.18 | val loss  1.45\n",
      "| epoch   0 | [ 4200/287113] | ms/batch 323.12 | train acc 77.77 | train acc 76.09 |loss  1.17 | val loss  1.45\n",
      "| epoch   0 | [ 4400/287113] | ms/batch 323.21 | train acc 77.81 | train acc 76.28 |loss  1.17 | val loss  1.43\n",
      "| epoch   0 | [ 4600/287113] | ms/batch 323.19 | train acc 77.88 | train acc 76.61 |loss  1.16 | val loss  1.40\n",
      "| epoch   0 | [ 4800/287113] | ms/batch 323.17 | train acc 77.94 | train acc 76.87 |loss  1.16 | val loss  1.38\n",
      "| epoch   0 | [ 5000/287113] | ms/batch 323.13 | train acc 77.99 | train acc 77.00 |loss  1.15 | val loss  1.36\n",
      "| epoch   0 | [ 5200/287113] | ms/batch 323.38 | train acc 78.04 | train acc 77.21 |loss  1.15 | val loss  1.34\n",
      "| epoch   0 | [ 5400/287113] | ms/batch 323.11 | train acc 78.08 | train acc 77.39 |loss  1.14 | val loss  1.32\n",
      "| epoch   0 | [ 5600/287113] | ms/batch 323.26 | train acc 78.13 | train acc 77.52 |loss  1.14 | val loss  1.31\n",
      "| epoch   0 | [ 5800/287113] | ms/batch 323.27 | train acc 78.18 | train acc 77.65 |loss  1.14 | val loss  1.30\n",
      "| epoch   0 | [ 6000/287113] | ms/batch 323.21 | train acc 78.21 | train acc 77.84 |loss  1.13 | val loss  1.28\n",
      "| epoch   0 | [ 6200/287113] | ms/batch 323.11 | train acc 78.25 | train acc 77.90 |loss  1.13 | val loss  1.27\n",
      "| epoch   0 | [ 6400/287113] | ms/batch 323.19 | train acc 78.29 | train acc 77.99 |loss  1.13 | val loss  1.27\n",
      "| epoch   0 | [ 6600/287113] | ms/batch 323.35 | train acc 78.32 | train acc 77.98 |loss  1.12 | val loss  1.26\n",
      "| epoch   0 | [ 6800/287113] | ms/batch 323.09 | train acc 78.36 | train acc 77.92 |loss  1.12 | val loss  1.26\n",
      "| epoch   0 | [ 7000/287113] | ms/batch 323.18 | train acc 78.39 | train acc 78.04 |loss  1.12 | val loss  1.25\n",
      "| epoch   0 | [ 7200/287113] | ms/batch 323.04 | train acc 78.41 | train acc 78.03 |loss  1.11 | val loss  1.24\n",
      "| epoch   0 | [ 7400/287113] | ms/batch 323.15 | train acc 78.45 | train acc 78.12 |loss  1.11 | val loss  1.23\n",
      "| epoch   0 | [ 7600/287113] | ms/batch 323.00 | train acc 78.48 | train acc 78.18 |loss  1.11 | val loss  1.23\n",
      "| epoch   0 | [ 7800/287113] | ms/batch 323.20 | train acc 78.50 | train acc 78.23 |loss  1.11 | val loss  1.22\n",
      "| epoch   0 | [ 8000/287113] | ms/batch 323.13 | train acc 78.51 | train acc 78.26 |loss  1.11 | val loss  1.22\n",
      "| epoch   0 | [ 8200/287113] | ms/batch 323.23 | train acc 78.54 | train acc 78.30 |loss  1.10 | val loss  1.21\n",
      "| epoch   0 | [ 8400/287113] | ms/batch 323.06 | train acc 78.57 | train acc 78.40 |loss  1.10 | val loss  1.20\n",
      "| epoch   0 | [ 8600/287113] | ms/batch 323.25 | train acc 78.59 | train acc 78.48 |loss  1.10 | val loss  1.20\n",
      "| epoch   0 | [ 8800/287113] | ms/batch 323.09 | train acc 78.62 | train acc 78.54 |loss  1.10 | val loss  1.19\n",
      "| epoch   0 | [ 9000/287113] | ms/batch 323.28 | train acc 78.65 | train acc 78.59 |loss  1.09 | val loss  1.18\n",
      "| epoch   0 | [ 9200/287113] | ms/batch 323.26 | train acc 78.67 | train acc 78.67 |loss  1.09 | val loss  1.18\n",
      "| epoch   0 | [ 9400/287113] | ms/batch 323.21 | train acc 78.69 | train acc 78.64 |loss  1.09 | val loss  1.17\n",
      "| epoch   0 | [ 9600/287113] | ms/batch 323.28 | train acc 78.72 | train acc 78.67 |loss  1.09 | val loss  1.17\n",
      "| epoch   0 | [ 9800/287113] | ms/batch 323.22 | train acc 78.74 | train acc 78.70 |loss  1.09 | val loss  1.17\n",
      "| epoch   0 | [10000/287113] | ms/batch 323.30 | train acc 78.76 | train acc 78.74 |loss  1.09 | val loss  1.16\n",
      "| epoch   0 | [10200/287113] | ms/batch 322.99 | train acc 78.78 | train acc 78.70 |loss  1.08 | val loss  1.16\n",
      "| epoch   0 | [10400/287113] | ms/batch 323.29 | train acc 78.79 | train acc 78.65 |loss  1.08 | val loss  1.16\n",
      "| epoch   0 | [10600/287113] | ms/batch 323.16 | train acc 78.81 | train acc 78.62 |loss  1.08 | val loss  1.16\n",
      "| epoch   0 | [10800/287113] | ms/batch 323.23 | train acc 78.83 | train acc 78.67 |loss  1.08 | val loss  1.16\n",
      "| epoch   0 | [11000/287113] | ms/batch 323.02 | train acc 78.84 | train acc 78.74 |loss  1.08 | val loss  1.15\n",
      "| epoch   0 | [11200/287113] | ms/batch 323.31 | train acc 78.86 | train acc 78.78 |loss  1.08 | val loss  1.15\n",
      "| epoch   0 | [11400/287113] | ms/batch 323.24 | train acc 78.87 | train acc 78.78 |loss  1.08 | val loss  1.14\n",
      "| epoch   0 | [11600/287113] | ms/batch 323.28 | train acc 78.89 | train acc 78.82 |loss  1.07 | val loss  1.14\n",
      "| epoch   0 | [11800/287113] | ms/batch 323.26 | train acc 78.91 | train acc 78.87 |loss  1.07 | val loss  1.14\n",
      "| epoch   0 | [12000/287113] | ms/batch 323.18 | train acc 78.92 | train acc 78.86 |loss  1.07 | val loss  1.13\n",
      "| epoch   0 | [12200/287113] | ms/batch 323.34 | train acc 78.94 | train acc 78.86 |loss  1.07 | val loss  1.13\n",
      "| epoch   0 | [12400/287113] | ms/batch 323.24 | train acc 78.95 | train acc 78.82 |loss  1.07 | val loss  1.13\n",
      "| epoch   0 | [12600/287113] | ms/batch 323.32 | train acc 78.97 | train acc 78.84 |loss  1.07 | val loss  1.13\n",
      "| epoch   0 | [12800/287113] | ms/batch 323.31 | train acc 78.98 | train acc 78.82 |loss  1.07 | val loss  1.13\n",
      "| epoch   0 | [13000/287113] | ms/batch 323.41 | train acc 78.99 | train acc 78.79 |loss  1.07 | val loss  1.13\n",
      "| epoch   0 | [13200/287113] | ms/batch 324.32 | train acc 79.00 | train acc 78.80 |loss  1.07 | val loss  1.13\n",
      "| epoch   0 | [13400/287113] | ms/batch 323.56 | train acc 79.01 | train acc 78.76 |loss  1.06 | val loss  1.13\n",
      "| epoch   0 | [13600/287113] | ms/batch 323.27 | train acc 79.02 | train acc 78.77 |loss  1.06 | val loss  1.13\n",
      "| epoch   0 | [13800/287113] | ms/batch 323.60 | train acc 79.04 | train acc 78.79 |loss  1.06 | val loss  1.13\n",
      "| epoch   0 | [14000/287113] | ms/batch 323.57 | train acc 79.05 | train acc 78.79 |loss  1.06 | val loss  1.12\n",
      "| epoch   0 | [14200/287113] | ms/batch 323.40 | train acc 79.06 | train acc 78.77 |loss  1.06 | val loss  1.12\n",
      "| epoch   0 | [14400/287113] | ms/batch 323.52 | train acc 79.07 | train acc 78.80 |loss  1.06 | val loss  1.12\n",
      "| epoch   0 | [14600/287113] | ms/batch 323.42 | train acc 79.08 | train acc 78.85 |loss  1.06 | val loss  1.12\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   0 | [14800/287113] | ms/batch 323.56 | train acc 79.09 | train acc 78.84 |loss  1.06 | val loss  1.12\n",
      "| epoch   0 | [15000/287113] | ms/batch 323.37 | train acc 79.10 | train acc 78.87 |loss  1.06 | val loss  1.12\n",
      "| epoch   0 | [15200/287113] | ms/batch 326.07 | train acc 79.11 | train acc 78.87 |loss  1.06 | val loss  1.11\n",
      "| epoch   0 | [15400/287113] | ms/batch 323.32 | train acc 79.13 | train acc 78.88 |loss  1.06 | val loss  1.11\n",
      "| epoch   0 | [15600/287113] | ms/batch 323.56 | train acc 79.13 | train acc 78.92 |loss  1.05 | val loss  1.11\n",
      "| epoch   0 | [15800/287113] | ms/batch 323.50 | train acc 79.14 | train acc 78.88 |loss  1.05 | val loss  1.11\n",
      "| epoch   0 | [16000/287113] | ms/batch 323.65 | train acc 79.15 | train acc 78.87 |loss  1.05 | val loss  1.11\n",
      "| epoch   0 | [16200/287113] | ms/batch 323.51 | train acc 79.16 | train acc 78.93 |loss  1.05 | val loss  1.11\n",
      "| epoch   0 | [16400/287113] | ms/batch 323.68 | train acc 79.17 | train acc 78.94 |loss  1.05 | val loss  1.10\n",
      "| epoch   0 | [16600/287113] | ms/batch 323.58 | train acc 79.17 | train acc 78.97 |loss  1.05 | val loss  1.10\n",
      "| epoch   0 | [16800/287113] | ms/batch 323.47 | train acc 79.19 | train acc 78.96 |loss  1.05 | val loss  1.10\n",
      "| epoch   0 | [17000/287113] | ms/batch 323.66 | train acc 79.20 | train acc 79.01 |loss  1.05 | val loss  1.10\n",
      "| epoch   0 | [17200/287113] | ms/batch 323.40 | train acc 79.20 | train acc 79.03 |loss  1.05 | val loss  1.10\n",
      "| epoch   0 | [17400/287113] | ms/batch 323.62 | train acc 79.21 | train acc 79.00 |loss  1.05 | val loss  1.10\n",
      "| epoch   0 | [17600/287113] | ms/batch 323.51 | train acc 79.22 | train acc 79.01 |loss  1.05 | val loss  1.10\n",
      "| epoch   0 | [17800/287113] | ms/batch 330.67 | train acc 79.23 | train acc 79.00 |loss  1.05 | val loss  1.10\n"
     ]
    }
   ],
   "source": [
    "EPOCHS = 1\n",
    "log_interval = 200\n",
    "for epoch in range(EPOCHS):\n",
    "    # reset metrics\n",
    "    train_loss.reset_states()\n",
    "    train_accuracy.reset_states()\n",
    "    \n",
    "    val_loss.reset_states()\n",
    "    val_accuracy.reset_states()\n",
    "    \n",
    "    val_batches = iter(val_ds)\n",
    "    \n",
    "    start_time = time.time()\n",
    "    for i, (input_ids, input_mask, y) in enumerate(train_ds):\n",
    "        # training\n",
    "        train_step(input_ids, input_mask, y)\n",
    "        \n",
    "        # validation\n",
    "        if i % log_interval == 0:\n",
    "            x_val, x_mask_val, y_val = next(val_batches)\n",
    "            val_step(x_val, x_mask_val, y_val)\n",
    "            elapsed = time.time() - start_time\n",
    "            print('| epoch {:3d} | [{:5d}/{:5d}] | '\n",
    "                  'ms/batch {:5.2f} | '\n",
    "                  'train acc {:5.2f} | val acc {:5.2f} |'\n",
    "                  'loss {:5.2f} | val loss {:5.2f}'.format(\n",
    "                    epoch, i, int(len_train/BATCH_SIZE),\n",
    "                    elapsed * 1000 / log_interval,\n",
    "                    train_accuracy.result() * 100, val_accuracy.result() * 100, \n",
    "                    train_loss.result(),  val_loss.result()))\n",
    "            start_time = time.time()\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define Rouge Score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from rouge_score import rouge_scorer\n",
    "from rouge_score import scoring\n",
    "\n",
    "class RougeScore:\n",
    "    '''\n",
    "    mostly from https://github.com/google-research/text-to-text-transfer-transformer/blob/master/t5/evaluation/metrics.py \n",
    "    '''\n",
    "    \n",
    "    def __init__(self, score_keys=None)-> None:\n",
    "        super().__init__()\n",
    "        if score_keys is None:  \n",
    "            self.score_keys = [\"rouge1\", \"rouge2\", \"rougeLsum\"]\n",
    "        \n",
    "        self.scorer = rouge_scorer.RougeScorer(self.score_keys)\n",
    "        self.aggregator = scoring.BootstrapAggregator()\n",
    "        \n",
    "        \n",
    "    @staticmethod\n",
    "    def prepare_summary(summary):\n",
    "            # Make sure the summary is not bytes-type\n",
    "            # Add newlines between sentences so that rougeLsum is computed correctly.\n",
    "            summary = summary.replace(\" . \", \" .\\n\")\n",
    "            return summary\n",
    "    \n",
    "    def __call__(self, target, prediction):\n",
    "        \"\"\"Computes rouge score.''\n",
    "        Args:\n",
    "        targets: string\n",
    "        predictions: string\n",
    "        \"\"\"\n",
    "\n",
    "        target = self.prepare_summary(target)\n",
    "        prediction = self.prepare_summary(prediction)\n",
    "        \n",
    "        self.aggregator.add_scores(self.scorer.score(target=target, prediction=prediction))\n",
    "\n",
    "        return \n",
    "    \n",
    "    def reset_states(self):\n",
    "        self.rouge_list = []\n",
    "\n",
    "    def result(self):\n",
    "        result = self.aggregator.aggregate()\n",
    "        \n",
    "        for key in self.score_keys:\n",
    "            score_text = \"%s = %.2f, 95%% confidence [%.2f, %.2f]\"%(\n",
    "                key,\n",
    "                result[key].mid.fmeasure*100,\n",
    "                result[key].low.fmeasure*100,\n",
    "                result[key].high.fmeasure*100\n",
    "            )\n",
    "            print(score_text)\n",
    "        \n",
    "        return {key: result[key].mid.fmeasure*100 for key in self.score_keys}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compute Rouge Score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rouge1 = 36.45, 95% confidence [35.93, 36.91]\n",
      "rouge2 = 16.47, 95% confidence [16.02, 16.93]\n",
      "rougeLsum = 34.05, 95% confidence [33.53, 34.55]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'rouge1': 36.4493063524424,\n",
       " 'rouge2': 16.471916808516053,\n",
       " 'rougeLsum': 34.052609516834295}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions = []\n",
    "rouge_score = RougeScore()\n",
    "for i, (input_ids, input_mask, y) in enumerate(test_ds):\n",
    "    summaries = model.generate(input_ids=input_ids, attention_mask=input_mask)\n",
    "\n",
    "    pred = [tokenizer.decode(g, skip_special_tokens=True, clean_up_tokenization_spaces=False) for g in summaries]\n",
    "    real = [tokenizer.decode(g, skip_special_tokens=True, clean_up_tokenization_spaces=False) for g in y]\n",
    "    \n",
    "    for pred_sent, real_sent in zip(pred, real):\n",
    "        rouge_score(pred_sent, real_sent)\n",
    "        predictions.append(str(\"pred sentence: \" + pred_sent + \"\\n\\n real sentence: \" + real_sent))\n",
    "        \n",
    "    if i > 100:\n",
    "        # otherwise it will take ages\n",
    "        break\n",
    "\n",
    "\n",
    "rouge_score.result()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Predict some Sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------\n",
      "pred sentence: A cave photographer, john spies, 59, captured the sheer magnificence of the vast, yet intricate, underground wonderland . xe bang fai river caves feature imposing stalagmitemade of mineral deposits . the cave is only able to be safely accessed during the dry season from november to april and during this time the water is clear and deep with a rich green hue . in 2008, an expedition led to the mysterious caves being \n",
      "\n",
      " real sentence: the tham khoun ex cave has 15km of spectacular caves waiting to be explore by kayak . explorers can witness the incredible caverns, lake and even the vibrant forest at the entrance . cave photographer john spies captured the labyrinthine chambers to unfold the mystery .\n",
      "------\n",
      "------\n",
      "pred sentence: houston couple, who have not been named, were arguing in the parking lot of an apartment complex . husband admitted to police that during the heat of the argument he tried to drive off in his green chevrolet pickup truck when his wife grabbed the door handle, slipped, and he ran her over . police say that they questioned the man extensively but believe his story and have not filed charges against him . the woman was rushed to memorial hermann hospital southwest where doctors successfully delivered her baby by cesarean section . a friend of the woman's 17-year-old son was told of his mother's death during a phone call at school . she came to the driver's door, she fell and the husband ran her, but they have not yet been named . they were questioned about the incident . but they believe he was crying, and the baby was saved \n",
      "\n",
      " real sentence: her husband admits the pair had been arguing minutes before the accident . she came to the driver's door while her husband drove away and she fell . police say they believe the husband's story and have not charged him . doctors were able to deliver her 36-week-old baby in houston, texas . the pregnant woman, who has not been named, died later in hospital .\n",
      "------\n",
      "------\n",
      "pred sentence: osborne said he wanted a 1980s-style property revolution after years of declining home ownership . he pledged to double the number of people buying their first home . since 2010 there have been 1.2 million first-time purchases and 2.4 million more over the next five years . david cameron said: i believe britain should be a home-owning democracy, that’s the goal English pm: we have acted on every front, we’ll continue to help first time buyers get on the housing ladder every year by 2020 - 'home ownership is an absolutely core conservative belief and aspiration that we support a 80s style property revolution . chancellor wants to double number using government help-to-by schemes and wants at least 2.4m more over five years, mr o'british people can buy a house .\n",
      "\n",
      " real sentence: chancellor pledged to double the number of people buying their first home . since 2010 there have been 1.2 million first-time buyers getting first homes . mr osborne wants at least 2.4 million more over the next five years .\n",
      "------\n",
      "------\n",
      "pred sentence: hannah wilson's body was found in rural brown county, indiana, on friday morning after a night out . daniel messel, 49, was arrested hours later and has been charged in her death . circumstances of her disappearance are 'eerily similar' to the case of 20-year-old spierer, police said . she was last seen at kilroy's sports bar in bloomington with a group of friends on monday . it is not known if she was a student . a cellphone was found near her feet and no criminal charges have been filed in connection with the death of lauren messel . the body of the student has never been found in brown county . police will be looking into the case again . there is no sign of sexual assault . her body has not been found and she was\n",
      "\n",
      " real sentence: police said the circumstances of the disappearances of hannah wilson and lauren spierer are eerily similar spierer, 20, went missing in 2011 after a night partying with friends; her body has never been found and no criminal charges have been filed . wilson's body was found on rural land about 10 miles away last friday and it is believed she died of blunt force trauma . daniel messel, 49, has been arrested in her death after a cellphone at her feet was traced back to him and he had blood inside his car\n",
      "------\n",
      "------\n",
      "pred sentence: zeynab daghastani, 13, was shot and killed on tuesday as she tried to flee a militant-held area of damascus . she was reportedly gunned down by isis snipers in al-yarmouk camp . the teenager was trying to make her way to a nearby neighbourhood controlled by the syrian army . thousands of palestinians have been trying to escape the area since the area fell under is\n",
      "\n",
      " real sentence: zeynab daghastani, 13, reportedly gunned down as she fled yarmouk camp . teen trying to get to nearby yalda district which is not under isis control . yarmouk, on the outskirts of damascus, has been under siege since 2012 . isis seized it ten days ago and have been butchering those trapped there .\n",
      "------\n",
      "------\n",
      "pred sentence: israeli tourist captured the moment a group of baboons robbed a supply truck in tanzania . the 40-second-clip shows the driver standing next to his vehicle . man is forced to retreat as one makes a lunge at him and another jumps into the back of the truck to search for supplies . he makes an attempt to scare the monkey away and eventually it re-emerges . it carries snacks in both its hands and its mouth before scarping\n",
      "\n",
      " real sentence: video was captured at the ngorongoro conservation area in tanzania . israeli tourist filmed a group of baboons approaching a supply truck . as one distracts the driver by lunging at it another jumps into vehicle . before long it re-emerges with its hands and mouth full of snacks . the group of thieves reconvene and run off into the bushes together .\n",
      "------\n",
      "------\n",
      "pred sentence: england captain alastair cook completed a much-needed century on the second morning of englands opening tour match in the west indies . cook resumed on 95 and reached three figures with minimal fuss before retiring out . ian bell arrived at the crease, with batting time more important to the tourists than attempting to force a result in this two-day fixture . ivan bell walked off on 101 from 200 deliveries during day two of the st kitts and nevis invitational xi vs saturday's opening match . the tourists will be able to reach three figures on tuesday . captain resumed a century before retiring on 95 . it was the second day of the tour match at the end of the day - the tourists had to wait until the tourists are unable to make a decision \n",
      "\n",
      " real sentence: alastair cook completed his century on the second morning of action . england captain resumed on 95 and reached three figures before retiring . that allowed ian bell to arrive at the crease as tourists continued to bat .\n",
      "------\n",
      "------\n",
      "pred sentence: kenneth morgan stancil iii, 20, told raleighs wral-tv on wednesday that he killed ron lane earlier this week in north carolina because lane had made sexual advances towards his 16-year-old brother . in a telephone interview from a daytona beach jail, he said lane tried to take advantage of his brother and he was in the proposal to try to and i wasnt going to give a damn if i get life, 'i did what i did, i do anything necessary to take care of my family . he is awaiting extradition from florida to north america . the neo-nazi has no reports of unsolved homicides . police did not immediately return a phone message on th\n",
      "\n",
      " real sentence: suspect kenneth morgan stancil claimed he killed ron lane because the college employee made sexual advances to his 16-year-old brother . the murder suspect gave himself fascist face tattoos . stancil also said he is a neo-nazi who is concerned about the future of white children and hates race-mixing\n",
      "------\n",
      "------\n",
      "pred sentence: a new australian fashion report has named and shamed some of the worst aussie clothing brands . amongst the best performers were etiko, audrey blue, cotton on, h&m and zara . only two of the 59 companies could prove they were paying a full living wage to the workers in two of three production stages of their clothing . a living wage ensures that an employee has enough money to cover the necessities - like food, water, electricity and shelter . the report assessed the labour rights management system of 59 brands and 219 brands operating in australia . it named the worst performers by the 2015 australian clothing brands and their companies - which includes just jeans, portmans and dotti - among the best - including cotton on - and the just group - a country's legally set minimum wage - the only two could prove the\n",
      "\n",
      " real sentence: australian fashion report revealed the australian-sold brands and companies that ignore the exploitation of their overseas workers . lowes, industrie, best & less and the just group - which includes just jeans, portmans and dotti - were some of the worst performers . etiko, audrey blue, cotton on, h&m and zara had some of the best scores . 75 per cent of companies don't know the source of all their fabrics and inputs .\n",
      "------\n",
      "------\n",
      "pred sentence: lindsay lohan, 28, has been living in london for the past year and says she feels 'at home' in the uk . the flame-haired beauty recently spent a stint on the stage, starring in a west end production of david mamet's speed the plow . she has no plans to return to the states for the foreseeable future . ms lohan says she's going back to nyc and la for family and work . it's a big budget for lingerie and a sex appeal, but she has been in the us since she moved across the pond to uk for more than a year . but she doesn't see herself returning to the u.s. and wants to be a part of a series of racy ensembles by moschino .\n",
      "\n",
      " real sentence: lindsay, 28, moved to the uk last spring before making her london stage debut . the former child star has become more famous for her run-ins with the police and the media than anything else in recent years . but lindsay is now vowing to stay in london to grow up she appears in a provocative spread for homme style magazine, having been shot by famed fashion photographer rankin .\n",
      "------\n"
     ]
    }
   ],
   "source": [
    "for pred in predictions[:10]:\n",
    "    print(\"------\")\n",
    "    print(pred)\n",
    "    print(\"------\")     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "name": "test_huggingface.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "textsummary",
   "language": "python",
   "name": "venv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "016bc870a66b4b9b8a6a75962bef9d87": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "0374645206be4041bb7446fa8e5ff399": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "IntProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "IntProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Dl Completed...: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_7ec12b01c04b4f01bdcfe54c232c3f3f",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_b2c0a44a936c4713bffbc635e1eaf26e",
      "value": 1
     }
    },
    "0b17614a42b249ffaf5b6dee2cbaf2e3": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "0ffe8b9161f04645b1efe34ef7eae8fa": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "134aa7d0201e4bb4b89ab1c233ab7e05": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "1e6b2553ef864a19a59bf9d5e936e898": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "224cdc484e4840afb58817c86a6ee205": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_6232d0ebbbfa44ecb95f00e5d8ead87f",
      "placeholder": "​",
      "style": "IPY_MODEL_785065aa28264ab78f374aace63247c1",
      "value": " 5/5 [01:15&lt;00:00, 15.03s/ url]"
     }
    },
    "2397d41cf49b4598a38cebb786783536": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "IntProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "IntProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "info",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_a0160ed7e1074046b912be35937478f0",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_9786bef8c4f44fd4a67f36dce0921ab4",
      "value": 1
     }
    },
    "2adfcf3adc484714a9be55ed23c793d6": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_8b55482761a446daa6ccfb2cd2e9bc4a",
      "placeholder": "​",
      "style": "IPY_MODEL_87606d2b0fda450687d66c226c8877d1",
      "value": " 557/? [01:15&lt;00:00,  7.42 MiB/s]"
     }
    },
    "2d6af80aa66043de8f383a33836a9980": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_f8f1b594fb344a34a4be3c0f35dd4d12",
      "placeholder": "​",
      "style": "IPY_MODEL_5263262f561149b981ffebb3c341cbb1",
      "value": " 283730/287113 [00:05&lt;00:00, 50075.83 examples/s]"
     }
    },
    "307b983aca0049308008999ac3adb30d": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "32aff92a4044485fae3ee1f79c2a7c29": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "37d07d9e8bdc488e99395ffc56b8dba2": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_e301d715fe9043299869ac8807b6b733",
       "IPY_MODEL_9166f632078044cda8d4589042860180"
      ],
      "layout": "IPY_MODEL_ff651d29638e4d55b491eb50ac9abaf0"
     }
    },
    "3ba8fe9b8aee4ccb90d9dfe64a777b0e": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "419429f21a0e4166a2c1c89af4e2badc": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "422ef5f2294b46f7ab4c9badc6d89480": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "42bf617e559746fbb14ded624e992cac": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "4bad82a79e474cb2ad5545e2182238ef": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "50dd294dbcc6448f9cc0b7899b58b6b6": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_8fdb82b31dd4406c86c8ed2ed5873a57",
       "IPY_MODEL_7cdb9b751dff473ca5e70a919cdfe150"
      ],
      "layout": "IPY_MODEL_6c1e205cd773412483caa305b2e66129"
     }
    },
    "516554f0a5dd439abed88e1ef3b1df4e": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "5263262f561149b981ffebb3c341cbb1": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "5595f863787d48a1ad37456868c412a0": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "586a5811447c4180943013ee1d255066": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "58c2ee2b115045a49d0d297354456e84": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_62d4332d052c4ad8a1f4e9e983944f95",
       "IPY_MODEL_2d6af80aa66043de8f383a33836a9980"
      ],
      "layout": "IPY_MODEL_6942cfd7f0964f2db9c97bf95775926f"
     }
    },
    "59a2bf0c1b0d4512afe698d135537534": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "5d77a35f61fd47a087c67776ad6ed41f": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_0374645206be4041bb7446fa8e5ff399",
       "IPY_MODEL_224cdc484e4840afb58817c86a6ee205"
      ],
      "layout": "IPY_MODEL_42bf617e559746fbb14ded624e992cac"
     }
    },
    "60013e94ef89481ea6e8ad78689446fb": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "6232d0ebbbfa44ecb95f00e5d8ead87f": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "62bf0ecaccd14d48b8b38b7c3b1e7da1": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "62d4332d052c4ad8a1f4e9e983944f95": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "IntProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "IntProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "danger",
      "description": " 99%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_f44cdbcc689c4fbaa3848fcbfc097faa",
      "max": 287113,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_c75f830ef1804e1c87e0f50c97687be3",
      "value": 283730
     }
    },
    "6942cfd7f0964f2db9c97bf95775926f": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "6c1e205cd773412483caa305b2e66129": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "6fde331cda954c588707c46eca4e9bc9": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "74b176ec2fc944389fc19f0d85ee4ba4": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "76c2cbf2fee74bec9314b92ed8652cc5": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "7848ae7c618c4387ad7815b8c965d5d6": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "IntProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "IntProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "danger",
      "description": " 83%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_307b983aca0049308008999ac3adb30d",
      "max": 11490,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_76c2cbf2fee74bec9314b92ed8652cc5",
      "value": 9498
     }
    },
    "785065aa28264ab78f374aace63247c1": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "7ba6854529a64443b9761f26e75f60cd": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "IntProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "IntProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Dl Size...: ",
      "description_tooltip": null,
      "layout": "IPY_MODEL_516554f0a5dd439abed88e1ef3b1df4e",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_5595f863787d48a1ad37456868c412a0",
      "value": 1
     }
    },
    "7cdb9b751dff473ca5e70a919cdfe150": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_419429f21a0e4166a2c1c89af4e2badc",
      "placeholder": "​",
      "style": "IPY_MODEL_c9f2d8e2e176482aa307f45fa8e344c3",
      "value": " 9386/13368 [00:00&lt;00:00, 93859.87 examples/s]"
     }
    },
    "7ec12b01c04b4f01bdcfe54c232c3f3f": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "7f6cdc48bbcb4fa38faac6e7e12e6d4f": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_7ba6854529a64443b9761f26e75f60cd",
       "IPY_MODEL_2adfcf3adc484714a9be55ed23c793d6"
      ],
      "layout": "IPY_MODEL_90b113917f1f47da8ff6e8c3b6396a48"
     }
    },
    "8737fdc83ec346a8a757335a5143ff4d": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "87606d2b0fda450687d66c226c8877d1": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "8b55482761a446daa6ccfb2cd2e9bc4a": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "8dc3ae85b5674482827a9e79799f16e6": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_59a2bf0c1b0d4512afe698d135537534",
      "placeholder": "​",
      "style": "IPY_MODEL_4bad82a79e474cb2ad5545e2182238ef",
      "value": " 792k/792k [00:51&lt;00:00, 15.2kB/s]"
     }
    },
    "8f2c8a6bfca04fb3972f0f81c9e6424b": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "8fdb82b31dd4406c86c8ed2ed5873a57": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "IntProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "IntProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "danger",
      "description": " 70%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_8f2c8a6bfca04fb3972f0f81c9e6424b",
      "max": 13368,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_134aa7d0201e4bb4b89ab1c233ab7e05",
      "value": 9386
     }
    },
    "90b113917f1f47da8ff6e8c3b6396a48": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "9166f632078044cda8d4589042860180": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_8737fdc83ec346a8a757335a5143ff4d",
      "placeholder": "​",
      "style": "IPY_MODEL_1e6b2553ef864a19a59bf9d5e936e898",
      "value": " 2/2 [01:15&lt;00:00, 37.54s/ file]"
     }
    },
    "9786bef8c4f44fd4a67f36dce0921ab4": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "9b98165875e14228b84b159690bd11f8": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_7848ae7c618c4387ad7815b8c965d5d6",
       "IPY_MODEL_ea5fc83661d14aca880b64a25cbf5384"
      ],
      "layout": "IPY_MODEL_3ba8fe9b8aee4ccb90d9dfe64a777b0e"
     }
    },
    "9f7e7c4e542047baa0ab3b5243b70902": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "9fb029de2c8c4ac1a99256e580d58938": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_b792afc29f2d4b4ba6df23bc688adb3e",
       "IPY_MODEL_bcb4fa71770d40f694244a6551c8208f"
      ],
      "layout": "IPY_MODEL_586a5811447c4180943013ee1d255066"
     }
    },
    "a0160ed7e1074046b912be35937478f0": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "a0ccd8ef5139495c95d8ecb55dacaf0c": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "a0fb3a8862d2482cb718e05a55c95e75": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "a1df937387d74c85bdf401d513643587": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_d8ee845e5f8c47b98ca3bfdd9d36f57a",
       "IPY_MODEL_8dc3ae85b5674482827a9e79799f16e6"
      ],
      "layout": "IPY_MODEL_0b17614a42b249ffaf5b6dee2cbaf2e3"
     }
    },
    "abc0758352014fb4a58e64480ab64bba": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "ad77770d1ebb4d73a0b3038cc65dc0c1": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "b222ab334ad1411abdf989aa3f4efeef": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "b2c0a44a936c4713bffbc635e1eaf26e": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "b777595dc2554bdca40193880a51c9aa": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_abc0758352014fb4a58e64480ab64bba",
      "placeholder": "​",
      "style": "IPY_MODEL_a0ccd8ef5139495c95d8ecb55dacaf0c",
      "value": " 13368/0 [00:08&lt;00:00, 1683.60 examples/s]"
     }
    },
    "b792afc29f2d4b4ba6df23bc688adb3e": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "IntProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "IntProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "info",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_f79bdbbc8123453b8cd49bec3de46e5f",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_c8ba21378f21430dae451410a722413d",
      "value": 1
     }
    },
    "bcb4fa71770d40f694244a6551c8208f": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_422ef5f2294b46f7ab4c9badc6d89480",
      "placeholder": "​",
      "style": "IPY_MODEL_016bc870a66b4b9b8a6a75962bef9d87",
      "value": " 287113/0 [02:59&lt;00:00, 1519.02 examples/s]"
     }
    },
    "bf2bf4f333a043aeb87c5e6cb957d825": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "c75f830ef1804e1c87e0f50c97687be3": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "c8ba21378f21430dae451410a722413d": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "c9f2d8e2e176482aa307f45fa8e344c3": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "d5ab4da3f61448ba91fa4a7cbc2c5dec": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_a0fb3a8862d2482cb718e05a55c95e75",
      "placeholder": "​",
      "style": "IPY_MODEL_e70beb853fa348f8bf2e9019a1b4ae5d",
      "value": " 11490/0 [00:06&lt;00:00, 1656.55 examples/s]"
     }
    },
    "d8ee845e5f8c47b98ca3bfdd9d36f57a": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "IntProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "IntProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_62bf0ecaccd14d48b8b38b7c3b1e7da1",
      "max": 791656,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_0ffe8b9161f04645b1efe34ef7eae8fa",
      "value": 791656
     }
    },
    "ddada92688e74e079b791b3b6b253a07": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_e82ed2feedad4d4687314bf88bfe7e1d",
       "IPY_MODEL_d5ab4da3f61448ba91fa4a7cbc2c5dec"
      ],
      "layout": "IPY_MODEL_6fde331cda954c588707c46eca4e9bc9"
     }
    },
    "e301d715fe9043299869ac8807b6b733": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "IntProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "IntProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Extraction completed...: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_9f7e7c4e542047baa0ab3b5243b70902",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_32aff92a4044485fae3ee1f79c2a7c29",
      "value": 1
     }
    },
    "e70beb853fa348f8bf2e9019a1b4ae5d": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "e82ed2feedad4d4687314bf88bfe7e1d": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "IntProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "IntProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "info",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_74b176ec2fc944389fc19f0d85ee4ba4",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_b222ab334ad1411abdf989aa3f4efeef",
      "value": 1
     }
    },
    "ea5fc83661d14aca880b64a25cbf5384": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_ad77770d1ebb4d73a0b3038cc65dc0c1",
      "placeholder": "​",
      "style": "IPY_MODEL_60013e94ef89481ea6e8ad78689446fb",
      "value": " 9498/11490 [00:00&lt;00:00, 94969.90 examples/s]"
     }
    },
    "f24137e06248452c8d80a8e066d6bd7c": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_2397d41cf49b4598a38cebb786783536",
       "IPY_MODEL_b777595dc2554bdca40193880a51c9aa"
      ],
      "layout": "IPY_MODEL_bf2bf4f333a043aeb87c5e6cb957d825"
     }
    },
    "f44cdbcc689c4fbaa3848fcbfc097faa": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "f79bdbbc8123453b8cd49bec3de46e5f": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "f8f1b594fb344a34a4be3c0f35dd4d12": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "ff651d29638e4d55b491eb50ac9abaf0": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}