03b - LSTM_Model_w_Gridsearch_Chest_Device.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "view-in-github"
   },
   "source": [
    "<a href=\"https://colab.research.google.com/github/santteegt/om-fol-timeseries/blob/master/LSTM_Model_w_gridsearch_Chest_device.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 459
    },
    "colab_type": "code",
    "id": "zWa2GDwbVEpS",
    "outputId": "78804263-d4b2-4b82-d69a-e88742338e2d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting torch-lr-finder\n",
      "  Downloading https://files.pythonhosted.org/packages/3b/a1/b0745ba7312465342f8480b88bf55c67c2ca035cad77146f91db4843f2e2/torch_lr_finder-0.2.0-py3-none-any.whl\n",
      "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from torch-lr-finder) (20.4)\n",
      "Requirement already satisfied: matplotlib in /usr/local/lib/python3.6/dist-packages (from torch-lr-finder) (3.2.2)\n",
      "Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from torch-lr-finder) (4.41.1)\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from torch-lr-finder) (1.18.5)\n",
      "Requirement already satisfied: torch>=0.4.1 in /usr/local/lib/python3.6/dist-packages (from torch-lr-finder) (1.5.1+cu101)\n",
      "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from packaging->torch-lr-finder) (1.12.0)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->torch-lr-finder) (2.4.7)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->torch-lr-finder) (1.2.0)\n",
      "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib->torch-lr-finder) (0.10.0)\n",
      "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->torch-lr-finder) (2.8.1)\n",
      "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from torch>=0.4.1->torch-lr-finder) (0.16.0)\n",
      "Installing collected packages: torch-lr-finder\n",
      "Successfully installed torch-lr-finder-0.2.0\n",
      "Collecting skorch\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/42/21/4936b881b33de285faa0b36209afe4f9724a0875b2225abdc63b23d384a3/skorch-0.8.0-py3-none-any.whl (113kB)\n",
      "\u001b[K     |████████████████████████████████| 122kB 3.4MB/s \n",
      "\u001b[?25hRequirement already satisfied, skipping upgrade: numpy>=1.13.3 in /usr/local/lib/python3.6/dist-packages (from skorch) (1.18.5)\n",
      "Requirement already satisfied, skipping upgrade: scikit-learn>=0.19.1 in /usr/local/lib/python3.6/dist-packages (from skorch) (0.22.2.post1)\n",
      "Requirement already satisfied, skipping upgrade: tqdm>=4.14.0 in /usr/local/lib/python3.6/dist-packages (from skorch) (4.41.1)\n",
      "Requirement already satisfied, skipping upgrade: tabulate>=0.7.7 in /usr/local/lib/python3.6/dist-packages (from skorch) (0.8.7)\n",
      "Requirement already satisfied, skipping upgrade: scipy>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from skorch) (1.4.1)\n",
      "Requirement already satisfied, skipping upgrade: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-learn>=0.19.1->skorch) (0.16.0)\n",
      "Installing collected packages: skorch\n",
      "Successfully installed skorch-0.8.0\n"
     ]
    }
   ],
   "source": [
    "!pip install torch-lr-finder\n",
    "!pip install -U skorch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "xsvTkUtynjfO"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.preprocessing import StandardScaler, MinMaxScaler\n",
    "import skorch\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.utils.data import Dataset\n",
    "from torch.utils.data.dataloader import DataLoader\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 85
    },
    "colab_type": "code",
    "id": "xaZHaL8rpWKQ",
    "outputId": "bba06969-7386-4cee-9b5d-11db335eba02"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading...\n",
      "From: https://drive.google.com/uc?id=1_56HAZc2XSXQCAR4iZODq9VAoyq5CvAs\n",
      "To: /content/all_subjects.csv\n",
      "172MB [00:01, 129MB/s]\n"
     ]
    }
   ],
   "source": [
    "# !wget https://github.com/santteegt/om-fol-timeseries/archive/master.zip -O om-fol-timeseries.zip && unzip om-fol-timeseries.zip && rm om-fol-timeseries.zip\n",
    "# !wget https://github.com/santteegt/om-fol-timeseries/raw/master/segmented_data/WESAD_segmented.zip && unzip -d data WESAD_segmented.zip && rm WESAD_segmented.zip\n",
    "# from google.colab import files\n",
    "# uploaded = files.upload()\n",
    "\n",
    "!gdown --id 1_56HAZc2XSXQCAR4iZODq9VAoyq5CvAs  # segmented dataset on my Google drive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 68
    },
    "colab_type": "code",
    "id": "hL3XC4qKpa4W",
    "outputId": "5b3fd4a9-78a3-4d60-873a-521e87cdae7c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total 167928\n",
      "-rw-r--r-- 1 root root 171953449 Jul 14 14:46 all_subjects.csv\n",
      "drwxr-xr-x 1 root root      4096 Jul 10 16:29 sample_data\n"
     ]
    }
   ],
   "source": [
    "!ls -l"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "vkjh0j6rsaUu"
   },
   "source": [
    "## Load Subject Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "NQQm1-SIpdGB"
   },
   "outputs": [],
   "source": [
    "# BASE_PATH = './data'\n",
    "# [subject for subject in os.listdir(BASE_PATH) if subject.endswith('.feather')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "eJ0PTSZts2eI",
    "outputId": "efcd9277-9084-4e6a-e836-2f368aed6984"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(121813, 79)"
      ]
     },
     "execution_count": 5,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('all_subjects.csv', index_col=0)\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "kn05bY8QPvb-"
   },
   "source": [
    "Transform labels from range 0... TODO: This should be done directly in the dataset:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "LnejF7Kd2172"
   },
   "outputs": [],
   "source": [
    "df.label = df.label - 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "eBFzzX0sIeIg"
   },
   "source": [
    "## Data Loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Zq0UvOYj60q6"
   },
   "outputs": [],
   "source": [
    "class WESADDataset(Dataset):\n",
    "    #Constructor is mandatory\n",
    "    def __init__(self, dataframe, transform=None):\n",
    "        # normalizer = StandardScaler()\n",
    "        # Since different subjects have different responses on the signal values, min-max normalisation was conducted across different signals to normalize them to the same scale\n",
    "        normalizer = MinMaxScaler()\n",
    "        self.dataframe = dataframe.drop(columns=['subject','label'])\n",
    "        # self.X = self.dataframe.astype(np.float32).to_numpy()\n",
    "        # Normalize features\n",
    "        self.X = normalizer.fit_transform(self.dataframe.astype(np.float32))\n",
    "        self.labels = dataframe['label']\n",
    "        self.transform = transform # e.g. torch.Tensor\n",
    "    \n",
    "    def to_torchtensor(self):            \n",
    "        self.dataframe = torch.from_numpy(self.dataframe)\n",
    "        self.labels = torch.from_numpy(self.labels)\n",
    "    \n",
    "    def __len__(self):\n",
    "        #Mandatory\n",
    "        '''Returns:\n",
    "                Length [int]: Length of Dataset/batches\n",
    "        '''\n",
    "        return self.dataframe.shape[0]\n",
    "\n",
    "    def __getitem__(self, idx): \n",
    "        #Mandatory \n",
    "        \n",
    "        '''Returns:\n",
    "                    Data [Torch Tensor]: \n",
    "                    Target [ Torch Tensor]:\n",
    "        '''\n",
    "        sample = self.X[idx]\n",
    "        # sample = self.X[idx].reshape(1, -1)\n",
    "        target = self.labels[idx]\n",
    "                \n",
    "        if self.transform:\n",
    "            sample = self.transform(sample)\n",
    "\n",
    "        return sample, target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "yQZ-fX0SIj5L"
   },
   "outputs": [],
   "source": [
    "def get_data_loader(subject, train_batch_size=128, val_batch_size=5):\n",
    "\n",
    "    train = WESADDataset(df[df['subject'] != subject].reset_index(drop=True))\n",
    "    test = WESADDataset(df[df['subject'] == subject].reset_index(drop=True))\n",
    "\n",
    "    train_dl = torch.utils.data.DataLoader(train, batch_size=train_batch_size, shuffle=True, pin_memory=True, num_workers=4)\n",
    "    val_dl = torch.utils.data.DataLoader(test, batch_size=val_batch_size, shuffle=False, pin_memory=True, num_workers=4)\n",
    "    \n",
    "    return train_dl, val_dl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "PLPy0--FptDt"
   },
   "outputs": [],
   "source": [
    "def get_default_device():\n",
    "    \"\"\"Pick GPU if available, else CPU\"\"\"\n",
    "    if torch.cuda.is_available():\n",
    "        return torch.device('cuda')\n",
    "    else:\n",
    "        return torch.device('cpu')\n",
    "\n",
    "def to_device(data, device):\n",
    "    \"\"\"Move tensor(s) to chosen device\"\"\"\n",
    "    if isinstance(data, (list,tuple)):\n",
    "        return [to_device(x, device) for x in data]\n",
    "    return data.to(device, non_blocking=True)\n",
    "\n",
    "\n",
    "class DeviceDataLoader():\n",
    "    \"\"\"Wrap a dataloader to move data to a device\"\"\"\n",
    "    def __init__(self, dl, device):\n",
    "        self.dl = dl\n",
    "        self.device = device\n",
    "        \n",
    "    def __iter__(self):\n",
    "        \"\"\"Yield a batch of data after moving it to device\"\"\"\n",
    "        for b in self.dl: \n",
    "            yield to_device(b, self.device)\n",
    "\n",
    "    def __len__(self):\n",
    "        \"\"\"Number of batches\"\"\"\n",
    "        return len(self.dl)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "mVvM8CGVAzRC"
   },
   "source": [
    "## Methods & utils for model training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "zsinJ1Z-7BCU"
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import f1_score\n",
    "\n",
    "def accuracy(outputs, labels):\n",
    "    _, preds = torch.max(outputs, dim=1)\n",
    "    return torch.tensor(torch.sum(preds == labels).item() / len(preds))\n",
    "\n",
    "def f1(output, label, threshold=0.5, beta=1):\n",
    "    probs = torch.argmax(output.data.to('cpu'), dim=1)\n",
    "    preds = label.data.to('cpu')\n",
    "    return f1_score(preds, probs, average='macro')\n",
    "\n",
    "def plot_scores(history):\n",
    "    t_acc = [x['train_acc'] for x in history]\n",
    "    plt.plot(t_acc, '-x', label='Train Acc')\n",
    "\n",
    "    accuracies = [x['val_acc'] for x in history]\n",
    "    f1 = [x['val_f1'] for x in history]\n",
    "    plt.plot(accuracies, '-x', label='Valid Acc')\n",
    "    plt.plot(f1, '-x', label='Val F1')\n",
    "    plt.xlabel('epoch')\n",
    "    plt.ylabel('Score')\n",
    "    plt.title('Score vs. No. of epochs')\n",
    "    plt.legend();\n",
    "\n",
    "def plot_losses(history):\n",
    "    train_losses = [x.get('train_loss') for x in history]\n",
    "    val_losses = [x['val_loss'] for x in history]\n",
    "    plt.plot(train_losses, '-bx')\n",
    "    plt.plot(val_losses, '-rx')\n",
    "    plt.xlabel('epoch')\n",
    "    plt.ylabel('loss')\n",
    "    plt.legend(['Training', 'Validation'])\n",
    "    plt.title('Loss vs. No. of epochs');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "3bKM-vZh6WDN"
   },
   "outputs": [],
   "source": [
    "class BaseModel(nn.Module):\n",
    "    def training_step(self, batch):\n",
    "        data, labels = batch \n",
    "        out = self(data)                  # Generate predictions\n",
    "        loss = F.cross_entropy(out, labels) # Calculate loss\n",
    "        return loss\n",
    "    \n",
    "    def validation_step(self, batch):\n",
    "        data, labels = batch \n",
    "        out = self(data)                    # Generate predictions\n",
    "        # print('out', out.shape)\n",
    "        loss = F.cross_entropy(out, labels)   # Calculate loss\n",
    "        # print('loss', loss.shape)\n",
    "        out = F.softmax(out, dim=1)           # Apply Softmax\n",
    "        acc = accuracy(out, labels)           # Calculate accuracy\n",
    "        f1_ = f1(out, labels)                  # Calculate F1\n",
    "        return {'val_loss': loss.detach(), 'val_acc': acc, 'val_f1': f1_}\n",
    "        \n",
    "    def validation_epoch_end(self, outputs):\n",
    "        batch_losses = [x['val_loss'] for x in outputs]\n",
    "        epoch_loss = torch.stack(batch_losses).mean()   # Combine losses\n",
    "        batch_accs = [x['val_acc'] for x in outputs]\n",
    "        batch_f1 = [x['val_f1'] for x in outputs]\n",
    "        # print('batch_f1', len(batch_f1))\n",
    "        epoch_acc = torch.stack(batch_accs).mean()      # Combine accuracies\n",
    "        epoch_f1 = np.mean(np.array(batch_f1))         # Combine F1 scores\n",
    "        return {'val_loss': epoch_loss.item(), 'val_acc': epoch_acc.item(), 'val_f1': epoch_f1.item()}\n",
    "    \n",
    "    def epoch_end(self, epoch, result):\n",
    "        print(\"Epoch [{}], train_loss: {:.4f}, train_acc: {:.4f}, val_loss: {:.4f}, val_acc: {:.4f}, val_f1: {:.4f}\".format(\n",
    "            epoch, result['train_loss'], result['train_acc'], result['val_loss'], result['val_acc'], result['val_f1']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ewGiwm2H6DOg"
   },
   "outputs": [],
   "source": [
    "@torch.no_grad()\n",
    "def evaluate(model, val_loader):\n",
    "    model.eval()\n",
    "    outputs = [model.validation_step(batch) for batch in val_loader]\n",
    "    return model.validation_epoch_end(outputs)\n",
    "\n",
    "def fit(epochs, lr, model, train_loader, val_loader, optimizer):\n",
    "    history = []\n",
    "    for epoch in range(epochs):\n",
    "        # Training Phase \n",
    "        model.train()\n",
    "        train_accs = []\n",
    "        train_losses = []\n",
    "        for batch in tqdm(train_loader):\n",
    "            loss = model.training_step(batch)\n",
    "            train_losses.append(loss)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            optimizer.zero_grad()\n",
    "\n",
    "            data, labels = batch\n",
    "            train_out = F.softmax(model(data), dim=1)\n",
    "            train_accs.append(accuracy(train_out, labels))\n",
    "\n",
    "\n",
    "        # Validation phase\n",
    "        result = evaluate(model, val_loader)\n",
    "        result['train_loss'] = torch.stack(train_losses).mean().item()\n",
    "        result['train_acc'] = torch.stack(train_accs).mean().item()\n",
    "        history.append(result)\n",
    "\n",
    "        # if epoch < 5 or (epoch + 1) % 5 == 0:\n",
    "        model.epoch_end(epoch, result)\n",
    "            \n",
    "    return history"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "sbWcJEn7IEAo"
   },
   "source": [
    "## Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "-fDiM6gn4jdN"
   },
   "outputs": [],
   "source": [
    "class WesadFeedForward(BaseModel):\n",
    "    def __init__(self, input_dim, output_dim=3):\n",
    "        super().__init__()\n",
    "        self.network = nn.Sequential(\n",
    "            nn.Linear(input_dim, 128),\n",
    "            nn.Dropout(0.5),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(128, 256),\n",
    "            nn.Dropout(0.5),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(256, 128),\n",
    "            nn.Linear(128, output_dim),\n",
    "            #nn.Dropout(0.5),\n",
    "            # nn.Softmax(dim=1)\n",
    "        )\n",
    "        \n",
    "    def forward(self, x):\n",
    "        return self.network(x)\n",
    "\n",
    "\n",
    "class WesadLSTM(BaseModel):\n",
    "    def __init__(self, input_dim, hidden_dim, output_dim=3, lstm_layers=1, dropout=0.2):\n",
    "        super().__init__()\n",
    "        self.input_dim = input_dim\n",
    "        self.hidden_dim = hidden_dim\n",
    "        self.classes = output_dim\n",
    "        # self.linear1 = nn.Linear(input_dim, input_dim)\n",
    "        # self.lstm = nn.LSTM(input_dim, hidden_dim, num_layers=lstm_layers, batch_first=True, dropout=0.2)\n",
    "        self.lstm = nn.LSTM(input_size=input_dim, hidden_size=input_dim, num_layers=lstm_layers, dropout=dropout)\n",
    "        self.fc = nn.Linear(hidden_dim, output_dim)\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "\n",
    "    def forward(self, x):\n",
    "        xb = x.view(-1, 1, self.input_dim)\n",
    "        # print('x', xb.shape)\n",
    "        # out = self.linear1(x)\n",
    "        # out = x.view(x.shape, -1)\n",
    "        # print(out.shape)\n",
    "        # out = self.dropout(out)\n",
    "        # lstm_out, (ht, ct) = self.lstm(out)\n",
    "        # out = self.fc(ht[-1])\n",
    "        lstm_out, _ = self.lstm(xb)\n",
    "        # print(lstm_out.view(-1, self.hidden_dim).shape)\n",
    "        out = self.fc(lstm_out.view(-1, self.hidden_dim))\n",
    "        # out = self.dropout(out)\n",
    "        # out = self.softmax(out)\n",
    "        # out = F.softmax(out, dim=1)\n",
    "        return out.view(-1, self.classes)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "RqmrXK4O_tCX"
   },
   "source": [
    "## General Model Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "UBeX47-b_rZP"
   },
   "outputs": [],
   "source": [
    "subjects = df['subject'].unique()\n",
    "subjects.sort()\n",
    "device = get_default_device()\n",
    "\n",
    "train_batch_size = 25\n",
    "val_batch_size = 5\n",
    "input_dim = df.drop(columns=['subject', 'label']).shape[1]\n",
    "output_dim = 3\n",
    "lstm_layers = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "65FyNs92F5eX"
   },
   "outputs": [],
   "source": [
    "# lrs = [0.5, 0.1, 0.01, 0.001]\n",
    "# epochs = [20, 20, 20,  20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 366,
     "referenced_widgets": [
      "de18359779d24b8aa02ede2c144b53fe",
      "d07c1f93fe1349ec94f5a173060c8da9",
      "80c2efd5d3794a448fd0c22448d325ee",
      "69571c1e37ba428fb85c5da858eb3640",
      "8ab0c7d18e824e76839d7e2df636c410",
      "b5e8f1950fc640488d94e3109b0a237d",
      "e5ac081f09ba4ec18b09dcc6eb2e37d2",
      "e5562350386343909e87db299389e7ef"
     ]
    },
    "colab_type": "code",
    "id": "8Nxfyv_GYUw2",
    "outputId": "a7c7115f-a53b-41b6-dabc-a0a84cac6966"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "de18359779d24b8aa02ede2c144b53fe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1000.0), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stopping early, the loss has diverged\n",
      "Learning rate search finished. See the graph with {finder_name}.plot()\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEKCAYAAAAfGVI8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXhcd33v8fd3Fu2LbVmy5d1JnNXFhBiHlAAOSwgpCZTtUlpKgJJCLwW6cFt6+1Dovb3Q24fQhnAJaQkkEEIgUK4JONxAAiGbE9lxHG9JjPcltqxdlkaa5Xv/mCNZViRbtnVGMzqf1/PMozlzzpzz/VnWfOe3nN/P3B0REYmu2FQHICIiU0uJQEQk4pQIREQiTolARCTilAhERCJOiUBEJOISUx3A6Zo9e7YvWbJkqsMQESkp69evP+rujWPtK7lEsGTJElpaWqY6DBGRkmJme8bbp6YhEZGIUyIQEYk4JQIRkYhTIhARiTglAhGRiFMiEBGJOCUCEZES8MDWw7xwuCeUcysRiIiUgD+7az0/evpAKOcOLRGYWYWZPWlmz5jZFjP7/BjH3GBmrWa2MXj8SVjxiIiUqlzOSWedsng4H9lh3lk8ALze3XvNLAk8YmZr3f2JUcfd4+4fDzEOEZGSNpjNAVCWKLFE4Pk1MHuDzWTw0LqYIiKnaSgRlIeUCELtIzCzuJltBI4AD7j7ujEOe6eZbTKze81s4TjnudHMWsyspbW1NcyQRUSKzmAm3BpBqInA3bPu/nJgAbDKzJaPOuQnwBJ3fxnwAHDHOOe5zd1XuvvKxsYxJ88TEZm2hhNBSH0EBRk15O6dwEPANaNeb3P3gWDzP4DLChGPiEgpKdkagZk1mtmM4Hkl8CZg+6hjmkdsXg9sCyseEZFSVbKdxUAzcIeZxcknnO+7+31m9o9Ai7uvAT5hZtcDGaAduCHEeERESlLYTUNhjhraBFw6xuufHfH8M8BnwopBRGQ6GCjVpiEREZkcJdtHICIik6Ok7yMQEZGzd7yPIB7K+ZUIRESKnJqGREQibjCbBZQIREQiSzUCEZGImxZTTIiIyJm7+cEdgGoEIiKRlM05rT35Kdk0fFREJIIGMtnh50oEIiIRNJDO9w987rqLMbNQrqFEICJSxIbmGSpPhnMzGSgRiIgUtaGmobCahUCJQESkqA3XCBKqEYiIRNJQH4FqBCIiETXcNJRUIhARiaRUWk1DIiKRps5iEZGIOz58VIlARCSSjtcI1DQkIhJJGjUkIhJxQ01DFbqzWEQkmtRZLCIScWoaEhGJuMFsDjNIhLQ6GSgRiIgUtUzOScTCmX56iBKBiEgRy+WcWEjrEAxRIhARKWKqEYiIRFw258RKNRGYWYWZPWlmz5jZFjP7/BjHlJvZPWa2w8zWmdmSsOIRESlFOXfipZoIgAHg9e6+Ang5cI2ZvWrUMR8GOtz9PODLwD+HGI+ISMkp6aYhz+sNNpPBw0cd9jbgjuD5vcAbLKzVmUVESlDJdxabWdzMNgJHgAfcfd2oQ+YD+wDcPQN0AQ1jnOdGM2sxs5bW1tYwQxYRKSrZXGk3DeHuWXd/ObAAWGVmy8/wPLe5+0p3X9nY2Di5QYqIFLGSTwRD3L0TeAi4ZtSuA8BCADNLAPVAWyFiEhEpBdlS7iw2s0YzmxE8rwTeBGwfddga4APB83cBD7r76H4EEZHIyuaceMh9BIkQz90M3GFmcfIJ5/vufp+Z/SPQ4u5rgG8A3zazHUA78N4Q4xERKTmFaBoKLRG4+ybg0jFe/+yI5yng3WHFICJS6qZNH4GIiJyZnJf48FERETk7mZyTiCsRiIhEVrbUbygTEZGzU+pzDYmIyFnKZJUIREQiLefh30egRCAiUsQ0fFREJOKUCEREIq6k5xoSEZGzl82h4aMiIlGWzeVKd4UyERE5e+ojEBGJuJxDTIlARCS6MmoaEhGJtpw6i0VEoi2bc9UIRESiLJNz9RGIiERZfvbRcK+hRCAiUsTyTUPhflQrEYiIFDEtTCMiEnH5G8rCvYYSgYhIEctPOqemIRGRyFKNQEQk4rI5rVAmIhJZuZwDqGlIRCSqMsOJINzrKBGIiBSpVCYLQEUyHup1lAhERIpUKp1PBOVKBCIi0TSQzgFQkSjRPgIzW2hmD5nZVjPbYmafHOOY1WbWZWYbg8dnw4pHRKTUDNUIwm4aSoR47gzwV+6+wcxqgfVm9oC7bx113G/c/a0hxiEiUpL6C5QIQqsRuPshd98QPO8BtgHzw7qeiMh0kwqahipLNRGMZGZLgEuBdWPsvsLMnjGztWZ2yTjvv9HMWsyspbW1NcRIRUSKx/GmoRLtIxhiZjXAD4FPuXv3qN0bgMXuvgL4CvDjsc7h7re5+0p3X9nY2BhuwCIiRaJQfQShJgIzS5JPAne5+49G73f3bnfvDZ7/DEia2ewwYxIRKRWpTDBqqFRrBGZmwDeAbe5+0zjHzA2Ow8xWBfG0hRWTiEgpSQ0G9xEkSnfU0KuB9wPPmtnG4LW/AxYBuPutwLuAj5lZBugH3uvuHmJMIiIlY+jO4sqyEk0E7v4IcNIp89z9FuCWsGIQESll06KPQEREzlyq1O8sFhGRs5NKZ0nEjETI048qEYiIFKnBTI7ykGsDoEQgIlK00tkcSSUCEZHoGsw6ybBXpWGCicDMqs0sFjw/38yuD24WExGRkKSzOcqKJREADwMVZjYf+H/k7w/4VlhBiYhI0DQUD3fheph4IjB37wPeAfwfd383MOYEcSIiMjnyiaB4agRmZlcAfwj8NHgt3DscREQibjDjoQ8dhYkngk8BnwH+0923mNk5wEPhhSUiIvk+gvCbhiY0xYS7/xr4NUDQaXzU3T8RZmAiIlFXVE1DZvZdM6szs2pgM7DVzD4dbmgiItFWVIkAuDhYVObtwFpgKfmRQyIiEpLBrBfVDWXJ4L6BtwNr3D0NaLpoEZEQpTOF6SOYaCL4OrAbqAYeNrPFwOhlJ0VEZBIVqmloop3FNwM3j3hpj5ldFU5IIiICRdZHYGb1ZnaTmbUEjy+Rrx2IiEhI0sU01xBwO9ADvCd4dAPfDCsoEREJ7iNIFMl9BMC57v7OEdufH7EOsYiIhKComoaAfjO7cmjDzF5NfrF5EREJSaGahiZaI/gocKeZ1QfbHcAHwglJREQABots1NAzwAozqwu2u83sU8CmMIMTEYkqdy/YXEOnlWrcvTu4wxjgL0OIR0REgGzOcaeoZh8dS/hpSkQkogazOYCi6iwei6aYEBEJyY4jvQDMm1ER+rVO2kdgZj2M/YFvQGUoEYmICI//tg2AK85tCP1aJ00E7l4begQiIvISBzr7qa9M0lQbfo0g/MYnERE5bYW6hwCUCEREilKhho5CiInAzBaa2UNmttXMtpjZJ8c4xszsZjPbYWabzOwVYcUjIlJK0tlcQRalgYnfWXwmMsBfufsGM6sF1pvZA+6+dcQxbwGWBY/Lga8FP0VEIi2TdRKxEq8RuPshd98QPO8BtgHzRx32NuBOz3sCmGFmzWHFJCJSKgo1vQQUqI/AzJYAlwLrRu2aD+wbsb2flyYLEZHIyU9BPU0SgZnVAD8EPjVieorTPceNQ4vitLa2Tm6AIiJFaFo0DQEEC97/ELjL3X80xiEHgIUjthcEr53A3W9z95XuvrKxsTGcYEVEisi0aBoyMwO+AWxz95vGOWwN8MfB6KFXAV3ufiismERESkUhm4bCHDX0auD9wLMjVjP7O2ARgLvfCvwMuBbYAfQBHwwxHhGRklHIpqHQEoG7P8IpZih1dwf+a1gxiIiUqkItUwm6s1hEpCgNFvCGMiUCEZEilMk6yekwakhERM6MmoZERCKukHMNKRGIiBShdNYpU41ARCS60tnc9LizWEREzkwm62oaEhGJKnefHlNMiIjImcnkHEDDR0VEoiqTDRKBmoZERKJpMJsDUNOQiEhUZYJEoFFDIiIRlfV801BMiUBEJJpy+QoBcVMiEBGJpKEaQYG6CJQIRESKTS4YPhpTjUBEJJpywzUCJQIRkUjKqkYgIhJtOY0aEhGJtqxGDYmIRNtQ05BGDYmIRNRw05BqBCIi0XS8RqBEICISSZpiQkQk4oZuKFNnsYhIRAV5QE1DIiJRNdRHUKAKgRKBiEixGZ5iQk1DIiLRNG1GDZnZ7WZ2xMw2j7N/tZl1mdnG4PHZsGIRESklhR41lAjx3N8CbgHuPMkxv3H3t4YYg4hIyZk2o4bc/WGgPazzi4hMV9OmaWiCrjCzZ8xsrZldMsWxiIgUhUJPMRFm09CpbAAWu3uvmV0L/BhYNtaBZnYjcCPAokWLChehiMgUGJ59dLrXCNy92917g+c/A5JmNnucY29z95XuvrKxsbGgcYqIFFpk1iw2s7lm+XqPma0KYmmbqnhERIqF+9ANZSXeNGRmdwOrgdlmth/4ByAJ4O63Au8CPmZmGaAfeK8PlV5EJMKyBR41FFoicPc/OMX+W8gPLxURkRGiNmpIRERG0ZrFIiIRpzWLRUQi7vgUE4W5nhKBiEiRmTZTTIiIyJlRZ7GISMSps1hEJOIKPdeQEoGISJHRqCERkYjLadSQiEi0FXqKCSUCEZEio1FDIiIRl3PHrHCzjyoRiIgUmWzOC9YsBEoEIiJFJ+tesHsIQIlARKTo5FQjEBGJtpxDASsESgQiIsUmm1PTkIhIpLT2DPDOrz3GloNduDste9qhgAv3hrZUpYiITMz3W/axfk8Hf3730zTVlrP5QDdLGqoKdn0lAhGRKdQ/mOXbj+8BYGfrMXa2HuMjr1nKn77u3ILFoKYhEZEQ7T56jL//8bM8tuMoj+04yt62PrYe7CaVzrK3rY8vrN3G4Z4U37zhlcPv+fSbL2R2TXnBYlSNQERkkrj7S+4G/tdfPM+PNx7kO0/sHfd9v3/pfK66sIl7P3oFL3anKEsU9ju6EoGIyCT46kM7+OpDO7hs8UyWz69nw54OLmqu46fPHuIty+dycXMdT+3p4OHnW4ffU1UWp28wy5svmQPAyiWzpiR2JQIRkbOUSmf5l58/B8BvXjjKb144CsC6Xe0A/PWbL+DcxhoAjnSnqCpP0HFskOb6Cjbu6+SyxTOnJvCAEoGIyFn6yTMHAfjuRy7nvKYavvzACxzo7Ofh51tZ0lA1nAQAmuoqAKgpz3/8TlUtYCQlAhGRs3Drr3/LF9du56LmOq44pwEz4wvv+B0AntrdzpKG6imO8NQikwie3tvBHY/tJuf5Ob4TMSOZiFEWj5GMG8l4jEQ8hpGfAjabO/7I5JycBz9zTn1VkgUzKqksS1CeiFGRjFOeiA0/z7rT1Z+mN5UhGY8RM+jsTwNQnogxp66Cxtpy6iuT1FcmScZj9A1m6B3IUFeRpCIZn9p/LBGZkN1Hj/HFtdspT8T41gdf+ZKO4lcWwbf9iYhMIujsS/P0vs7ggx7S2RzprAc/c8PbAGaQiBkxyyeMWJA44rHgQ70vzeDQoqKToCwRI5PNEaxFQVk8Rm1FgrrKJHUVCWorkvQOZEils5zTWM3CmVU01JQxq7qcXM7B4GUL6lnSUM1AOofjzKgqm7T4ROREn1uzhQe2HuZAZz8AH75yKXOCJp9SFJlEcNWFTVx1YdNJj3EfSgQnn+Mjm3PaegdIpXMMZLIMZHKk0vmfA5ksZkZ9ZZLa8gQDmRzuMKMqiVm+U+lQV4q23kG6U2m6+tL0DmSoSMaZXVNGdypDTypDTypNdypDV3+anlSa2ooEDdVlbD7QzS+2HjlpIjKDi+bWsbihilVLZ/G68xtZ0lBd0LlLRKarR3cc5VuP7T7htXevXDg1wUyS0BKBmd0OvBU44u7Lx9hvwL8B1wJ9wA3uviGseCZioqsBxWM23OFzJs5rqj3j90I+YfUOZGg/Nkgm57jD5gNd7GnroyeVpqo8Qcvudp7Z18nazS8CcP6cGt540RzesryZ/R19fPOx3dRVJPjbt1x41vGIRIG788iOo3z+J1tpqi3nux+5nG8+uptPvGFZSdcGAGzoW/Ckn9jstUAvcOc4ieBa4M/JJ4LLgX9z98tPdd6VK1d6S0vLZIc7LWVzzrpdbTy5q53vPLGHo72Dw/sWN1TRfmyQxtpyPnDFEpY11RCPGUd6Brh86SziMaO2IjnujS09qTQ15YmCLaU31dydgUyO8kSMjr40773tcV65ZBZvvmQu58+pZW59aX8QyKndu34/f/2DZ6hMxrntjy/jNcsapzqk02Jm69195Zj7wkoEwYWXAPeNkwi+DvzK3e8Otp8DVrv7oZOdU4ngzB3pSfGr51qJm/HWFc08uuMo/7BmC/va+8c8fnZNGTe95+U01JTx5QeeJ5tzdrf1cfnSWfxg/X4uXzqLve19dPenqa9KMq++kssWz2T1BU2sWjrrhLssW3a3899+uIlL5tXzjkvn85pls4cX5t7f0c+Wg10c7R3kdec3Mre+gmQ8xrGBDFVl8dCSTf9glk37O0nEY8PjuI8NZFi3q4223kHmz6zkF1uP8NTudtp6BzjYlWJefQWxmLG/4/i/WUUyxp0fupxVS0/sGOwbzHCoK8WMyiR1lcn8lALtfVQm4zTUlFNXkU+kuWDK4Y6ghtdYO/GpBTJB31Zl2YkDDA509rPtYDe7245RXZ4gm3MOd6e4fsU8ls3J1wD3tfcBsHDW8cnNDnb209Wf5qLmuvxyiSOaE3sHMnzvyb0011dy9SVziJmxv6OPhTOrcMBguPnxzsd3870n91FTkeDqi+dwsDPF7Noyrl3eTO9AhuXz6ydcxrOVzTltxwaYXV0+4ebRrr40lWXxE74IXfeVR+joG+T+T712eOhnKSnWRHAf8EV3fyTY/iXwN+5+0k95JYLJ5e7s7+hn0/4uBjJZFs6q4v7NL1JXkeQ76/bQ2jNw0vfPrEqSyTlzg6rxzqPHyOacptpycg5XntdAKp3j0d8epSeVoSwRYzCTo74yyWAmx2A2RzZ3/P+gGbjDnLpyDncPsKyphtk15cybUck//f7yCY2o2rC3gyPdKa5Z3sy+9j62HeomZsbrgz6i9Xs7eHpvBzc98DypdL6vZf6MSqrK4uxo7WX0n8SKBfWUJ+Kc21TNpv1dZHPO3//exQxms9zx2B6e3ttBdyrDO1+xgHdeNp971++ntWeAdbvaGczkzx+P2QnlBHjt+Y10HBvkaO8A775sAV9/eCfZnPPZ6y6mobqc+zYd5JNvXMaFc+sA6Dg2yN1P7WXl4lmUJWJ854k9rNvVRt9Alr940/m8b9UiBjI5/v03O/nKgy8MD34Y7Xfm13Pdimb+9/3Pkck5t7zvUjr70qzdfIgnd7WTzjrnzK5mT3sfV543mw9fuZS/+eEmDnWlTjhPImZkcp7/P5B16iqTfP39l3H/5he55aEdJ/0dzauv4NrfaeZlC2dw3cuaMbPhPrpUOveSxHY6ntnXScueDna29vKyBfV8/eGd7Gw9BsAl8+q46oImBrM5yuIxPvjqJTSMmtNnzTMH+Yt7NlKZjHPFuQ1c3FzHEzvbWLernb+++nw+/vplZxzbVCr5RGBmNwI3AixatOiyPXv2hBazHNfWO8AjO46y9VA3ly6cwWuWNZJ1Z93Odi6eV0ddRYKKZJxEzIa/tXen0vzHwzvZ9mIP21/sZl97P+WJGFdfMpdPvP48FjdU8+D2I/xow34qknEWzqqkbzBLXUWS1Rc08tBzrexs7eW+Tccrhuc11bDjSC8VyRjnNtZQU56gO5Vh4cxKzm2qYfm8et6yfC6xmPHg9sN86Fv5/0IzqpJ09qVPKFNteYKegQwAr1g0gzddPJdN+zvpTqUpi8dYNKuK112Q71zf297HeU01LJh58umAu1NpbnlwB3c+vns4scysSnLVhU3UVSTZ39HHuU01NFSX0VhbzrZDPdzz1D56UmlmVZcNN9mtXDyTdM55Zl/n8LmTceOqC5o43J1i88HuE5JJXUWChppyXuxK0Z/OnhDTG4LBEZfMqyMZj1FdniARM3657TBfeXAHbcfy14wZjMxPb75kDhfMqeWJne3MrE7y8y2Hh/etXDyT91+xmNqKBOt2tnOwK8WlC2ew+WAX2w71sONIz3DyWTCzkl/85evoH8xybDBDbUWSTfs72X6oh9sf3XVCUrluxTw+9OolfGHtdp7c1Y4ZvOmiOSxtzI+/37Svi4ua6/ijVy3inBE3ZgH8340HuOOx3dzyvlfQVFvOU7s7+MDtT54wmCIZN977ykXc07JvODGP1FxfQX86y6JZVbjDswe6mFNXzqqlDTz+2zaO9h7/MrT2k6/houa6k/xvKF7FmgjUNBQBY03CNRFd/Wlae1LDHdk/3XSItZsP0TuQYcvBbjLZHN2p/Ad6Nucsa6ohGY/x/OEeFjVU8ZblcznaM8j5c2tZsaCejfs6+fXzrcyfUckrl8zi/Dm1LJ9fN6nNTuv3tPPP9z/HH71qMdevmHfK4/sHs1QkY/QOZPht6zGWz6sjZsbG/Z3sa+/j4uY6vv3EHn71XCsxg9UXNLH6gkb2tPVxqCvFn772HGZWl+HufPH+7Tyw9TB1FUnes3Ihf7Bq4bhl6x3I0N47yMJZleQc/vPpA/ygZR//8+3Lh5uNhqx99hA7jx5jxYIZXLls9knLs6+9j9se3snihire8YoFzKoeewjzUFNY+7FB/vUXz3Pv+v30DR5PZItmVdE3mDmhTwvyw6xfvnAGL3al+Pz1l3DJvDre8KVf0zOQoTIZH65dNtdXcOeHVlFbkaSjLz+Nw9Bwandnw94ODnamqKtM8tSudra/2M3h7gFS6SwvHOll4axK7vrwq1jUUEUmm+PB7Udorq9kZnXylF8KilmxJoLfAz7O8c7im9191anOqUQgQ4a+Hf+P+7YOD+c7Z3Y1t77/Ms6fo5FQpaKrL81ffn8jFzbX8vuXzh9O/lsPdvPU7nYWN1Rx4dw6/u2Xz7Nm40GODWaZUZWkpjzBkZ4B/uVdL+Px37Yxu6acBTMref1FTTTVnn7n/USHj5eqKUkEZnY3sBqYDRwG/gFIArj7rcHw0VuAa8gPH/3gqfoHQIlAxrevvY8FMyun7R+y5Dvz735yL3et20tlMs4n3rCMa5bPneqwSsKU1QjCoEQgInL6TpYItEKZiEjEKRGIiEScEoGISMQpEYiIRJwSgYhIxCkRiIhEnBKBiEjEKRGIiERcyd1QZmatQCfQNeLl+hHbQ89nA0fP8nIjz3s2x421/1Svjd6vMp45lXHix033Mo63LwplXObuY8//7e4l9wBuG2976DnQMtnXOdPjxtp/qtdURpVRZZz8Mo63L6plHHqUatPQT06yPXrfZF7nTI8ba/+pXlMZJ4/KOPHjpnsZx9sX1TICJdg0NFFm1uLjzKsxXaiM04PKOD2UchlLtUYwEbdNdQAFoDJODyrj9FCyZZy2NQIREZmY6VwjEBGRCVAiEBGJOCUCEZGIi2QiMLPVZvYbM7vVzFZPdTxhMbNqM2sxs7dOdSxhMLOLgt/hvWb2samOJwxm9nYz+3czu8fMrp7qeMJgZueY2TfM7N6pjmUyBX9/dwS/vz+c6nhOpuQSgZndbmZHzGzzqNevMbPnzGyHmf3tKU7jQC9QAewPK9YzNUllBPgb4PvhRHl2JqOM7r7N3T8KvAd4dZjxnolJKuOP3f0jwEeB/xJmvGdiksq4090/HG6kk+M0y/sO4N7g93d9wYM9DSU3asjMXkv+Q/xOd18evBYHngfeRP6D/SngD4A48IVRp/gQcNTdc2Y2B7jJ3YsqW09SGVcADeST3VF3v68w0U/MZJTR3Y+Y2fXAx4Bvu/t3CxX/RExWGYP3fQm4y903FCj8CZnkMt7r7u8qVOxn4jTL+zZgrbtvNLPvuvv7pijsU0pMdQCny90fNrMlo15eBexw950AZvY94G3u/gXgZM0iHUB5GHGejckoY9DkVQ1cDPSb2c/cPRdm3Kdjsn6P7r4GWGNmPwWKKhFM0u/RgC+S/0ApqiQAk/73WPROp7zkk8ICYCNF3vpScolgHPOBfSO29wOXj3ewmb0DeDMwA7gl3NAmzWmV0d3/O4CZ3UBQAwo1uslxur/H1eSr3+XAz0KNbPKcVhmBPwfeCNSb2XnufmuYwU2S0/09NgD/BFxqZp8JEkYpGa+8NwO3mNnvMbnTUEy66ZIITou7/wj40VTHUQju/q2pjiEs7v4r4FdTHEao3P1m8h8o05a7t5HvA5lW3P0Y8MGpjmMiirq6choOAAtHbC8IXptOVMbpQWWcfkq+vNMlETwFLDOzpWZWBrwXWDPFMU02lXF6UBmnn5Ivb8klAjO7G3gcuMDM9pvZh909A3wc+DmwDfi+u2+ZyjjPhsqoMpaKKJRxpOla3pIbPioiIpOr5GoEIiIyuZQIREQiTolARCTilAhERCJOiUBEJOKUCEREIk6JQKYNM+st8PUeK/D1ZpjZnxXymhINSgQi4zCzk87F5e6/W+BrzgCUCGTSKRHItGZm55rZ/Wa23vKr0l0YvH6dma0zs6fN7BfB2hSY2efM7Ntm9ijw7WD7djP7lZntNLNPjDh3b/BzdbD/XjPbbmZ3BdNHY2bXBq+tN7Obzewl60KY2Q1mtsbMHgR+aWY1ZvZLM9tgZs+a2duCQ78InGtmG83sX4L3ftrMnjKzTWb2+TD/LWUac3c99JgWD6B3jNd+CSwLnl8OPBg8n8nxO+v/BPhS8PxzwHqgcsT2Y+Snup4NtAHJkdcDVgNd5Ccbi5GfguBK8osC7QOWBsfdDdw3Row3kJ+6eFawnQDqguezgR2AAUuAzSPedzVwW7AvBtwHvHaqfw96lN4jktNQSzSYWQ3wu8APgi/ocHwhogXAPWbWDJQBu0a8dY2794/Y/qm7DwADZnYEmMNLlzh90t33B9fdSP5DuxfY6e5D574buHGccB9w9/ah0IH/FayGlSM/3/2cMd5zdfB4OtiuAZYBD49zDZExKRHIdBYDOt395WPs+wr5ZUrXBAvcfG7EvmOjjh0Y8TzL2H83EznmZEZe8w+BRuAyd0+b2W7ytYvRDPiCu3/9NK8lcgL1Eci05e7dwC4zezfkl300sxXB7nqOzxn/gXaVLBgAAADgSURBVJBCeA44Z8TShhNdfL4eOBIkgauAxcHrPUDtiON+DnwoqPlgZvPNrOmso5bIUY1AppMqMxvZZHMT+W/XXzOzvweSwPeAZ8jXAH5gZh3Ag8DSyQ7G3fuD4Z73m9kx8vPWT8RdwE/M7FmgBdgenK/NzB41s83k1zD+tJldBDweNH31An8EHJnsssj0pmmoRUJkZjXu3huMIvoq8IK7f3mq4xIZSU1DIuH6SNB5vIV8k4/a86XoqEYgIhJxqhGIiEScEoGISMQpEYiIRJwSgYhIxCkRiIhEnBKBiEjE/X93dSPcOFHDGQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light",
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7f6fe3490b70>"
      ]
     },
     "execution_count": 21,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from torch_lr_finder import LRFinder\n",
    "\n",
    "max_lr = 1e-5\n",
    "# grad_clip = 0.001\n",
    "# weight_decay = 1e-4\n",
    "opt_func = torch.optim.Adam\n",
    "criterion = F.cross_entropy\n",
    "train_dl, val_dl = get_data_loader(subject='S2', train_batch_size=train_batch_size, val_batch_size=val_batch_size)\n",
    "\n",
    "# model = WesadFeedForward(input_dim, output_dim)\n",
    "model = WesadLSTM(input_dim=input_dim, hidden_dim=input_dim, output_dim=output_dim, lstm_layers=lstm_layers)\n",
    "# optimizer = opt_func(model.parameters(), lr=max_lr, weight_decay=weight_decay)\n",
    "optimizer = opt_func(model.parameters(), lr=max_lr)\n",
    "lr_finder = LRFinder(model, optimizer, criterion, device=device)\n",
    "lr_finder.range_test(train_dl, end_lr=10000, num_iter=1000)\n",
    "lr_finder.plot() # to inspect the loss-learning rate graph\n",
    "# lr_finder.reset() # to reset the model and optimizer to their initial state"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "tMur3N7FIBS0"
   },
   "source": [
    "## Running models - LOSO CV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "LuzlSLBypEDg"
   },
   "outputs": [],
   "source": [
    "# def get_loso_cv(subjects, to_device=False):\n",
    "#     for subject in subjects:\n",
    "#         history = []\n",
    "#         print('LOSO', subject)\n",
    "#         train_dl, val_dl = get_data_loader(subject, train_batch_size=train_batch_size, val_batch_size=val_batch_size)\n",
    "#         if to_device:\n",
    "#             train_dl = DeviceDataLoader(train_dl, device)\n",
    "#             val_dl = DeviceDataLoader(val_dl, device)\n",
    "\n",
    "#         yield train_dl, val_dl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "P_qs92RNGITw",
    "outputId": "7d16960d-3100-4f7e-f99f-4aa635e52e96"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LOSO S10\n"
     ]
    }
   ],
   "source": [
    "# epochs = 20\n",
    "# lr = 1e-4\n",
    "\n",
    "# models = []\n",
    "# histories = []\n",
    "# val_histories = []\n",
    "# for subject in subjects:\n",
    "#     history = []\n",
    "#     print('LOSO', subject)\n",
    "#     train_dl, val_dl = get_data_loader(subject, train_batch_size=train_batch_size, val_batch_size=val_batch_size)\n",
    "#     train_ddl = DeviceDataLoader(train_dl, device)\n",
    "#     val_ddl = DeviceDataLoader(val_dl, device)\n",
    "#     # model = to_device(WesadFeedForward(input_dim, output_dim), device)\n",
    "#     model = to_device(WesadLSTM(input_dim=input_dim, hidden_dim=input_dim, \n",
    "#                                 output_dim=output_dim, lstm_layers=lstm_layers), device)\n",
    "\n",
    "#     # optimizer = torch.optim.Adam(model.parameters(), lr=lr)\n",
    "\n",
    "#     # history += fit(epochs, lr, model, train_ddl, val_ddl, optimizer)\n",
    "#     break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "4LfpiXbjEQTQ"
   },
   "outputs": [],
   "source": [
    "def try_batch(model, dl):\n",
    "    model.eval()\n",
    "    for data, labels in dl:\n",
    "        # print(data)\n",
    "        # print(labels)\n",
    "        print('data.shape:', data.shape)\n",
    "        out = model(data)\n",
    "        print('out[0]', out[0])\n",
    "        print('softmax(out[0]):', F.softmax(out, dim=1)[0])\n",
    "        _, p = torch.max(F.softmax(out, dim=1), dim=1)\n",
    "        print('Max', p[0])\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "MXCusL0C-M_E"
   },
   "outputs": [],
   "source": [
    "# try_batch(model, train_ddl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 68
    },
    "colab_type": "code",
    "id": "Nc7XHp4RGK5J",
    "outputId": "c206f7ff-8376-4135-9068-cf4c775c3ca8"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'val_acc': 0.31704410910606384,\n",
       "  'val_f1': 0.3168419036267665,\n",
       "  'val_loss': 1.1131939888000488}]"
      ]
     },
     "execution_count": 20,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# history = [evaluate(model, val_ddl)]\n",
    "# history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Ti_YihcVJ7BI"
   },
   "outputs": [],
   "source": [
    "# epochs = 5\n",
    "# lr = 5e-3\n",
    "# optimizer = torch.optim.Adam(model.parameters(), lr=lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 347,
     "referenced_widgets": [
      "7df734534de34697af24898a2f195b9e",
      "f16ad6145b1943fc997f67d408c50fca",
      "d6468069d66d43fa961fdcf24b10f10e",
      "892e164f1d744c1cb546e04cc85164fa",
      "088b20767d6e4d71b51412890969dc76",
      "0a134d32013545eda0bd5d5dcfc36707",
      "f60febd9a8c643be8d9cef045cce4828",
      "e0e86e49b779404897aab3b4ae85edfd",
      "d819f1d1baea4290aec116d61e358adf",
      "4457224890204d60b868f1ea8f4f2dc8",
      "7d051de8c8eb4470b64145f58dfdca55",
      "395517cb7770477cadfd082dfa542a66",
      "2e5127384bce43d9afffcdbaa164206d",
      "0c51ac7d534c4a9f82d0f16d32c269f1",
      "9acbca14de074bd89da12c2f1fbdf6cf",
      "15f9d1856c5d4ca79070fccec70aa5fc",
      "d344f2b329634552ad3577c978f2dd52",
      "345ac6be53ec48ba867d0a4132bf9f8e",
      "c5e77a4b28b84512a40b76302b22717d",
      "9395e8ff5b7e43edabadf3ff766ea1d8",
      "7e7dbf922e5d4286a5e53cbfdd84c523",
      "fb06ea6f38d74306afbcc22d4cb4adcf",
      "bdea7d80b440433cbc09943fb26d83e3",
      "3b9cc8d292c6406a9bbd1c6d8d4c41c8",
      "df4136bbe3d7481cb7c2ae651d1cd0d7",
      "0d9defd150074322bfc6b8a527fb6e16",
      "7e89d00dac1343e182dc0c50ef5ffa39",
      "74cdddbd2f004d7886894e4297474708",
      "6d6d9f0b2a6e48de83eaef9caa9ebd02",
      "175dc9ca4cf941b3a42566b44a0d4f99",
      "91317de663ce44999df501c8b0d75a47",
      "210889d450ce4292af0cc8ddf4d46eae",
      "ad112087ff3b4cc583a60a610891b1de",
      "506e4435462a4d4292a719819a0c8e17",
      "6cb1f020061a471ba03d5eec27d2d35a",
      "b955083d78f2426db949d80f09ee20f2",
      "6b1d165de1c14f72a289fc9e863492d7",
      "51b233b4756a47579d9253d9446ad465",
      "073d47da916f411ca281934f2765d067",
      "43540220a99d43fa9434c6641c983779"
     ]
    },
    "colab_type": "code",
    "id": "1i9617j0DOuG",
    "outputId": "5687c477-d1f3-4161-b73b-8415b9106973"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7df734534de34697af24898a2f195b9e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=4537.0), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch [0], train_loss: 0.4630, train_acc: 0.8153, val_loss: 0.3250, val_acc: 0.8965, val_f1: 0.8817\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d819f1d1baea4290aec116d61e358adf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=4537.0), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch [1], train_loss: 0.1342, train_acc: 0.9572, val_loss: 0.5303, val_acc: 0.8629, val_f1: 0.8468\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d344f2b329634552ad3577c978f2dd52",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=4537.0), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch [2], train_loss: 0.0932, train_acc: 0.9718, val_loss: 0.6836, val_acc: 0.8338, val_f1: 0.8255\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "df4136bbe3d7481cb7c2ae651d1cd0d7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=4537.0), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch [3], train_loss: 0.0777, train_acc: 0.9771, val_loss: 0.4298, val_acc: 0.8890, val_f1: 0.8769\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ad112087ff3b4cc583a60a610891b1de",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=4537.0), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch [4], train_loss: 0.0615, train_acc: 0.9816, val_loss: 0.4548, val_acc: 0.8970, val_f1: 0.8842\n"
     ]
    }
   ],
   "source": [
    "history += fit(epochs, lr, model, train_ddl, val_ddl, optimizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 295
    },
    "colab_type": "code",
    "id": "wFZKOpppLwXf",
    "outputId": "b6250572-eb23-437c-bb7a-f9fa1964bd54"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEWCAYAAABxMXBSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd3hUVfrA8e+bHhIIJUBCS+hFYCmRJs2GKIgiFsBeQN1FWZFV1NVl3fVnQxZd20oVERBRxC6KdJRQpYsQWgIBQiCEhPTz++PehElImcBMJuX9PM88mbn3njvnTpL7zulijEEppZRylpenM6CUUqpi0cChlFKqVDRwKKWUKhUNHEoppUpFA4dSSqlS0cChlFKqVDRwKFWJiEh9EVkpIski8oan8wMgIgdE5BpP50O5jgYOVWZEpLeIrBWRJBFJFJE1InK5p/PlaiIyUUSMiNzusM3H3hbp5rcfDSQANYwxT7r5vVQVpYFDlQkRqQF8DfwXqA00BP4JpLv4fbxdeb5LkAj80wP5iQB2Gh3Zq9xIA4cqK60AjDHzjDHZxphzxpglxpituQeIyCgR2WVXs+wUkS729rYislxETovIDhEZ4pBmloi8JyLfikgKcKWINBCRz0TkhIjsF5HHC8uQiHQXkXjHm7uIDBWRrfbzbiKyQUTOiMgxEZlciuv9HsgA7irivUNEZLadx4Mi8ncRcer/UUR6ich6u+S2XkR65X4WwL3AUyJytrDqIRHxF5FJInLIvqb3RSTQ3tdfRGJF5FkRSbCrmO50Ns9F/f5snURkq53nT0QkwE4TKiJf27/bRBFZ5eznoDzIGKMPfbj9AdQATgIfAtcDtQrsvw2IAy4HBGiB9e3ZF9gLPAv4AVcByUBrO90sIAm4AuuLUDVgI/CCfXwzIAa4roh87QOudXj9KTDBfv4LcLf9PBjo4eS1TgTmAEPs9/YFfAADRNrHzAYWA9WBSGAP8KAT564NnALuts85wn5dx+Hz+Hcx6f8DfGmfpzrwFfCyva8/kAVMBvyBfkCKw2ddZJ6L+v3Z+w4A0UAD+313AY/Y+14G3rc/I1+gDyCe/nvVRwl/h57OgD6qzgNoa9/YYu0b1JdAfXvfD8DYQtL0AeIBL4dt84CJ9vNZwGyHfd2BQwXO8Qwws4g8/RuYYT+vbt8oI+zXK7Gq00JLeZ0TgTn283XAo46BA/DGKo20c0jzMLDciXPfDUQX2PYLcJ/D51Fo4LBv6ClAc4dtPYH99vPcwBHksH8B8HxJeS7q92fvOwDc5fD6NeB9+/mLdjBq4em/T304/9AioSozxphdxpj7jDGNgPZY30Cn2LsbY337L6gBcNgYk+Ow7SBWG0muww7PI4AGdtXHaRE5jVVaqV9EtuYCt4iIP3ALsMkYc9De9yBWFdtuu0posNMXe97fgeeAAIdtoVjfrg86bCt4TUVpUCBdadLWxS6ROXw239vbc50yxqQUOHcDJ/Jc1O8vV7zD81SsEhzA61glyiUiEiMiE5y4DuVhGjiURxhjdmN9O25vbzoMNC/k0CNA4wL13k2wqkXyTufw/DDWN+iaDo/qxpgbisjHTqwb4PXASKxAkrvvD2PMCKAe8CqwUESCSnGZGGN+xLox/tlhcwKQiRXkirqmohwpkK40aROAc8BlDp9NiDEm2OGYWgWusYn9niXluajfX7GMMcnGmCeNMc2wqvbGicjVpT2PKlsaOFSZEJE2IvKkiDSyXzfGqp//1T5kGjBeRLqKpYWIRGBV9aRiNfj6ikh/4EZgfhFvFQ0ki8jTIhIoIt4i0r6Ebr9zgbFAX6w2jtw83yUide3Szml7c04h6UvyHPBU7gtjTDZWFdBLIlLdvs5xWO0iJfkWaCUiI+0uvncA7bB6rBXLvo6pwH9EpB6AiDQUkesKHPpPEfETkT7AYOBTJ/Jc1O+vWCIy2D5WsNqqsrm4z1iVIQ0cqqwkY7U/rLN7P/0KbAeeBDDGfAq8hHUTTwa+AGobYzKwAsX1WN963wXusUssF7BvcIOBTsB+O800IKSYvM3Dagj+2RiT4LB9ILBDRM4CbwLDjTHnAOxeS32cuXBjzBqsgOboMaz2hhhgtX3dM+xzPysi3xVxrpP29T2J1dngKWBwgXwX52msEtCvInIG+Alo7bA/Hqux/QjwMVYjdu5nXWSei/r9OZGflnYezmK11bxrjFnm5LUoDxFjtLu3UsrqjovVqN/I03lR5ZuWOJRSSpWKBg6llFKlolVVSimlSkVLHEoppUrFx9MZKAuhoaEmMjLS09lQSqkKZePGjQnGmLoFt1eJwBEZGcmGDRs8nQ2llKpQRKTgLAWAVlUppZQqJQ0cSimlSkUDh1JKqVLRwKGUUqpUNHAopZQqFQ0cSilViby/Yh9r9+Wf83LtvgTeX1Hccimlo4FDKaUqkY6NQhgzd3Ne8Fi7L4ExczfTsVFxE0SXTpUYx6GUUpVVelY2SecySUrN5PS5TFLSs7mtayMe+nAD7cJrsO/EWd65swu9moe67D01cCilVDmQlpnN6dRMTp/LsH6mZpKU+/xcJqdTz28/fS6TpNQMTp/LJDUju8hzbjh4iju7N3Fp0AANHEop5TLGGFIzsvNu9El5N30rICSlZl4QHHKfp2cVvfChr7dQs5ofNQN9qVnNl4Y1A7msQY281yEO+2oG+hFz4iwTv9rBXT0i+HjdIQZ1DNcSh1JKuZMxhrPpWRfc3PO+6TsEhKR8+zLJyC46APj5eFHLvrmHVPMlMrQaNQNr2jd/a7t187df2wGhmp831uq6JVu7L4F/fr0zr3qqZ/M6jJm7mbdHdnZZ8NDAoZQq195fsY+OjULy3fTW7ktga2wSj/RrXmzanBxDclpWvht//iofx1JBRt7N//S5TLJzil5yopqft31zt27sLeoFWzf/QD8rMNjPa1Y7XwqoWc2XAF9vl30uRdkam5QvSPRqHsrbIzuzNTbJZYGjSqzHERUVZXSSQ6UqprX7Ehjz8WZeHdaB5vWCWbU3gde//517ekZQt7q//a0/k1OpGRfU/yedy6S4W1x1fx/7m/35UoBjlU/u61pBfnmlgJBAX/x93B8AygMR2WiMibpguzsDh4gMBN4EvIFpxphXCuyPwFrsvi6QCNxljIkVkSuB/zgc2gYYboz5QkRmAf2AJHvffcaYLcXlQwOHUuVLdo7hVGoGiSkZnDxr/UxMSedkir0tJYPEsw7PU9IppgBAjQAfalazvu3nr+8v8NqhJBAS6Iuvt45IKE5RgcNtVVUi4g28A1wLxALrReRLY8xOh8MmAbONMR+KyFXAy8DdxphlQCf7PLWBvcASh3R/M8YsdFfelVKlk5GVw6nU80HgZEq6HQwKBgFr++liSgIhgb7UCfKjdpAfEXWq0SWiJrWD/Ngam8SqPxIY8qcG3H9FZF79f41AX7y9nKv/V67hzjaObsBeY0wMgIjMB24CHANHO2Cc/XwZ8EUh57kV+M4Yk+rGvCqlHJzLyM67yRf89n/KoRSQuy05LavQ83gJ1KpmBYHaQX60DqtuP/fPCw51gvyoHWw9r1XNr9BSwNp9CcyLPszjV7VgzrpDDO/WmM6hQe7+GFQR3Bk4GgKHHV7HAt0LHPMbcAtWddZQoLqI1DHGnHQ4ZjgwuUC6l0TkBWApMMEYk17wzUVkNDAaoEmTJpdyHUpVaLk9hIqqAsqtHnKsNjqXWfjYAF9vyQsEdYL96FCrZl4AyAsC9r7aQf6EuKA0kDvyObfBt4cbegmp0vF0r6rxwNsich+wEogD8v5iRSQc6AD84JDmGSAe8AM+AJ4GXix4YmPMB/Z+oqKiKn8PAFVhXEovIbB6Cp1JyzzfHlBIG0G+7akZZBQxRiDA14s6Qf55N/4WdYOt58G5QcCf2kG+9k8/agT4ON0t1FXKopeQKh13Bo44oLHD60b2tjzGmCNYJQ5EJBgYZow57XDI7cAiY0ymQ5qj9tN0EZmJFXyUqjBy5xLKvRmu/iOBMfM28ewNbfll38kSG4pPpWYU2VU02N8nLwiEhwRwWYMa+YJAvtJBsB/V/Dz93bFkhQXTXs1DNWh4kDv/atYDLUWkKVbAGA6MdDxAREKBRGNMDlZJYkaBc4ywtzumCTfGHBXra8/NwHY35V8plzudmkFaZjZ9WoRyz/RofLyFtEyrNPDUwq0XHF9UQ3FhQaBWNb8yGSeglNsChzEmS0TGYFUzeQMzjDE7RORFYIMx5kugP/CyiBisqqq/5KYXkUisEsuKAqf+WETqAgJsAR5x1zUodSmMMexPSGHDwVNsOniKDQdPsff4WQB8vIQ6QX4cS06nW2RtBnUMd7qhWClP0wGASrlIWmY22+KS2HDgFBsPnmLToVMkpmQAVsmha0StvEdaRjbjPv2Nu7o3Yc66Q9rQq8qlMh/HoVRldyI5nY0HE9lolya2xyWRmW19EWsWGsTVberRNaIWUZG1aBYajJfdu2jtvgTGffqb9hJSFZYGDqWckJNj2HM8mQ0Hzlc7HUq0hhb5+Xjxp0YhPNC7KVERtenSpCZ1gv2LPJf2ElIVnVZVKVWIlPQsthw+nVea2HzoVN4gt9Bgf6Jyq50ia3FZgxpVZu4iVbVoVZVSxYg7fY6NB0+x8UAiGw6eYtfRM+QYEIHW9atz458a5AWLJrWrlflYBqXKEw0cqsrJzM5h19EzeaWJTQdPcTQpDbCmy+7UuCZjrmxB18jadGpck5BAXw/nWKnyRQOHqvSSUjPZdPgUG+3eTlsOn86bUqNhzUCiImvnlSbahFXHR7vAKlUsDRyqUjHGcPBkKhsOnsrr8bTnmDV2wttLaBdegzsub0xUpBUowkMCPZxjpSoeDRyqQkvLzGbHkfxjJxLOWmMnagT40CWiFkP+1IAuEbXo1LhmhZhiQ6nyTv+LVIWScDbdasS2H9tik/LWeI6sU41+rc6PnWhR9/zYCaWU62jgUOVWTo5h74mzeaWJjQcTOXDSHjvh7UWHRiHcf0UkXez2idBixk4opVxHA4cqN1Iz7LETB06x8ZDV2+mMPXaiTpAfXSNqMaJbE6Iia3FZgxCd0E8pD9HAoTzmaNI5h9LEKXYePZM3XXir+sEM6hhO1wirx1NEHR07oVR5oYFDXTJnFibKys5hd3wyGw4ksvHQaTYeSOSIPXYi0NcaO/Fov+Z0jaxFl8a1CKmmYyeUKq80cKhLVnBhorX7EvjLx5sY3bcZk5f8zgZ77ERqhjV2IqxGAF0jazEqohZREbVpE15dpw9XqgLRuaqUS6zdl8CjczbRuHYgu44kk23/XXkJtGtQg65NatE1sjZdI2rRsKaOnVCqItC5qpRb9WxWBz8fYXvcGZrUrsatXRvR1R47EeSvf2ZKVSZaP6BcYvrq/ZxIzuDK1vU4m55FVGQtrmgRqkFDqUpIA4e6ZGv3JfDq97upEeDDe3d14e2RnRkzdzNr9yV4OmtKKTfQwKEu2bLdx8nMNjzQuykBvt75FiZSSlU+Wo+gLllyWhb+Pl7c3SMib1uv5qG6mp1SlZSWONQlOZGczueb4xjWtVGxy6UqpSoPtwYOERkoIr+LyF4RmVDI/ggRWSoiW0VkuYg0ctiXLSJb7MeXDtubisg6+5yfiIifO69BFe+jXw6QmZ3Dg72bejorSqky4rbAISLewDvA9UA7YISItCtw2CRgtjGmI/Ai8LLDvnPGmE72Y4jD9leB/xhjWgCngAfddQ2qeOcysvno14Nc3aY+zesGezo7Sqky4s4SRzdgrzEmxhiTAcwHbipwTDvgZ/v5skL25yPWZEVXAQvtTR8CN7ssx6pUFm6K5VRqJqP7NvN0VpRSZcidgaMhcNjhday9zdFvwC3286FAdRGpY78OEJENIvKriOQGhzrAaWNMVjHnBEBERtvpN5w4ceJSr0UVkJ1jmLF6P39qFMLlkbU8nR2lVBnydOP4eKCfiGwG+gFxQLa9L8Ie6j4SmCIizUtzYmPMB8aYKGNMVN26dV2aaQU/7TrG/oQURvVtprPWKlXFuLM7bhzQ2OF1I3tbHmPMEewSh4gEA8OMMaftfXH2zxgRWQ50Bj4DaoqIj13quOCcqmxMWxVDw5qBDLwszNNZUUqVMXeWONYDLe1eUH7AcOBLxwNEJFREcvPwDDDD3l5LRPxzjwGuAHYaa0bGZcCtdpp7gcVuvAZViM2HTrH+wCke7N0UH53VVqkqx23/9XaJYAzwA7ALWGCM2SEiL4pIbi+p/sDvIrIHqA+8ZG9vC2wQkd+wAsUrxpid9r6ngXEisherzWO6u65BFW7aqv1UD/Dh9ssbl3ywUqrScevIcWPMt8C3Bba94PB8Ied7SDkesxboUMQ5Y7B6bCkPOJyYynfbjzK6b3OCdQJDpaokrWdQpTJ99X68RLivV6Sns6KU8hANHMppSamZLNhwmCGdGhAWEuDp7CilPEQDh3Lax9EHSc3I5qHeOuBPqapMA4dySkZWDrPWHKBPy1DaNajh6ewopTxIA4dyype/HeF4cjoP9dHShlJVnQYOVSJjDNNWxdC6fnX6ttQ1NpSq6jRwqBKt+iOB3fHJPNSnqU4vopTSwKFKNnVVDPWq+zOkUwNPZ0UpVQ5o4FDF2nX0DKv+SODeXpH4+3h7OjtKqXJAA4cq1tRVMVTz8+bO7k08nRWlVDmhgUMVKT4pja9+O8LtUY2pWU1X6FVKWTRwqCLNWnuA7BzDA1foeuJKqfM0cKhCnU3PYu66gwxsH0aTOtU8nR2lVDmigUMVasH6w5xJy2KUDvhTShWggUNdICs7hxlr9hMVUYvOTXQ9caVUfho41AW+3xFP7KlzjOqrpQ2l1IU0cKh8jDFMXRlDZJ1qXNO2vqezo5QqhzRwqHzWHzjFb7FJPNinGd5eOr2IUupCGjhUPlNXxVCrmi+3dmnk6awopcopDRwqT8yJs/y06xh394gg0E+nF1FKFc6tgUNEBorI7yKyV0QmFLI/QkSWishWEVkuIo3s7Z1E5BcR2WHvu8MhzSwR2S8iW+xHJ3deQ1UyffV+fL29uLtnpKezopQqx9wWOETEG3gHuB5oB4wQkXYFDpsEzDbGdAReBF62t6cC9xhjLgMGAlNEpKZDur8ZYzrZjy3uuoaq5OTZdBZujOWWzg2pW93f09lRSpVj7ixxdAP2GmNijDEZwHzgpgLHtAN+tp8vy91vjNljjPnDfn4EOA7UdWNeq7w5vx4iPSuHh/ro9CJKqeK5M3A0BA47vI61tzn6DbjFfj4UqC4idRwPEJFugB+wz2HzS3YV1n9ERL8eX6K0zGxm/3KAq9rUo0W96p7OjlKqnPN04/h4oJ+IbAb6AXFAdu5OEQkHPgLuN8bk2JufAdoAlwO1gacLO7GIjBaRDSKy4cSJE268hIpv0eY4TqZkaGlDKeUUdwaOOKCxw+tG9rY8xpgjxphbjDGdgefsbacBRKQG8A3wnDHmV4c0R40lHZiJVSV2AWPMB8aYKGNMVN26WstVlJwcw9RVMbRvWIOezeqUnEApVeW5M3CsB1qKSFMR8QOGA186HiAioSKSm4dngBn2dj9gEVbD+cICacLtnwLcDGx34zVUest+P07MiRRG9Wmm64krpZzitsBhjMkCxgA/ALuABcaYHSLyoogMsQ/rD/wuInuA+sBL9vbbgb7AfYV0u/1YRLYB24BQ4N/uuoaq4IOVMTQICeCGDuGezopSqoLwcefJjTHfAt8W2PaCw/OFwMJC0s0B5hRxzqtcnM0qa2vsadbtT+S5G9ri6+3p5i6lVEWhd4sqbOqq/VT392F4t8YlH6yUUjYNHFVU7KlUvt12lOHdGlM9wNfT2VFKVSAaOKqomWsOIMD9up64UqqUNHBUQUnnMpkffYhBHcNpUDPQ09lRSlUwGjiqoPnRh0jJyNb1xJVSF0UDRxWTkZXDzDUH6NmsDu0bhng6O0qpCkgDRxXzzbYjxJ9JY7SuJ66UukgaOKoQaz3x/bSoF0y/VjoNi1Lq4mjgqELW7jvJzqNnGNWnKV66nrhS6iJp4KhCpq6KITTYj5s6FZzdXimlnKeBo4rYcyyZ5b+f4N6ekQT46nriSqmLp4Gjipi2KoYAXy/u6hHh6awopSo4DRxVwPHkNL7YfITbujamVpCfp7OjlKrgNHBUAbPXHiQzJ4cHe+v0IkqpS6eBo5JLzcjio18PMqBdfSJDgzydHaVUJeB04BCRQBFp7c7MKNdbuDGWpHOZOr2IUsplnAocInIjsAX43n7dSUS+LD6V8rTsHMO0Vfvp3KQmXSNqeTo7SqlKwtkSx0SgG3AawBizBdAK83Lux53xHEpM1fXElVIu5WzgyDTGJBXYZlydGeVaH6yMoXHtQK67LMzTWVFKVSLOBo4dIjIS8BaRliLyX2CtG/OlLtHGg4lsOnSaB69oirdOL6KUciFnA8djwGVAOjAXSAL+6q5MqUs3deV+QgJ9uS1K1xNXSrlWiYFDRLyBb4wxzxljLrcffzfGpDmRdqCI/C4ie0VkQiH7I0RkqYhsFZHlItLIYd+9IvKH/bjXYXtXEdlmn/Mt0cr7Cxw8mcIPO+O5s3sTgvx9PJ0dpVQlU2LgMMZkAzkiUqpVf+yA8w5wPdAOGCEi7QocNgmYbYzpCLwIvGynrQ38A+iO1Sj/DxHJ7Rb0HjAKaGk/BpYmX1XB9NX78fES7usV6emsKKUqIWe/jp4FtonIj0BK7kZjzOPFpOkG7DXGxACIyHzgJmCnwzHtgHH282XAF/bz64AfjTGJdtofgYEishyoYYz51d4+G7gZ+M7J66j0TqVk8OmGWG7q1JB6NQI8nR2lVCXkbOD43H6URkPgsMPrWKwShKPfgFuAN4GhQHURqVNE2ob2I7aQ7RcQkdHAaIAmTZqUMusV18frDnIuU9cTV0q5j1ON48aYD4F5wEb7MdfedqnGA/1EZDPQD4gDsl1wXowxHxhjoowxUXXrlnK1u9VTYP/K/Nv2r7S2l2PpWdnMWnuQvq3q0jqsuqezo5TyhDK4fzk7crw/8AdWm8W7wB4R6VtCsjjAsUtPI3tbHmPMEWPMLcaYzsBz9rbTxaSNs58XeU6XaNgFPr3v/Ie/f6X1umEXl7+VKy3efISEs+mM1tKGUlWX4/0rOxP2rXD5/cvZqqo3gAHGmN8BRKQVVgmkazFp1gMtRaQp1s19ODDS8QARCQUSjTE5wDPADHvXD8D/OTSIDwCeMcYkisgZEekBrAPuAf7r5DU4r2lfuG0WzB8Jra6Dfcus101LipWeY4xh6qoY2oRV54oWdTydHaVUWcjOhDNxcPpQ/kf1MJh9E5gcCKwFt8926f3L2cDhmxs0AIwxe0TEt7gExpgsERmDFQS8gRnGmB0i8iKwwRjzJdAfeFlEDLAS+IudNlFE/oUVfABezG0oB/4MzAICsRrF3dMw3rQvBNSEbQshpDF4+7vlbVxl+Z4T/HH8LJNv/5NOL6JUZZGdCUmx+YNC0uHzz8/EWcEhj0CNhlCzCdRtC8d3wJ9GuvxLrxhT8swhIjIDyAHm2JvuBLyNMQ+4NDduEhUVZTZs2FC6RLnVUw06w96lgIGW18HVz0NYB3dk85LcOe1X9h4/y6qnrsLPR2fLV6pCKCwwOD6Sj+QPDOJ1PjCENLZ+Oj5qNAQfv/P3r6gHYcP0i64xEZGNxpiogtudLXE8ilUayO1+uwqrraNyyv3Qcz/sPUvg03vhwGp4vze0vxWufBbqNPd0TgHYcSSJNXtPMuH6Nho0lCpPsjLgTMHAcNi5wNC0T+GBwbvYyp4L719N++R/7QLOBg4f4E1jzGTIG9xXvutuLkXcpvwfcqsBMPITOLjG+obw63uwYxF0vgv6PQ0hhfYILjPTVu0nyM+bEd2qTrdjpcqFQgODw+PMEfLNByteUKORHRj62gGhcekCQ0kK3r9y22zjNrkscDhbVfUrcI0x5qz9OhhYYozp5ZJcuNlFVVUV5+xxWDkJNsyw/hC6jYLe4yCo7Buljyado8+ry7inZyQv3FhwYL5S6pJkpZfQxlAwMHifLzEU9qjR4NIDQxm61KqqgNygAWCMOSsi1VyWu4omuB7c8Br0/AuseBV+fRc2zoKeY6xtATXKLCuz1hwgxxjuvyKyzN5TqUqjsMCQr43hKBcEhpCGUDMCmvW/MDBUbwDelX9+OGevMEVEuhhjNgGISBRwzn3ZqiBqRcDN78IVY+Hnf8OKVyD6A+gzDi5/CHwD3fr2yWmZzF13iBs6hNO4dtWN46qSWz3FGoPgWM2yf6VV9dK7hEm68wLDwQvbF4oMDHZVUvMrq2xgKImzn8BfgU9F5Ij9Ohy4wz1ZqoDqtoY7PrL+kH/+Nyz5O/zyLvR7ymoHcVPR9JP1h0lOz9LpRVTlljugLbfe3rHx94LAULDEEE/RgeGqQgJDuAYGJxTbxiEilwOHjTHx9riNh7HmltoJvOAwtqJcc3kbR0kOrIalL8LhdVCrKVz5HLQfBl6u6/GUlZ1Dv9eX07BWIAse7umy8ypVLu1fCQvugbptIG4j1G4GaUl2icGBl8/5wBBSSBuDBoZSudg2jv8B19jPewLPYi3q1An4ALjVlZmsNCJ7wwM/wB9LrADy+UOwZgpc9XdoNRBcMEDv2+3xxJ0+x8Qhl7kgw0qVc35BVuni0C8QEGK1MzaKstoaCgYGL29P57bSKylweDuUKu4APjDGfAZ8JiJb3Ju1Ck7Emq6kxbWw43NY9hLMGw6NusHVL1h9qy+SMYapK2NoFhrE1W3quTDTSpVDu76CTx8Ak2UNaNv5BfT9W7meAqiyK6nuxFtEcoPL1cDPDvu0vOcMLy/ocCv8JRpufNOqj/1wMMy+2WoTuQjr9ieyLS6JB/s0xUvXE1eVlTGw9m345C4gx5pvafBkq23DcRJSVeZKChzzgBUishirF9UqABFpgbXuuHKWty90vQ8e3wQDXoKjv8HUK61/iuO7S3WqqStjqB3kx7AujUo+WKmKKDsLvh0PS56Deu1gxHxoe6O1z3FAm7rAjO0ziD4anW9b9NFoZmyfUUSK0is2cBhjXgKexJpUsLc535LuhdXWoUrLNxB6jYGxv0H/Z2DfcnivJyx6FE4dLDH53uNnWWWMj6AAACAASURBVLr7OHf3iCDAV+tyVSWUngzzR8D6aVZX90fWQMtr8x/TtG/JXXGrqPZ12jN+xfi84BF9NJrxK8bTvk57l71HidVNucu0Fti2x2U5qKoCakD/CXD5KFg9GaKnwrZPIep+6DMeqtcvNNn01TH4+3hxd8+IMs6wUmXgzBGYezsc2wmDp1j/D6pYWTlZJJxLID4lnqMpR4lPiadLvS488tMjhPiFkJmTyeT+k+kW3s1l76ntFIWYsX0G7eu0z/dBRx+NZvvJ7TzQ3sUTAgfVgetegh5/hpWvw/rpsHkOdH8ErnjcmkvflnA2nc82xTGsSyNCgyvvVGGqiorfBh/fbpU47lwALa4pOU0lZ4zhdPrpvIBwNOUox1KOnQ8SqfGcSD1Btsm/cGqQbxDBvsEkpCUwos0IlwYN0MBRqNyi3mv9XqNHeI+8ot6kfpPc96YhDeHGKdDrMVj+Mqz+jxVErngcejwKfkHM/uUgGVk5PNSnqfvyoZQn7FkCC++3uto+8D2Eua5apTxLzUzNV1KIT43n6FkrIMSnWI/07PR8aXy9fAkLCiMsKIxuYd2oX60+YUFhhAeF523fdXIX41eM5+GOD7Pg9wVc0+QalwYPpyY5rOguZgBg9NFo/rz0zzQKbsSJcyf4T///uDxqFyt+uzUKfc93EFSPjF7j6LM0gg4RdZl27+Vllw+l3G39NPj2b1C/PYxcADXCPZ0jl8jMzuRY6vnSgePz3J/JGcn50ghC3cC6hAWHEVYtfzAIDwqnflB9agfUxkuKbp52/KLbLbzbBa9L41InOaxyuoV3o0NoBzYc24Ag/Hz4Z1rWakmtgFolJ3aFsPYwcj4cjoalL+L34wQ+M6Gkhz8FOV10kJOq+HJy4Mfn4Ze3odX1MGwa+Ad7OldOyTE5JKYl5pUOCpYS4lPiSTiXgCH/F/MQ/xDCqoXRIKgBnet1viAw1K1WF1+vS5uiaPvJ7fmCRLfwbkzqN4ntJ7e77MuvljiKkBulBzUbxII9C8jMziTIN4gHOzzInW3vJNDHvRMYOsrJzuGp19/i4aw5tMz6A0Jbw1XPQdshLhmFrlSZy0iFz0fB7q+h28Mw8OVy82XIGENyZnK+IFCwpHAs9RhZOVn50gX6BBZabZT3qBZGNd+KNRmpljhKoWDR7srGV/LE8idoGtKUNze9ybzd8xjTaQxDmg/Buwz+2H/afZyFp1vSb/hiWvpttKqwFtwD4Z2sUejNr9IAoiqOs8etWRTiNsHAV6HHI2X69unZ6RcEg3yP1HhSMlPypfEW77yg0LFux/OBoVoY4cHhhFULI8Q/BKki/4da4ihEcb2qOoZ2ZPLGyWxL2EaLmi14ousT9GnYx61/MLe//wtxp8+x4m/98fH2gpxs2PoJLHsZkg5BRG8rgDTp7rY8KOUSx3fDx7dBagIMmw5tbigxSWl6OWbnZHPi3IliSwqJaRfOzVo7oHah7Qm57QyhgaFl8iWxvCmqxKGB4yIYY/jx4I+8uelNDiUf4vKwy3my65NcFur6CQe3HD7Nze+s4fnB7Xiwd4HeVFnpsPFDqxtvynFrAsWr/g5hHVyeD6UuWcxy+OQe8A2wlmJu0NmpZI41AK1rt2bJgSW8sfENbmp+E/4+/sSfjc9rXzieerzQrqn5qo4cSglhQWHUD6qPv7d2by+MRwKHiAwE3gS8gWnGmFcK7G8CfAjUtI+ZYIz5VkTuBP7mcGhHoIsxZouILMdaDyR3IakBxpjjxeXDXdOqZ2Zn8umeT/nf1v+RmJbI9ZHX81iXx2hcvbHL3uMvczexcs8JfnnmaoL9i6hZzEiBde/Dmjetqabb3wpXPgt1mrssH0pdks1z4KuxENrK6jlVs3T/I2vi1vD4ssfJyM7It92xa2puICjYxlDdr7orr6RKKfPAISLewB7gWiAWWA+MMMbsdDjmA2CzMeY9EWkHfGuMiSxwng7AF8aY5vbr5cB4Y4zTkcDd63GczTjLzB0zmb1jNlkmi+GthzO64+hL7oF1ODGVfq8vY1TfZjxzfduSE5w7BWv/C7++Z5VGutwNfZ+yxogo5QnGWG1yqyZBsyvh9g+tsRqlcCzlGOOWj2NrwlYABkQM4IH2DzjVNVVdmqIChzs/8W7AXmNMjDEmA5gP3FTgGAPkLtAdAhzhQiPstOVWsF8wj3V+jG9u+Yabmt/E3N1zueHzG5i2bRppWWkXfd4Za/bjJcJ9vSKdSxBYy2rreHyLtXTt5o/hrc7ww3OQcvKi86HURclMg88esoJGl3vhzk9LHTQ2H9/M8G+GsztxN0G+QTzc8WHWx68nJTOF0MBQDRoe4s5PvSFw2OF1rL3N0UTgLhGJBb6l8IkT78CapdfRTBHZIiLPSxGt0iIyWkQ2iMiGEydOXNQFlFa9avWY2Gsinw/5nKj6Uby56U0GLxrMoj8WkZ2TXfIJHCSlZvLJ+sMM+VMDwkNK2fW3en244TV4bKM1pfuv78KbHa3G9LQzpTuXUhcj5SR8dDNsXwjXTLSWFCjlEsoLfl/AAz88gCAE+ATw1pVvMabzGCb1m5RvEj9V9jwdrkcAs4wxjYAbgI9Ezn+FEJHuQKoxZrtDmjuNMR2APvbj7sJObIz5wBgTZYyJqlu3rvuuoBDNazbnv1f/lxnXzaBuYF1eWPsCt351K6tiV+Fs1eDc6EOkZmTz0KWsJ14rAm5+F/78q9Vld8Ur8OafrOqszHMlp1fqYpzcB9Ovsbrb3jYLej9Rqu7iGdkZ/POXf/KvX/9F9/Du3Nrq1nwzNzgOaFOe4c42jp7ARGPMdfbrZwCMMS87HLMDGGiMOWy/jgF65DZ2i8h/gBPGmP8r4j3uA6KMMWOKy0uZrznuwBjDkoNLeHPTmxxOPkz3sO48EfUEl9UpugdWRlYOfV77mZb1qjPnIRd2sY3bBD//C/b9DNUbQL+noPNdpf4mqFSRDv5iTYkuXtYaGo1LN1L5ROoJnlj+BL+d+I2HOjzEmE5jqmQ32PLCE20c64GWItJURPyA4cCXBY45hLWyICLSFggATtivvYDbcWjfEBEfEQm1n/sCg4Fy/bVDRLgu8joW37SYCd0msOfUHoZ/PZynVj5FbHJsoWm++u0Ix86ku34yw4Zd4O5FcO/XENIIvv4rvNMNti20pn9Q6lJsWwizh0C1UHjop1IHjS3Ht3DH13ew59QeJvWbxNguYzVolFNuCxzGmCxgDPADsAtYYIzZISIvisgQ+7AngVEi8htWO8Z9DotF9QUOG2NiHE7rD/wgIluBLUAcMNVd1+BKvt6+3Nn2Tr695VtGdRjFskPLuPGLG3k1+lVOp53OO84Yw9RVMbSuX51+rdxUxda0Dzy4BEZ8Ar7V4LMH4X994PfvrF4wSpWGMdZYos8ehEbdrL+t2qWrYv1sz2fc/8P9+Hv7M+eGOVwXeZ2bMqtcQQcAesixlGO8+9u7fLH3C4J8zs+BtX5/MndPj+a1Wztye5TrxoMUKScHdnwOy16CxBjrH//qF6zgolRJsjLg6ydgyxzoeAcM+S/4OD+YLjM7k1eiX2HBngX0atCL1/q+Roh/6XpeKffRkePlLHDk2ntqL1M2TWFF7ArqV6uP/9kbOHakPWuevhp/nzIspmdnwpaPYfmrkHzEaky/6nmrekupwpw7DQvuhv0rod8Ea0XLUjSCJ5xLYNzycWw+vpn729/P2M5aNVXeaOAop4Ej1/r49bz0y+vsO7OL2r4RvNRvAlc0uKLsJ03LPGctILXqDTiXCG1vtAJI3dZlmw9Vvp06aM05lRhjlTI6jShV8m0ntvHX5X8lOSOZF3u9yMCmA92UUXUpPNE4rkrh8rDLaZ75LNnxdxHgl8WjPz3KqB9HsfPkzpITu5JvIPQaA2N/g/7PwL7l8G4PWPSodbNQKnYjTLsazsZbnS1KGTQW/bGIe7+/F18vXz66/iMNGhWQBo5y4tiZNL787Qi3tRnE10O/YkK3Cfye+Dt3fH0HT698usgeWG4TUMOqehj7m7Ue+vbP4L9drZXako+VbV5U+bHrK5g1yOpU8eBPpWoLy8zJ5KVfX+KFtS/QtX5X5g+aT+vaWpKtiLSqqpx47fvdvL9iH8vG9yeiThAAyRnJzNw+k9k7Z5NjchjeZjijO4ymZkDNss9gUhysfA02fWQ1fnZ/xFoPPbCMVkRUnmUM/PIOLPk7NIqC4fMg2PlefwnnEnhy+ZNsOr6J+y67j7FdxuLjpcsBlXfaxlGOA0dKehY9X17KFS1Cee+urhfsj0+J590t77J43+J8PbACfALKPrMn98Hyl60++wE14Iqx1vogTXpA077nj9u/0hpw2PuvZZ9H5VrZWfD909ba4O1ugqH/s6o0nbQ9YTt/XfZXktKTmNhrIoOaDXJjZpUraRtHOfbphsOcSctiVN/C+76HBYXx4hUvsvDGhXSu35kpm6YweNFgFu9dXOo5sC5ZnebW2tCPrIYmPWHpi9Y30bnDYe9S65j9K+HT+7RHVmWQnmyNBF8/zfqScOusUgWNxXsXc+939+It3nx0w0caNCoJLXF4WHaOof+kZdSrHsBnj/ZyKk300Wgmb5zMjpM7aFWrFU90fcIzPbAADq2zgsfB1dY0Ey0HQOx6a44ixxKIqnjOHIG5t8OxnTDoDYi63+mkmTmZTFo/ibm759I9rDuv93v9kpcZUGVPSxzl1A874jmceI5RpZjMsFt4N+YOmstrfV8jJTPFcz2wwFqu9r6vrd41QfVgz/fWdNqxG6x+/qpiit8GU6+GxANw54JSBY2T504yeslo5u6ey93t7ub9a9/XoFHJaODwIGMMH6yMIaJONa5tV79Uab3Ei+ubXs+XN3/J05c/ndcDa8KqCcSdjXNTjosgAl4+kJMJHW6D7AxY+k/4z2Xw/TNw+lDZ5kddmj1LYMZA6/f6wPfQ4hqnk+44uYPh3wxnW8I2/q/3//HU5U9pI3glpIHDgzYePMWWw6d5qHdTvL0urprJz9uPu9rdxbe3fMtDHR7ip4M/ceOiG3l9/eskpSe5OMdFyG3TuG2W1f5x9+cQUNPqfRP9AbzZCRY+CEe2lE1+1MVbPw3m3WHNNfXQUghr73TSr/Z9xb3f3YsgzL5+Njc2v9GNGVWepG0cHjR69gaiDyTyy4SrCfRzzVQL8SnxvLPlHRbvXUywXzCjOoxiZNuR+Hs7P39Qqa2eYjWEF9arqsOt1nroG2ZBRjJE9rEaWVtcU6rpKZSb5eTAj8/DL29Dq+utLwD+wU4lzcrJ4o0NbzBn1xwuD7ucSf0mUTugtpszrMqCdsctZ4Fjf0IKV72xnDFXtuDJAa4fBLXn1B6mbJzCqrhVhAWF8VjnxxjUdJDn5gJKS4KNH1rroScfgbptrRHqHW4r1aR4yg0yUuHzUbD7a+j2MAx8GZz8O0lMS+RvK/5GdHw0d7W9i3FR4/D10vVdKgsNHOUscPz9i20sWB/L6glXUq+6+8ZjrDu6jskbJ7Pz5E5a12rNuK7j6NXQud5bbpGVATsWwdq34Nh2CA6D7g9bja86mLDsnT0O84ZbpcOBr0CPR5xOuuvkLsYuG8vJcyf5R69/MKT5kJITqQpFA0c5ChyJKRn0emUpN/2pIa/e2tHt75djcvh+//e8tfkt4s7G0SO8B+O6jqNtnbZuf+8iGQMxy6xlbPf9DL5B0OUe6PGoteStcr/ju62JClMTYNh0aHOD00m/ifmGiWsnEuIfwptXvslloUWvaKkqLg0c5ShwvLX0Dyb/uIcfn+hLy/rVy+x9M7Iz+OT3T/jf1v+RlJ7E4GaDeazzYzQIblBmeShU/DZY+zZsX2gFlMtuhl6PQYPOns1XZRazHD65B3wDYOQnTn/WWTlZTNk4hQ93fkjX+l2Z1G8SoYGh7s2r8hgNHOUkcKRlZtP71Z/p0DCEmfeXbmlNVzmTcYbp26bz8a6PyTE5jGwzklEdR3l+AZ2kOKshfeMsSD9jNaT3etxqSPfSDoAus3kOfDUWQlvByAVQ07kFw06nnWb8yvGsO7qOEW1G8LfL/6btGZWcBo5yEjjmRx9iwufbmPtQd3q18Ow3tfiUeN7e/DZf7vuSYL9gRncYzYi2I9zbA8sZaUmwabbVkH4mDuq2gZ5joOPt2pB+KYyBn/8NqyZBsyvh9g8hwLkvC78n/s7YZWM5nnqc53s8z9CWQ92cWVUeaOAoB4EjJ8cwYMpK/H28+Pqx3p6ZIqQQvyf+zpRNU1gdt5rwoHCrB1azQXiJh7/lZ2fC9s+tdpBj2yC4vt2Q/oA2pJdWZhos/otVHdjlHhg0GbydKy18t/87XljzAjX8azCl/xQ61O3g5syq8kIDRzkIHD/vPsYDszYw5Y5O3Ny5oaezc4Ffj/7K5A2T2ZW4iza12/BE1yfo1cCDPbByaUP6pUk5CZ/cCYd+gav/Ab2fcGoMTXZONm9uepOZO2bSpV4X3uj/hrZnVDEematKRAaKyO8isldEJhSyv4mILBORzSKyVURusLdHisg5EdliP953SNNVRLbZ53xLysvXdidMXbmf8JAABnUM93RWCtUjvAfzB8/nlT6vkJyRzMM/PszoJaPZnbjbsxkTsdZAv3sRPLLGWs52/VR4qxN8ej8c2ezZ/JVnJ/fB9Gus7ra3zoQ+45wKGknpSTz606PM3DGTO1rfwbQB0zRoqDxuK3GIiDewB7gWiAXWAyOMMTsdjvkA2GyMeU9E2gHfGmMiRSQS+NoYc8F8ByISDTwOrAO+Bd4yxnxXXF7KQ4lje1wSg/+7mmdvaMPovs09mhdnZGRnMH/3fD7Y9gFn0s8wuNlgxnQe4/keWLkKbUh/DFpcqw3puQ7+Yk2JLl7WwktNujuVbM+pPYz9eSzHUo/x9x5/55aWt7g5o6q8KqrE4c7Zx7oBe40xMXYG5gM3AY5TuBqghv08BDhS3AlFJByoYYz51X49G7gZKDZwlAdTV8UQ7O/D8G5NPJ0Vp/h5+3HPZfdwc8ubmb5tOnN2zuGHAz8wsu1IHurwkOd7YIU0hAH/gr5/g032iPS5t2tDeq5tC+GLR6FmE7jzU2vuKSd8f+B7XljzAtV9qzNz4Ez+VPdPbs7opcvMzCQ2Npa0tDRPZ6XCCggIoFGjRvj6Otfu5c7A0RA47PA6Fij4lWcisEREHgOCAMdpOJuKyGbgDPB3Y8wq+5yOi2/H2tvKtbjT5/h661Hu7xVJjYCK1X2xhl8Nnuj6BMNbD+ftLW/z4Y4P+fyPzxndcTTD2wz3fA+sgBpWSaP7I9aI9DVvwZdj4Od/Vc2GdGOsXlM//xsiroA75kC1kueNys7J5q3NbzFj+ww61e3E5P6TqVvN+aVhPSk2Npbq1asTGRlZbjqcVCTGGE6ePElsbCxNmzZ1Ko2ny/QjgFnGmEbADcBHIuIFHAWaGGM6A+OAuSJSo5jzXEBERovIBhHZcOLECZdnvDRmrdkPwP29nfullEfhweG81PslPr3xUzrU7cCkDZO4cdGNfLXvK6Zvm0700eh8x0cfjWbG9hlll0FvX6uU8cgquPsLqN/eWmBq8mXw3dNw6mDZ5cVTsjJg8RgraHS8w2oTciJoJKUn8Zelf2HG9hnc1uo2Zlw3o8IEDYC0tDTq1KmjQeMiiQh16tQpVYnNnYEjDnAcWdTI3uboQWABgDHmFyAACDXGpBtjTtrbNwL7gFZ2+kYlnBM73QfGmChjTFTdup77JziTlsm86MMM6hBOw5rOL7lZXrWu3Zr3r3mfD679gJr+NXl29bN8/sfnjF02Ni94RB+NZvyK8bSv4/yU3C4jAs2vtKZ2f2QNtBtiTRWe25Aet6ns81QWzp2Gj4fBljnQb4K1LrgTVXV/nPqDEd+MYF38Ol7o+QIv9HwBXye76ZYnGjQuTWk/P3dWVa0HWopIU6yb+3BgZIFjDgFXA7NEpC1W4DghInWBRGNMtog0A1oCMcaYRBE5IyI9sBrH7wH+68ZruGSfRB/mbHpWqVb4qwh6NuhJ9/DufLf/O97a9BZnM8/y8E8Pc2OzG1l+eDmT+k2iW7hnRsbnCWsPQ9+Hq56H6P/Bhpmw43OI6A1XPF55GtJPHbTmnEqMgZvfh04jnEr248EfeW71cwT5BjHzupl0qtfJzRlVlYXb/muMMVnAGOAHYBewwBizQ0ReFJHcaTSfBEaJyG/APOA+Y3Xz6gtsFZEtwELgEWNMop3mz8A0YC9WSaTcNoxnZucwY81+ejSrTYdGHm5MdgMv8WJQs0F8NfQrxkeNxwsvFu1dRLbJZu2Rtew8uZNyMU4opCFc+yI8sQMGvASnDlgN6e/2sEaoZ6V7OocXL3YjTLsazsZbVVNOBI3snGze2vQW45aPo2Wtlnwy+JMqEzTeX7GPtfsS8m1buy+B91fsu+hznjx5kk6dOtGpUyfCwsJo2LBh3uuMjIxi027YsIHHH3+81O+5ZcsWRITvv//+YrN9SXQAoBst3hLH2PlbmH5vFFe3Ld3SsBVN9NFonlzxJB1DO7L26FpycnLIIYdGwY0YEDmAAZEDaFe7XfmoUsjOPD+1e/w2a6303IZ0J9oEyo1dX8FnoyC4Hty5EOq2KjHJmYwzPL3yaVbHrWZYy2E82/1Z/Lz9yiCz7rNr1y7atnVupue1+xIYM3czb4/sTK/moRe8vlQTJ04kODiY8ePH523LysrCx8e1lTtPP/00a9eupVmzZnz44YcuOWdhn6OOHC/jwGGMYfB/V5OWmc2PT/TD6yKXhq0Icts0cqunoo9GM27FOIa2GMofp/5g3dF1ZJksGgY3ZECEFUQuq3OZ54OIMbB/hTUife9P9oj0u+0R6ZGezVtxjLFW6lvyvLU87/B5EFxyO96+0/sYu2wscclxPNP9GW5rdZvnfwcu4HjD++dXO9h55Eyxxyedy2Tv8bPUr+HPsTPptKgXTEhg0e067RrU4B83OjdtfG7g2L59OwEBAWzevJkrrriC4cOHM3bsWNLS0ggMDGTmzJm0bt2a5cuXM2nSJL7++msmTpzIoUOHiImJ4dChQ/z1r38ttDRijKF58+b8+OOP9OnTh5iYGAICrDV9Xn31VebMmYOXlxfXX389r7zyCnv37uWRRx7hxIkTeHt78+mnn9K8+YVjyUoTOHQVeTf5JeYkO46c4eVbOlTqoAGw/eT2fG0a3cK7MbnfZLaf3M77175PUnoSPx/6mR8O/sBHOz9i5o6ZNAxuyLUR1zIgYgDtQ9t75gYmAs36W49jO6yp3ddPt9ZJb3eT1c23Ydeyz1dxsrPgu6dgw3Qrj0P/B74ld7pYenApz65+lkCfQKZfN50u9buUQWbLp5BAX+rX8CfudBoNawYUGzQuRWxsLGvXrsXb25szZ86watUqfHx8+Omnn3j22Wf57LPPLkize/duli1bRnJyMq1bt+bRRx+9YGzF2rVradq0Kc2bN6d///588803DBs2jO+++47Fixezbt06qlWrRmKiVbt/5513MmHCBIYOHUpaWho5OTmXfG0aONxk6soYQoP9GFoO56RytQfaP3DBtm7h3fICSYh/CENbDmVoy6F5QWTJwSXM2TWHWTtm0SCogRVEIgfQIbSDZ4JI/ctg6Htw9fP2GukzreqsiN5WAGk5wPMN6enJVs+wvT9a67ZfPbHEPOWYHN7d8i7/2/o/OoR2YHL/yYQFhZVNfj3AmZJBbvXU41e1YM66Q4y9pqVLqqkKuu222/D2tpbgTUpK4t577+WPP/5ARMjMzCw0zaBBg/D398ff35969epx7NgxGjVqlO+YefPmMXz4cACGDx/O7NmzGTZsGD/99BP3338/1apVA6B27dokJycTFxfH0KHWbMa5JZNLpYHDDf44lsyy30/wxDWtCPD10Brf5VTBILLs8DKWHFjCx7s/5sOdHxIeFJ4XRDqGdiz7IFKjgdWQ3mc8bP4IfnkX5t0Boa3tNdJvtxY/KmtJcTD3Dji+EwZPsZbaLUFyRjLPrHqGFbErGNpiKM/1eM7zAzY9rGCbRo/mdVzaxuEoKCgo7/nzzz/PlVdeyaJFizhw4AD9+/cvNI2///nfj7e3N1lZWfn2Z2dn89lnn7F48WJeeumlvMF7ycnJLs17SSpBX8TyZ9qq/fj7eHF3T525tTgh/iHc3OJm3r3mXVbcsYKXer9Eq1qtmLd7Hnd9excDPhvAa+tfY8vxLeSYSy9el0pADej5Fxi7BW6ZBj5+8OVjMKUDrHwdUhNLPoerHN1q9Zw6dQDuXOBU0Ig5HcPIb0ayJm4Nz3Z/ln/2+meVDxoAW2OT8gWJXs1DeXtkZ7bGJrn1fZOSkmjY0Kp9mDVr1kWfZ+nSpXTs2JHDhw9z4MABDh48yLBhw1i0aBHXXnstM2fOJDU1FYDExESqV69Oo0aN+OKLLwBIT0/P238pNHC42PHkNBZtjuPWro2oHVSxe6uUpRp+NRjSfAhvX/02K+5Ywf/1/j/a1GrD/N3zufu7uxmwcACvRr/K5uObyzaIePtCx9vg4VVwz2II72iNzP7PZfDtU9bN3J32LIEZA62JCh/43loNsQTLDi1j5LcjOZNxhqkDpjKizYhK0QjuCo/0a35ByaJX81Ae6efeiUefeuopnnnmGTp37nxBKaI05s2bl1ftlGvYsGHMmzePgQMHMmTIEKKioujUqROTJk0C4KOPPuKtt96iY8eO9OrVi/j4+Eu6FtBeVS73xpLfeXvZXpaO60ezusFl8p6VWXJGMssPL2fJwSWsiVtDZk4m9arVy2tY71SvU9kvOHVsB/zyDmxdACbbfQ3p0VOthvD67a0lXmsUPx1/jsnh/d/e573f3uOyOpcx5coplbo9I1dpuuOqoml33ALKKnCcy8im5ytLuTyyNlPvueCzVpfobMZZlscuZ8kBK4hk5GRQL7Ae10Rcw4DIAXSu17lsg8iZI7DOHpGenmRNKtjr8UtvSM/Jhh9fsLrcnmGZ0wAAFdhJREFUtroehk0D/+K/hJzNOMszq59h+eHlDGk+hOd7PE+AjwfaYjxAA4draHdcD1m48TCnUzMZ3bdyTS9SXgT7BTO42WAGNxvM2YyzrIhdwZIDS1i4ZyFzd8+lbmBdK4hEWEHE28vNHRNqNIBr/wl9x59fI33eHRDayp7a/Y7SN6RnpMLno2D319DtYRj4MpRwHfuT9jN22VgOnTnEhG4TGNlmpFZNKbfSEoeLZOcYrn5jOSHV/Pjiz730H7cMpWSmsOLwCpYcXMLquNWkZ6cTGhjKNU2skkiXel3cH0TAGpG+czGseRPit9oj0kdD1IPOjUhPPgbzhlsrGg582RqIWILlh5fzzKpn8PXy5Y3+b3B52OUuuJCKRUscrqElDg/4cecxDpxM5Z3r2mjQKGNBvkHc0OwGbmh2A6mZqXklkUV7FzH/9/nUCajDNRHXcF3kde4NIt6+0OFWaD8M9q+0RqT//G9YNRk63wU9/gy1i5ha//gu+Ph2SE2A4R9Dm0HFvlWOyeGDrR/wzpZ3aFu7LW/+f3t3Hl1VfS1w/LsTEpLILEFjEggOBZEykzKUMMQgoEJ9pSJBbB4VlRal8njagVex1dW6Fu3DoU9FpWKLJCgqqCCBgCEMShBDRAZlCENIFYMCQhgS9vvjnMQQMt3kDonsz1p3ce+5v3vuvj9y7r7nd87Zv6FPEtWsYU5JbL5/LHF4yYtZe4lpHc5NN3y/a1I1dBEhEYzsOJKRHUdy6twp1uavJT0vnSW7l5C2K60siSR1SKL3Fb1pEuSDTUAErh7s3L7Y7hyr2PwPp7z79aOd2lLX3wodE5z2e9+HhcnO61Lehejqr+o+ee4kv8v6HasPruaWq2/hkf6PXDLHM0zDYInDC7Yc+JrN+7/mkVu70CTYznBuKCJCIhgRN4IRcSM4de4UWflZpOels3TPUtJ2pdEmrA2J7RMZHjecPlf08U0SuaIL/OT/nNLupVeknzkGm+fBkN9Csyvg7QcAgf+YW2PSyDuWx7Q109h/fD8P9X2IO6+/0/Zwjd/Zt5wXvJi1lxZhTbi9T2zNjU1ARIREcFPcTfx1yF95//b3+etg53jAO3vfYXL6ZBJfS+TRjY+y8fBGis/X/Tz7KrWIcg6kT/8UbvqzM53t6j8509xKsDM89cOx1a5i7aG1JL+bzNHTR5mbNJeJXSZa0vDUujnOMGJ5+9Y6y+to6NChrFix4oJlc+bMYcqUqo9RDRkyhNLjrqNGjeKbb765qM2sWbPKrsWoTI8ePcpKj/ibJY56OlB4ive2/ZsJ/TpwWVPbgWsMIkIiGB43nNmDZ5M5LpO/Dfkb8VfG8+7ed7ln5T0MWzSMWRtmseHwBu8nkabNof8vYfpO6Hyrs2zgA9BpZJUvUVXm5s5lasZUoptHk3pLauAnyWqsonvBaynfJY99a53HNezpVWf8+PGkpqZesCw1NZXx42s3odayZcto1aqVR++5Y8cOSkpKyMrK4uTJkx691hvsm66e5q3fR3CQkDIgLtChmDoIbxJOUockkjokUVRcxPr89aTnpbNs3zIWf76YVk1bOcNZHYbTN6ovIUFeqqR6YINzS3Ar3V495LtjHuWcOneKmetnsnL/SkZ1HMWsAbMIb9L4pyD2meW/ceZYqU7zKPjnbc6/JwogsjO8/4Rzq8yVP4SRf6lydWPHjmXmzJmcPXuW0NBQ8vLyOHz4MIMGDWLKlClkZ2dTVFTE2LFjefTRRy96fVxcHJs3b6Zt27Y8/vjjzJ8/n3bt2hEbG0vv3pVfVLpw4UImTpzIjh07WLJkCcnJzuSq2dnZTJs2jZMnT9K0aVMyMjKIiIjg4Ycf5r333iMoKIjJkydz//33V99HNbDEUQ/fnDpLWvZBRneP5ooWdnCysQtvEs6NHW7kxg43crr4NOvz17Ni/wqW71vO4s8X07JpSxLbJ5LUIYkfRf2o7kmk9Ffuz152kkXHQRc+dh04foBpa6ax99heZvSZwV1d7rKhKW8Ia+UkjWMHoWWs87ge2rRpQ3x8PMuXL2fMmDGkpqZy++23IyI8/vjjtGnThpKSEhITE8nNzaVbt26Vruejjz4iNTWVnJwciouL6dWrV5WJIy0tjZUrV7Jz506efvppkpOTOXv2LOPGjSMtLY2+ffty/PhxwsPDmTt3Lnl5eeTk5NCkSZOycuv1YYmjHhZ8eICicyXcPaiKUyxNoxXWJIzEDokkdkh0kshhZ09kRd4K3vj8DVqEtmBY+2EM7zCcflH9CAn2IInkb7kwSXRMcB7nbylbti5/HQ+tfYggCeK5G5+j/1X9vf4Zv5eq2TMoU5q4S/f2hjxc6d6eJ0qHq0oTx0svvQTAokWLmDt3LsXFxRQUFLB9+/YqE0dWVha33XZbWVn00aNHV9qudO+kffv2REdHM2nSJI4ePUp+fj5RUVH07etcy9OiRQsAVq1axX333Vc2C2GbNvWf5dISRx2dKS7h5Q15DLquLddHtQh0OMaHwpqEkdg+kcT2iZwpOcP6/PWs3L+SVftX8dbut2gR2oKhsUMZHjec/lH9a04iP/71xcs6JkDHBFSVl7a9xFNbnuIHrX/AnKFziGkec3F7Uze13Nvz1JgxY3jwwQfZsmULp06donfv3uzbt4/Zs2eTnZ1N69atSUlJ4fTp0/X+CAsXLmTnzp3ExcUBcPz4cRYvXky/fv3qve7asoPjdbQ05zBHTpxh8iArL3IpaRrclGHth/HnQX8mc1wmTw97msExg8k4kMGvMn7F4EWD+f2637P20FrOlVQ+WU9VTp07xYzMGTy55UluiruJV0a+YknD26rb26uHZs2aMXToUCZNmlR2UPz48eNcdtlltGzZki+++ILly5dXu46EhATeeustioqKOHHiBG+//fZFbc6fP8+iRYv45JNPyMvLIy8vjyVLlrBw4UI6depEQUEB2dnZAJw4cYLi4mKSkpJ4/vnny6ry2lBVgKgqL2bto/OVzRl0nfdnDjONQ2hwKENihzAkdghnS86y8fBG0vens+bAGpbuWUrzkOYMbT+U4R2G0/+q/oQGV11m/+CJg0xbM4093+xheu/ppNyQYsczfKGavb36Gj9+PLfddlvZGVbdu3enZ8+edO7cmdjYWAYOHFjt63v16sW4cePo3r077dq1KxtyKi8rK4vo6GiuuuqqsmUJCQls376dwsJC0tLSuP/++ykqKiI8PJxVq1Zx991389lnn9GtWzdCQkKYPHkyU6dOrddn9WmtKhEZATwJBAMvqupfKjzfHpgPtHLb/EZVl4lIEvAXIBQ4C/y3qq52X/M+EAUUuasZrqpfVheHt2tVZX52hJ/P28Tsn3VnbG/7RWgudLbkLB8UfMCKvBWsObCGE+dO0CykWdlw1udff073yO5lp9RuyN/Ag+8/SImW8NTQpxgQPSDAn6BxsVpV3tEgalWJSDDwdyAJOARki8hSVd1ertlMYJGqPisiXYBlQBzwFXCrqh4Wka7ACqD85N0TVNU/E2xU4oW1e7miRVNGd7+q5sbmkhMaHEpCTAIJMQmcKznHxoKNpOels/rgat7e+zZhwWGc1/Pc0+0eQoJDmPPRHIIkiMcGPmZJwzQKvhyqigd2q+peABFJBcYA5ROHAqVHllsChwFU9eNybT4FwkWkqaqe8WG8tbL98HHW7f6Kh0Z0IrSJHSIy1QsJDrkgiXxQ8AHp+9NJz0vnmZxnnDZBIcwZOoeEmPoPlxjjD7785osGDpZ7fIgL9xoAZgF3isghnL2Nyq5K+SmwpULS+IeI5IjI/0gVA8Eico+IbBaRzUeOHKnzh6joxay9RIQGMyHe5hM3ngkJDmFQzCD+NPBPrBu/jlEdRwEwqeskSxqmUQn0T+bxwMuqGgOMAv4p8t0UbiJyA/AEcG+510xQ1R8Cg9zbxMpWrKpzVbWPqvaJjIz0SrAFx4pYuvUwt/eJpWWEl64gNpekj7/4mI2HN3Jvt3tZtGsRmwo2BTokY2rNl4kjHyhf9S/GXVbeL4BFAKq6EQgD2gKISAzwJnCXqu4pfYGq5rv/ngBexRkS84uXN+RxXpVf/Ngu+DN1t6lgEzMyZzB78Gym9pzK7MGzmZE5w5KHaTR8mTiygetEpKOIhAJ3AEsrtDkAJAKIyPU4ieOIiLQC3sU5y2p9aWMRaSIipYklBLgF2ObDz1Dm2zPFvPrhAUZ2jSK2TYQ/3tJ8T20r3MbswbPLzqqKj4pn9uDZbCv0y5+yMfXms8ShqsXAVJwzonbgnD31qYj8UURKr6X/L2CyiGwFFgIp6pwfPBW4FviDeywjR0TaAU2BFSKSC+Tg7MG84KvPUF5a9kFOnC628iKm3iZ1nXRRddv4qHgmdZ0UoIguHfO2zbtoz25TwSbmbZtX53XWt6x6xeWdOnWiR48e9OjRg9dffx2ASZMm0a5dO7p27VrnOL3Jp8c4VHWZqv5AVa9R1cfdZX9Q1aXu/e2qOlBVu6tqD1VNd5c/pqqXuctKb1+q6klV7a2q3VT1BlWdpqolvvwMAMUl55m3bh9941rTs31rX7+dMcZHul7e9YJhwdJhw66X1/0Lub5l1StasGABOTk55OTkMHasM0dLSkoK7733Xp1j9Da7crwWlm/7N/nfFPHIrV0CHYoxphpPbHqCnUd3VtsmMiKSe1feS2REJEdOHeHqVlfz7NZneXbrs5W279ymMw/HP1zl+upbVr02EhISyMvLq9NrfSHQZ1U1eE55kb10bHsZN15v84kb09i1CG1BZEQkBScLiIyIpEVo/YqUli+rDlxUVn3z5s3k5uaSmZlJbm5ujeubMGFC2VBVYWFhvWLzFdvjqMGmfUfZeugYj/2kK0FBVjvImIasuj2DUqXDU6WnQk/pPqXeMyp6o6x6qQULFtCnz0VVPhoU2+OoxHOZe9iw5ysAXsjaR+uIEGJahfNc5p4aXmmMach8dSr0mDFjyMjIqLSsekZGBrm5udx8881eKaveEFjiqES3mJZMffVjXv/oIKt2fMGQTu2Y/tpWusW0DHRoxph68NWp0N4oq96YWOKoxIBr2vJMck9+/+Y2ggTW7PqSZ5J7MuAaK6FuTGPmy1Ohx48fz9atW8sSR/my6snJyTWWVa9p3f3792fXrl3ExMSUDYUFik/LqjcUdS2rnvzCB2zYU8gDw65l+vBOPojMGFNfVlbdOzwpq257HFXYsOcrdv77BA8Mu5Z/fXig7JiHMcZc6ixxVGLDnq+Y+urHPJPck+nDO/FMck+mvvqxJQ9jjMESR6VyDx274JhG6TGP3EPHAhyZMaYyl8KQuy952n92HUcl7ht8zUXLBlzT1g6OG9MAhYWFUVhYyOWXX27ztNeBqlJYWEhYWFitX2OJwxjTqMXExHDo0CG8OWHbpSYsLIyYmJhat7fEYYxp1EJCQujY0apW+5Md4zDGGOMRSxzGGGM8YonDGGOMRy6JK8dF5Aiwv44vbws0xAs4LC7PWFyesbg8832Nq4OqRlZceEkkjvoQkc2VXXIfaBaXZywuz1hcnrnU4rKhKmOMMR6xxGGMMcYjljhqNjfQAVTB4vKMxeUZi8szl1RcdozDGGOMR2yPwxhjjEcscRhjjPGIJQ6XiIwQkV0isltEflPJ801FJM19/kMRiWsgcaWIyBERyXFvd/shpnki8qWIVDpRsziecmPOFZFevo6plnENEZFj5frqD36KK1ZE1ojIdhH5VESmVdLG731Wy7j83mciEiYim0RkqxvXo5W08fv2WMu4/L49lnvvYBH5WETeqeQ57/aXql7yNyAY2ANcDYQCW4EuFdr8EnjOvX8HkNZA4koBnvFzfyUAvYBtVTw/ClgOCNAP+LCBxDUEeCcAf19RQC/3fnPgs0r+H/3eZ7WMy+995vZBM/d+CPAh0K9Cm0Bsj7WJy+/bY7n3ng68Wtn/l7f7y/Y4HPHAblXdq6pngVRgTIU2Y4D57v3XgUTxffH/2sTld6q6FjhaTZMxwCvq+ABoJSJRDSCugFDVAlXd4t4/AewAois083uf1TIuv3P74Fv3YYh7q3gWj9+3x1rGFRAiEgPcDLxYRROv9pclDkc0cLDc40NcvAGVtVHVYuAYcHkDiAvgp+7wxusiEuvjmGqjtnEHQn93qGG5iNzg7zd3hwh64vxaLS+gfVZNXBCAPnOHXXKAL4GVqlplf/lxe6xNXBCY7XEO8BBwvornvdpfljgav7eBOFXtBqzku18V5mJbcGrvdAeeBt7y55uLSDNgMfBrVT3uz/euTg1xBaTPVLVEVXsAMUC8iHT1x/vWpBZx+X17FJFbgC9V9SNfv1cpSxyOfKD8L4MYd1mlbUSkCdASKAx0XKpaqKpn3IcvAr19HFNt1KY//U5Vj5cONajqMiBERPwyH7CIhOB8OS9Q1TcqaRKQPqsprkD2mfue3wBrgBEVngrE9lhjXAHaHgcCo0UkD2c4e5iI/KtCG6/2lyUORzZwnYh0FJFQnINHSyu0WQr83L0/Flit7pGmQMZVYRx8NM44daAtBe5yzxTqBxxT1YJAByUiV5aO64pIPM7fv8+/bNz3fAnYoap/q6KZ3/usNnEFos9EJFJEWrn3w4EkYGeFZn7fHmsTVyC2R1X9rarGqGocznfEalW9s0Izr/aXTR2LM+YnIlOBFThnMs1T1U9F5I/AZlVdirOB/VNEduMcgL2jgcT1gIiMBorduFJ8HZeILMQ526atiBwCHsE5UIiqPgcswzlLaDdwCvhPX8dUy7jGAlNEpBgoAu7wQ/IH5xfhROATd3wc4HdA+3KxBaLPahNXIPosCpgvIsE4iWqRqr4T6O2xlnH5fXusii/7y0qOGGOM8YgNVRljjPGIJQ5jjDEescRhjDHGI5Y4jDHGeMQShzHGGI9Y4jCmgROnQu1FFU+NCRRLHMYYYzxiicMYLxGRO935GnJE5Hm3IN63IvK/7vwNGSIS6bbtISIfuMXw3hSR1u7ya0VklVtUcIuIXOOuvplbNG+niCzwQ2VmY6pkicMYLxCR64FxwEC3CF4JMAG4DOfq3RuATJyr2QFeAR52i+F9Um75AuDvblHBAUBp2ZGewK+BLjjzswz0+YcypgpWcsQY70jEKWiX7e4MhOOU3j4PpLlt/gW8ISItgVaqmukunw+8JiLNgWhVfRNAVU8DuOvbpKqH3Mc5QBywzvcfy5iLWeIwxjsEmK+qv71gocj/VGhX1xo/Z8rdL8G2XRNANlRljHdkAGNFpB2AiLQRkQ4429hYt00ysE5VjwFfi8ggd/lEINOdhe+QiPzEXUdTEYnw66cwphbsV4sxXqCq20VkJpAuIkHAOeBXwEmcCX9m4gxdjXNf8nPgOTcx7OW7argTgefdyqbngJ/58WMYUytWHdcYHxKRb1W1WaDjMMabbKjKGGOMR2yPwxhjjEdsj8MYY4xHLHEYY4zxiCUOY4wxHrHEYYwxxiOWOIwxxnjk/wF+JOAeXJ7kDgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light",
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot_scores(history[1:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 295
    },
    "colab_type": "code",
    "id": "0sBIoVu-gqTL",
    "outputId": "5eacf1c8-9aee-4bbc-e079-4a14ec228550"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd5hU5fXA8e9hacJSpClVICIqIm1BA2pAJMESsCCylohGUWNFDSqChahRwk8RO/aOqFFBUVRcS0zUBQVDEUWCulZEhEWKLJzfH2eGHZbZZRfmzp1yPs8zz87cuTNz9sLeM/ct5xVVxTnnXPaqFnYAzjnnwuWJwDnnspwnAuecy3KeCJxzLst5InDOuSznicA557KcJwLnUoiI7CYib4tIsYj8X9jxAIjIMhE5LOw4XHA8EbiEyKSThYhcIyIqIkNjtlWPbGsb8MePAH4E6qvqJQF/lnOAJwLnyvMTcK2I5CT5c/cAFqrP9HRJ5InABUpEaonIRBH5JnKbKCK1Is81EZEXReRnEflJRN4RkWqR5y4Tka8jTSSLRaR/nPc+QES+iz1Zi8gxIvJx5H4vEZktIqtF5HsRubkKob8C/AqcXM7v1UBEHhGR5SLyhYiMicZeiWPSW0QKRWRV5GfvyPaHgFOBUSKyJt4VVuR4ThCRLyO/090iskvkub4iUiQio0Xkx8hV2kmVjVlEzhSRRZFjvlBEusd8dFcR+TgS81MiUjvymnL/DV368H8wF7QrgQOBrkAXoBcwJvLcJUAR0BTYDRgNqIh0BM4DeqpqPeAPwLKyb6yq7wO/AIfGbD4ReCJy/1bgVlWtD/wGmFqFuBUYC1wtIjXiPH8b0ABoD/wO+BNw2vbeVEQaAS8Bk4DGwM3ASyLSWFWHA48D41U1V1Vfj/MWNwJ7YcdzT6AlcFXM87sDTSLbTwUmR45nhTGLyPHANZFt9YFBwIqY9x0KDATaAfsDwyPb4/4bbu84uNTiicAF7SRgnKr+oKrLgWuBUyLPbQSaA3uo6kZVfSfSJLIJqAXsKyI1VHWZqn5ezvs/CeQDiEg94IjItuj77ykiTVR1jaq+V5XAVXUasBw4I3Z75ApkGHCFqhar6jLg/2J+r4ocCXymqo+qaomqPgl8Avxxey8UEcH6EEaq6k+qWgzcEIkl1lhV3aCqb2FJZ2glYj4DS0CFapao6hcx7zlJVb9R1Z+A6VgigvL/DV0a8UTggtYCiD2hfBHZBvAPYAnwqogsFZHLAVR1CXAR9g31BxGZIiItiO8J4NhIc9OxwIcxJ7A/Y9+eP4k0wRy1A/GPwa5qasdsawLUiPN7tazE+5U9HlV5bVOgDjAn0hTzM9aE1TRmn5Wq+kuZ925RiZhbA+UlW4DvYu6vBXIj9+P+G7r04onABe0brAM0qk1kG5FvppeoanusKeLiaF+Aqj6hqgdFXqvATfHeXFUXYie0w9m6WQhV/UxV84Fmkdc/IyJ1qxK8qr6Gnej+ErP5R+ybcNnf6+tKvGXZ41GV1/4IrAM6qWrDyK2BqubG7LNrmd8xery3F/NXWPNZlVT0b+jShycCl0g1RKR2zK061kwzRkSaikgTrD37MQAROUpE9ow0eazCmoQ2i0hHETk08i1/PXby21zB5z4BXAgcAjwd3SgiJ4tIU1XdDPwc2VzR+5TnSmBU9IGqbsL6G64XkXoisgdwcfT32o4ZwF4icmJkSOoJwL7Ai9t7YeT3uBe4RUSaAYhISxH5Q5ldrxWRmiJyMHAU8HQlYr4PuFREeojZM7JPhcr7N6zEcXApxBOBS6QZ2Ek7ersGuA6YDXwM/Bf4MLINoAPwOrAG+A9wp6oWYP0DN2LfYr/DvtFfUcHnPol1fr6hqj/GbB8ILBCRNVjH8TBVXQcQGZVzcGV+KVV9F/igzObzsY7qpcC/sGT0QOS9R4vIy+W81wrs5HwJ1hk7CjiqTNwVuQy7QnlPRFZjx69jzPPfASuxq4DHgbNV9ZPtxayqTwPXR7YVA88DjSoRT3n/hi6NiPfrOJcZRKQv8Jiqtgo7Fpde/IrAOeeynCcC55zLct405JxzWc6vCJxzLstVDzuAqmrSpIm2bds27DCccy6tzJkz50dVbRrvubRLBG3btmX27Nlhh+Gcc2lFRMrOaN8i0KYhERkoVjlySbyp5yJyi4jMjdw+jUyZd845l0SBXRFEilzdAQzAqhMWisi0SEkAAFR1ZMz+5wPdgorHOedcfEFeEfQClqjqUlX9FZgCDK5g/3xKq0Y655xLkiD7CFpihayiioAD4u0YqWnSDnijnOdHYOV3adOmTWKjdM6FZuPGjRQVFbF+/fqwQ8kYtWvXplWrVtSoEW8ZjfhSpbN4GPBMpDDWNlR1MjAZIC8vzyc+OJchioqKqFevHm3btsXq1rmdoaqsWLGCoqIi2rVrV+nXBdk09DVW4zyqFeWX2h2GNwu5dDR+PBSUqbFWUGDb3XatX7+exo0bexJIEBGhcePGVb7CCjIRFAIdRKSdiNTETvbTyu4kInsDu2KVC51LLz17wtChpcmgoMAe9+wZblxpxJNAYu3I8QysaUhVS0TkPGAmkAM8oKoLRGQcMDuyDCBYgpjiy9u5tNSvH0ydCkOGQK9eUFgITz9t251LE4H2EajqDKxGfey2q8o8vibIGJwLlCr873/wyy/wyivQvj307h12VK6SVqxYQf/+tqDad999R05ODk2b2uTbDz74gJo1a5b72tmzZ/PII48wadKkCj+jd+/e/Pvf/05c0AFIlc5i59LPkiUwYoQ1B9WoYVcBBQUwYAC88QZU9z+vRBo/3lrcYi+2CgrsImzUqPJfV5HGjRszd+5cAK655hpyc3O59NJLtzxfUlJC9XL+HfPy8sjLy9vuZ6R6EgAvOudc1W3cCDfdBJ07wwcfQG4uvPyynfzPPRfeeQeOPBI2+4qNiZSs7pjhw4dz9tlnc8ABBzBq1Cg++OADfvvb39KtWzd69+7N4sWLAXjzzTc56qijAEsip59+On379qV9+/ZbXSXk5uZu2b9v374MGTKEvffem5NOOoloi/iMGTPYe++96dGjBxdccMGW900W/8riXFXMmQNnnAFz58Kxx8I++0D//qVfU2+/3ZqJHnoILrwQJk0C7wytlIsussNakRYt4A9/gObN4dtv7fBfe63d4unaFSZOrHosRUVF/Pvf/yYnJ4fVq1fzzjvvUL16dV5//XVGjx7Ns88+u81rPvnkEwoKCiguLqZjx46cc84524zl/+ijj1iwYAEtWrSgT58+vPvuu+Tl5XHWWWfx9ttv065dO/Lz86se8E7yROBcZaxdC1dfDTffDLvtBs8+a4kgngcegCZNYMIEqF8frr8+ubFmsF13tSTw5ZfQpo09DsLxxx9PTk4OAKtWreLUU0/ls88+Q0TYuHFj3NcceeSR1KpVi1q1atGsWTO+//57WrXaetXQXr16bdnWtWtXli1bRm5uLu3bt98y7j8/P5/JkycH84uVwxOBc9vz2mtw1lnWKTxihDULNWxY/v4i1qC9ejXccIMlg8suS168aaoy39yjzUFjx8Jdd1luDmKAVt26dbfcHzt2LP369eO5555j2bJl9O3bN+5ratWqteV+Tk4OJSUlO7RPGLyPwLnyrFgBw4fD739vncFvvQX33FNxEogSgTvvhPx8uPxyO2u5nRJNAlOnwrhx9jO2zyAoq1atomXLlgA89NBDCX//jh07snTpUpYtWwbAU089lfDP2B5PBM6VpQpTplgD9OOPw5VXwrx5cMghVXufnBx4+GH44x+tE/mxx4KJN0sUFtrJP3oFEJ3CUVgY7OeOGjWKK664gm7dugXyDX6XXXbhzjvvZODAgfTo0YN69erRoEGDhH9ORdJuzeK8vDz1hWlcYL78Ev7yF3jpJRuOct99sP/+O/ee69fbKKK33oJnnoGjj05MrBlg0aJF7LPPPmGHEbo1a9aQm5uLqnLuuefSoUMHRo4cuf0XliPecRWROaoad7yrXxE4B7BpE9x2G3TqZG0Nt9wC//nPzicBgNq14YUXIC8PTjgBXn9959/TZZR7772Xrl270qlTJ1atWsVZZ52V1M/3zmLnFiywIaHvvWdjE+++GxK9LnZuLsyYYe0ZgwdbB7TPQHYRI0eO3KkrgJ3lVwQue23YYMNOunWDzz6zNvyXX058Eohq1AhefRVatoQjjoCPPgrmc5yrIk8ELju9+67NNho3zpprFi2Ck04KfvLXbrtZ01D9+nb18cknwX6ec5XgicBll9WrrTP4oINg3TorFPfooxApNJYUbdrArFlQrRocdhhEhg06FxZPBC57TJsG++5rcwFGjoT58+1beRg6dLBmol9+sWTw7bfhxOEcnghcNvjuO5t5NHiwtdP/5z9WKiJSDCw0++9vVyTffWcVS1esCDeeLNSvXz9mzpy51baJEydyzjnnxN2/b9++RIevH3HEEfz888/b7HPNNdcwYcKECj/3+eefZ+HChVseX3XVVbwe4mgyTwQuc6la3Z999rGrgeuvt6JxvXqFHVmpAw6A6dOtpPXhh1vTlYsvgGVB8/PzmTJlylbbpkyZUqnCbzNmzKBhZWaZx1E2EYwbN47DDjtsh94rETwRuMy0ZIlVBf3zn+2b97x5MHq0lYpINf362apmH31ks5DXrQs7otQUQB3qIUOG8NJLL/Hrr78CsGzZMr755huefPJJ8vLy6NSpE1dffXXc17Zt25Yff/wRgOuvv5699tqLgw46aEuZarD5AT179qRLly4cd9xxrF27ln//+99MmzaNv/71r3Tt2pXPP/+c4cOH88wzzwAwa9YsunXrRufOnTn99NPZsGHDls+7+uqr6d69O507d+aTBA408HkELrNs3GjNPtdcAzVrWn/AGWdYx2wq++MfrdP6xBNt2cvnnrP4s0kIdagbNWpEr169ePnllxk8eDBTpkxh6NChjB49mkaNGrFp0yb69+/Pxx9/zP7lTC6cM2cOU6ZMYe7cuZSUlNC9e3d69OgBwLHHHsuZZ54JwJgxY7j//vs5//zzGTRoEEcddRRDhgzZ6r3Wr1/P8OHDmTVrFnvttRd/+tOfuOuuu7jooosAaNKkCR9++CF33nknEyZM4L777qv4eFVSiv91OFcF0Wafyy+3ZpZFi6xaaKongahhwyxxzZgBJ59ss53d1mLrUDdvnpA61LHNQ9FmoalTp9K9e3e6devGggULtmrGKeudd97hmGOOoU6dOtSvX59BgwZteW7+/PkcfPDBdO7cmccff5wFCxZUGMvixYtp164de+21FwCnnnoqb7/99pbnj42UPu/Ro8eWInWJ4FcELv1VZa2AVHfmmdZPcOmlUK8e3Htv+iSynRVSHerBgwczcuRIPvzwQ9auXUujRo2YMGEChYWF7LrrrgwfPpz169fv0HsPHz6c559/ni5duvDQQw/x5ptv7lSs0TLWiS5hnSX/w1zGeu012G8/WwTmjDNg4cL0TQJRl1wCV11lHd0XX2yd3i6wOtS5ubn069eP008/nfz8fFavXk3dunVp0KAB33//PS+//HKFrz/kkEN4/vnnWbduHcXFxUyfPn3Lc8XFxTRv3pyNGzfy+OOPb9ler149iouLt3mvjh07smzZMpYsWQLAo48+yu9+97ud+v0qwxOBS0+xawVUrw5vvln5tQLSwTXXWJv5rbfafRdoHer8/HzmzZtHfn4+Xbp0oVu3buy9996ceOKJ9OnTp8LXdu/enRNOOIEuXbpw+OGH0zOm8/pvf/sbBxxwAH369GHvvffesn3YsGH84x//oFu3bnz++edbtteuXZsHH3yQ448/ns6dO1OtWjXOPvvsnf79tsfLULv0ogpPPQUXXAArV8KoUdZMULt22JElnqpd5TzwgF3xXHJJ2BElnJehDkZKlaEWkYEislhElojI5eXsM1REForIAhF5Ish4XJr78ksbXZOfb4Xh5syxuQGZmATA6h5NngzHH299BvfeG3ZELkMF1lksIjnAHcAAoAgoFJFpqrowZp8OwBVAH1VdKSLNgorHpbFNm2zZx9GjYfNmWyvg/PNtBbBMl5NjVVHXrLF1k+vVs9FFziVQkFcEvYAlqrpUVX8FpgCDy+xzJnCHqq4EUNUfAozHpaMFC6xA3AUXQJ8+9viii7IjCUTVrGkjoQ45BE45BV58MeyIEirdmqdT3Y4czyATQUvgq5jHRZFtsfYC9hKRd0XkPREZGO+NRGSEiMwWkdnLly8PKFyXUsquFfDoo8GuFZDqdtnFymR062YTzoJesT1JateuzYoVKzwZJIiqsmLFCmpXsbk07HkE1YEOQF+gFfC2iHRW1a0qOanqZGAyWGdxsoN0Sfbuu9ZJ+sknNrHq5puTWyY6VdWvb8nwd7+zvpJZs6xWURpr1aoVRUVF+Be8xKlduzatWrWq0muCTARfA61jHreKbItVBLyvqhuB/4nIp1hi2PnxYC79rF5ts4Lvugv22MNOegPjXiRmr8aNbe7EwQfb7Ok330zMusohqVGjBu3atQs7jKwXZNNQIdBBRNqJSE1gGDCtzD7PY1cDiEgTrKloaYAxuVQVXSvg7rutD2D+fE8C5Wne3FY5q1PH5lF89lnYEbk0F1giUNUS4DxgJrAImKqqC0RknIhEi3HMBFaIyEKgAPirqnpR9mwSb62AW24Jf62AVNe2rSWDzZttYZsvvww7IpfGfEKZC4cqPPigTZJat85KKvz1r6lZJjqVzZ0LfftCs2bwzjtWa8m5OEKbUOZcXOm0VkCq69rVqpV+/bU1E61cGXZELg15InDJs3Ej3HQTdO5ss4LvuceGQXbsGHZk6a13b3jhBRtldfjhEKeYmXMV8UTgkiPd1wpIdYcdZjWYZs+Go4+GHSyb7LKT/xW6YK1da23/vXrB99/bDNl//tNWmnKJdfTR8NBD8MYb1gG/cWPYEbk04YnABScT1wpIdSefbHWZpk+HU0/1Vc5cpYQ9s9hlohUrbDTQww9Dhw426SkJi2u4iHPOKZ2cV6+ezc0QCTsql8I8EbjEKbtWwOjRmbtWQKq77DJLBjfcYKUpxo/3ZODK5YnAJcaXX8Jf/gIvvQQ9e9pkpzQufZARrrvOksGECdCgAYwZE3ZELkV5InA7p+xaATffbFcE2VQmOlWJ2FKXq1fblVm9enDhhWFH5VKQJwK34xYssE7g996zyUx33w1eQCy1VKsG999vC9tcdJE1E512WthRuRTjo4Zc1cVbK+CVVzwJpKrq1eGJJyxZn3EGPP102BG5FONXBK5qYtcKOOkkKxDnawWkvlq1bP7GwIH275abaxP7nMOvCFxlrV5tncEHHWSTxGbMsLV0PQmkj7p1bZnLzp1tPsdbb4UdkUsRngjc1saP33YZxOuug9atrQ/gwgutb8C/TaanBg1Km/H++EcrSeGynicCt7WePa08QUGBrRXQt6+NOImuFTBxoq8VkO6aNrVZ302awB/+YIsAuazmicBtrV8/mDrV6ta0bWvNB6efDosXp/36uC5Gy5Y216NWLRgwAD7/POyIXIg8EbitqcLHH1ufwIYNVq7g/vuhZs2wI3OJ1r69JYONG616aVFR2BG5kHgicKVKSmwy2EUX2Yn/sstsqGHZPgOXOfbdF2bOtPpQAwbA8uVhR+RC4InAmeJiWzf49tthl12sVMSNN1ozUbTPwGWmHj3s3/uLL6zP4Oefw47IJZknAmdNAgcfbN8MjznGTgqHHWbPRfsMCgvDjdEF6+CDbZ7B/Plw5JHwyy9hR+SSyCeUZbsPP7RhhMXFlgD+8Idt9+nXz24usw0cCE8+aVeAxxxjaxrUqhV2VC4J/Iogm02fbt8Ec3JsxnC8JOCyy3HHwQMP2PDS/HzrN3IZL9BEICIDRWSxiCwRkcvjPD9cRJaLyNzI7Ywg43ERqlaVcvBg6yx8/32bbeoc2MpmkybBc8/Z0OHNm8OOyAUssKYhEckB7gAGAEVAoYhMU9WFZXZ9SlXPCyoOV0ZJCYwcaZ3CRx9tZSLq1g07Kpdqzj/fhhCPGWPlq2+/3Re2yWBB9hH0Apao6lIAEZkCDAbKJgKXLMXFMGyY1Qm65BK46SZfN8CVb/RoSwbjx1v56r//PeyIXECCTAQtga9iHhcB8aamHicihwCfAiNV9as4+7idVVQERx1lo0LuugvOPjvsiFyqE7EhxKtX28/69eGKK8KOygUg7FFD04EnVXWDiJwFPAwcWnYnERkBjABo06ZNciPMBJUZGeRcPCJwxx32f2f0aEsG554bdlQuwYLsLP4aaB3zuFVk2xaqukJVN0Qe3gf0iPdGqjpZVfNUNa+plz2uGh8Z5HZWtWrw4IM2uOC88+CRR8KOyCVYkImgEOggIu1EpCYwDJgWu4OINI95OAhYFGA82WfSJB8Z5BKjRg2YMgX697elLp97LuyIXAIFlghUtQQ4D5iJneCnquoCERknIoMiu10gIgtEZB5wATA8qHiySkmJjfq48EJLBG++Cc2bb/dlzlWodm14/nmrQjtsGLz6atgRuQQRVQ07hirJy8vT2b6YRvl8ZJAL2sqVNtP8009t4lmfPmFH5CpBROaoal6853xmcSaJrRl0110wYYInAZd4u+5qVwNt2sARR9hgBJfWPBFkig8/tEv2pUttZJAPD3VBatbMrgYaNrQBCIu8ey+deSLIBD4yyIWhdWuYNQuqV7e1DP73v7Ajcjso4xNBvLXYCwpse0bwkUEuTHvuac1Ea9da6fJvvgk7IrcDMj4RxK7FDvZz6FDbntZ8ZJBLFZ07wyuvwA8/2JXBjz+GHZGrooxPBNF1VYYMsXLrQ4fa47Qurx+7mtgll8Azz3jhOBeuXr2siXLpUvtDW7067IhcFWR8IgA76XfubINpDjkkzZOAjwxyqapvX/tSMm+e1bVauzbsiDJDEtq3syIRFBTAggXQsqVNiHziibAj2kE+MsiluiOPtNLm//qXLXLz669hR5T+ktC+HXbRucBFj9nUqdCuHXTqZOtuNG1qzZlpY/p0WzGqUSMbGeSdwi5VnXACrFkDZ5wBJ51ky19Wz/hTTWKsXg1ff22d7l9/XXp/773h97+3JUQLChLevp3x/zqFhVsfswcftP+nf/97GiWCSZNsMZnu3WHaNO8Udqnvz3+2k9rFF0NuLtx/vxWvy1YbN8K338Y/ycfeX7Nm29c2bAgtWthw3aefhrFjE96+nfGJYNSorR8PHWrzYO6/H954Aw7dpuh1CvHVxFw6GznSksE111j56okTM2+VM1X46aftn+B/+MH2jVWzpp3gW7SALl1slnaLFtaG3bJl6XN165Y2bYwda32D/fr5FcHOmjjRmjBPPtn6tVKysrXXDHKZ4KqrLBncfDM0aADjxoUdUeWtW2cn8diTeryT/IYN2762adPSk3pe3tYn+OhJvkmTyiXG2PbtaAJI8PDHrEwEdetaRd0DDrCKutOnp9gXFV9NzGUKERvZtno1/O1vdmVw6aXhxrR5s31D394J/qeftn1tnTqlJ/Lf/rb0fuwJvnlzqFUrcfGWbd+OjokvLExYIsjq6qO3325zsiZOtHlZKSF2NbGnn/ZyES4zbNpkHcdPPQX33AMjRgTzOcXF22+m+fZba3aNVa0a7L57/BN77P0GDVLsW2PlVVR9NCuvCKLOPdf6C0aNsvkF3bqFHJCPDHKZKicHHn3UOkPPOgu+/BKuu670+YIC+4ZbtlMvauNG+O678k/w0cfFxdu+tkGD0hP5oYfGP8HvtltWj2zK6isCgBUrrJ+mbl2YM8cGOITCRwa5bLBunTWpzJtnTUVXXmn/3087DS6/3NrN432L//77bTtba9QoPaFX9C3eB1gAFV8RZH0iACvTc+ihNr/gwQcT+tbb5yODXLYpLraSFJ98YlcKmzZtu0+TJtue1Ms+btw4u4ekVpE3DW1H374wZox9QRkwAE48MUkf7CODXDaqV8+aPo84wirm/v73Nu8geoJPdGer2y5PpxFXXWUr7p19Nnz+eRI+0GsGuWw2b579oY0dawMkmja1P8C2bT0JhMATQUT16vD443Yuzs8PuESK1wxy2Sx2XPy4cfYztpaOSzpPBDH22APuu88GL1x1VUAfMn26DVHy1cRctqpoXLwLhXcWx3H22TbU+dVXE1yPyEcGOedCUlFnsV8RxHHzzVal9JRTbALiTotdTWzQIF9NzDmXUgJNBCIyUEQWi8gSEbm8gv2OExEVkbjZKtnq1LESFKtW2ZDSzZt34s2Ki21YqK8m5pxLUYElAhHJAe4ADgf2BfJFZN84+9UDLgTeDyqWHbHffnZl8MorVoJih0RHBr3yio8Mcs6lrCCvCHoBS1R1qar+CkwBBsfZ72/ATcD6AGPZIWefbetAXH65zTquko8+8pFBzrm0EGQiaAl8FfO4KLJtCxHpDrRW1ZcqeiMRGSEis0Vk9vLlyxMfabmfa6OIdt/d5n3FK2MS1/TpdiXgI4Occ2kgtM5iEakG3Axcsr19VXWyquapal7TJC8e0KiRzS9YutSK1G3XpEnWJ7DPPjZr0gvHOedSXJCJ4GugdczjVpFtUfWA/YA3RWQZcCAwLVU6jGMdfLDNK3j0UbvF5SODnHNpqlKJQEQuFJH6Yu4XkQ9F5PfbeVkh0EFE2olITWAYMC36pKquUtUmqtpWVdsC7wGDVDXYSQI76MorLSH85S/w2WdlnvSRQc65NFbZK4LTVXU18HtgV+AU4MaKXqCqJcB5wExgETBVVReIyDgRGbQTMYciWoKiRo0yJSh8ZJBzLs1VtvpodEmeI4BHIyf07S7To6ozgBlltsUt3qCqfSsZS2hat4YHHrCRRKNHw4STPrIlJYuLbWSQdwo759JQZa8I5ojIq1gimBkZ+78z06zS1tFHW/PQ4v+bTklvHxnknEt/lb0i+DPQFViqqmtFpBFwWnBhpbZb2k8ih5F8XNKdltOn0ayzdwo759JXZa8IfgssVtWfReRkYAywKriwUtSmTXDBBdS89EJ+6T+IAdXf5OS/Nt+5EhTOOReyyiaCu4C1ItIFG/f/OfBIYFGlouJiGDwYbrsNLrmE+jOf4YZb6/Laa9Y/7C0B9usAABUDSURBVJxz6aqyiaBErV71YOB2Vb0DmweQHcoZGXTmmTBkiA0t/eCDsIN0zrkdU9lEUCwiV2DDRl+KzAquEVxYKaSCmkEiMHmyraednw+rV4cYp3PO7aDKJoITgA3YfILvsFnC/wgsqlRRiZpBu+4KTz4JX3xhOSLN1vlxzrnKJYLIyf9xoIGIHAWsV9XM7iOoQs2g3r3hmmssITyS2UfFOZeBKltiYijwAXA8MBR4X0SGBBlYaCIjg6paM+iKK6BvXytM9+mngUfpnHMJU9mmoSuBnqp6qqr+CVtrYGxwYYWkzMigqtQMysmBxx6DWrWsZPWGDQHH6pxzCVLZRFBNVWNX711RhdemhwTUDGrZEh580PqXLy93YU7nnEstlZ1Z/IqIzASejDw+gTI1hNLaR4mrGTRokFWjnjgRDjsMjjwygXE651wARCs5zEVEjgP6RB6+o6rPBRZVBfLy8nT27ARWqn7xRWvLadTIkkACFpJZvx4OPBC+/hrmzbPhpc45FyYRmaOqcdd7qXTzjqo+q6oXR26hJIGEmzTJ+gQSvJpY7dowZQqsXQunnGL9z845l6oqTAQiUiwiq+PcikUkfadP7eDIoKrYe2/LM2+8AePHJ/StnXMuoSpMBKpaT1Xrx7nVU9X6yQpyp4wfDwUFpY/XrIE+fXZoZFBVnX46nHACjB0L770XyEc459xOy6yRP/H07AlDh1oyKCqCrl2tGeiiiwJfTUwE7rnHFrTJz4effw7so5xzbodlfiLo1w+mToVjj7W+gKVL4aab4JZbkvLxDRrYjOOvvvISFM651JT5iQAsGfTubc1CI0bAqFFJ/fgDD4S//Q2eesqWunTOuVSSHYmgoMDqRI8aBc8+u3WfQZJcdhn072991IsWJf3jnXOuXJmfCAoKrI9g6lRrEpo6tbTPIImqVbOCdHXq2LSF9euT+vHOOVeuzE8EhYV28u/Xzx5H+wwKC5MeSosW8PDD8PHHSW+dcs65clV6ZnGqSPjM4hCMHGklKF54waYxOOdc0BIys3gHP3igiCwWkSUisk0ZNhE5W0T+KyJzReRfIrJvkPGkihtvhG7d4LTTrAyFc86FKbBEICI5wB3A4cC+QH6cE/0TqtpZVbsC44Gbg4onldSqZSUoNmyAk07yEhTOuXAFeUXQC1iiqktV9VdgCjA4dgdVjS1TURdIr3aqnbDXXnD77fDWW/D3v4cdjXMumwWZCFoCX8U8Lops24qInCsin2NXBBfEeyMRGSEis0Vk9vLlywMJNgynngonnmjLXL77btjROOeyVeijhlT1DlX9DXAZMKacfSarap6q5jVt2jS5AQZIxNbA2WMPSwgrV4YdkXMuGwWZCL4GWsc8bhXZVp4pwNEBxpOS6te3EhTffANnnuklKJxzyRdkIigEOohIOxGpCQwDpsXuICIdYh4eCXwWYDwpq1cvuP56m/R8771hR+OcyzaBJQJVLQHOA2YCi4CpqrpARMaJSHT0/HkiskBE5gIXA6cGFU+qu/RSGDDAlkhYsCDsaJxz2cQnlKWQ776DLl2gWTMrjbTLLmFH5JzLFKFNKHNVs/vuVoJi/ny7QnDOuWTwRJBiBg60hdPuvBOey4yVoZ1zKc4TQQq64Qbo0QP+/Gdb0MY554LkiSAF1axpJSg2brQSFCUlYUfknMtknghS1J572mSzd96xoaXOORcUTwQp7OST4ZRTYNw4ePvtsKNxzmUqTwQp7o47oH17ayL66aewo3HOZSJPBCmuXj3rL/j+e+s8TrNpH865NOCJIA306GGlqp9/Hu6+O+xonHOZxhNBmhg50uYYjBwJ//1v2NE45zKJJ4I0Ua0aPPQQNGwIw4bB2rVhR+ScyxSeCNLIbrvBo4/CwoV2ZeCcc4ngiSDNDBgAo0bB5MnwzDNhR+OcywSeCNLQddfZGgZnnglffBF2NM65dOeJIA3VqGGrmm3a5CUonHM7zxNBmmrfHu65xxa9Hzcu7Gicc+nME0Eay8+H4cOtqejNN8OOxjmXrjwRpLnbboMOHayJ6Mcfw47GOZeOPBGkudxcK0Hx449w+ulegsI5V3WeCDJAt24wfjxMn25F6pxzrio8EWSICy6AI4+0tY7nzQs7GudcOvFEkCFE4MEHoVEjK0Hxyy9hR+ScSxeeCDJI06ZWgmLxYrjwwrCjcc6li0ATgYgMFJHFIrJERC6P8/zFIrJQRD4WkVkiskeQ8WSD/v3h8svh/vvhqafCjsY5lw4CSwQikgPcARwO7Avki8i+ZXb7CMhT1f2BZ4DxQcWTTa69Fg48EEaMgP/9L+xonHOpLsgrgl7AElVdqqq/AlOAwbE7qGqBqkYLKr8HtAownqxRowY88YTdP/FE2Lgx3Hicc6ktyETQEvgq5nFRZFt5/gy8HO8JERkhIrNFZPby5csTGGLmatfOKpS+9x5cfXXY0TjnUllKdBaLyMlAHvCPeM+r6mRVzVPVvKZNmyY3uDR2wgm2zvGNN8KsWWFH45xLVUEmgq+B1jGPW0W2bUVEDgOuBAap6oYA48lKt94KHTvCKaeAX0w55+IJMhEUAh1EpJ2I1ASGAdNidxCRbsA9WBL4IcBYslbdulaC4qefrECdl6BwzpUVWCJQ1RLgPGAmsAiYqqoLRGSciAyK7PYPIBd4WkTmisi0ct7O7YQuXWDCBJgxw64QnHMulmiafUXMy8vT2bNnhx1G2lGFo4+Gl1+2DuTu3cOOyDmXTCIyR1Xz4j2XEp3FLngi8MAD0KyZrWOwZk3YETnnUoUngizSuDE89hh89hmcf37Y0TjnUoUngizTty+MGQMPPVQ66cw5l908EWShq66C3r3h7LPh88/DjsY5FzZPBFmoenW7GsjJsf6CX38NOyLnXJg8EWSpPfaAe++FwkIYOzbsaJxzYfJEkMWGDLEKpePHw6uvhh2Ncy4sngiy3C23wL77wp/+BN9/H3Y0zrkweCLIcnXqWAmKVausBMXmzWFH5JxLNk8Ejs6d4eab4ZVX7ArBOZddPBE4wIaSHnMMXHEFeAUP57KLJwIHWAmK++6D3XazIaXFxWFH5JxLFk8EbotGjeCoo2yS2bnnlm4vKLCRRc65zOSJwG1l6FDYZRd49FG7FRTYtp49w47MOReU6mEH4FJLv37wwgtw+OE2iqhaNRg4EObPt+f32w98tVDnMosnAreNww6z6qS33AKtWsG778KLL5Y+36yZJYT99oNOnUp/NmgQXszOuR3nicBto6DAmoXGjoW77oJnnoF99rGrggUL7Of8+XD//fDLL6Wva9162wSxzz42V8E5l7o8EbitRPsEpk61ZqJ+/UofDxhgt6jNm+HLL0sTQzRJvPEGbNhg+4jAb36zbYLYay+oWTOc39E5tzVPBG4rhYWlSQDs59Sptj26LapaNWjb1m5HHVW6vaTERh6VTRDTp8OmTbZP9erQseO2CaJ9e6uK6pxLHl+z2CXNhg2wePG2CWLp0tJ9ate22kfRBBFNEq1b29WFc27HVLRmsV8RuKSpVQv2399usdasgUWLShPE/PkwaxY88kjpPvXrl141xN6aNUvu7+BcJvIrApeyVq7cunM6eluxonSfJk22TQ6dOkHDhuHF7Vwq8isCl5Z23RUOOshuUarwww/bJoeHHrIri6iWLbdNEPvsA3XrJv3XcC7lBZoIRGQgcCuQA9ynqjeWef4QYCKwPzBMVZ8JMh6X/kSsHtJuu0H//qXbVeGrr7ZNELffvvUIpnbttk0QHTv6CCaX3QJLBCKSA9wBDACKgEIRmaaqC2N2+xIYDlwaVBwuO4hAmzZ2O+KI0u2bNllndNkEMWOGjW4CG8HUocO2CeI3v/ERTC47BHlF0AtYoqpLAURkCjAY2JIIVHVZ5DlfDsUFIifHTvIdOliZ7ahff4VPP906OXz4oU2ei3ab1aplzUllE0SbNqUjmMaPtzpMsUNrCwpsuO2oUcn7PZ3bGUEmgpbAVzGPi4ADduSNRGQEMAKgTZs2Ox+Zy3o1a5ae2GOtXbvtCKY334THHivdp14965Du1MmSxQ032CzrY4+1faMT8JxLF2nRWayqk4HJYKOGQg7HZbA6daBHD7vF+vlnG8EUO4rphRfgxx/t+SFDLCmUlMCee8KNN8Ldd1uHd8OGW/8su61hQ2ueci4sQf73+xpoHfO4VWSbc2mnYUPo08dusaIjmMaPh5kzSye/rVwJX3xhP1euhI0bK37/3Nz4SWN7P3fd1ZKXT7ZzOyPIRFAIdBCRdlgCGAacGODnOZd0zZrZSXjOnNIifbfdtnWfgSqsW2dXFStXlv6MvV/25xdfwNy59nj16opjqF59x5KIX424qMD+C6hqiYicB8zEho8+oKoLRGQcMFtVp4lIT+A5YFfgjyJyrap2Ciom5xKtoiJ90WQgYt/a69SBFi2q/hklJbBqVflJI16CWbas9H50dFR56tXbsSSyo1cj3sGeegL9LqCqM4AZZbZdFXO/EGsyci4tVaVI346qXh0aN7ZbValaB3hVrkb+97/Sx9tbu7pGjdIri3j9H/F+tmgBxx8PU6bY2hexydSFw0tMOOfKVVJiCaEqVyOx27Z3NVKzpu3Trh3ssUdpMmnUaOufZbfVq2fVb13leYkJ59wOqV7d6jk1aVL116rawkXlJY1//hPefhs6d7a5GStXwsKF8NNPdv/XX8t/72rV7AqjooRRXkLxzvVteSJwzgVCxEZD5ebakqexCgrguutKO9hvvXXbDva1a0uvMqLJIfZ+2W2ff176eHMFU1Rr1qxcwoh3P1NLkXgicM4lVWU72OvWtVvZJLI9mzdb30a85BEvkXzzjQ0BXrly+yO06tatevJo1MjW897RciXJ6Fz3ROCcS6qgO9irVbMTb4MG1vdQFdE+kfKSR9lEsmRJ6f116yp+7wYNdqwpKy9v60QZROe6dxY751wCbNhQcdNVRfcrmnCYk2NXIr/8AoMHW79KbCKtLO8sds65gNWqBbvvbreqiHaqb+/q4+23rYN97NjEDU2O8kTgnHMhiu1Ub906/j4FBVYZN9q5Hu1bSRQfieuccykstk9g3Dj7OXSobU8UTwTOOZfCKupcTxTvLHbOuSxQUWexXxE451yW80TgnHNZzhOBc85lOU8EzjmX5TwROOdclku7UUMishz4Ygdf3gT4MYHhJIrHVTUeV9WlamweV9XsTFx7qGrTeE+kXSLYGSIyu7zhU2HyuKrG46q6VI3N46qaoOLypiHnnMtyngiccy7LZVsimBx2AOXwuKrG46q6VI3N46qaQOLKqj4C55xz28q2KwLnnHNleCJwzrksl5GJQEQGishiEVkiIpfHeb6WiDwVef59EWmbInENF5HlIjI3cjsjSXE9ICI/iMj8cp4XEZkUiftjEemeInH1FZFVMcfrqiTE1FpECkRkoYgsEJEL4+yT9ONVybjCOF61ReQDEZkXievaOPsk/e+xknGF8vcY+ewcEflIRF6M81zij5eqZtQNyAE+B9oDNYF5wL5l9vkLcHfk/jDgqRSJazhwewjH7BCgOzC/nOePAF4GBDgQeD9F4uoLvJjkY9Uc6B65Xw/4NM6/Y9KPVyXjCuN4CZAbuV8DeB84sMw+Yfw9ViauUP4eI599MfBEvH+vII5XJl4R9AKWqOpSVf0VmAIMLrPPYODhyP1ngP4iIikQVyhU9W3gpwp2GQw8ouY9oKGINE+BuJJOVb9V1Q8j94uBRUDLMrsl/XhVMq6kixyDNZGHNSK3siNUkv73WMm4QiEirYAjgfvK2SXhxysTE0FL4KuYx0Vs+wexZR9VLQFWAY1TIC6A4yLNCc+ISDkrmCZdZWMPw28jl/cvi0inZH5w5JK8G/ZtMlaox6uCuCCE4xVp5pgL/AC8pqrlHq8k/j1WJi4I5+9xIjAK2FzO8wk/XpmYCNLZdKCtqu4PvEZp1nfxfYjVT+kC3AY8n6wPFpFc4FngIlVdnazP3Z7txBXK8VLVTaraFWgF9BKR/ZLxudtTibiS/vcoIkcBP6jqnKA/K1YmJoKvgdjM3SqyLe4+IlIdaACsCDsuVV2hqhsiD+8DegQcU2VV5pgmnaqujl7eq+oMoIaINAn6c0WkBnayfVxV/xlnl1CO1/biCut4xXz+z0ABMLDMU2H8PW43rpD+HvsAg0RkGdZ8fKiIPFZmn4Qfr0xMBIVABxFpJyI1sc6UaWX2mQacGrk/BHhDIz0vYcZVph15ENbOmwqmAX+KjIY5EFilqt+GHZSI7B5tGxWRXtj/50BPIJHPux9YpKo3l7Nb0o9XZeIK6Xg1FZGGkfu7AAOAT8rslvS/x8rEFcbfo6peoaqtVLUtdo54Q1VPLrNbwo9X9Z15cSpS1RIROQ+YiY3UeUBVF4jIOGC2qk7D/mAeFZElWGfksBSJ6wIRGQSUROIaHnRcACLyJDaipImIFAFXY51nqOrdwAxsJMwSYC1wWorENQQ4R0RKgHXAsCQk9D7AKcB/I+3LAKOBNjFxhXG8KhNXGMerOfCwiORgiWeqqr4Y9t9jJeMK5e8xnqCPl5eYcM65LJeJTUPOOeeqwBOBc85lOU8EzjmX5TwROOdclvNE4JxzWc4TgXNJJFYBdJuKks6FyROBc85lOU8EzsUhIidH6tXPFZF7IgXK1ojILZH69bNEpGlk364i8l6kONlzIrJrZPueIvJ6pMjbhyLym8jb50aKmH0iIo8nofKtcxXyROBcGSKyD3AC0CdSlGwTcBJQF5vd2Ql4C5vpDPAIcFmkONl/Y7Y/DtwRKfLWG4iWmegGXATsi61P0SfwX8q5CmRciQnnEqA/VmCsMPJlfResVPFm4KnIPo8B/xSRBkBDVX0rsv1h4GkRqQe0VNXnAFR1PUDk/T5Q1aLI47lAW+Bfwf9azsXnicC5bQnwsKpesdVGkbFl9tvR+iwbYu5vwv8OXci8aci5bc0ChohIMwARaSQie2B/L0Mi+5wI/EtVVwErReTgyPZTgLciq4QVicjRkfeoJSJ1kvpbOFdJ/k3EuTJUdaGIjAFeFZFqwEbgXOAXbAGTMVhT0QmRl5wK3B050S+ltNroKcA9kcqRG4Hjk/hrOFdpXn3UuUoSkTWqmht2HM4lmjcNOedclvMrAuecy3J+ReCcc1nOE4FzzmU5TwTOOZflPBE451yW80TgnHNZ7v8BwN01l2v4GWgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light",
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot_losses(history[1:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "lsmpvyb7sb2-"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "bvp2Yi5Cjoa8"
   },
   "source": [
    "## GridSearch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "d0t6TKFRjo1F"
   },
   "outputs": [],
   "source": [
    "from scipy import stats\n",
    "from sklearn.model_selection import cross_validate, LeaveOneGroupOut, RandomizedSearchCV\n",
    "from skorch import NeuralNetClassifier\n",
    "from skorch.dataset import CVSplit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "uYrAEVCFUjTD"
   },
   "outputs": [],
   "source": [
    "class WesadLSTM(BaseModel):\n",
    "    def __init__(self, input_dim, hidden_dim, output_dim=3, lstm_layers=1, dropout=0.2):\n",
    "        super(WesadLSTM, self).__init__()\n",
    "        self.input_dim = input_dim\n",
    "        self.hidden_dim = hidden_dim\n",
    "        self.classes = output_dim\n",
    "        # self.linear1 = nn.Linear(input_dim, input_dim)\n",
    "        # self.lstm = nn.LSTM(input_dim, hidden_dim, num_layers=lstm_layers, batch_first=True, dropout=0.2)\n",
    "        self.lstm = nn.LSTM(input_size=input_dim, hidden_size=input_dim, num_layers=lstm_layers, dropout=dropout)\n",
    "        self.fc = nn.Linear(hidden_dim, output_dim)\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "\n",
    "    def forward(self, x):\n",
    "        xb = x.view(-1, 1, self.input_dim)\n",
    "        # print('x', xb.shape)\n",
    "        # out = self.linear1(x)\n",
    "        # out = x.view(x.shape, -1)\n",
    "        # print(out.shape)\n",
    "        # out = self.dropout(out)\n",
    "        # lstm_out, (ht, ct) = self.lstm(out)\n",
    "        # out = self.fc(ht[-1])\n",
    "        lstm_out, _ = self.lstm(xb)\n",
    "        # print(lstm_out.view(-1, self.hidden_dim).shape)\n",
    "        out = self.fc(lstm_out.view(-1, self.hidden_dim))\n",
    "        # out = self.dropout(out)\n",
    "        out = F.softmax(out.view(-1, self.classes), dim=1)\n",
    "        # out = F.softmax(out, dim=1)\n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "PxZZeIGuMmLp"
   },
   "outputs": [],
   "source": [
    "NUM_CV_STEPS = 10\n",
    "input_dim = df.drop(columns=['subject', 'label']).shape[1]\n",
    "output_dim = 3\n",
    "\n",
    "optimizer = torch.optim.Adam\n",
    "\n",
    "params = {\n",
    "    # 'net__module__input_dim': [input_dim],\n",
    "    # 'net__module__hidden_dim': [input_dim],\n",
    "    # 'net__module__output_dim': [output_dim],\n",
    "    # 'net__module__lstm_layers': [1, 2, 3, 4, 5],\n",
    "    # 'net__module__dropout': stats.uniform(0, 0.9),\n",
    "    # 'net__lr': [10**(-stats.uniform(1, 5).rvs()) for _ in range(NUM_CV_STEPS)],\n",
    "    # 'net__max_epochs': [5, 10]\n",
    "    'module__input_dim': [input_dim],\n",
    "    'module__hidden_dim': [input_dim],\n",
    "    'module__output_dim': [output_dim],\n",
    "    'module__lstm_layers': [2, 3, 4, 5],\n",
    "    'module__dropout': [0., 0.2, 0.5, 0.7],\n",
    "    # 'lr': [10**(-stats.uniform(1, 5).rvs()) for _ in range(NUM_CV_STEPS)],\n",
    "    'lr': [1e-3, 5e-3, 1e-4, 5e-4, 1e-5],\n",
    "    'max_epochs': [5, 10]\n",
    "}\n",
    "\n",
    "net = NeuralNetClassifier(module=WesadLSTM,\n",
    "                    train_split=None,\n",
    "                    iterator_train__batch_size=train_batch_size,\n",
    "                    iterator_valid__batch_size=val_batch_size,\n",
    "                    # max_epochs=epochs,\n",
    "                    # lr=lr,\n",
    "                    device=device,\n",
    "                    optimizer=optimizer,\n",
    "                    callbacks=[\n",
    "                              ('accuracy', skorch.callbacks.EpochScoring(scoring='accuracy', lower_is_better=False,\n",
    "                                                                         on_train=True))  \n",
    "                    ]\n",
    "                    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "-0VugtFCQljX"
   },
   "outputs": [],
   "source": [
    "search_cv = RandomizedSearchCV(net, params, n_iter=NUM_CV_STEPS, verbose=2, refit=False, scoring='accuracy', \n",
    "                               cv=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "Ez9TSieeQl77",
    "outputId": "1974a6fe-1b04-4055-8a4a-0a6735758406"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 2 folds for each of 10 candidates, totalling 20 fits\n",
      "[CV] module__output_dim=3, module__lstm_layers=4, module__input_dim=77, module__hidden_dim=77, module__dropout=0.0, max_epochs=10, lr=0.001 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  epoch    accuracy    train_loss      dur\n",
      "-------  ----------  ------------  -------\n",
      "      1      \u001b[36m0.7215\u001b[0m        \u001b[32m0.7908\u001b[0m  35.8401\n",
      "      2      0.6949        0.8260  35.7870\n",
      "      3      \u001b[36m0.7486\u001b[0m        \u001b[32m0.7280\u001b[0m  35.7331\n",
      "      4      \u001b[36m0.7829\u001b[0m        \u001b[32m0.6704\u001b[0m  35.8111\n",
      "      5      \u001b[36m0.7978\u001b[0m        \u001b[32m0.6454\u001b[0m  35.9507\n",
      "      6      \u001b[36m0.8131\u001b[0m        \u001b[32m0.6057\u001b[0m  36.0602\n",
      "      7      \u001b[36m0.8208\u001b[0m        \u001b[32m0.5783\u001b[0m  35.8568\n",
      "      8      \u001b[36m0.8255\u001b[0m        \u001b[32m0.5484\u001b[0m  36.0967\n",
      "      9      \u001b[36m0.8415\u001b[0m        \u001b[32m0.5012\u001b[0m  35.8221\n",
      "     10      0.7807        0.5978  36.1035\n",
      "[CV]  module__output_dim=3, module__lstm_layers=4, module__input_dim=77, module__hidden_dim=77, module__dropout=0.0, max_epochs=10, lr=0.001, total= 6.4min\n",
      "[CV] module__output_dim=3, module__lstm_layers=4, module__input_dim=77, module__hidden_dim=77, module__dropout=0.0, max_epochs=10, lr=0.001 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:  6.4min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  epoch    accuracy    train_loss      dur\n",
      "-------  ----------  ------------  -------\n",
      "      1      \u001b[36m0.7560\u001b[0m        \u001b[32m0.7513\u001b[0m  36.4822\n",
      "      2      \u001b[36m0.7675\u001b[0m        \u001b[32m0.7103\u001b[0m  36.5184\n",
      "      3      \u001b[36m0.8155\u001b[0m        \u001b[32m0.6107\u001b[0m  36.6670\n",
      "      4      0.7849        0.6693  36.6295\n",
      "      5      \u001b[36m0.8360\u001b[0m        \u001b[32m0.5401\u001b[0m  36.6003\n",
      "      6      0.8001        0.6445  36.7480\n",
      "      7      0.8217        0.5603  36.6189\n",
      "      8      \u001b[36m0.8548\u001b[0m        \u001b[32m0.4935\u001b[0m  36.6582\n",
      "      9      0.8404        \u001b[32m0.4730\u001b[0m  35.9626\n",
      "     10      0.8517        \u001b[32m0.4457\u001b[0m  35.8302\n",
      "[CV]  module__output_dim=3, module__lstm_layers=4, module__input_dim=77, module__hidden_dim=77, module__dropout=0.0, max_epochs=10, lr=0.001, total= 6.5min\n",
      "[CV] module__output_dim=3, module__lstm_layers=3, module__input_dim=77, module__hidden_dim=77, module__dropout=0.0, max_epochs=10, lr=0.0001 \n",
      "  epoch    accuracy    train_loss      dur\n",
      "-------  ----------  ------------  -------\n",
      "      1      \u001b[36m0.6706\u001b[0m        \u001b[32m0.7780\u001b[0m  27.7524\n",
      "      2      \u001b[36m0.7257\u001b[0m        \u001b[32m0.7099\u001b[0m  27.7379\n",
      "      3      \u001b[36m0.7502\u001b[0m        \u001b[32m0.6269\u001b[0m  27.7795\n",
      "      4      0.7357        0.6560  28.1195\n",
      "      5      0.6987        0.7067  28.3594\n",
      "      6      0.6906        0.7371  28.2456\n",
      "      7      0.7098        0.6956  28.3802\n",
      "      8      \u001b[36m0.7899\u001b[0m        \u001b[32m0.5730\u001b[0m  28.2946\n",
      "      9      \u001b[36m0.7950\u001b[0m        0.6077  28.2933\n",
      "     10      \u001b[36m0.8018\u001b[0m        \u001b[32m0.5556\u001b[0m  28.2654\n",
      "[CV]  module__output_dim=3, module__lstm_layers=3, module__input_dim=77, module__hidden_dim=77, module__dropout=0.0, max_epochs=10, lr=0.0001, total= 5.0min\n",
      "[CV] module__output_dim=3, module__lstm_layers=3, module__input_dim=77, module__hidden_dim=77, module__dropout=0.0, max_epochs=10, lr=0.0001 \n",
      "  epoch    accuracy    train_loss      dur\n",
      "-------  ----------  ------------  -------\n",
      "      1      \u001b[36m0.6600\u001b[0m        \u001b[32m0.8157\u001b[0m  28.3040\n",
      "      2      \u001b[36m0.7003\u001b[0m        \u001b[32m0.7255\u001b[0m  28.3173\n",
      "      3      \u001b[36m0.7243\u001b[0m        \u001b[32m0.6428\u001b[0m  28.2331\n",
      "      4      0.6740        0.7588  28.2522\n",
      "      5      0.7223        \u001b[32m0.6426\u001b[0m  28.4492\n",
      "      6      \u001b[36m0.7628\u001b[0m        \u001b[32m0.6105\u001b[0m  28.2651\n",
      "      7      0.7075        0.7498  28.3324\n",
      "      8      0.7064        0.7405  28.2151\n",
      "      9      0.7392        0.6942  28.1411\n",
      "     10      0.7301        0.7144  27.9438\n",
      "[CV]  module__output_dim=3, module__lstm_layers=3, module__input_dim=77, module__hidden_dim=77, module__dropout=0.0, max_epochs=10, lr=0.0001, total= 5.0min\n",
      "[CV] module__output_dim=3, module__lstm_layers=3, module__input_dim=77, module__hidden_dim=77, module__dropout=0.7, max_epochs=5, lr=1e-05 \n",
      "  epoch    accuracy    train_loss      dur\n",
      "-------  ----------  ------------  -------\n",
      "      1      \u001b[36m0.4461\u001b[0m        \u001b[32m1.0542\u001b[0m  37.7694\n",
      "      2      \u001b[36m0.5436\u001b[0m        \u001b[32m0.9836\u001b[0m  37.6517\n",
      "      3      \u001b[36m0.6643\u001b[0m        \u001b[32m0.8510\u001b[0m  37.3865\n",
      "      4      \u001b[36m0.7560\u001b[0m        \u001b[32m0.7190\u001b[0m  37.5119\n",
      "      5      \u001b[36m0.7730\u001b[0m        \u001b[32m0.6620\u001b[0m  37.5884\n",
      "[CV]  module__output_dim=3, module__lstm_layers=3, module__input_dim=77, module__hidden_dim=77, module__dropout=0.7, max_epochs=5, lr=1e-05, total= 3.5min\n",
      "[CV] module__output_dim=3, module__lstm_layers=3, module__input_dim=77, module__hidden_dim=77, module__dropout=0.7, max_epochs=5, lr=1e-05 \n",
      "  epoch    accuracy    train_loss      dur\n",
      "-------  ----------  ------------  -------\n",
      "      1      \u001b[36m0.4700\u001b[0m        \u001b[32m1.0526\u001b[0m  37.5623\n",
      "      2      \u001b[36m0.5371\u001b[0m        \u001b[32m1.0042\u001b[0m  37.5464\n",
      "      3      \u001b[36m0.5382\u001b[0m        \u001b[32m0.9955\u001b[0m  37.3638\n",
      "      4      \u001b[36m0.5389\u001b[0m        \u001b[32m0.9789\u001b[0m  37.5275\n",
      "      5      \u001b[36m0.6044\u001b[0m        \u001b[32m0.8840\u001b[0m  37.6425\n",
      "[CV]  module__output_dim=3, module__lstm_layers=3, module__input_dim=77, module__hidden_dim=77, module__dropout=0.7, max_epochs=5, lr=1e-05, total= 3.4min\n",
      "[CV] module__output_dim=3, module__lstm_layers=5, module__input_dim=77, module__hidden_dim=77, module__dropout=0.2, max_epochs=10, lr=1e-05 \n",
      "  epoch    accuracy    train_loss      dur\n",
      "-------  ----------  ------------  -------\n",
      "      1      \u001b[36m0.4890\u001b[0m        \u001b[32m1.0557\u001b[0m  72.9313\n",
      "      2      \u001b[36m0.5488\u001b[0m        \u001b[32m1.0047\u001b[0m  73.1283\n",
      "      3      \u001b[36m0.5506\u001b[0m        \u001b[32m0.9234\u001b[0m  73.1509\n",
      "      4      \u001b[36m0.6766\u001b[0m        \u001b[32m0.8161\u001b[0m  73.2566\n",
      "      5      \u001b[36m0.7435\u001b[0m        \u001b[32m0.7254\u001b[0m  73.3895\n",
      "      6      \u001b[36m0.7722\u001b[0m        \u001b[32m0.6858\u001b[0m  73.1358\n",
      "      7      \u001b[36m0.7834\u001b[0m        \u001b[32m0.6674\u001b[0m  73.2326\n",
      "      8      \u001b[36m0.7925\u001b[0m        \u001b[32m0.6474\u001b[0m  73.2648\n",
      "      9      \u001b[36m0.7946\u001b[0m        \u001b[32m0.6419\u001b[0m  73.0830\n",
      "     10      \u001b[36m0.7985\u001b[0m        \u001b[32m0.6314\u001b[0m  73.1610\n",
      "[CV]  module__output_dim=3, module__lstm_layers=5, module__input_dim=77, module__hidden_dim=77, module__dropout=0.2, max_epochs=10, lr=1e-05, total=12.6min\n",
      "[CV] module__output_dim=3, module__lstm_layers=5, module__input_dim=77, module__hidden_dim=77, module__dropout=0.2, max_epochs=10, lr=1e-05 \n",
      "  epoch    accuracy    train_loss      dur\n",
      "-------  ----------  ------------  -------\n",
      "      1      \u001b[36m0.4892\u001b[0m        \u001b[32m1.0505\u001b[0m  73.2278\n",
      "      2      \u001b[36m0.5488\u001b[0m        \u001b[32m1.0104\u001b[0m  73.0681\n",
      "      3      0.5488        \u001b[32m1.0018\u001b[0m  72.9076\n",
      "      4      0.5488        \u001b[32m0.9910\u001b[0m  72.7939\n",
      "      5      0.5484        \u001b[32m0.9227\u001b[0m  72.8259\n",
      "      6      \u001b[36m0.6714\u001b[0m        \u001b[32m0.8133\u001b[0m  72.9152\n",
      "      7      \u001b[36m0.7107\u001b[0m        \u001b[32m0.7655\u001b[0m  73.0554\n",
      "      8      \u001b[36m0.7384\u001b[0m        \u001b[32m0.7153\u001b[0m  72.9428\n",
      "      9      \u001b[36m0.7469\u001b[0m        \u001b[32m0.6986\u001b[0m  72.8477\n",
      "     10      \u001b[36m0.7536\u001b[0m        \u001b[32m0.6857\u001b[0m  72.8959\n",
      "[CV]  module__output_dim=3, module__lstm_layers=5, module__input_dim=77, module__hidden_dim=77, module__dropout=0.2, max_epochs=10, lr=1e-05, total=12.6min\n",
      "[CV] module__output_dim=3, module__lstm_layers=4, module__input_dim=77, module__hidden_dim=77, module__dropout=0.0, max_epochs=5, lr=0.0001 \n",
      "  epoch    accuracy    train_loss      dur\n",
      "-------  ----------  ------------  -------\n",
      "      1      \u001b[36m0.6573\u001b[0m        \u001b[32m0.8022\u001b[0m  36.2681\n",
      "      2      \u001b[36m0.7012\u001b[0m        \u001b[32m0.7249\u001b[0m  36.4183\n",
      "      3      \u001b[36m0.7399\u001b[0m        \u001b[32m0.6702\u001b[0m  36.2841\n",
      "      4      0.7064        0.7101  36.3204\n",
      "      5      0.6447        0.8354  36.2230\n",
      "[CV]  module__output_dim=3, module__lstm_layers=4, module__input_dim=77, module__hidden_dim=77, module__dropout=0.0, max_epochs=5, lr=0.0001, total= 3.4min\n",
      "[CV] module__output_dim=3, module__lstm_layers=4, module__input_dim=77, module__hidden_dim=77, module__dropout=0.0, max_epochs=5, lr=0.0001 \n",
      "  epoch    accuracy    train_loss      dur\n",
      "-------  ----------  ------------  -------\n",
      "      1      \u001b[36m0.6617\u001b[0m        \u001b[32m0.7876\u001b[0m  36.2909\n",
      "      2      \u001b[36m0.7026\u001b[0m        \u001b[32m0.7278\u001b[0m  36.2304\n",
      "      3      \u001b[36m0.7180\u001b[0m        \u001b[32m0.6808\u001b[0m  36.2916\n",
      "      4      0.6673        0.7537  36.3242\n",
      "      5      0.5997        0.9195  36.4547\n",
      "[CV]  module__output_dim=3, module__lstm_layers=4, module__input_dim=77, module__hidden_dim=77, module__dropout=0.0, max_epochs=5, lr=0.0001, total= 3.4min\n",
      "[CV] module__output_dim=3, module__lstm_layers=4, module__input_dim=77, module__hidden_dim=77, module__dropout=0.0, max_epochs=5, lr=0.005 \n",
      "  epoch    accuracy    train_loss      dur\n",
      "-------  ----------  ------------  -------\n",
      "      1      \u001b[36m0.8907\u001b[0m        \u001b[32m0.5306\u001b[0m  36.3583\n",
      "      2      \u001b[36m0.9076\u001b[0m        \u001b[32m0.4615\u001b[0m  36.3136\n",
      "      3      0.9058        \u001b[32m0.4501\u001b[0m  36.2634\n",
      "      4      \u001b[36m0.9110\u001b[0m        \u001b[32m0.4493\u001b[0m  36.2351\n",
      "      5      0.9035        0.4836  36.1840\n",
      "[CV]  module__output_dim=3, module__lstm_layers=4, module__input_dim=77, module__hidden_dim=77, module__dropout=0.0, max_epochs=5, lr=0.005, total= 3.4min\n",
      "[CV] module__output_dim=3, module__lstm_layers=4, module__input_dim=77, module__hidden_dim=77, module__dropout=0.0, max_epochs=5, lr=0.005 \n",
      "  epoch    accuracy    train_loss      dur\n",
      "-------  ----------  ------------  -------\n",
      "      1      \u001b[36m0.8889\u001b[0m        \u001b[32m0.5282\u001b[0m  36.2509\n",
      "      2      0.8887        \u001b[32m0.5188\u001b[0m  36.2819\n",
      "      3      0.8606        0.5684  36.3404\n",
      "      4      0.8731        0.5728  36.2799\n",
      "      5      \u001b[36m0.9006\u001b[0m        0.5192  36.1428\n",
      "[CV]  module__output_dim=3, module__lstm_layers=4, module__input_dim=77, module__hidden_dim=77, module__dropout=0.0, max_epochs=5, lr=0.005, total= 3.4min\n",
      "[CV] module__output_dim=3, module__lstm_layers=5, module__input_dim=77, module__hidden_dim=77, module__dropout=0.0, max_epochs=10, lr=0.0005 \n",
      "  epoch    accuracy    train_loss      dur\n",
      "-------  ----------  ------------  -------\n",
      "      1      \u001b[36m0.7430\u001b[0m        \u001b[32m0.7436\u001b[0m  44.8497\n",
      "      2      \u001b[36m0.7524\u001b[0m        \u001b[32m0.6661\u001b[0m  44.7709\n",
      "      3      0.7288        \u001b[32m0.6159\u001b[0m  44.7506\n",
      "      4      0.6963        0.7287  44.8502\n",
      "      5      0.7216        0.7039  44.9586\n",
      "      6      0.6846        0.7643  44.7701\n",
      "      7      0.7462        0.6359  44.7595\n",
      "      8      0.7056        0.6851  44.7905\n",
      "      9      0.6531        0.8091  44.8168\n",
      "     10      0.5799        0.9065  44.8436\n",
      "[CV]  module__output_dim=3, module__lstm_layers=5, module__input_dim=77, module__hidden_dim=77, module__dropout=0.0, max_epochs=10, lr=0.0005, total= 7.9min\n",
      "[CV] module__output_dim=3, module__lstm_layers=5, module__input_dim=77, module__hidden_dim=77, module__dropout=0.0, max_epochs=10, lr=0.0005 \n",
      "  epoch    accuracy    train_loss      dur\n",
      "-------  ----------  ------------  -------\n",
      "      1      \u001b[36m0.7612\u001b[0m        \u001b[32m0.7092\u001b[0m  44.8165\n",
      "      2      0.7311        \u001b[32m0.6504\u001b[0m  44.5959\n",
      "      3      0.7337        0.6543  44.6550\n",
      "      4      0.6414        0.8181  44.7810\n",
      "      5      0.7213        0.6717  44.8022\n",
      "      6      0.5791        0.9143  44.8080\n",
      "      7      0.7056        0.7120  44.8147\n",
      "      8      0.6591        0.7870  44.8487\n",
      "      9      0.5869        0.9457  44.7049\n",
      "     10      \u001b[36m0.8036\u001b[0m        \u001b[32m0.5552\u001b[0m  44.8234\n",
      "[CV]  module__output_dim=3, module__lstm_layers=5, module__input_dim=77, module__hidden_dim=77, module__dropout=0.0, max_epochs=10, lr=0.0005, total= 7.9min\n",
      "[CV] module__output_dim=3, module__lstm_layers=4, module__input_dim=77, module__hidden_dim=77, module__dropout=0.5, max_epochs=5, lr=0.0005 \n",
      "  epoch    accuracy    train_loss      dur\n",
      "-------  ----------  ------------  -------\n",
      "      1      \u001b[36m0.7236\u001b[0m        \u001b[32m0.7717\u001b[0m  53.7450\n",
      "      2      0.7219        \u001b[32m0.7338\u001b[0m  54.1532\n",
      "      3      0.6787        0.7522  54.0692\n",
      "      4      0.5949        0.9315  54.0258\n",
      "      5      0.6724        0.8390  54.2038\n",
      "[CV]  module__output_dim=3, module__lstm_layers=4, module__input_dim=77, module__hidden_dim=77, module__dropout=0.5, max_epochs=5, lr=0.0005, total= 4.8min\n",
      "[CV] module__output_dim=3, module__lstm_layers=4, module__input_dim=77, module__hidden_dim=77, module__dropout=0.5, max_epochs=5, lr=0.0005 \n",
      "  epoch    accuracy    train_loss      dur\n",
      "-------  ----------  ------------  -------\n",
      "      1      \u001b[36m0.7364\u001b[0m        \u001b[32m0.7448\u001b[0m  54.2590\n",
      "      2      0.6910        \u001b[32m0.7015\u001b[0m  54.2317\n",
      "      3      0.5509        0.9921  54.1903\n",
      "      4      0.6043        0.9251  54.3504\n",
      "      5      0.6283        0.9062  54.3862\n",
      "[CV]  module__output_dim=3, module__lstm_layers=4, module__input_dim=77, module__hidden_dim=77, module__dropout=0.5, max_epochs=5, lr=0.0005, total= 4.9min\n",
      "[CV] module__output_dim=3, module__lstm_layers=5, module__input_dim=77, module__hidden_dim=77, module__dropout=0.7, max_epochs=10, lr=0.001 \n",
      "  epoch    accuracy    train_loss      dur\n",
      "-------  ----------  ------------  -------\n",
      "      1      \u001b[36m0.7499\u001b[0m        \u001b[32m0.7504\u001b[0m  74.1137\n",
      "      2      \u001b[36m0.7512\u001b[0m        \u001b[32m0.7444\u001b[0m  73.6907\n",
      "      3      0.7483        0.7454  73.9592\n",
      "      4      \u001b[36m0.7769\u001b[0m        \u001b[32m0.7032\u001b[0m  73.9648\n",
      "      5      0.7473        0.7500  73.8826\n",
      "      6      0.7588        0.7288  73.6867\n",
      "      7      0.7282        0.7863  73.8846\n",
      "      8      0.7352        0.7681  73.8521\n",
      "      9      0.7403        0.7680  73.8242\n",
      "     10      0.7293        0.7816  73.9298\n",
      "[CV]  module__output_dim=3, module__lstm_layers=5, module__input_dim=77, module__hidden_dim=77, module__dropout=0.7, max_epochs=10, lr=0.001, total=12.7min\n",
      "[CV] module__output_dim=3, module__lstm_layers=5, module__input_dim=77, module__hidden_dim=77, module__dropout=0.7, max_epochs=10, lr=0.001 \n",
      "  epoch    accuracy    train_loss      dur\n",
      "-------  ----------  ------------  -------\n",
      "      1      \u001b[36m0.7426\u001b[0m        \u001b[32m0.7799\u001b[0m  74.2671\n",
      "      2      0.6780        0.8528  74.1960\n",
      "      3      0.7362        \u001b[32m0.7644\u001b[0m  74.1711\n",
      "      4      \u001b[36m0.7688\u001b[0m        \u001b[32m0.7175\u001b[0m  74.3752\n",
      "      5      \u001b[36m0.7721\u001b[0m        \u001b[32m0.7079\u001b[0m  74.6580\n",
      "      6      0.7559        0.7470  74.2504\n",
      "      7      0.7335        0.7773  74.3824\n",
      "      8      0.7352        0.7758  74.7545\n",
      "      9      0.7034        0.7969  74.7843\n",
      "     10      0.6891        0.8260  74.9516\n",
      "[CV]  module__output_dim=3, module__lstm_layers=5, module__input_dim=77, module__hidden_dim=77, module__dropout=0.7, max_epochs=10, lr=0.001, total=12.8min\n",
      "[CV] module__output_dim=3, module__lstm_layers=2, module__input_dim=77, module__hidden_dim=77, module__dropout=0.7, max_epochs=10, lr=0.001 \n",
      "  epoch    accuracy    train_loss      dur\n",
      "-------  ----------  ------------  -------\n",
      "      1      \u001b[36m0.7215\u001b[0m        \u001b[32m0.7966\u001b[0m  23.3395\n",
      "      2      \u001b[36m0.7284\u001b[0m        \u001b[32m0.7908\u001b[0m  23.3015\n",
      "      3      \u001b[36m0.7600\u001b[0m        \u001b[32m0.7337\u001b[0m  23.3442\n",
      "      4      \u001b[36m0.7808\u001b[0m        \u001b[32m0.6812\u001b[0m  23.2974\n",
      "      5      \u001b[36m0.7829\u001b[0m        \u001b[32m0.6442\u001b[0m  23.2741\n",
      "      6      0.7712        0.6526  23.3125\n",
      "      7      \u001b[36m0.7899\u001b[0m        \u001b[32m0.6016\u001b[0m  23.3757\n",
      "      8      0.7593        0.6632  23.3686\n",
      "      9      0.7800        0.6368  23.3271\n",
      "     10      \u001b[36m0.8331\u001b[0m        \u001b[32m0.5286\u001b[0m  23.4704\n",
      "[CV]  module__output_dim=3, module__lstm_layers=2, module__input_dim=77, module__hidden_dim=77, module__dropout=0.7, max_epochs=10, lr=0.001, total= 4.2min\n",
      "[CV] module__output_dim=3, module__lstm_layers=2, module__input_dim=77, module__hidden_dim=77, module__dropout=0.7, max_epochs=10, lr=0.001 \n",
      "  epoch    accuracy    train_loss      dur\n",
      "-------  ----------  ------------  -------\n",
      "      1      \u001b[36m0.7783\u001b[0m        \u001b[32m0.7145\u001b[0m  23.3917\n",
      "      2      0.7527        0.7365  23.3628\n",
      "      3      0.7459        0.7644  23.3357\n",
      "      4      0.7311        0.7703  23.3014\n",
      "      5      0.7458        0.7486  23.2844\n",
      "      6      0.7709        \u001b[32m0.6726\u001b[0m  23.2682\n",
      "      7      \u001b[36m0.7933\u001b[0m        \u001b[32m0.6101\u001b[0m  23.3166\n",
      "      8      \u001b[36m0.8119\u001b[0m        \u001b[32m0.5721\u001b[0m  23.3495\n",
      "      9      0.7955        0.5921  23.2691\n",
      "     10      \u001b[36m0.8144\u001b[0m        \u001b[32m0.5580\u001b[0m  23.2729\n",
      "[CV]  module__output_dim=3, module__lstm_layers=2, module__input_dim=77, module__hidden_dim=77, module__dropout=0.7, max_epochs=10, lr=0.001, total= 4.2min\n",
      "CPU times: user 1h 47min 39s, sys: 20min 2s, total: 2h 7min 42s\n",
      "Wall time: 2h 7min 59s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done  20 out of  20 | elapsed: 128.0min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "RandomizedSearchCV(cv=2, error_score=nan,\n",
       "                   estimator=<class 'skorch.classifier.NeuralNetClassifier'>[uninitialized](\n",
       "  module=<class '__main__.WesadLSTM'>,\n",
       "),\n",
       "                   iid='deprecated', n_iter=10, n_jobs=None,\n",
       "                   param_distributions={'lr': [0.001, 0.005, 0.0001, 0.0005,\n",
       "                                               1e-05],\n",
       "                                        'max_epochs': [5, 10],\n",
       "                                        'module__dropout': [0.0, 0.2, 0.5, 0.7],\n",
       "                                        'module__hidden_dim': [77],\n",
       "                                        'module__input_dim': [77],\n",
       "                                        'module__lstm_layers': [2, 3, 4, 5],\n",
       "                                        'module__output_dim': [3]},\n",
       "                   pre_dispatch='2*n_jobs', random_state=None, refit=False,\n",
       "                   return_train_score=False, scoring='accuracy', verbose=2)"
      ]
     },
     "execution_count": 31,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "normalizer = MinMaxScaler()\n",
    "# data = df[df['subject'] != 'S3'].copy()\n",
    "data = df.copy()\n",
    "X = normalizer.fit_transform(data.drop(columns=['subject','label']).copy().astype(np.float32))\n",
    "y = data['label'].copy()\n",
    "%time search_cv.fit(X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "8BT2guko1AqC"
   },
   "source": [
    "### Cross-validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "BOqVUnxj0fG4"
   },
   "outputs": [],
   "source": [
    "def run_cv(clf, X, y, groups, cv, scoring=['accuracy', 'f1_macro'], return_train_score=True,\n",
    "          return_estimator=True, n_jobs=-1):\n",
    "    \"\"\"\n",
    "        More on cross validation: https://scikit-learn.org/stable/modules/cross_validation.html#\n",
    "        More on scoring: https://scikit-learn.org/stable/modules/model_evaluation.html#scoring-parameter\n",
    "    \"\"\"\n",
    "    return cross_validate(clf, X, y, groups=groups, cv=cv, scoring=scoring, return_train_score=return_train_score,\n",
    "                          return_estimator=return_estimator, n_jobs=n_jobs)\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 85
    },
    "colab_type": "code",
    "id": "IrxC44i6-bEH",
    "outputId": "963cebee-326e-45ea-93ad-137845a2f662"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data.shape: torch.Size([25, 77])\n",
      "out[0] tensor([0.3756, 0.3181, 0.3063], grad_fn=<SelectBackward>)\n",
      "softmax(out[0]): tensor([0.3476, 0.3281, 0.3243], grad_fn=<SelectBackward>)\n",
      "tensor(0)\n"
     ]
    }
   ],
   "source": [
    "# input_dim = df.drop(columns=['subject', 'label']).shape[1]\n",
    "# output_dim = 3\n",
    "# lstm_layers = 5\n",
    "# train_dl, _ = get_data_loader('S3', train_batch_size=25, val_batch_size=5)\n",
    "# model = WesadLSTM(input_dim, input_dim, output_dim, lstm_layers)\n",
    "# try_batch(model, train_dl)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Zh4iFCBXcXug"
   },
   "outputs": [],
   "source": [
    "# def get_losso_cv(ds, y, **fit_params):\n",
    "#     print('X', type(ds.X))\n",
    "#     print('y', type(y))\n",
    "#     print('params', fit_params)\n",
    "#     X = ds.X\n",
    "#     for subject in fit_params['subjects']:\n",
    "#         print('LOSO', subject)\n",
    "#         train = X[X['subject'] != subject].reset_index(drop=True)#.copy()\n",
    "#         train_ds = skorch.dataset.Dataset(train.drop(columns=['subject','label']).copy(), train['label'].copy())\n",
    "#         val = X[X['subject'] == subject].reset_index(drop=True)#.copy()\n",
    "#         val_ds = skorch.dataset.Dataset(val.drop(columns=['subject','label']).copy(), val['label'].copy())\n",
    "\n",
    "#         yield train_ds, val_ds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "O4BgnnIdj4-u"
   },
   "outputs": [],
   "source": [
    "optimizer = torch.optim.Adam\n",
    "loso_cv = LeaveOneGroupOut()\n",
    "epochs = 5\n",
    "lr = 1e-4\n",
    "\n",
    "subjects = df['subject'].unique()\n",
    "subjects.sort()\n",
    "device = get_default_device()\n",
    "\n",
    "train_batch_size = 25\n",
    "val_batch_size = 5\n",
    "input_dim = df.drop(columns=['subject', 'label']).shape[1]\n",
    "output_dim = 3\n",
    "lstm_layers = 4\n",
    "dropout = 0.\n",
    "\n",
    "net = NeuralNetClassifier(module=WesadLSTM,\n",
    "                    module__input_dim=input_dim,\n",
    "                    module__hidden_dim=input_dim,\n",
    "                    module__output_dim=output_dim,\n",
    "                    module__lstm_layers=lstm_layers,\n",
    "                    module__dropout=dropout,\n",
    "                    # criterion=criterion, # using NLL_Loss so model need to apply softmax manually\n",
    "                    # train_split=CVSplit(cv=get_loso_cv(subjects=subjects, to_device=False)),\n",
    "                    # train_split=get_losso_cv,\n",
    "                    train_split=None,\n",
    "                    iterator_train__batch_size=train_batch_size,\n",
    "                    iterator_valid__batch_size=val_batch_size,\n",
    "                    max_epochs=epochs,\n",
    "                    lr=lr,\n",
    "                    device=device,\n",
    "                    optimizer=optimizer,\n",
    "                    callbacks=[\n",
    "                            #    ('progress_bar', skorch.callbacks.ProgressBar()),\n",
    "                               ('f1_macro', skorch.callbacks.EpochScoring(scoring='f1_macro', lower_is_better=False,\n",
    "                                                                         on_train=True)),\n",
    "                               ('accuracy', skorch.callbacks.EpochScoring(scoring='accuracy', lower_is_better=False,\n",
    "                                                                         on_train=True)),\n",
    "                               ('accuracy_val', skorch.callbacks.EpochScoring(scoring='accuracy', lower_is_better=False,\n",
    "                                                                         on_train=False)),\n",
    "                            #    ('checkpoint', skorch.callbacks.Checkpoint(dirname='lstm_model_rs')),\n",
    "                            #    ('lr_sched', skorch.callbacks.LRScheduler()),\n",
    "\n",
    "                    ]\n",
    "                    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Ix4F0g54VnxF"
   },
   "outputs": [],
   "source": [
    "normalizer = MinMaxScaler()\n",
    "X = normalizer.fit_transform(df.drop(columns=['subject','label']).copy().astype(np.float32))\n",
    "y = df['label'].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "RIzkBN0nS70y",
    "outputId": "85ad7fa8-2846-4e1e-fb83-1d3631a9885d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  epoch    accuracy    f1_macro    train_loss      dur\n",
      "-------  ----------  ----------  ------------  -------\n",
      "      1      \u001b[36m0.6881\u001b[0m      \u001b[32m0.5802\u001b[0m        \u001b[35m0.7634\u001b[0m  66.0398\n",
      "      2      \u001b[36m0.7453\u001b[0m      \u001b[32m0.6450\u001b[0m        \u001b[35m0.6183\u001b[0m  65.9664\n",
      "      3      0.7189      0.6281        0.6803  65.9524\n",
      "      4      0.6606      0.5428        0.7768  65.9011\n",
      "      5      0.6465      0.5143        0.8053  66.0517\n",
      "  epoch    accuracy    f1_macro    train_loss      dur\n",
      "-------  ----------  ----------  ------------  -------\n",
      "      1      \u001b[36m0.6827\u001b[0m      \u001b[32m0.5826\u001b[0m        \u001b[35m0.7588\u001b[0m  66.0745\n",
      "      2      \u001b[36m0.7392\u001b[0m      \u001b[32m0.6307\u001b[0m        \u001b[35m0.6295\u001b[0m  66.2088\n",
      "      3      0.6890      0.5826        0.7414  66.5275\n",
      "      4      0.6489      0.5302        0.8071  66.6150\n",
      "      5      0.6341      0.5047        0.8095  66.6128\n",
      "  epoch    accuracy    f1_macro    train_loss      dur\n",
      "-------  ----------  ----------  ------------  -------\n",
      "      1      \u001b[36m0.6918\u001b[0m      \u001b[32m0.5863\u001b[0m        \u001b[35m0.7393\u001b[0m  66.6015\n",
      "      2      \u001b[36m0.7224\u001b[0m      \u001b[32m0.6146\u001b[0m        \u001b[35m0.6688\u001b[0m  66.5212\n",
      "      3      0.6505      0.5203        0.8299  66.7784\n",
      "      4      0.6466      0.5097        0.8173  66.5110\n",
      "      5      0.6251      0.4626        0.8431  66.5576\n",
      "  epoch    accuracy    f1_macro    train_loss      dur\n",
      "-------  ----------  ----------  ------------  -------\n",
      "      1      \u001b[36m0.7012\u001b[0m      \u001b[32m0.5930\u001b[0m        \u001b[35m0.7426\u001b[0m  66.3890\n",
      "      2      \u001b[36m0.7318\u001b[0m      \u001b[32m0.6116\u001b[0m        \u001b[35m0.6514\u001b[0m  66.4918\n",
      "      3      0.6748      0.5492        0.7469  66.5323\n",
      "      4      0.6285      0.4820        0.8298  66.8445\n",
      "      5      0.5917      0.4142        0.8904  66.4193\n",
      "  epoch    accuracy    f1_macro    train_loss      dur\n",
      "-------  ----------  ----------  ------------  -------\n",
      "      1      \u001b[36m0.6759\u001b[0m      \u001b[32m0.5704\u001b[0m        \u001b[35m0.7600\u001b[0m  66.5079\n",
      "      2      \u001b[36m0.7292\u001b[0m      \u001b[32m0.6024\u001b[0m        \u001b[35m0.6562\u001b[0m  66.3049\n",
      "      3      0.7279      \u001b[32m0.6319\u001b[0m        0.6706  66.2772\n",
      "      4      0.6836      0.5822        0.7375  66.5133\n",
      "      5      0.6341      0.5047        0.8272  66.3279\n",
      "  epoch    accuracy    f1_macro    train_loss      dur\n",
      "-------  ----------  ----------  ------------  -------\n",
      "      1      \u001b[36m0.7255\u001b[0m      \u001b[32m0.6381\u001b[0m        \u001b[35m0.6994\u001b[0m  66.9775\n",
      "      2      \u001b[36m0.7568\u001b[0m      \u001b[32m0.6744\u001b[0m        \u001b[35m0.6325\u001b[0m  67.2907\n",
      "      3      0.6863      0.5847        0.7281  66.6514\n",
      "      4      0.6593      0.5401        0.7804  66.8406\n",
      "      5      0.6292      0.4796        0.8384  66.8812\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "ignored",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_scorer.py\u001b[0m in \u001b[0;36m_cached_call\u001b[0;34m(cache, estimator, method, *args, **kwargs)\u001b[0m\n\u001b[1;32m     54\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 55\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mcache\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mmethod\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     56\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: 'predict'",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-51-1913ce4fe5d1>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# net.fit(X, y, subjects=subjects)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mcv_net\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrun_cv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnet\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgroups\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'subject'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcv\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mLeaveOneGroupOut\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_jobs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-32-f76642911c29>\u001b[0m in \u001b[0;36mrun_cv\u001b[0;34m(clf, X, y, groups, cv, scoring, return_train_score, return_estimator, n_jobs)\u001b[0m\n\u001b[1;32m      6\u001b[0m     \"\"\"\n\u001b[1;32m      7\u001b[0m     return cross_validate(clf, X, y, groups=groups, cv=cv, scoring=scoring, return_train_score=return_train_score,\n\u001b[0;32m----> 8\u001b[0;31m                           return_estimator=return_estimator, n_jobs=n_jobs)\n\u001b[0m\u001b[1;32m      9\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_validation.py\u001b[0m in \u001b[0;36mcross_validate\u001b[0;34m(estimator, X, y, groups, scoring, cv, n_jobs, verbose, fit_params, pre_dispatch, return_train_score, return_estimator, error_score)\u001b[0m\n\u001b[1;32m    234\u001b[0m             \u001b[0mreturn_times\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreturn_estimator\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mreturn_estimator\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    235\u001b[0m             error_score=error_score)\n\u001b[0;32m--> 236\u001b[0;31m         for train, test in cv.split(X, y, groups))\n\u001b[0m\u001b[1;32m    237\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    238\u001b[0m     \u001b[0mzipped_scores\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mscores\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/joblib/parallel.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, iterable)\u001b[0m\n\u001b[1;32m   1030\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_iterating\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_original_iterator\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1031\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1032\u001b[0;31m             \u001b[0;32mwhile\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdispatch_one_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1033\u001b[0m                 \u001b[0;32mpass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1034\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/joblib/parallel.py\u001b[0m in \u001b[0;36mdispatch_one_batch\u001b[0;34m(self, iterator)\u001b[0m\n\u001b[1;32m    845\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    846\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 847\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dispatch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtasks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    848\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    849\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/joblib/parallel.py\u001b[0m in \u001b[0;36m_dispatch\u001b[0;34m(self, batch)\u001b[0m\n\u001b[1;32m    763\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    764\u001b[0m             \u001b[0mjob_idx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jobs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 765\u001b[0;31m             \u001b[0mjob\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply_async\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallback\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    766\u001b[0m             \u001b[0;31m# A job can complete so quickly than its callback is\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    767\u001b[0m             \u001b[0;31m# called before we get here, causing self._jobs to\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/joblib/_parallel_backends.py\u001b[0m in \u001b[0;36mapply_async\u001b[0;34m(self, func, callback)\u001b[0m\n\u001b[1;32m    206\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mapply_async\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallback\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    207\u001b[0m         \u001b[0;34m\"\"\"Schedule a func to be run\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 208\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mImmediateResult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    209\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mcallback\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    210\u001b[0m             \u001b[0mcallback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/joblib/_parallel_backends.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, batch)\u001b[0m\n\u001b[1;32m    570\u001b[0m         \u001b[0;31m# Don't delay the application, to avoid keeping the input\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    571\u001b[0m         \u001b[0;31m# arguments in memory\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 572\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    573\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    574\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/joblib/parallel.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    251\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mparallel_backend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_jobs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_n_jobs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    252\u001b[0m             return [func(*args, **kwargs)\n\u001b[0;32m--> 253\u001b[0;31m                     for func, args, kwargs in self.items]\n\u001b[0m\u001b[1;32m    254\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    255\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__reduce__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/joblib/parallel.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m    251\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mparallel_backend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_jobs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_n_jobs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    252\u001b[0m             return [func(*args, **kwargs)\n\u001b[0;32m--> 253\u001b[0;31m                     for func, args, kwargs in self.items]\n\u001b[0m\u001b[1;32m    254\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    255\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__reduce__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_validation.py\u001b[0m in \u001b[0;36m_fit_and_score\u001b[0;34m(estimator, X, y, scorer, train, test, verbose, parameters, fit_params, return_train_score, return_parameters, return_n_test_samples, return_times, return_estimator, error_score)\u001b[0m\n\u001b[1;32m    545\u001b[0m         \u001b[0mscore_time\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mstart_time\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mfit_time\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    546\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mreturn_train_score\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 547\u001b[0;31m             \u001b[0mtrain_scores\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_score\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mestimator\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mscorer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    548\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mverbose\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    549\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest_scores\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_validation.py\u001b[0m in \u001b[0;36m_score\u001b[0;34m(estimator, X_test, y_test, scorer)\u001b[0m\n\u001b[1;32m    589\u001b[0m         \u001b[0mscores\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mscorer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mestimator\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    590\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 591\u001b[0;31m         \u001b[0mscores\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mscorer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mestimator\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    592\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    593\u001b[0m     error_msg = (\"scoring must return a number, got %s (%s) \"\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_scorer.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, estimator, *args, **kwargs)\u001b[0m\n\u001b[1;32m     85\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mscorer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_BaseScorer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     86\u001b[0m                 score = scorer._score(cached_call, estimator,\n\u001b[0;32m---> 87\u001b[0;31m                                       *args, **kwargs)\n\u001b[0m\u001b[1;32m     88\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     89\u001b[0m                 \u001b[0mscore\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mscorer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mestimator\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_scorer.py\u001b[0m in \u001b[0;36m_score\u001b[0;34m(self, method_caller, estimator, X, y_true, sample_weight)\u001b[0m\n\u001b[1;32m    203\u001b[0m         \"\"\"\n\u001b[1;32m    204\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 205\u001b[0;31m         \u001b[0my_pred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmethod_caller\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mestimator\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"predict\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    206\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0msample_weight\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    207\u001b[0m             return self._sign * self._score_func(y_true, y_pred,\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_scorer.py\u001b[0m in \u001b[0;36m_cached_call\u001b[0;34m(cache, estimator, method, *args, **kwargs)\u001b[0m\n\u001b[1;32m     55\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mcache\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mmethod\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     56\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 57\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mestimator\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     58\u001b[0m         \u001b[0mcache\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mmethod\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     59\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/skorch/classifier.py\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(self, X)\u001b[0m\n\u001b[1;32m    208\u001b[0m         \"\"\"\n\u001b[1;32m    209\u001b[0m         \u001b[0my_preds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 210\u001b[0;31m         \u001b[0;32mfor\u001b[0m \u001b[0myp\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward_iter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtraining\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    211\u001b[0m             \u001b[0myp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0myp\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0myp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0myp\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    212\u001b[0m             \u001b[0my_preds\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mto_numpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0myp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/skorch/net.py\u001b[0m in \u001b[0;36mforward_iter\u001b[0;34m(self, X, training, device)\u001b[0m\n\u001b[1;32m    915\u001b[0m         \u001b[0mdataset\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_dataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    916\u001b[0m         \u001b[0miterator\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_iterator\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtraining\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtraining\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 917\u001b[0;31m         \u001b[0;32mfor\u001b[0m \u001b[0mdata\u001b[0m \u001b[0;32min\u001b[0m \u001b[0miterator\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    918\u001b[0m             \u001b[0mXi\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0munpack_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    919\u001b[0m             \u001b[0myp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mevaluation_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mXi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtraining\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtraining\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    343\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    344\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__next__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 345\u001b[0;31m         \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_next_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    346\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_yielded\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    347\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dataset_kind\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0m_DatasetKind\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mIterable\u001b[0m \u001b[0;32mand\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m_next_data\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    383\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_next_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    384\u001b[0m         \u001b[0mindex\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_next_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# may raise StopIteration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 385\u001b[0;31m         \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dataset_fetcher\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# may raise StopIteration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    386\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_pin_memory\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    387\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_utils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpin_memory\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpin_memory\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/utils/data/_utils/fetch.py\u001b[0m in \u001b[0;36mfetch\u001b[0;34m(self, possibly_batched_index)\u001b[0m\n\u001b[1;32m     42\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mfetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     43\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mauto_collation\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 44\u001b[0;31m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0midx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0midx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     45\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     46\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/utils/data/_utils/fetch.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m     42\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mfetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     43\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mauto_collation\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 44\u001b[0;31m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0midx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0midx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     45\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     46\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/skorch/dataset.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, i)\u001b[0m\n\u001b[1;32m    207\u001b[0m         \u001b[0mXi\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmulti_indexing\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mX_indexing\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    208\u001b[0m         \u001b[0myi\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmulti_indexing\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0my_indexing\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 209\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mXi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0myi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    210\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    211\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/skorch/dataset.py\u001b[0m in \u001b[0;36mtransform\u001b[0;34m(self, X, y)\u001b[0m\n\u001b[1;32m    192\u001b[0m         \u001b[0;31m# anyway, which results in a dummy ``y`` value for each row in\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    193\u001b[0m         \u001b[0;31m# the batch.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 194\u001b[0;31m         \u001b[0my\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0my\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    195\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    196\u001b[0m         \u001b[0;31m# pytorch cannot convert sparse matrices, for now just make it\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# net.fit(X, y, subjects=subjects)\n",
    "cv_net = run_cv(net, X, y, groups=df['subject'], cv=LeaveOneGroupOut(), n_jobs=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "uvYslDaAjmh5"
   },
   "source": [
    "## TODOs:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "VZ5hbzj0O8AJ"
   },
   "source": [
    "TODOs\n",
    "\n",
    "* Use [scorch](https://github.com/skorch-dev/skorch) for hyperparameter tuning\n",
    "* Apply SMOTE oversampling technique to tackle class imbalance and naturally adapt it as a data augmentation\n",
    "* a model combining a CNN to automatically extracts local features and RNN to extract temporal dependencies\n",
    "* apply a scalogram to the entire signal in order to get an equivalent image dataset\n",
    "    * *The scalogram is basically a visible time-frequency illustration of a signal using wavelet transform.*\n",
    "* Experiment with fusion schemes to apply different signals for Affect Recognition\n",
    "\n",
    "Related papers:\n",
    "\n",
    "* Psychological Stress Detection Using Deep Convolutional Neural Networks\n",
    "* An Explainable Deep Fusion Network for Affect Recognition Using Physiological Signals.pdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "2HHMPQMgPCUH"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "authorship_tag": "ABX9TyPjOSbXrN6Z1a/pDSHBR4ki",
   "collapsed_sections": [
    "tMur3N7FIBS0"
   ],
   "include_colab_link": true,
   "name": "LSTM Model w-gridsearch - Chest device.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "073d47da916f411ca281934f2765d067": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "088b20767d6e4d71b51412890969dc76": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "0a134d32013545eda0bd5d5dcfc36707": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "0c51ac7d534c4a9f82d0f16d32c269f1": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "0d9defd150074322bfc6b8a527fb6e16": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "15f9d1856c5d4ca79070fccec70aa5fc": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "175dc9ca4cf941b3a42566b44a0d4f99": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "210889d450ce4292af0cc8ddf4d46eae": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "2e5127384bce43d9afffcdbaa164206d": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "345ac6be53ec48ba867d0a4132bf9f8e": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "395517cb7770477cadfd082dfa542a66": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_15f9d1856c5d4ca79070fccec70aa5fc",
      "placeholder": "​",
      "style": "IPY_MODEL_9acbca14de074bd89da12c2f1fbdf6cf",
      "value": " 4537/4537 [17:33&lt;00:00,  4.31it/s]"
     }
    },
    "3b9cc8d292c6406a9bbd1c6d8d4c41c8": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "43540220a99d43fa9434c6641c983779": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "4457224890204d60b868f1ea8f4f2dc8": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "506e4435462a4d4292a719819a0c8e17": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "51b233b4756a47579d9253d9446ad465": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "69571c1e37ba428fb85c5da858eb3640": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_e5562350386343909e87db299389e7ef",
      "placeholder": "​",
      "style": "IPY_MODEL_e5ac081f09ba4ec18b09dcc6eb2e37d2",
      "value": " 608/1000 [00:40&lt;00:15, 24.77it/s]"
     }
    },
    "6b1d165de1c14f72a289fc9e863492d7": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "6cb1f020061a471ba03d5eec27d2d35a": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_51b233b4756a47579d9253d9446ad465",
      "max": 4537,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_6b1d165de1c14f72a289fc9e863492d7",
      "value": 4537
     }
    },
    "6d6d9f0b2a6e48de83eaef9caa9ebd02": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "74cdddbd2f004d7886894e4297474708": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_210889d450ce4292af0cc8ddf4d46eae",
      "placeholder": "​",
      "style": "IPY_MODEL_91317de663ce44999df501c8b0d75a47",
      "value": " 4537/4537 [08:22&lt;00:00,  9.04it/s]"
     }
    },
    "7d051de8c8eb4470b64145f58dfdca55": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_0c51ac7d534c4a9f82d0f16d32c269f1",
      "max": 4537,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_2e5127384bce43d9afffcdbaa164206d",
      "value": 4537
     }
    },
    "7df734534de34697af24898a2f195b9e": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_d6468069d66d43fa961fdcf24b10f10e",
       "IPY_MODEL_892e164f1d744c1cb546e04cc85164fa"
      ],
      "layout": "IPY_MODEL_f16ad6145b1943fc997f67d408c50fca"
     }
    },
    "7e7dbf922e5d4286a5e53cbfdd84c523": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "7e89d00dac1343e182dc0c50ef5ffa39": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_175dc9ca4cf941b3a42566b44a0d4f99",
      "max": 4537,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_6d6d9f0b2a6e48de83eaef9caa9ebd02",
      "value": 4537
     }
    },
    "80c2efd5d3794a448fd0c22448d325ee": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "danger",
      "description": " 61%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_b5e8f1950fc640488d94e3109b0a237d",
      "max": 1000,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_8ab0c7d18e824e76839d7e2df636c410",
      "value": 608
     }
    },
    "892e164f1d744c1cb546e04cc85164fa": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_e0e86e49b779404897aab3b4ae85edfd",
      "placeholder": "​",
      "style": "IPY_MODEL_f60febd9a8c643be8d9cef045cce4828",
      "value": " 4537/4537 [22:11&lt;00:00,  3.41it/s]"
     }
    },
    "8ab0c7d18e824e76839d7e2df636c410": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "91317de663ce44999df501c8b0d75a47": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "9395e8ff5b7e43edabadf3ff766ea1d8": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_3b9cc8d292c6406a9bbd1c6d8d4c41c8",
      "placeholder": "​",
      "style": "IPY_MODEL_bdea7d80b440433cbc09943fb26d83e3",
      "value": " 4537/4537 [12:58&lt;00:00,  5.83it/s]"
     }
    },
    "9acbca14de074bd89da12c2f1fbdf6cf": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "ad112087ff3b4cc583a60a610891b1de": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_6cb1f020061a471ba03d5eec27d2d35a",
       "IPY_MODEL_b955083d78f2426db949d80f09ee20f2"
      ],
      "layout": "IPY_MODEL_506e4435462a4d4292a719819a0c8e17"
     }
    },
    "b5e8f1950fc640488d94e3109b0a237d": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "b955083d78f2426db949d80f09ee20f2": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_43540220a99d43fa9434c6641c983779",
      "placeholder": "​",
      "style": "IPY_MODEL_073d47da916f411ca281934f2765d067",
      "value": " 4537/4537 [04:26&lt;00:00, 17.01it/s]"
     }
    },
    "bdea7d80b440433cbc09943fb26d83e3": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "c5e77a4b28b84512a40b76302b22717d": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_fb06ea6f38d74306afbcc22d4cb4adcf",
      "max": 4537,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_7e7dbf922e5d4286a5e53cbfdd84c523",
      "value": 4537
     }
    },
    "d07c1f93fe1349ec94f5a173060c8da9": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "d344f2b329634552ad3577c978f2dd52": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_c5e77a4b28b84512a40b76302b22717d",
       "IPY_MODEL_9395e8ff5b7e43edabadf3ff766ea1d8"
      ],
      "layout": "IPY_MODEL_345ac6be53ec48ba867d0a4132bf9f8e"
     }
    },
    "d6468069d66d43fa961fdcf24b10f10e": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_0a134d32013545eda0bd5d5dcfc36707",
      "max": 4537,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_088b20767d6e4d71b51412890969dc76",
      "value": 4537
     }
    },
    "d819f1d1baea4290aec116d61e358adf": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_7d051de8c8eb4470b64145f58dfdca55",
       "IPY_MODEL_395517cb7770477cadfd082dfa542a66"
      ],
      "layout": "IPY_MODEL_4457224890204d60b868f1ea8f4f2dc8"
     }
    },
    "de18359779d24b8aa02ede2c144b53fe": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_80c2efd5d3794a448fd0c22448d325ee",
       "IPY_MODEL_69571c1e37ba428fb85c5da858eb3640"
      ],
      "layout": "IPY_MODEL_d07c1f93fe1349ec94f5a173060c8da9"
     }
    },
    "df4136bbe3d7481cb7c2ae651d1cd0d7": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_7e89d00dac1343e182dc0c50ef5ffa39",
       "IPY_MODEL_74cdddbd2f004d7886894e4297474708"
      ],
      "layout": "IPY_MODEL_0d9defd150074322bfc6b8a527fb6e16"
     }
    },
    "e0e86e49b779404897aab3b4ae85edfd": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "e5562350386343909e87db299389e7ef": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "e5ac081f09ba4ec18b09dcc6eb2e37d2": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "f16ad6145b1943fc997f67d408c50fca": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "f60febd9a8c643be8d9cef045cce4828": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "fb06ea6f38d74306afbcc22d4cb4adcf": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}