mdlm_test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: JAX_PLATFORMS=cpu\n"
     ]
    }
   ],
   "source": [
    "%env JAX_PLATFORMS=cpu\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "import jax\n",
    "import jax.numpy as jnp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiMAAAGdCAYAAADAAnMpAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/TGe4hAAAACXBIWXMAAA9hAAAPYQGoP6dpAABC6UlEQVR4nO3dd1wUd/4/8NfsLrsLSFOkKYrYKygqdS+NiwkGNbEba2IvmHC5RBNLuiZniig2jCXFHjUqnCkkJiCoEcSKFRUsgKiwFGFhd35/3O+4L4lGFoHZ8no+HvPHjTO7r/2cD/eVee/OCqIoiiAiIiKSiEzqAERERGTdWEaIiIhIUiwjREREJCmWESIiIpIUywgRERFJimWEiIiIJMUyQkRERJJiGSEiIiJJKaQOUBsGgwE3btyAg4MDBEGQOg4RERHVgiiKKC4uhpeXF2SyB1//MIsycuPGDXh7e0sdg4iIiOogJycHLVu2fOCfm0UZcXBwAPCfF+Po6ChxGiIiIqoNrVYLb2/v6vfxBzGLMvLf0YyjoyPLCBERkZl52Ecs+AFWIiIikhTLCBEREUmKZYSIiIgkxTJCREREkmIZISIiIkmxjBAREZGkWEaIiIhIUiwjREREJCmWESIiIpKU0WXkt99+Q2RkJLy8vCAIAnbv3v3Qcw4cOIBevXpBpVKhXbt22LBhQx2iEhERkSUyuoyUlpbCz88PsbGxtTr+8uXL6N+/P5544glkZGTglVdewcSJE/H9998bHZaIiIgsj9G/TfPss8/i2WefrfXxq1atQps2bfDJJ58AADp37ozk5GR89tln6Nevn7FPT0RERBamwT8zkpqaivDw8Br7+vXrh9TU1AeeU1FRAa1WW2NrCN+fzkXU5mMoLq9skMcnIiKih2vwMpKbmwt3d/ca+9zd3aHVanHv3r37nrNo0SI4OTlVb97e3vWe655Ojzd3nsSe4zcQuSwZp64X1ftzEBER0cOZ5Ldp5s6di6KiouotJyen3p/DVinHmrEB8HJS48rtMrywIgVfpV6BKIr1/lxERET0YA1eRjw8PJCXl1djX15eHhwdHWFra3vfc1QqFRwdHWtsDSGgdVPER2kQ3tkNOr0B8787jRmb0qHl2IaIiKjRNHgZCQ4ORmJiYo19P/74I4KDgxv6qWvFxV6JuLG9Ma9/Z9jIBSSczEX/mCQczymUOhoREZFVMLqMlJSUICMjAxkZGQD+89XdjIwMZGdnA/jPiGXs2LHVx0+dOhVZWVl4/fXXcfbsWaxYsQLbtm3Dq6++Wj+voB4IgoCJGl9snxqCli62yLlzD0NWpWBd8mWObYiIiBqY0WXk6NGj6NmzJ3r27AkAiI6ORs+ePbFgwQIAwM2bN6uLCQC0adMG8fHx+PHHH+Hn54dPPvkEa9euNcmv9fp7OyM+SoNnunqgUi/i3X1nMPmrNBSVcWxDRETUUATRDP7TX6vVwsnJCUVFRQ32+ZH/SxRFfJl6FR/EZ0KnN6CFsy2WjeqJXq1cGvy5iYiILEVt379N8ts0UhMEAeNCfLBzeghaN7PD9cJ7GLYqFWt+uwSDweS7GxERkVlhGfkL3Vo4Yd+sMDzXwxNVBhEfJpzFxC+P4m6pTupoREREFoNl5CEc1DZYNrInPni+G5QKGX4+m4+ImCT8fuWO1NGIiIgsAstILQiCgBcDW2P39FD4utrjZlE5Rqw5hNhfLnJsQ0RE9IhYRozQxcsRe2aFYZC/F/QGEf/6/hzGrT+CgpIKqaMRERGZLZYRIzVRKfDZcH98PLgH1DYyJF0oQMTSJKReui11NCIiIrPEMlIHgiBgWB9vfDcjDO3cmiC/uAIvrj2EpT9dgJ5jGyIiIqOwjDyCjh4O2DMzFEMCWsIgAp/9dB5jvjiM/OJyqaMRERGZDZaRR2SnVGDJUD98MtQPtjZypFy6jYilyUi+UCB1NCIiIrPAMlJPBge0xN5ZYejk4YCCkgqMWXcYn/xwDlV6g9TRiIiITBrLSD1q59YEu2eEYmTfVhBFYNnPFzFq7WHkFnFsQ0RE9CAsI/VMbSPHohe6I2ZkT9gr5Thy+Q4iYpJw4Fy+1NGIiIhMEstIAxng54V9URp08XTEnVIdxq//HYv/fRaVHNsQERHVwDLSgNq42mPn9BCMDW4NAFj16yWMWHMI1wvvSZyMiIjIdLCMNDC1jRzvDuyGFS/2goNKgbSrd9E/Jgk/ncmTOhoREZFJYBlpJBHdPREfpUGPlk4oLKvExC+P4v19Z6Cr4tiGiIisG8tII2rVzA7bpwbjpdA2AIC1yZcxdHUqcu6USZyMiIhIOiwjjUylkGNBZBesGRMAR7UCx3MKERGThP2ncqWORkREJAmWEYk83dUDCbM16NnKGcXlVZj6dRre3nMaFVV6qaMRERE1KpYRCbV0scO2KcGY8jdfAMCGlCsYvDIFV2+XSpyMiIio8bCMSMxGLsPciM5YN743XOxscOq6Fs/FJCP+xE2poxERETUKlhET8WQndyTM1qCPjwuKK6owY1M65u0+ifJKjm2IiMiysYyYEE8nW2yeFITpj7cFAHx9KBvPr0hB1q0SiZMRERE1HJYRE6OQy/D6M52w8aW+aGavROZNLSKXJeO7jOtSRyMiImoQLCMm6rEOzZEwW4Mg36Yo1ekxe0sG3thxAvd0HNsQEZFlYRkxYe6OanwzMQhRT7WHIABbj+ZgUOxBXMwvljoaERFRvWEZMXFymYDov3fA1y8HwrWJCufyihG57CC2H82ROhoREVG9YBkxE6HtXJEwOwyh7ZrhXqUe/9xxAtHbMlBaUSV1NCIiokfCMmJG3BzU+PKlQPzj7x0gE4Cd6dcxYHkyzuZqpY5GRERUZywjZkYuEzDrqfbYNCkI7o4qXLpVioHLD2LLkWyIoih1PCIiIqOxjJipIN9mSIjS4LEOzVFRZcCcnScxe0sGSji2ISIiM8MyYsaaNVFh/fg+eOOZTpDLBOw5fgORy5Jx+kaR1NGIiIhqjWXEzMlkAqY93hbbpgTBy0mNywWleH5FCr46dJVjGyIiMgssIxYioHVTxEdpEN7ZDboqA+bvPoWZm45BW14pdTQiIqK/xDJiQVzslYgb2xvz+neGQiYg/uRNPBeTjBPXCqWORkRE9EAsIxZGEARM1Phix7QQtHSxRfadMgxemYL1By9zbENERCaJZcRC+Xs7Iz5Kg35d3VGpF/HO3jOY8lUaiso4tiEiItPCMmLBnGxtsGp0AN4Z0BVKuQw/nMlDREwSjmXflToaERFRNZYRCycIAsaF+ODbaSFo3cwO1wvvYeiqVMT9lgWDgWMbIiKSHsuIleje0gl7Z4Whfw9PVBlEfJCQiYlfHsXdUp3U0YiIyMqxjFgRR7UNlo/sifcHdYNSIcPPZ/MREZOE36/ckToaERFZMZYRKyMIAkYHtcbu6aHwdbXHzaJyjFhzCLG/XOTYhoiIJMEyYqW6eDliz6wwDPL3gt4g4l/fn8P4Db+joKRC6mhERGRlWEasWBOVAp8N98dHg7tDbSPDb+dvIWJpElIv3ZY6GhERWRGWESsnCAKG92mF72aEoZ1bE+QXV+DFtYew9KcL0HNsQ0REjYBlhAAAHT0csGdmKIYEtIRBBD776TzGrjuM/OJyqaMREZGFYxmhanZKBZYM9cMnQ/1gayPHwYu3EbE0GckXCqSORkREFoxlhP5kcEBL7J0Vio7uDigoqcCYdYfx6Q/nUKU3SB2NiIgsEMsI3Vc7Nwd8NzMUI/p4QxSBmJ8vYtTaw8jTcmxDRET1i2WEHkhtI8fiwT2wdIQ/7JVyHLl8B88uTcKBc/lSRyMiIgvCMkIPNdC/BfbOCkNnT0fcKdVh/Prf8dH+sxzbEBFRvWAZoVrxbd4Eu6aHYHRQKwDAygOXMGLNIdwovCdxMiIiMncsI1Rrahs53h/UHbGjesFBpcDRq3cREZOExMw8qaMREZEZYxkho/Xv4Yl9UWHo3sIJhWWVeHnjUXwQfwa6Ko5tiIjIeCwjVCetm9ljx7RgTAj1AQDEJV3GsNWpyLlTJm0wIiIyOywjVGcqhRwLI7ti9ZgAOKoVyMgpRP+YJHx/OlfqaEREZEZYRuiR9evqgYTZGvh7O0NbXoUpX6Xh7T2nUVGllzoaERGZgTqVkdjYWPj4+ECtViMwMBBHjhz5y+M///xzdOzYEba2tvD29sarr76K8nLePMuStHSxw/apwZj8N18AwIaUKxiyMhVXb5dKnIyIiEyd0WVk69atiI6OxsKFC5Geng4/Pz/069cP+fn3vxHWpk2bMGfOHCxcuBCZmZn44osvsHXrVrz55puPHJ5Mi41chjcjOmPd+N5wtrPByetFeC4mGfEnbkodjYiITJggiqJRvxMfGBiIPn36YPny5QAAg8EAb29vzJo1C3PmzPnT8TNnzkRmZiYSExOr9/3jH//A4cOHkZycXKvn1Gq1cHJyQlFRERwdHY2JSxK5UXgPUZuP4ejVuwCA0UGtMK9/F6ht5BInIyKixlLb92+jrozodDqkpaUhPDz8fw8gkyE8PBypqan3PSckJARpaWnVo5ysrCwkJCQgIiLigc9TUVEBrVZbYyPz4uVsiy2TgzD98bYAgK8PZeP5FSm4dKtE4mRERGRqjCojBQUF0Ov1cHd3r7Hf3d0dubn3/wbFqFGj8O677yIsLAw2NjZo27YtHn/88b8c0yxatAhOTk7Vm7e3tzExyUQo5DK8/kwnbHypL5rZK5F5U4vIZcnYfey61NGIiMiENPi3aQ4cOIAPP/wQK1asQHp6Onbu3In4+Hi89957Dzxn7ty5KCoqqt5ycnIaOiY1oMc6NEfCbA0C2zRFmU6PV7Zm4I0dJ3BPx2/bEBERoDDmYFdXV8jlcuTl1bz9d15eHjw8PO57zvz58zFmzBhMnDgRANC9e3eUlpZi8uTJeOuttyCT/bkPqVQqqFQqY6KRiXN3VOObiYGI+fkilv18AVuP5iAjpxCxL/ZEOzcHqeMREZGEjLoyolQqERAQUOPDqAaDAYmJiQgODr7vOWVlZX8qHHL5fz7EaORnZ8nMKeQyRP+9A75+ORCuTVQ4l1eMyGUHsSPtmtTRiIhIQkaPaaKjoxEXF4eNGzciMzMT06ZNQ2lpKSZMmAAAGDt2LObOnVt9fGRkJFauXIktW7bg8uXL+PHHHzF//nxERkZWlxKyLqHtXJEwOwyh7ZrhXqUer20/juhtGSjTVUkdjYiIJGDUmAYAhg8fjlu3bmHBggXIzc2Fv78/9u/fX/2h1uzs7BpXQubNmwdBEDBv3jxcv34dzZs3R2RkJD744IP6exVkdtwc1PjypUDE/nIRn/90HjvTr+N4TiFiX+yFTh78+jYRkTUx+j4jUuB9RizboazbmL3lGPK0FVApZHhnQFcM7+MNQRCkjkZERI+gQe4zQtQQgnybISFKg791aI6KKgPm7DyJV7ZmoKSCYxsiImvAMkImoVkTFTaM74PXn+kIuUzAdxk3MGBZMk7fKJI6GhERNTCWETIZMpmA6Y+3w9bJQfB0UiOroBTPr0jBV4eu8ptXREQWjGWETE5vn6ZIiNLgqU5u0FUZMH/3KczcfAza8kqpoxERUQNgGSGT5GKvxNpxvfFWRGcoZALiT9zEczHJOHmNYxsiIkvDMkImSxAETPqbL7ZNDUYLZ1tk3ynD4JUp2HDwMsc2REQWhGWETF6vVi5IiNLg6S7u0OkNeHvvGUz9Og1FZRzbEBFZApYRMgtOdjZYPSYACyO7wEYu4PvTeei/LAnHsu9KHY2IiB4RywiZDUEQMCG0Db6dFoJWTe1w7e49DF2VirVJWRzbEBGZMZYRMjs9WjpjX1QY+nf3RJVBxPvxmZi48SjuluqkjkZERHXAMkJmyVFtg+WjeuL9Qd2gVMiQeDYf/WOSkHb1jtTRiIjISCwjZLYEQcDooNbYNT0EbVztcaOoHMNWH8LKA5dgMHBsQ0RkLlhGyOx19XLC3llhGOjvBb1BxEf7z2LCht9xu6RC6mhERFQLLCNkEZqoFPh8uD8+GtwdKoUMv56/hYiYJBzOui11NCIiegiWEbIYgiBgeJ9W2DMzDG2b2yNPW4GRcYewLPEC9BzbEBGZLJYRsjgdPRywd1YYBvdqCYMIfPLjeYxddxj5xeVSRyMiovtgGSGLZKdU4JNhflgy1A+2NnIcvHgbEUuTcfBigdTRiIjoD1hGyKINCWiJvbNC0dHdAQUlFRj9xWF8+sM5jm2IiEwIywhZvHZuDtg9IxQj+nhDFIGYny9iVNwh5Gk5tiEiMgUsI2QVbJVyLB7cA0tH+MNeKcfhy3fw7NIk/Hr+ltTRiIisHssIWZWB/i2wd1YYOns64k6pDuPWHcFH+8+iSm+QOhoRkdViGSGr49u8CXZND8HooFYAgJUHLmHEmkO4UXhP4mRERNaJZYSsktpGjvcHdcfyUT3RRKXA0at3ERGThJ/P5kkdjYjI6rCMkFV7rocX4qPC0L2FEwrLKvHShqP4MCETlRzbEBE1GpYRsnqtm9ljx7RgjA/xAQCs+S0Lw1an4trdMmmDERFZCZYRIgAqhRxvD+iKVaMD4KhW4Fh2ISKWJuGH07lSRyMisngsI0T/xzPdPBAfpYGftzO05VWY/FUa3tl7Groqjm2IiBoKywjRH3g3tcP2KcGYpGkDAFh/8AqGrEpB9m2ObYiIGgLLCNF9KBUyvNW/C74Y1xvOdjY4ca0I/WOSkHDyptTRiIgsDssI0V94qrM7EqI06N3aBcUVVZj+TTrm7z6F8kq91NGIiCwGywjRQ3g522Lz5CBMe7wtAOCrQ1cxeGUKLheUSpyMiMgysIwQ1YKNXIY3numEDRP6oKm9EqdvaPFcTBL2HL8hdTQiIrPHMkJkhMc7uuHfszUIbNMUpTo9ojYfw9ydJzm2ISJ6BCwjREZyd1Tjm4mBiHqyHQQB2HwkG4NiD+JifonU0YiIzBLLCFEdKOQyRD/dEV+/HAjXJiqczS3GgOXJ2Jl+TepoRERmh2WE6BGEtnNFwuwwhLZrhjKdHtHbjuO17cdRpquSOhoRkdlgGSF6RG4Oanz5UiCi/94BMgHYkXYNA5YfxLncYqmjERGZBZYRonoglwmIeqo9Nk0KgpuDChfzSzAwNhlbf8+GKIpSxyMiMmksI0T1KMi3GRJma/C3Ds1RXmnAG9+exKtbM1BSwbENEdGDsIwQ1TPXJipsGN8Hrz/TEXKZgN0ZNzBgWTLO3NBKHY2IyCSxjBA1AJlMwPTH22HL5CB4OqmRVVCKQSsO4utDVzm2ISL6A5YRogbUx6cpEqI0eLKTG3RVBszbfQozNx9DcXml1NGIiEwGywhRA3OxV2Lt2N54K6IzFDIB8Sdu4rllyTh1vUjqaEREJoFlhKgRyGQCJv3NF9umBqOFsy2u3i7DCytSsDHlCsc2RGT1WEaIGlGvVi5IiNLg6S7u0OkNWLjnNKZ9nY6iexzbEJH1YhkhamROdjZYPSYACyO7wEYuYP/pXPSPSUJGTqHU0YiIJMEyQiQBQRAwIbQNvp0WglZN7XDt7j0MXZWCtUlZHNsQkdVhGSGSUI+WztgXFYaI7h6o1It4Pz4Tk748isIyndTRiIgaDcsIkcQc1TaIHdUL7w3sCqVChp8y8xGxNAlpV+9IHY2IqFGwjBCZAEEQMCbYB7umh6CNqz1uFJVj2OpDWPXrJRgMHNsQkWVjGSEyIV29nLB3VhgG+HlBbxCx+N9n8dLG33G7pELqaEREDYZlhMjENFEpsHSEPxa/0B0qhQwHzt1CREwSjlzm2IaILBPLCJEJEgQBI/q2wnczQ9G2uT3ytBUYsSYVy3++wLENEVkclhEiE9bJwxF7ZobhhV4tYBCBJT+cx7j1R3CrmGMbIrIcLCNEJs5epcCnw/yxZKgfbG3kSLpQgIiYJKRcLJA6GhFRvWAZITITQwJaYu+sUHRwb4JbxRV48YvD+OzH89BzbENEZo5lhMiMtHNzwHczwjCijzdEEViaeAGj1x5GvrZc6mhERHVWpzISGxsLHx8fqNVqBAYG4siRI395fGFhIWbMmAFPT0+oVCp06NABCQkJdQpMZO1slXIsHtwDS0f4w14pR2rWbTy7NAm/nb8ldTQiojoxuoxs3boV0dHRWLhwIdLT0+Hn54d+/fohPz//vsfrdDr8/e9/x5UrV7Bjxw6cO3cOcXFxaNGixSOHJ7JmA/1bYO+sMHT2dMTtUh3GrT+Cf31/FlV6g9TRiIiMIohG/ipXYGAg+vTpg+XLlwMADAYDvL29MWvWLMyZM+dPx69atQr/+te/cPbsWdjY2NQppFarhZOTE4qKiuDo6FinxyCyVOWVery37wy+OZwNAOjj44KYkT3h6WQrcTIisna1ff826sqITqdDWloawsPD//cAMhnCw8ORmpp633P27NmD4OBgzJgxA+7u7ujWrRs+/PBD6PX6Bz5PRUUFtFptjY2I7k9tI8cHz3fH8lE90USlwO9X7iJiaRJ+OXv/q5VERKbGqDJSUFAAvV4Pd3f3Gvvd3d2Rm5t733OysrKwY8cO6PV6JCQkYP78+fjkk0/w/vvvP/B5Fi1aBCcnp+rN29vbmJhEVum5Hl6IjwpDtxaOuFtWiQkbfseihExUcmxDRCauwb9NYzAY4ObmhjVr1iAgIADDhw/HW2+9hVWrVj3wnLlz56KoqKh6y8nJaeiYRBahdTN7fDstBONDfAAAq3/LwrDVqbh2t0zaYEREf8GoMuLq6gq5XI68vLwa+/Py8uDh4XHfczw9PdGhQwfI5fLqfZ07d0Zubi50Ot19z1GpVHB0dKyxEVHtqBRyvD2gK1aN7gUHtQLHsgvRPyYZP5y+/9VLIiKpGVVGlEolAgICkJiYWL3PYDAgMTERwcHB9z0nNDQUFy9ehMHwv0vF58+fh6enJ5RKZR1jE9HDPNPNEwlRGvh5O6PoXiUmf5WGd/aehq6KYxsiMi1Gj2mio6MRFxeHjRs3IjMzE9OmTUNpaSkmTJgAABg7dizmzp1bffy0adNw584dzJ49G+fPn0d8fDw+/PBDzJgxo/5eBRHdl3dTO2yfEoyJYW0AAOsPXsGQVSnIvs2xDRGZDoWxJwwfPhy3bt3CggULkJubC39/f+zfv7/6Q63Z2dmQyf7Xcby9vfH999/j1VdfRY8ePdCiRQvMnj0bb7zxRv29CiJ6IKVChnnPdUGQbzP8Y/txnLhWhP4xSfh4SA88291T6nhERMbfZ0QKvM8IUf24XngPUZuPIe3qXQDA2ODWeDOiM9Q28oecSURkvAa5zwgRmbcWzrbYMjkIUx9rCwD4MvUqBq9MweWCUomTEZE1YxkhsjI2chnmPNsJ6yf0QVN7JU7f0CJyWTL2HL8hdTQislIsI0RW6omObkiI0qCvT1OUVFQhavMxzN15EuWVD747MhFRQ2AZIbJiHk5qbJoUiFlPtoMgAJuPZGNQ7EFczC+ROhoRWRGWESIrp5DL8I+nO+LLl/rCtYkSZ3OLMWB5MnamX5M6GhFZCZYRIgIAaNo3R0KUBiFtm6FMp0f0tuP45/bjKNNVSR2NiCwcywgRVXNzVOOrlwPxangHyARge9o1DFx+EOfziqWORkQWjGWEiGqQywTMDm+PbyYGwc1BhQv5JRiwPBnbjubADG5LRERmiGWEiO4ruG0zJMzWQNPeFeWVBry+4wSitx1HaQXHNkRUv1hGiOiBXJuosHFCX7z+TEfIZQJ2HbuOyOXJyLyplToaEVkQlhEi+ksymYDpj7fDlslB8HRSI+tWKQbGHsSmw9kc2xBRvWAZIaJa6ePTFPFRGjzZyQ26KgPe3HUSUVsyUFxeKXU0IjJzLCNEVGtN7ZVYO7Y33ozoBIVMwN7jNxC5LBmnrhdJHY2IzBjLCBEZRSYTMPlvbbFtajBaONviyu0yvLAiBV+mXuHYhojqhGWEiOqkVysXJERp8Pcu7tDpDVjw3WlM/yYdRfc4tiEi47CMEFGdOdnZYM2YACx4rgts5AL+fSoX/WOSkJFTKHU0IjIjLCNE9EgEQcBLYW2wY2oIvJva4trdexi6KgVrk7I4tiGiWmEZIaJ64eftjPgoDSK6e6BSL+L9+ExM+jINhWU6qaMRkYljGSGieuOotkHsqF54b2BXKOUy/JSZh4ilSUi7elfqaERkwlhGiKheCYKAMcE+2Dk9BD7N7HCjqBzDVqdi1a+XYDBwbENEf8YyQkQNolsLJ+yL0mCAnxf0BhGL/30WL238HXdKObYhoppYRoiowTRRKbB0hD8WvdAdKoUMB87dQsTSJBy5fEfqaERkQlhGiKhBCYKAkX1bYfeMUPg2t0euthwj1qRi+c8XOLYhIgAsI0TUSDp7OmLvzDC80LMFDCKw5IfzGLf+CG4VV0gdjYgkxjJCRI3GXqXAp8P98a8hPaC2kSHpQgEiYpKQcqlA6mhEJCGWESJqdEN7e2PvzDC0d2uCW8UVGL32MD7/6Tz0HNsQWSWWESKSRHt3B+yZGYZhvVvCIAKf/3QBo9ceRr62XOpoRNTIWEaISDK2Sjk+HuKHz4b7wU4pR2rWbUTEJCHpwi2poxFRI2IZISLJPd+zJfbOCkMnDwcUlOgwdt0RLPn+HKr0BqmjEVEjYBkhIpPQtnkT7J4RilGBrSCKwPJfLmJU3GHcLLondTQiamAsI0RkMtQ2cnz4fHcsG9kTTVQKHLlyBxFLk/DLuXypoxFRA2IZISKTE+nnhX2zwtCthSPullViwvrfsejfmajk2IbIIrGMEJFJ8nG1x7fTQjA+xAcAsPrXLAxfnYrrhRzbEFkalhEiMlkqhRxvD+iKVaN7wUGtQHp2ISKWJuHHM3lSRyOiesQyQkQm75lunkiI0sDP2xlF9yox6cujeG/fGeiqOLYhsgQsI0RkFryb2mH7lGBMDGsDAPgi+TKGrk5Fzp0yiZMR0aNiGSEis6FUyDDvuS5YO7Y3nGxtcDynEBExSdh/6qbU0YjoEbCMEJHZCe/ijoTZGvRq5Yzi8ipM/TodC787hfJKvdTRiKgOWEaIyCy1cLbF1inBmPKYLwBgY+pVDF6ZgisFpRInIyJjsYwQkdmykcsw99nOWD++D1zsbHD6hhbPLUvG3uM3pI5GREZgGSEis/dEJzckzNagr09TlFRUYdbmY3hz10mObYjMBMsIEVkETydbbJoUiJlPtIMgAJsOZ2NQ7EFculUidTQiegiWESKyGAq5DK/164gvX+oL1yZKnM0tRuSyZOw6dk3qaET0F1hGiMjiaNo3R0KUBsG+zVCm0+PVrcfx+o7juKfj2IbIFLGMEJFFcnNU4+uJgXglvD0EAdh29BoGLE/GhbxiqaMR0R+wjBCRxZLLBLwS3gHfTAxEcwcVLuSXIHJ5MrYdzYEoilLHI6L/j2WEiCxeSFtXJERpoGnvivJKA17fcQL/2HYcpRVVUkcjIrCMEJGVaO6gwsYJffHa0x0gE4Cdx65jwPJkZN7USh2NyOqxjBCR1ZDJBMx8sj22TA6Gu6MKl26VYlDsQWw6nM2xDZGEWEaIyOr0bdMUCVEaPN6xOSqqDHhz10lEbclAcXml1NGIrBLLCBFZpWZNVFg3rg/mPNsJcpmAvcdvIHJZMk5dL5I6GpHVYRkhIqslkwmY+lhbbJsSBC8nNa7cLsMLK1LwVeoVjm2IGhHLCBFZvYDWTZEwW4Pwzu7Q6Q2Y/91pzNiUDi3HNkSNgmWEiAiAs50ScWMDMP+5LrCRC0g4mYv+MUk4nlModTQii8cyQkT0/wmCgJfD2mDH1BC0dLFFzp17GLIqBV8kX+bYhqgBsYwQEf2Bn7cz4qM0eLabByr1It7bdwaTvkxDYZlO6mhEFollhIjoPpxsbbDixV54d2BXKOUy/JSZh/4xyUjPvit1NCKLU6cyEhsbCx8fH6jVagQGBuLIkSO1Om/Lli0QBAGDBg2qy9MSETUqQRAwNtgHO6eHoHUzO1wvvIdhq1Kx+tdLMBg4tiGqL0aXka1btyI6OhoLFy5Eeno6/Pz80K9fP+Tn5//leVeuXMFrr70GjUZT57BERFLo1sIJ+2aF4bkenqgyiFj077N4eePvuFPKsQ1RfTC6jHz66aeYNGkSJkyYgC5dumDVqlWws7PDunXrHniOXq/Hiy++iHfeeQe+vr6PFJiISAoOahssG9kTHz7fHSqFDL+cu4WIpUk4cvmO1NGIzJ5RZUSn0yEtLQ3h4eH/ewCZDOHh4UhNTX3gee+++y7c3Nzw8ssv1+p5KioqoNVqa2xERFITBAGjAlth94xQ+Da3R662HCPjDiH2l4sc2xA9AqPKSEFBAfR6Pdzd3Wvsd3d3R25u7n3PSU5OxhdffIG4uLhaP8+iRYvg5ORUvXl7exsTk4ioQXX2dMTemWF4oWcL6A0i/vX9OYxbfwQFJRVSRyMySw36bZri4mKMGTMGcXFxcHV1rfV5c+fORVFRUfWWk5PTgCmJiIxnr1Lgk2F++HhID6htZEi6UIBnlyYh5VKB1NGIzI7CmINdXV0hl8uRl5dXY39eXh48PDz+dPylS5dw5coVREZGVu8zGAz/eWKFAufOnUPbtm3/dJ5KpYJKpTImGhFRoxMEAcN6e8Pf2xkzvknHhfwSjF57GFFPtcesJ9tDLhOkjkhkFoy6MqJUKhEQEIDExMTqfQaDAYmJiQgODv7T8Z06dcLJkyeRkZFRvQ0YMABPPPEEMjIyOH4hIovQwd0Be2aGYVjvljCIwOc/XcCYLw4jX1sudTQis2DUlREAiI6Oxrhx49C7d2/07dsXn3/+OUpLSzFhwgQAwNixY9GiRQssWrQIarUa3bp1q3G+s7MzAPxpPxGRObNVyvHxED8E+TbDvN2nkHLpNiJikvDZcH9o2jeXOh6RSTO6jAwfPhy3bt3CggULkJubC39/f+zfv7/6Q63Z2dmQyXhjVyKyTi/0aokeLZ0xc1M6zuYWY+y6I5jxeDu8Et4eCjn/bSS6H0E0g19/0mq1cHJyQlFRERwdHaWOQ0T0UOWVeryz9ww2H8kGAPT1aYqYkT3h4aSWOBlR46nt+zdrOhFRA1DbyLHohe6IGdkT9ko5jly5g4iYJPxy7q/vVk1kjVhGiIga0AA/L+yL0qCLpyPulOowYf3vWPTvTFTqDVJHIzIZLCNERA2sjas9dk4Pwdjg1gCA1b9mYcSaQ7heeE/iZESmgWWEiKgRqG3keHdgN6x4sRccVAqkXb2LiKVJ+OlM3sNPJrJwLCNERI0oorsn4qM08GvphKJ7lZj45VG8v+8MdFUc25D1YhkhImpkrZrZYfvUELwU2gYAsDb5MoauTkXOnTKJkxFJg2WEiEgCSoUMCyK7YM2YADiqFTieU4iImCTsP3VT6mhEjY5lhIhIQk939UDCbA16tnJGcXkVpn6djoXfnUJFlV7qaESNhmWEiEhiLV3ssG1KMKY85gsA2Jh6FYNXpuBKQanEyYgaB8sIEZEJsJHLMPfZzlg/vg9c7Gxw6roWzy1Lxr4TN6SORtTgWEaIiEzIE53ckDBbgz4+LiipqMLMTcfw5q6TKK/k2IYsF8sIEZGJ8XSyxeZJQZjxRFsIArDpcDYGxR7EpVslUkcjahAsI0REJkghl+Gf/Trhy5f6opm9EmdzixG5LBm7jl2TOhpRvWMZISIyYZr2zfHv2RoE+zZDmU6PV7cex+s7juOejmMbshwsI0REJs7NUY2vJwbilfD2kAnAtqPXMDA2GRfyiqWORlQvWEaIiMyAXCbglfAO+HpiIJo7qHA+rwSRy5Ox/WiO1NGIHhnLCBGRGQlp64qEKA007V1RXmnAP3ecQPTWDJRWVEkdjajOWEaIiMxMcwcVNk7oi9ee7gCZAOw8dh0DlifjbK5W6mhEdcIyQkRkhmQyATOfbI/Nk4Lg7qjCpVulGLj8IDYfyYYoilLHIzIKywgRkRkL9G2GhCgNHuvQHBVVBszdeRJRWzJQXF4pdTSiWmMZISIyc82aqLB+fB/MebYT5DIBe4/fQOSyZJy6XiR1NKJaYRkhIrIAMpmAqY+1xbYpQfByUuPK7TK8sCIFX6Ve4diGTB7LCBGRBQlo3RTxURqEd3aDTm/A/O9OY8amdGg5tiETxjJCRGRhXOyViBvbG/P6d4ZCJiDhZC76xyThxLVCqaMR3RfLCBGRBRIEARM1vtg+NRgtXWyRc+ceBq9MwbrkyxzbkMlhGSEismA9W7kgPkqDfl3dUakX8e6+M5jyVRqKyji2IdPBMkJEZOGcbG2wanQA3hnQFUq5DD+cyUNETBLSs+9KHY0IAMsIEZFVEAQB40J88O20ELRuZofrhfcwbFUq1vx2CQYDxzYkLZYRIiIr0r2lE/bOCkP/Hp6oMoj4MOEsJn55FHdLdVJHIyvGMkJEZGUc1TZYPrInPni+G5QKGX4+m4+ImCT8fuWO1NHISrGMEBFZIUEQ8GJga+yeHgpfV3vcLCrHiDWHEPvLRY5tqNGxjBARWbEuXo7YOysMg/y9oDeI+Nf35zB+w+8oKKmQOhpZEZYRIiIrZ69S4LPh/vh4cA+obWT47fwtRCxNwqGs21JHIyvBMkJERBAEAcP6eGPPzDC0c2uC/OIKjIo7hKU/XYCeYxtqYCwjRERUrYO7A/bMDMWQgJYwiMBnP53HmC8OI7+4XOpoZMFYRoiIqAY7pQJLhvrh02F+sLWRI+XSbUQsTULyhQKpo5GFYhkhIqL7eqFXS+ydFYZOHg4oKNFhzLrD+OSHc6jSG6SORhaGZYSIiB6onVsT7J4RipF9vSGKwLKfL2LU2sPILeLYhuoPywgREf0ltY0ci17ogaUj/GGvlOPI5TuIiEnCgXP5UkcjC8EyQkREtTLQvwX2RWnQxdMRd0p1GL/+dyz+91lUcmxDj4hlhIiIaq2Nqz12Tg/BmKDWAIBVv17CiDWHcL3wnsTJyJyxjBARkVHUNnK8N6gbVrzYCw4qBdKu3kX/mCT8dCZP6mhkplhGiIioTiK6eyI+SoMeLZ1QWFaJiV8exfv7zkBXxbENGYdlhIiI6qxVMztsnxqMCaE+AIC1yZcxdHUqcu6USRuMzArLCBERPRKVQo6FkV2xekwAHNUKHM8pRERMEvafypU6GpkJlhEiIqoX/bp6IGG2Bv7ezigur8LUr9Pw9p7TqKjSSx2NTBzLCBER1ZuWLv8Z20z+my8AYEPKFQxemYKrt0slTkamjGWEiIjqlY1chjcjOmPd+N5wtrPBqeta9I9Jxr4TN6SORiaKZYSIiBrEk53ckRClQe/WLiipqMLMTcfw1q6TKK/k2IZqYhkhIqIG4+Vsiy2TgzD98bYAgG8OZ+P5FSnIulUicTIyJSwjRETUoBRyGV5/phM2vtQXzeyVyLypxXPLkrH72HWpo5GJYBkhIqJG8ViH5kiYrUGQb1OU6fR4ZWsG3thxAvd0HNtYO5YRIiJqNO6OanwzMQizn2oPQQC2Hs3BoNiDuJhfLHU0khDLCBERNSq5TMCrf++Ab14ORHMHFc7lFSNy2UHsSLsmdTSSCMsIERFJIqSdKxKiNAhr54p7lXq8tv04ordloExXJXU0amQsI0REJJnmDipsfKkvXnu6A2QCsDP9OiKXJeNsrlbqaNSIWEaIiEhScpmAmU+2x+ZJQXB3VOHSrVIMXH4QW45kQxRFqeNRI6hTGYmNjYWPjw/UajUCAwNx5MiRBx4bFxcHjUYDFxcXuLi4IDw8/C+PJyIi6xTo2wwJURo81qE5KqoMmLPzJF7ZmoGSCo5tLJ3RZWTr1q2Ijo7GwoULkZ6eDj8/P/Tr1w/5+fn3Pf7AgQMYOXIkfvnlF6SmpsLb2xtPP/00rl/n98uJiKimZk1UWD++D954phPkMgHfZdxA5LJknL5RJHU0akCCaOQ1sMDAQPTp0wfLly8HABgMBnh7e2PWrFmYM2fOQ8/X6/VwcXHB8uXLMXbs2Fo9p1arhZOTE4qKiuDo6GhMXCIiMlNpV+9g1qZjuFFUDqVChvn9O2N0UGsIgiB1NKql2r5/G3VlRKfTIS0tDeHh4f97AJkM4eHhSE1NrdVjlJWVobKyEk2bNn3gMRUVFdBqtTU2IiKyLgGtmyI+SoPwzm7QVRkw/7vTmLEpHdrySqmjUT0zqowUFBRAr9fD3d29xn53d3fk5ubW6jHeeOMNeHl51Sg0f7Ro0SI4OTlVb97e3sbEJCIiC+Fir0Tc2N6Y178zFDIBCSdz8VxMMk5cK5Q6GtWjRv02zeLFi7Flyxbs2rULarX6gcfNnTsXRUVF1VtOTk4jpiQiIlMiCAImanyxfWowWjjbIvtOGQavTMG65Mv8to2FMKqMuLq6Qi6XIy8vr8b+vLw8eHh4/OW5S5YsweLFi/HDDz+gR48ef3msSqWCo6NjjY2IiKxbz1YuSIjSoF9Xd1TqRby77wymfJWGojKObcydUWVEqVQiICAAiYmJ1fsMBgMSExMRHBz8wPM+/vhjvPfee9i/fz969+5d97RERGTVnOxssGp0AN6O7AKlXIYfzuQhIiYJx7LvSh2NHoHRY5ro6GjExcVh48aNyMzMxLRp01BaWooJEyYAAMaOHYu5c+dWH//RRx9h/vz5WLduHXx8fJCbm4vc3FyUlJTU36sgIiKrIQgCxoe2wbfTQtC6mR2uF97D0FWpiPstCwYDxzbmyOgyMnz4cCxZsgQLFiyAv78/MjIysH///uoPtWZnZ+PmzZvVx69cuRI6nQ5DhgyBp6dn9bZkyZL6exVERGR1urd0wt5ZYejfwxNVBhEfJGRi4pdHcbdUJ3U0MpLR9xmRAu8zQkREDyKKIr45nI13952BrsoATyc1lo3sid4+D76FBDWOBrnPCBERkakRBAGjg1pj1/QQtHG1x82icgxfcwgrDlzk2MZMsIwQEZFF6Or1n7HNQH8v6A0iPt5/DuM3/I6Ckgqpo9FDsIwQEZHFaKJS4PPh/vhocHeoFDL8dv4WIpYm4VDWbamj0V9gGSEiIosiCAKG92mFPTPD0La5PfKLKzAq7hBiEi9Az7GNSWIZISIii9TRwwF7Z4VhcK+WMIjApz+ex9h1h5FfXC51NPoDlhEiIrJYdkoFPhnmhyVD/WBrI8fBi7cRsTQZBy8WSB2N/g+WESIisnhDAlpi76xQdHR3QEFJBUZ/cRif/nCOYxsTwTJCRERWoZ2bA76bGYqRfb0hikDMzxcxKu4Q8rQc20iNZYSIiKyG2kaORS/0wNIR/rBXynH48h08uzQJv56/JXU0q8YyQkREVmegfwvsnRWGLp6OuFOqw7h1R/DR/rOo0hukjmaVWEaIiMgq+TZvgp3TQzAmqDUAYOWBSxix5hBuFN6TOJn1YRkhIiKrpbaR471B3RA7qhccVAocvXoXETFJSMzMkzqaVWEZISIiq9e/hyf2RYWhewsnFJZV4uWNR/H+///hPWp4LCNEREQAWjezx45pwZgQ6gMAWJt8GUNXpyLnTpm0wawAywgREdH/p1LIsTCyK1aPCYCjWoHjOYXoH5OE/adypY5m0VhGiIiI/qBfVw/ER2ng7+0MbXkVpn6dhrf3nEZFlV7qaBaJZYSIiOg+vJvaYduUYEzStAEAbEi5giErU3H1dqnEySwPywgREdEDKBUyvNW/C74Y1xvOdjY4eb0Iz8UkI/7ETamjWRSWESIiood4qrM7EqI06N3aBcUVVZixKR3zdp9EeSXHNvWBZYSIiKgWvJxtsXlyEKY/3hYA8PWhbDy/IgVZt0okTmb+WEaIiIhqyUYuw+vPdMLGl/qimb0SmTe1iFyWjO8yrksdzayxjBARERnpsQ7NkTBbg8A2TVGq02P2lgzM+fYE7uk4tqkLlhEiIqI6cHdU45uJgYh6sh0EAdjyew4GxR7ExfxiqaOZHZYRIiKiOlLIZYh+uiO+eikQrk1UOJdXjMhlB7Ej7ZrU0cwKywgREdEjCmvvioTZYQht1wz3KvV4bftx/GPbcZTpqqSOZhZYRoiIiOqBm4MaX74UiOi/d4BMAL5Nv4YByw/iXC7HNg/DMkJERFRP5DIBUU+1x6ZJQXBzUOFifgkGLE/G1t+zIYqi1PFMFssIERFRPQvybYaE2Rr8rUNzVFQZ8Ma3J/Hq1gyUVHBscz8sI0RERA3AtYkKG8b3wevPdIRcJmB3xg0MWJaMMze0UkczOSwjREREDUQmEzD98XbYOjkInk5qZBWUYtCKg/jq0FWObf4PlhEiIqIG1tunKRKiNHiqkxt0VQbM330KMzcfg7a8UupoJoFlhIiIqBG42CuxdlxvzOvfGQqZgPgTNxG5LBknrxVJHU1yLCNERESNRBAETNT4YvvUYLRwtsXV22UYvDIFGw5etuqxDcsIERFRI+vZygUJURo83cUdOr0Bb+89g6lfp6GozDrHNiwjREREEnCys8HqMQFYGNkFNnIB35/OQ/9lSTiWfVfqaI2OZYSIiEgigiBgQmgbfDstBK2a2uHa3XsYuioVcb9lWdXYhmWEiIhIYj1aOmNfVBj6d/dElUHEBwmZmLjxKO6W6qSO1ihYRoiIiEyAo9oGy0f1xPuDukGpkCHxbD4iYpJw9ModqaM1OJYRIiIiEyEIAkYHtcau6SFo42qPm0XlGL7mEFYcuAiDwXLHNiwjREREJqarlxP2zgrDQH8v6A0iPt5/DuM3/I7bJRVSR2sQLCNEREQmqIlKgc+H+2PxC92hUsjw2/lbiIhJwqGs21JHq3csI0RERCZKEASM6NsK380MRdvm9sjTVmBU3CHEJF6A3oLGNiwjREREJq6ThyP2zgrD4F4tYRCBT388j7HrDiO/uFzqaPWCZYSIiMgM2CkV+GSYH5YM9YOtjRwHL95GxNJkHLxYIHW0R8YyQkREZEaGBLTEnpmh6ODeBAUlFRj9xWF8+uN5sx7bsIwQERGZmfbuDvhuRhhG9PGGKAIxiRcwKu4Q8rTmObZhGSEiIjJDtko5Fg/ugaUj/GGvlOPw5TuIWJqEX8/fkjqa0VhGiIiIzNhA/xbYOysMnT0dcbtUh3HrjuCj/WdRpTdIHa3WWEaIiIjMnG/zJtg1PQSjg1oBAFYeuIQRaw7hRuE9iZPVDssIERGRBVDbyPH+oO5YPqonHFQKHL16FxExSfj5bJ7U0R6KZYSIiMiCPNfDC/uiwtC9hRMKyyrx0oaj+DAhE5UmPLZhGSEiIrIwrZvZY8e0YIwP8QEArPktC0NXpeLa3TJpgz0AywgREZEFUinkeHtAV6weEwBHtQIZOYWIWJqE70/nSh3tT1hGiIiILFi/rh6Ij9LA39sZ2vIqTPkqDe/sPQ1dlemMbVhGiIiILJx3UztsmxKMSZo2AID1B69gyKoUZN82jbENywgREZEVUCpkeKt/F3wxrjec7Wxw4loR+sckIeHkTamjsYwQERFZk6c6uyMhSoPerV1QXFGF6d+kY/7uUyiv1EuWqU5lJDY2Fj4+PlCr1QgMDMSRI0f+8vjt27ejU6dOUKvV6N69OxISEuoUloiIiB6dl7MtNk8OwvTH2wIAvjp0Fd9lXJcsj9FlZOvWrYiOjsbChQuRnp4OPz8/9OvXD/n5+fc9PiUlBSNHjsTLL7+MY8eOYdCgQRg0aBBOnTr1yOGJiIiobmzkMrz+TCdsfKkvXujVAkMDvCXLIoiiaNRvDgcGBqJPnz5Yvnw5AMBgMMDb2xuzZs3CnDlz/nT88OHDUVpain379lXvCwoKgr+/P1atWlWr59RqtXByckJRUREcHR2NiUtEREQSqe37t1FXRnQ6HdLS0hAeHv6/B5DJEB4ejtTU1Puek5qaWuN4AOjXr98DjweAiooKaLXaGhsRERFZJqPKSEFBAfR6Pdzd3Wvsd3d3R27u/W+ikpuba9TxALBo0SI4OTlVb97e0l06IiIiooZlkt+mmTt3LoqKiqq3nJwcqSMRERFRA1EYc7Crqyvkcjny8mr+AmBeXh48PDzue46Hh4dRxwOASqWCSqUyJhoRERGZKaOujCiVSgQEBCAxMbF6n8FgQGJiIoKDg+97TnBwcI3jAeDHH3984PFERERkXYy6MgIA0dHRGDduHHr37o2+ffvi888/R2lpKSZMmAAAGDt2LFq0aIFFixYBAGbPno3HHnsMn3zyCfr3748tW7bg6NGjWLNmTf2+EiIiIjJLRpeR4cOH49atW1iwYAFyc3Ph7++P/fv3V39INTs7GzLZ/y64hISEYNOmTZg3bx7efPNNtG/fHrt370a3bt3q71UQERGR2TL6PiNS4H1GiIiIzE+D3GeEiIiIqL6xjBAREZGkWEaIiIhIUiwjREREJCmWESIiIpKU0V/tlcJ/v/DDH8wjIiIyH/99337YF3fNoowUFxcDAH8wj4iIyAwVFxfDycnpgX9uFvcZMRgMuHHjBhwcHCAIQr09rlarhbe3N3Jycnj/kgbGtW48XOvGxfVuPFzrxlNfay2KIoqLi+Hl5VXjhqh/ZBZXRmQyGVq2bNlgj+/o6Mi/2I2Ea914uNaNi+vdeLjWjac+1vqvroj8Fz/ASkRERJJiGSEiIiJJWXUZUalUWLhwIVQqldRRLB7XuvFwrRsX17vxcK0bT2OvtVl8gJWIiIgsl1VfGSEiIiLpsYwQERGRpFhGiIiISFIsI0RERCQpiy8jsbGx8PHxgVqtRmBgII4cOfKXx2/fvh2dOnWCWq1G9+7dkZCQ0EhJzZ8xax0XFweNRgMXFxe4uLggPDz8of/f0P8Y+/f6v7Zs2QJBEDBo0KCGDWhBjF3rwsJCzJgxA56enlCpVOjQoQP/HTGCsev9+eefo2PHjrC1tYW3tzdeffVVlJeXN1Ja8/Tbb78hMjISXl5eEAQBu3fvfug5Bw4cQK9evaBSqdCuXTts2LChfkOJFmzLli2iUqkU161bJ54+fVqcNGmS6OzsLObl5d33+IMHD4pyuVz8+OOPxTNnzojz5s0TbWxsxJMnTzZycvNj7FqPGjVKjI2NFY8dOyZmZmaK48ePF52cnMRr1641cnLzY+xa/9fly5fFFi1aiBqNRhw4cGDjhDVzxq51RUWF2Lt3bzEiIkJMTk4WL1++LB44cEDMyMho5OTmydj1/uabb0SVSiV+88034uXLl8Xvv/9e9PT0FF999dVGTm5eEhISxLfeekvcuXOnCEDctWvXXx6flZUl2tnZidHR0eKZM2fEZcuWiXK5XNy/f3+9ZbLoMtK3b19xxowZ1f9br9eLXl5e4qJFi+57/LBhw8T+/fvX2BcYGChOmTKlQXNaAmPX+o+qqqpEBwcHcePGjQ0V0WLUZa2rqqrEkJAQce3ateK4ceNYRmrJ2LVeuXKl6OvrK+p0usaKaFGMXe8ZM2aITz75ZI190dHRYmhoaIPmtCS1KSOvv/662LVr1xr7hg8fLvbr16/ecljsmEan0yEtLQ3h4eHV+2QyGcLDw5Gamnrfc1JTU2scDwD9+vV74PH0H3VZ6z8qKytDZWUlmjZt2lAxLUJd1/rdd9+Fm5sbXn755caIaRHqstZ79uxBcHAwZsyYAXd3d3Tr1g0ffvgh9Hp9Y8U2W3VZ75CQEKSlpVWPcrKyspCQkICIiIhGyWwtGuO90Sx+KK8uCgoKoNfr4e7uXmO/u7s7zp49e99zcnNz73t8bm5ug+W0BHVZ6z9644034OXl9ae/8FRTXdY6OTkZX3zxBTIyMhohoeWoy1pnZWXh559/xosvvoiEhARcvHgR06dPR2VlJRYuXNgYsc1WXdZ71KhRKCgoQFhYGERRRFVVFaZOnYo333yzMSJbjQe9N2q1Wty7dw+2traP/BwWe2WEzMfixYuxZcsW7Nq1C2q1Wuo4FqW4uBhjxoxBXFwcXF1dpY5j8QwGA9zc3LBmzRoEBARg+PDheOutt7Bq1Sqpo1mkAwcO4MMPP8SKFSuQnp6OnTt3Ij4+Hu+9957U0chIFntlxNXVFXK5HHl5eTX25+XlwcPD477neHh4GHU8/Udd1vq/lixZgsWLF+Onn35Cjx49GjKmRTB2rS9duoQrV64gMjKyep/BYAAAKBQKnDt3Dm3btm3Y0GaqLn+vPT09YWNjA7lcXr2vc+fOyM3NhU6ng1KpbNDM5qwu6z1//nyMGTMGEydOBAB0794dpaWlmDx5Mt566y3IZPzv7frwoPdGR0fHerkqAljwlRGlUomAgAAkJiZW7zMYDEhMTERwcPB9zwkODq5xPAD8+OOPDzye/qMuaw0AH3/8Md577z3s378fvXv3boyoZs/Yte7UqRNOnjyJjIyM6m3AgAF44oknkJGRAW9v78aMb1bq8vc6NDQUFy9erC58AHD+/Hl4enqyiDxEXda7rKzsT4Xjv0VQ5M+u1ZtGeW+st4/CmqAtW7aIKpVK3LBhg3jmzBlx8uTJorOzs5ibmyuKoiiOGTNGnDNnTvXxBw8eFBUKhbhkyRIxMzNTXLhwIb/aW0vGrvXixYtFpVIp7tixQ7x582b1VlxcLNVLMBvGrvUf8ds0tWfsWmdnZ4sODg7izJkzxXPnzon79u0T3dzcxPfff1+ql2BWjF3vhQsXig4ODuLmzZvFrKws8YcffhDbtm0rDhs2TKqXYBaKi4vFY8eOiceOHRMBiJ9++ql47Ngx8erVq6IoiuKcOXPEMWPGVB//36/2/vOf/xQzMzPF2NhYfrXXWMuWLRNbtWolKpVKsW/fvuKhQ4eq/+yxxx4Tx40bV+P4bdu2iR06dBCVSqXYtWtXMT4+vpETmy9j1rp169YigD9tCxcubPzgZsjYv9f/F8uIcYxd65SUFDEwMFBUqVSir6+v+MEHH4hVVVWNnNp8GbPelZWV4ttvvy22bdtWVKvVore3tzh9+nTx7t27jR/cjPzyyy/3/ff3v2s7btw48bHHHvvTOf7+/qJSqRR9fX3F9evX12smQRR5LYuIiIikY7GfGSEiIiLzwDJCREREkmIZISIiIkmxjBAREZGkWEaIiIhIUiwjREREJCmWESIiIpIUywgRERFJimWEiIiIJMUyQkRERJJiGSEiIiJJsYwQERGRpP4fEOpL7uGsQ28AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from src.jif.diffusion import MDLMDiffusion\n",
    "n_classes = 8\n",
    "seq_len = 4\n",
    "batch = 6\n",
    "diffusion = MDLMDiffusion(n_classes, 1e-7)\n",
    "data = jax.random.randint(jax.random.PRNGKey(0), (batch, seq_len), 0, n_classes)\n",
    "\n",
    "\n",
    "from matplotlib import pyplot as plt\n",
    "t = jnp.arange(0, 1, 0.01)\n",
    "plt.plot(t, diffusion.alpha(t))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "\n",
    "\n",
    "# https://github.com/neverix/Score-Entropy-Discrete-Diffusion/blob/f7221e3b835045f75444c7429955aa420111cc7d/model/transformer.py#L80\n",
    "def timestep_embedding(t, dim, max_period=10000):\n",
    "    # https://github.com/openai/glide-text2im/blob/main/glide_text2im/nn.py\n",
    "    half = dim // 2\n",
    "    freqs = jnp.exp(\n",
    "        -math.log(max_period) * jnp.arange(0, half, dtype=jnp.float32) / half\n",
    "    )\n",
    "    args = t[..., None].astype(jnp.float32) * freqs[None]\n",
    "    embedding = jnp.concatenate([jnp.cos(args), jnp.sin(args)], axis=-1)\n",
    "    if dim % 2:\n",
    "        embedding = jnp.cat([embedding, jnp.zeros_like(embedding[:, :1])], dim=-1)\n",
    "    return embedding\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/8192 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 8192/8192 [02:07<00:00, 64.09it/s, loss=0.009358923] \n"
     ]
    }
   ],
   "source": [
    "from functools import partial\n",
    "import optax\n",
    "from tqdm import trange\n",
    "key_params = jax.random.key(3)\n",
    "key_w1, key_w2, key_w3, key_b1, key_b2, key_b3 = jax.random.split(key_params, 6)\n",
    "n_time = 128\n",
    "in_dim = (n_classes + 1) * seq_len + n_time\n",
    "d = 1024\n",
    "out_dim = (n_classes + 1) * seq_len\n",
    "scale_in = math.sqrt(in_dim)\n",
    "scale_out = math.sqrt(out_dim)\n",
    "w1, w2, w3 = jax.random.normal(key_w1, (in_dim, d)) / scale_in, jax.random.normal(key_w2, (in_dim, d)) / scale_in, jax.random.normal(key_w3, (d, out_dim)) / scale_out\n",
    "b1, b2, b3 = jax.random.normal(key_b1, (d,)) * 0.1, jax.random.normal(key_b2, (d,)) * 0.1, jax.random.normal(key_b3, (out_dim,)) * 0.1\n",
    "params = w1, w2, w3, b1, b2, b3\n",
    "\n",
    "def run_net(params, data_perturbed, total_noise):\n",
    "    x = jnp.concatenate([jax.nn.one_hot(data_perturbed, n_classes + 1).reshape(*data_perturbed.shape[:-1], -1) * 2 - 1, timestep_embedding(total_noise[..., 0], n_time).reshape(*data_perturbed.shape[:-1], -1)], -1)\n",
    "    w1, w2, w3, b1, b2, b3 = params\n",
    "    # h = (x @ w1 + b1) * (x @ w2 + b2)\n",
    "    # h = jax.nn.relu(x @ w1 + b1)\n",
    "    h = jax.nn.relu(x @ w1 + b1) * (x @ w2 + b2)\n",
    "    logits = (h @ w3 + b3).reshape(*data_perturbed.shape, n_classes + 1)\n",
    "    return logits\n",
    "\n",
    "def get_loss(params, data, key):\n",
    "    loss = diffusion.get_loss(key, partial(run_net, params), data)\n",
    "    return loss.mean()\n",
    "\n",
    "@partial(jax.jit, donate_argnums=(0, 3))\n",
    "def update_step(params, data, key, opt_state):\n",
    "    loss, grad = jax.value_and_grad(get_loss)(params, data, key)\n",
    "    updates, opt_state = optimizer.update(grad, opt_state)\n",
    "    params = optax.apply_updates(params, updates)\n",
    "    return params, opt_state, loss\n",
    "\n",
    "loss_and_grad = jax.value_and_grad(get_loss)\n",
    "optimizer = optax.chain(optax.clip_by_global_norm(1.0), optax.adam(4e-4))\n",
    "ema_params = jax.tree.map(jnp.copy, params)\n",
    "opt_state = optimizer.init(params)\n",
    "losses = []\n",
    "train_chunk = 128\n",
    "for i in (bar := trange(8192)):\n",
    "    key = jax.random.PRNGKey(i)\n",
    "    # loss, grad = loss_and_grad(params, data.repeat(16, 0), key)\n",
    "    # updates, opt_state = optimizer.update(grad, opt_state, params)\n",
    "    # params = optax.apply_updates(params, updates)\n",
    "    params, opt_state, loss = update_step(params, data.repeat(128, 0), key, opt_state)\n",
    "    ema_params = jax.tree.map(lambda x, y: x * 0.99 + y * 0.01, ema_params, params)\n",
    "    bar.set_postfix(loss=loss)\n",
    "    losses.append(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAi8AAAGhCAYAAACphlRxAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/TGe4hAAAACXBIWXMAAA9hAAAPYQGoP6dpAABGg0lEQVR4nO3deVzUdf4H8NfMcMl9KcjtbYhCIZCpiUaZmd13eVXWFlYbXfardNtt17Yt161ms7Sy2mrNDluzzMQzLwQFD1REUREEOYThkAFmvr8/kIGRGZgZZuY78+X1fDx4PJjv9zvf7xu/6rz4fD+HTBAEAUREREROQi52AURERETmYHghIiIip8LwQkRERE6F4YWIiIicCsMLERERORWGFyIiInIqDC9ERETkVFzELsDatFotSktL4ePjA5lMJnY5REREZAJBEFBXV4ewsDDI5d23rUguvJSWliIyMlLsMoiIiMgCxcXFiIiI6PYYyYUXHx8fAG0/vK+vr8jVEBERkSlUKhUiIyN1n+PdkVx4aX9U5Ovry/BCRETkZEzp8sEOu0RERORUGF6IiIjIqUgmvCiVSsTGxiIpKUnsUoiIiMiGZIIgCGIXYU0qlQp+fn6ora1lnxciIiInYc7nt2RaXoiIiKhvYHghIiIip8LwQkRERE6F4YWIiIicCsMLERERORXJhBcOlSYiIuobOFSaiIiIRMeh0kRERCRZDC9ERETkVBheiIiIyKkwvBAREZFTYXghIiIip8LwQkRERE5FMuGF87wQERH1DZznhYiIiETHeV6IiIhIshheiIiIyKkwvBAREZFTYXghIiIip8LwQkRERE6F4YWIiIiciovYBdjKZzuL0M/Lx6rnVMhluGKgLxIi/eHhqrDquYmIiMg0kg0v//i1AHJ3T5uc200hR3ykH5IHBSJ5UBASowPg7S7ZP0oiIiKHItlJ6v7w8Ta4e3pb9dyNzRrkFtfgfJ1ab7tCLkNcmK8uzCTFBMDf082q1yYiIpIycyapk0x4USqVUCqV0Gg0KCgosNkMu4Ig4HRVI7KKqrGnqBpZp6pQXH2xy3EjQ32QPCgQKYOCkDQoAAN8PKxeCxERkVT0yfDSTozlAUprLnaEmaIqnKho6HLM4GCvSy0zbV8RAbZ5pEVEROSMGF5EXtuook6N7FNtYWZPUTWOlqlw+Z9yuH8/pHQKM4OCvSCTyUSpl4iISGwMLw62MGNtYwuyT1frWmcOltRCo9X/Yw/2dkfKoECkDG4LM8MH+EAuZ5ghIqK+geHFwcLL5RrUrdh35oIuzOQW16C5Vat3jF8/VyTFBOpaZ0aF+cJFwWl5iIhImhheHDy8XK6pRYMDZ2uRVVSFPUXVyDl9AY3NGr1jvNwUSOwUZsZE+MHdhXPNEBGRNDC8OFl4uVyLRovDpSpkFVUhq6jtcZOqqVXvGG93F8y+JhrzJg7msGwiInJ6DC9OHl4up9UKOFZehz0nq5B1qi3MVNY3A2gLMQ+Pj8EjEwbDz9NV5EqJiIgsw/AisfByOUEQsCG/HEs3HseRcyoAgI+HCx6ZMAgPTxgEXw+GGCIici4MLxIPL+20WgG/Hi7D0o3Hcay8DgDg6+GCeRMHY874GPgwxBARkZNgeOkj4aWdVivg50Pn8K+Nx3H8fD0AwN/TFfMmDsbsa2K47hIRETk8hpc+Fl7aabQCfjpQin9lHsfJS7P8Bni64vFJQzBrXDQ83RhiiIjIMTG89NHw0k6jFfC/vBK8m1mIosq2EBPk5YY/TBqCh66ORj83DrEmIiLHwvDSx8NLu1aNFj/mluLdTcdxuqoRQNtMvk+kDsGDKVHwcGWIISIix8DwwvCip0WjxQ/7S/Bu5nGcvdC2AvYAH3c8mToE9yUzxBARkfj6ZHhRKpVQKpXQaDQoKChgeDGgRaPFdzln8d6mQpTUtIWYUF8PpE8egnuSIjljLxERiaZPhpd2bHnpWXOrFqtzivH+pkKcq20CAIT5eeDepCjMiB+Iwf29Ra6QiIj6GoYXhheTqFs1+GZvMd7fXIhylVq3fVSYL2bEh2H66IGIDPQUsUIiIuorGF4YXszS1KLBTwfO4acDpdh+vBIabcdfiSuj/DFjTBimjxmIEF8PEaskIiIpY3hheLFYdUMz1h8qw9q8UuwuqkL73w6ZDEgZFIgZ8WGYFjcQgV5cDJKIiKyH4YXhxSrOq5qw7uA5rM0rxb4zNbrtCrkME4YG4+YxA3HDqFD49eMyBERE1DsMLwwvVnf2QiPWHTiHtQdKcahEpdvuppAjZXAgxkYHIjE6AAlR/lyOgIiIzMbwwvBiU0WVDfgprxT/yyvVraXUTi4DRoT6IjHaH4nRAUiMCkRkYD/IZDKRqiUiImfA8MLwYjcF5XXYfbIKOacvIOf0Bd0keJ0Fe7t3hJnoAIwK8+PEeEREpIfhheFFNOWqJuy7FGRyzlzAoZJatGj0/4q5KeSIC/fVhZmrogMwwIcjmYiI+jKGF4YXh9HUosGhklpdy0zO6QuoamjuclxkYD8kRrWFmfFDgzlRHhFRH8PwwvDisARBwJnqRr0wc6y8Dp3/FspkwOLbR+O+5CjxCiUiIrsy5/Obw0LIrmQyGaKDvBAd5IU7rooAAKiaWpBXXIOc0xews7AKWaeqseD7g5DLZLgnKVLkiomIyNGw5YUciiAIeH1tPlbuPAWZDPj7nWNwz1gGGCIiqTPn81tup5qITCKTybBoRixmjYuGIAAvfXcA3+acFbssIiJyIAwv5HBkMhlev2UUHro6CoIAvPBtHr7fxwBDRERtGF7IIclkMvz5ljg8mNIWYJ5bnYcf9jPAEBERwws5MLlchr/cGof7ky8FmG/y8GNuidhlERGRyBheyKHJ5TL89bY43JcUCa0APLsqF1/sOoXG5laxSyMiIpFwtBE5Ba1WwILvD+Cb7LZHR+4uckwa3h9TR4Ui7YoQ+HlyZWsiImfGSeoYXiRJqxXw7y2F+Cb7LM5UN+q2u8hlGDckCDfEhiA6yAte7i7wdneBt4cLvN1c4OWugIuCjYxERI7M6cPLTz/9hOeeew5arRYvvfQSHn30UZPfy/AifYIg4Mi5Ovx6uAy/Hi7D0bK6bo/3cJXjTzNGccZeIiIH5tThpbW1FbGxsdi8eTP8/PyQmJiInTt3IigoyKT3M7z0PacqG/Dr4TJsLahAdUMzGppb0aDWoF7diuZWLYC21pmvH7saSTGBIldLRESGOPXyAFlZWRg1ahTCw8MBANOmTcOGDRtw//33i1wZOaqYYC88PmkIHp80pMu+5lYtnludh7V5pUj/ch9+enoCV7AmInJyVu8IsG3bNsyYMQNhYWGQyWRYs2ZNl2OUSiViYmLg4eGBlJQUZGVl6faVlpbqggsAhIeHo6SEw2PJMm4ucrx5x2gMG+CN83VqzP9qP1o1WrHLIiKiXrB6eGloaEB8fDyUSqXB/atWrUJGRgYWLVqEffv2IT4+HlOnTsX58+etXQoRAMDL3QUfPJQIb3cXZBVV461fj4ldEhER9YLVw8u0adPwxhtv4Pbbbze4f8mSJZg3bx7mzp2L2NhYLFu2DJ6envjkk08AAGFhYXotLSUlJQgLCzN6PbVaDZVKpfdFdLmhA7zxj7vGAAA+2nYSvxw8J3JFRERkKbuOH21ubkZOTg7S0tI6CpDLkZaWhl27dgEAkpOTcejQIZSUlKC+vh6//PILpk6davScixcvhp+fn+4rMpIrEJNh00YPxGPXDgYAvPDtAZyoqBe5IiIisoRdw0tlZSU0Gg1CQkL0toeEhKCsrAwA4OLignfeeQeTJ09GQkICnnvuuW5HGr388suora3VfRUXF9v0ZyDn9uLUEUgZFIh6dStmfZyFI+fYUkdE5GwccuauW265BQUFBSgsLMRjjz3W7bHu7u7w9fXV+yIyxkUhx/sPXIXBwV4oqbmIOz/YiQ2Hy8Qui4iIzGDX8BIcHAyFQoHy8nK97eXl5QgNDe3VuZVKJWJjY5GUlNSr85D09fdxxw9Pjsf4oUFobNbg8f/k4N9bCuFgUx4REZERdg0vbm5uSExMRGZmpm6bVqtFZmYmxo0b16tzp6enIz8/H3v37u1tmdQH+Hm6YuXcZMy8OhqCALy1/hgyvsnDjsJK7CisxM7CSuw8UYmqerXYpRIR0WWsPkldfX09CgsLda+LioqQm5uLwMBAREVFISMjA7Nnz8bYsWORnJyMpUuXoqGhAXPnzrV2KUTdclXI8Zfb4jA8xBt/WpuPH/aX4If9+nMKhfp6YONzk+Dt7nDzORIR9VlW/x85OzsbkydP1r3OyMgAAMyePRsrV67Evffei4qKCixcuBBlZWVISEjA+vXru3TiJbKXmeNiMLi/N5ZuLIDqYisAQICAczVNKFM14aOtJ5BxwwiRqyQionYOt7ZRb3FtI7KWXw6ewxNf7oOHqxybn0/FQL9+YpdERCRZ5nx+O+RoI0uwwy5Z241xoUiKCUBTixbvbCgQuxwiIrqELS9E3dh/5gJu//dOyGTAT09NwKgwP7FLIiKSpD7Z8kJkC1dGBWBGfBgEAfjruiMcTk1E5AA4hIKoBy9OHYFfD5dh54kqfJNdjEAvd5yqbEBp7UUkRPpj6qhQeLgqxC6TiKjPYHgh6kFkoCfmjo/Bh1tP4qXvDnbZ7+vhgtuuDMdDV0djeIiPCBUSEfUtknlsxA67ZEtPpg7FkP5e8HRTYFSYL6aPGYhZ46IR5ucBVVMrPt91GtPf3Y6dJyrFLpWISPLYYZfIDIIgQCaT6V5rtAJ2nqjEB1tOYOeJKvj1c8Wa9PEYFOwlYpVERM6HHXaJbKRzcAEAhVyGicP645M5SUiI9EftxRY8snIvahtbRKqQiEj6GF6IrMDDVYGPZiUi3L8fTlY24Ikvc9CgbhW7LCIiSWJ4IbKSAT4eWDF7LLzcFNh5ogrXvrUZK3cUQd2qEbs0IiJJYXghsqIrBvri4zlJGBTshaqGZvxpbT6mvL0VH2w5gYo6rlBNRGQNkumwq1QqoVQqodFoUFBQwA67JKoWjRars89i6cYCnL8UWlzkMkwdFYonUocgLpwz9RIRdWZOh13JhJd2HG1EjuRiswZr80rxVdYZ5BbX6LZPGTkAL904EiNCOS8MERHA8MLwQg4pv1SFj7adwP/ySqEVgGBvd+xcMAVuLnx6S0TEodJEDig2zBdL77sSGzMmIdjbDZX1auw+WSV2WURETofhhcjOBvf3xvWxIQCADfllIldDROR8GF6IRHDDqFAAwG/55dBqJfXklojI5hheiERwzZAgeLu7oFylRt7ZGrHLISJyKpIJL1yYkZyJu4sCqSP6AwA25JcDAFo1WmjYCkNE1CPJhJf09HTk5+dj7969YpdCZJL2R0e/Hi7D1oIKXL14E6a/ux3nVU0iV0ZE5NgkE16InM3kEf3hqpDhZEUDZn+Shcp6NY6W1eHBFXtQWnMRX+w6hce/yEZBeZ3YpRIRORQXsQsg6qt8PFxxzZBgbC2oAADceVUEdhRW4vj5eoz/+ya0z8BUVtuENenju6xoTUTUV7HlhUhEf5g0BGMi/PDO3fF45554fDUvBcHe7hAEINy/HzzdFMg7W4t1B8+JXSoRkcPgDLtEDqastgmHSmpx7fD++GDLCfxzYwGiAj2xMWMSZ+MlIsniDLtETizUzwNpsSFwc5Hj0YmD0N/HHWeqG/Hh1hNil0ZE5BAYXogcmJe7C567fjgA4J3fCrBi+0mRKyIiEp9kwgvneSGpujcpEumThwAA3lh3BMu3McAQUd/GPi9ETuLdzONY8lsBZDJg2UOJmHppnhhBEKBu1cLDVSFyhURElmOfFyIJevq6YZg1LhqCAPzxv7nILa5Bq0aL+V/vR9yiX7H/zAUIgoDvcs7iaJlK771NLRpI7PcUIurDGF6InMjCm2MxcVgwLrZocM+Hu/DAij1Yd+AcWrUC/rP7DH4+WIbnVufhj//N1b1nw+EyjHxtPb7JLhavcCIiK2J4IXIiLgo5lA9ehbQrBqC5VYusomrdvl8Pl+GzXacAAEfL6lDb2AIAWJNbAgDIPHLe7vUSEdkCwwuRk/H1cMXyWWPx2s2xGDrAG/+6LwHh/v1Qr27VCzO5Z2sgCAL2nroAADhV1SBWyUREVsXlAYickEwmwyMTBuGRCYMAAMfK6vDvLfrzwOw7fQHRgZ6oqFMDAE5XNUKrFSCXc5kBInJubHkhkoBbE8J134+NDgAA7C+uwd5THS0x6lYtznHFaiKSAIYXIgkYEeqDO64Kx/ihQXj5ppEAgP1nLug9RgKAogo+OiIi58fHRkQSseSeBABAq0YLD1c56ppadQs6erop0NisQVFVAyYMCxaxSiKi3mPLC5HEuCjkGBPhDwBobNbARS7DtLiBAIBTlWx5ISLnJ5nwwuUBiDo8kByFYG83TBk5AB/OTERClD8A/fBS3dCMY2V1utcVdeouk9sRETkiLg9A1AfsKKzEgyv2YHB/L/z27CS8+O0B/C+vBC0aAa/dHIvZ46Jxw9JtKKpswOcPJ2PisP4mn/uH/Wfx88EyvH13PPz6udrwpyAiKePyAESkJybYCwBQXN2IPSer8N2+s2jRtP3e8pef8pHxTR5OVjRAEIBX1xxCU4vG5HM/uyoPv+WX4631R21SOxHR5RheiPqAgb4e8PVwQYtGwJLfCgAAM+LDdPPE/C+vVHfs6apGfJtz1qTzdm643Xa8wooVExEZx/BC1AfI5TLcGNe2CnX26bYZdyeP6I9Xp1+BOdfEAAD6uSowa1w0AGD/mRrsPlmFslrD88KsP1SGB1fsxuHSjj4yxdUXUdfUYsOfgoioDYdKE/URt8SH45vsjhaVicP6QyaTYdGMWFwZ5Y+Bfv1Q09iMz3edxnf7zuK7fWfh6+GCT+cmI/HSxHft/vCfHADAE1/m6G0/XKrC1YODbP/DWNmvh8sQEdAPo8L8xC6FiEzAlheiPmLckCAEe7sDAGIH+qK/T9v3MpkMtyaEI3lQIEaF6394q5pa8chne9GgbjV4zuLqi3qvLzQ091hHZb0aN7+3HSu2n7Tkx7C6QyW1ePyLHEx/93exSyEiEzG8EPURCrkMd1zVtozADaNCDB4T5ucBf0/9EUM1jS34fl9bi82nO4pwz4e7jF6j6lJ4KapswLR/bce6A+e6HLM2rxSHSlR4Y90RnL3QCABoatHg812nUFJzscvxtlZ4vt7u1ySi3mF4IepDnrthOJbPGosnU4ca3C+TyRDm10/3+snUIQCAT3eewhe7T+P1tfldlhzorLqhGR9tO4HJb2/BkXMqpH+1r8sx7QtFAsCPuW0dhf/5WwEW/ngYtyt3WPRz9QYXqiRyPgwvRH2Iu4sC18eGwM3F+D99H4+OrnBPTh4KHw8XnKxowGtrDhl9T/vjqMyj5/G3n/WHTJ+s0G/ZOF3VqPu+Pcj8dqQcAHC+U7CxF4WM4YXI2TC8EJGeRTNGIdTXA2/dNQbe7i54dfoVPb4nZVAgACCvuKbLvk1Hz+u9PlXVMcvvhcZLfWREnCqTDS9EzofhhYj0xIb5Yvf/XYd7xkYCAO4ZG4m7EyMwMtQH79wdb/A9Vw8ONHq+M9UdLS2CIOi1vFxobBtaba3scqiktktLT726Fc+uysXG/HKD7+n82EijldSE40SSxaHSRNQtmUyGf1wKLYY61Lq5yBER4Gn0/WW1TdBqBfx86BzOXriI+k4jl2outbxYY5WSqno1bn6vbcTQqTen67Z/uPUEfthfgh/2l+htb9f5sVGLRguFXNHrWojIthheiMhkA309umwL9HRDoJdbl+3e7i6oV7eiXNWEDfllmP/V/i7HtD826hxdBEGAzIJ+KGcvdAQrjVaA4lKLynlV9/1oFJ1aXpo1Wni4MryQZerVrdAKAnw9uMaXrfGxERGZzNDInP4+7nrhJSbIE7kLr8dnD7et8F6uUmP3ScMjlGoaW6Bu1aBzw8vFTusqqVs1mP7udjy/Os/g+7/ccxqf7TzVVlunwKNu7TiHQtF9EOqck1patd0eS2SMVisgbtGvGPOnDWatDUaWkUx4USqViI2NRVJSktilEEnaY9cOhkIuw4ShwXCRy/DGbXEI6BRekgcFwt/TDZGXHiWVqZqwOrtY7xzh/m3DseuaWjHytfV6/WLqmjoeK+05WY3DpSp8m3MW2sv6ozQ2t+KVHw5h0f8O6x4/tWtq6QghLj30yO0cnNoXqzR+LPvEkGFNnQJzhQij5voayTw2Sk9PR3p6um5JbSKyjZenjcRTU4bCx8MVjc2t8HRz0ftQb19KIOjS8GkAaGjW/000KtATpbUXIQj64QEAVBdbEGLg8VTtxRZdSDpcWqsXUNStWrRoO153/s1X0UN46dxJt0WjRXOrFr8cOodxQ4IwwKejjpzT1fjDf/bhtZtjcUt8WLfnJCLbkkzLCxHZh0wmg8+lZ/qebi66bY9PGozrRg7ArQlts/h2FxoCvd2M9gtQdVrcsXMIqahv+222trEFtyt34s4Pdur2tWi0UHcKM5uPndcFqp5aXjSCfnhZtvUEnvlvLm5X7tQ7bu6ne1FRp8bTX7f13ckrrsFN/9qOnYWV3Z6fiKyP4YWIrOLlaVfg4zlJBju8Rgb2wydzxupeB3i6Iti7aydfoK2FpZ2q0yOk9qb44guNaNbo901pbtXqbXvlh0PYcqwCAKCQd/w319Siwb+3FOqtq6Tf8iLgl0NlALqOrOpcCwA8tGIP8s+p8MCKPQZ/DiKyHck8NiIix/PJnLH4X24p/nTLKL1w4evhimEDfHCioqHLe9YfKsOUkW1rL3UOMgXldXjr12MI9OzaYqNu1UJ9WSfJ346UQysIWLb1hG7byNfW675PHdEfQwf46IWXkxX1OHJOZdLPVmdksUoiY4w9kiTzseWFiGxmysgQLL3vSvh7uiHAU7+lZXiIt8H3rM45i60Fba0me05W6bYv3XgcecU12HypRaWzB1fs0RulBLTNnPvIZ9lGa0tbsg3lqiZoOz02eum7Az3/UEQWen/TcTzz31zc8p791/CSGra8EJFduCo6flfSCALCOy0ACQAThwUjzK8fVmUX49MdRXCVy7Ch06y4nVthLlfd0Iy9p/SHY5fVNvVYU8rfMvVeX/5oiMia2v8+l6l6/rtJ3WN4ISK78/VwxZWRAXrbJg3vj6sHB2FVdjG2HKvQ9Vkx1X92n9F7fc6E8GIpQzMNE5H9MLwQkd28fsso/HzwHGaNi4aPhys+nJmIqvpm1F5swZxrYqx6rTobtqKMf3OTScdptQIE9Dxcm5wfpwCyL4YXIrKb2dfEYHankDJ1VGiXYx6dMAgrfi/q9bUam603y6klLS2CIGD6e7+jqUWDjRmTGGDIIW0+eh7fZBfjr7ePNrjMh6Nih10iciiv3hyLTc9N0r2+Z2wEvv3DON3ricOC8acZsT2ep15tvI+MOQ6X1prc0tJZY7MGR86pUFTZgHO1Fy9ta8XHvxehuNOMwkRimrtyL345VIY3fzkidilmYcsLETmcwf29cUNsCEpqLuLPt8ahqqFj+v8b40IxKqznWbQ7z8DbG2vzzlnlPADw5i9H8fmu03hnwzFMGBqM0eF+eOq6YVY7P/VdrRotLjS2oL+Pe88HG1DewwKmjoYtL0TkkD6aNRbrnp4ID1cF+ndaakArAN7uxld+7mlGXXMJ6LkzQ8yCdais1//Pv/O7thZU4KcDpdhxaTbexmYNNuSX453fCqxZKvVh9320G0l/3YiDZ2vFLsUuGF6IyOG5uXT8V6Vu0eiWJTDksWsHG9w+ONjLsoub2BHz1R8O6b3uPPndKz8cwvyv9uO8k/12S84j+/QFAMDqnLZFUKW+iCjDCxE5hZGhPgCAtCtC4NvP8LpIABAXbviR0p2JEWZf83h5ncnHHilT4VBJLeourc10+SrYAGflJfv4OusMrl6ciWNlpv/9dTYML0TkFNakj8ful69DTLAXvN1d8NHMRIPHJQ8KNLg9Kcbw9u7c9O52k489XdWIm9/7HXcv2wVAf8FH6tuaW7Wo7tRvyxyCIOBwaS1aNKb34Xr5+4MoV6nx4rd5Fl3TGbDDLhE5BQ9XBUL9Ovq63DAqFB6ucl3H3PfuvxKuCjmCvbt2WHwydQiuGOije33dyAHIPHq+x2u2aEzp8aLv6KXfdg21vJB0db7bssu6XaUt2Yoz1Y3w66bF0JgV24vw15+PYOqoEHw4c2zPb+jEnADtbH9b2fJCRE7ri0dSMKS/F/7zSApmxIfhxriu88YAwIs3joS7S0fw8XJ30T2G6knumRqLamPLC7U7c2lofHdLXBjz0aUV0H89XN7DkX0LwwsROa2kmEBkPpeKCcOC9ba7Kbr+1+aq6Ph12MNVjiX3JMDTzfiopXZZl62ZZIrmVq1eh11zNbVocM+Hu/DvLYUWn4P6pr4yFSLDCxFJzuePJHfZJuvUlt/PVYHYMF8c/NNUm1z/2W9yMeHvmy1+/+rsYmQVVeOt9cf0tguCgO3HK0xadNJZHC+vw10f7NQNI++r8ktVmPSPzVibV2r0mHp1K05U1Jt97p2FlSYNoS6tudhlyL+jYnghIsm5enAQvN2Nd+nzcG1rcbHVlP3rDpg+sZ26tW0ZA0EQcKKiHks2HEPhecMfUJuPncfMj7Nw9eJMg/ud0eP/yUH26Qt4cMUes94nCAJ2n6yyuCOsparq1dh5otLqQ5Hnf70Pp6sa8dTX+40eM+mtzbjuna3IK64x+bxltU14YMUezHj/926Pq29qwTVvbsLYNzaafG4xOWR4uf322xEQEIC77rpL7FKISILawwsAxAR5AtCfB6Z9mz2MeHU9Nh0tx1Nf78d172zFu5sK8dmu07r9R86pdN9vP268dUKjFZBzuloXhpxFZZ1lv+n/ergc9320G5Pf3mLdgnqQ+o8teGD5Hqw/VGbV86pNmBG6fabpzCOm938prTVtXa7iC861UrpDhpdnnnkGn3/+udhlEJET665NJTbMV/f9F4+kYNa4aKyc2/GoKT7S33aFGfDwymz8ZKS1Ztq/tqOgvA7nVU3YWVhl9BxLNxbgzg924Zmvc21UpWP5w39yAFjWCbY32ufqMWW0mqU+33XK4tFqssuHOkmUQw6VTk1NxZYtW8Qug4icmYH/w797YhwOnq3FDbEhum2RgZ74861xesf1prOtLew+WYXX1+Z3W9eK7W0rca8/bN0WAbKdTUfLkVtci2fT9Ne3WvjjYXi4KnDP2EiRKnN8Zre8bNu2DTNmzEBYWBhkMhnWrFnT5RilUomYmBh4eHggJSUFWVlZ1qiViMhkhn7/TIwOxJzxg3r87VQrCHjpxpG2KcwCf/v5SJfgcqGhGUs3FnCFapFYo8vLwyuz8W7mcYPDoPNLVQbe0TNVk31bosRidnhpaGhAfHw8lEqlwf2rVq1CRkYGFi1ahH379iE+Ph5Tp07F+fMdTWwJCQmIi4vr8lVaaryXNRGRvQwP8cETqUOwJn08Rob6YMrIAXr7f356Ir58NMVu9RhaIfv51XlYuvE47vxgJ5SbC3Gxxbn6ukjZiu1FZo0IKzOxX0pnX+05g/FvburSufv7fSW672USHjht9mOjadOmYdq0aUb3L1myBPPmzcPcuXMBAMuWLcO6devwySefYMGCBQCA3Nxcy6o1QK1WQ63u6PClUlmWVolIWix59r92/gT8ll+GP0waAgBIiPTH+j9eiwNna7CpUx+H2DBfnFeJO1x5+6Whxefr1PjHr8d6OJqsrbu5l1fuPIXNx85j6wuTLT7/2UsdaA39LX53U8f8P6/8cBCrHh9nvE7HegJqNVbtsNvc3IycnBykpaV1XEAuR1paGnbt2mXNS+ksXrwYfn5+uq/ISD4jJCJg5tXRAIBrh/c3+T2jI/yQccMIvdFIADA63K/LjLwDfD3w89MTu7TK2EurGWvdkP2drmpEZb3a4iHVG4+Uo8CEhUFbrdQ/y9naaKwaXiorK6HRaBASEqK3PSQkBGVlpnciS0tLw913342ff/4ZERER3Qafl19+GbW1tbqv4uJii+snIul4Jm0YPn84GR88eFWvzyWTyfDDk+MRFeiJ2xLCdNtjw3yx9L6EXp/fEg7Wp5gM+L/vD2LOp3stfv/mo+dFWXNIEARUNzTjgp3n0DGHQ4422rjR9Ely3N3d4e7edSE2IurbXBVys1pdetLPTYEtz6dCftnEdr4e+ovtLb03AX9clWu16/YlG/PLERvmizD/flY7Z1ltE55fnYfZ18Tg+tiQnt9gKhNSRVFlA44bmXDQ2pPcWZO6VYur/vIbAKDwr9Pg0mm5Da1WwJnqRkQHeYo6LNuqLS/BwcFQKBQoL9fvOV1eXo7QUMMLphEROYvLg0u7zrP5WrJysK3kFdfgwNmabo+x16R2tRdb8GNuCRqbW40e8+jn2bjmzU1Wve7CHw/h98JKzPs8Gw8s3435X+2z6vkt9fHvRSYd19t4sK2gwqTjOkepC40dLS6Nl3UEf2XNQaS+vQWf7jjVy8p6x6rhxc3NDYmJicjM7Ji6WqvVIjMzE+PGGe9QZA1KpRKxsbFISkqy6XWIiC63YvZY3fe2WnLAXI3NrbhVuQO3vL8DTUZGIq0/VIYRr67HF7tPG9xvTY9/kY1n/puL//v+oMXneH3tYdz0r+1Gfx5Dqjo9+th5ogo/HTiHez7cpTdzsRjezTyu+95WLRjnapvwr07XMVV3jUJfZ7V1zfjnbwWWlmUVZoeX+vp65Obm6kYMFRUVITc3F2fOnAEAZGRkYPny5fjss89w5MgRPPHEE2hoaNCNPrKV9PR05OfnY+9ey58vEhFZIikmEHHhvpg0vD/kDjLDaX1TRwvHxWbDH/ZPfNk2S+1raw6ZdW6tVjC7w/Duk22rc6/JtXxKjE93nEL+ORV+Ptj92lFnLzTir+vyUVJjeAhyVlE1Zn/Su/nHLv98N/cxkKrJeAuUsWsYknP6gtF9ly+y2NSiweHSWoO19vS39uyFRqN/j8Rgdp+X7OxsTJ7cMfwrIyMDADB79mysXLkS9957LyoqKrBw4UKUlZUhISEB69ev79KJl4hIKhRyGdbOnwCg+w8TR2Npt4u7lu1ESc1FbHtxMtxdFD0e391U95a0OvTUWXnWx1k4WdmArQUV8PEw/BjvvJE1lVo0WpyraUKUHde3spf7PtqN3OIaLLknHndcFWH0uM5/vNcv2YoPHkrEHf/eiWBvx+lfanZ4SU1N7TFhzp8/H/Pnz7e4KCIiZ9P+IZwYHYAZ8WFYmyfdSTf3nakBABwqUSExOqDH47vrwGzq2kTd9ZW53MnKBgBAQXk9wrvp/LuzsBLbjlfidFUDxg8NxkNXR+PBFXuQVVSNT+ckYXI3w+AducOtMbmXVqP+Jru4S3jpnCE7/2zlKjUe/SwbQNeWHDE55MKMlmCfFyJyBDKZDO/dfyW2v2j5BGXWYGhWXrH8zwpB7umv9+u+Nyc4GHt0BAAPrNiDZVtP4JdDZXj10qOzrKK2x1tf7mnrCnGyoh4vfXsApy4FIqlq0XT8mZ6o0P9ZHW2tL0BC4YV9XojIkUQGivfY4ffjlbj2H5t1rxuaW3vdUqBu1ZjUUVarFXDOhOnuWzRao31XDpytwfOr81DeaRbjjUdst4pzd+5fvhursotx17Ku840JgoCDZ2t7vZ6QsSdn9mzcqe7UsfmF1Xl6++y9crcpJBNeiIicxW/PXmvT8z/08R691xP+vhmPXGr6b2xu7bYPiiFarYCxb2xE/Osb0KLXUbfjPOdqL+KZ/+7H9f/cinGLN+HH3JKuJ+rkgy0n8OSXhoct3/L+DnybcxbPfZNncL89lavaHpVU1qv1loQQAGwtqMCM93/H9Uu2GnyvsTleHJ0pHXPr1KY/xrMFhhciIjsbFuLT80FWtunoeVTUqRG78FfctWynWe9tbNGgrqkV6lYtKjp1dF2dfVbXGvPcN3n4MbdU98jh3R6G6K4/1POs6ycqHOvD/6cD+i1F7T9De8CxNlsPXPtv1hmD29VOsPQEwwsRkY0MCvYCALxy0xUiV9JmQ37bh217h9ve+u/eYiy5NN9H0WV9Qhyvl0Tv2ftnKqttMjoqqjeqG5rx8vcHscDInDvNrQwvdsMOu0TkaNY9PQG/PXstkgcFdtm395U0DO7vJUJVvXP5B3jmkXKDxxljTmuCuAN6xI9fK3eeMu24HabN1tuuoLweXxtpdXEWkgkv7LBLRI7G080Fw0J8MDrcr8u+/j7u+ODBRBGqarN0o/4Mqfmlqi6tJ7YgCMCa/d33hzHX5qPnMfPjPd2OLLKG7jo9ixl1/rQ2X8Sri0My4YWIyFHJ5TLd0OkbOi0OOCLUB8oHer/qtSWWbtTvk3LTu9sx+e0tPb7v8oYTwcj2kxUNWLLhmMFzmLpwpdFROJe9nrtyL7Yfr8SC7w6YdF5T9DQDrjVbhRxjTmbnwvBCRGQHkYGeyP/zVHw4U7+1ZfqYgXarwZxRRlqtgIvNmi5rAJnzmf3upkIzju7K3IBQYcX+Ie3zvZBjMnuGXSIisoynm7j/5b7242GTjvto2wkoN5+AqqkFggAsvTdBt6/Lo5NLL82d5t8Wj1mOltXZ4KwdOv/o4veI6dvY8kJEJLJ1T08QuwSdclUT/vbzUdRebNF9WH9v5T4qtrTnZJVdrrM2rxT/3Vtsl2tRV5IJLxxtRETOalSYHz54sGvfF3cX+/8Xfe1bm7ts69ymYu6jHHMnxNO7rgWdQTYds91MvALbWxyGZMILRxsRkTO7MS4Uz6YNR+xAX922V6bbf34YtZlzfPT0cW5suK8pucSiTrE2zBfd1cNOt/YlmfBCROTMZDIZnkkbhuuu6FjJeMaYMBEr6tCbPPBtzlmrn1NybD2VrgQxvBAROZBJw/sDADzdFAjwckPBG9NErkifVtyZ40TV3U/ed/9UxMHRRkREDmRsTCD+N388IgPaVqV2E6Hfy+WOl3eM4nGI7NJNDb0pz5pDrc1RXd/c80GkR/x/FUREpGdMhD8CvNx0r0N9PUSsBjhX27Ga8uUjj9qHTkvhyUfSXzeKct1/XjbbMfWM4YWIyME5UjDoabXoy+VfNsmdM3OIVicCIKHwwqHSRETU3fpDJB2SCS8cKk1EfcmXj6Zg8R2jxS5Dx5Fah2yF87w4DsmEFyIiqfrb7W0h5enrhum2+Xu64tYE8YdS96WPczbqOA6GFyIiBzd55AAc+fONyLh+uG6bu4scnm4u2JgxCb88M1G02mz5gW6speNASY3R9+xx0gUVW7XmTQ7Y13GoNBGRE+jnpgAAvHFbHM6rmjB0gA8AYOgAbwBAkJcbqhqcZ8htb/qm/Gf3GaP7Dpyttfi8vdHbEFeuEmeYtrNieCEiciIPXR1tcPsPT47Ht/vOYsaYgbj+n9vsVs+Z6kYAgMzMCfJNWQHa3HNS38HwQkQkAVFBnnqPlcj6OJLJcbDPCxGRxLxyk30XdCy+1PpibY42uufH3FKj+0prLtqxEpJMeOE8L0REbR6eMMiu15v41uY+MVT6+Pl6o/tmfrzHjpWQZMIL53khImqjkEsjSThTn5dKrk9kV5IJL0RE1OHaS6tTOzNHe2xEjoPhhYhIgt69LwF/uXWU3a7HlZHJnhheiIgkyN/TDTPHxdjtenXqVpuct7axBZ/tPGWTc5Pz4lBpIiJySOUqNeL/vEHsMsgBseWFiIiInArDCxERkQG3vP+72CWQEQwvRER9xLv3X4kfnrxG7DKchljrJFHPGF6IiCRsoJ+H7nsZgCujAnDqzenY+0oa4sJ9uxw/fcxAO1ZHZBmGFyIiCfv2iY6Wlv4+7nrf//TURFwzJEjv+KenDLNbbUSWksxoI6VSCaVSCY1GI3YpREQOI9y/H5Y9lIjC83VIGRTYZf/law32hWn+yflJJrykp6cjPT0dKpUKfn5+YpdDROQwbowLBRBqcB9nsSVnxMdGRERE5FQYXoiI+rAuj40AeLopuhz3+0uTkRzT9bETkRgYXoiISM/eV9Kwc8EUvW0RAZ5IGhQgUkVE+hheiIj6MEM9XrzcXRDm36/L9qc4EokcBMMLEVFf1s1oo9uvDAcAPDphEADAw7Xr4yQiMUhmtBEREfVe54Dy9zvH4MGUKCRE+otXEJEBDC9ERH2Yu2tHA/zzNwxHRICn7rWbixxj2UmXHBAfGxER9WF/vjUOg4K98Pc7R2M++7SQk2DLCxFRHzYo2Aubn08Vuwwis7DlhYiIiJwKwwsREZns+tgQsUsgYnghIiLTffhQIva/dj2igzx7PpjIRhheiIjIZHK5DAFebmKXQX2cZMKLUqlEbGwskpKSxC6FiEjyFJ1nsyOyM8mEl/T0dOTn52Pv3r1il0JEJHkD/T103z9zXccQ6/+7aaTBfjGThve3S13UN0gmvBARkf3I0NHy8uz1w3Xfy2UyvP/Alfjy0RRkvXIdACAmyBOuCn7ckPXwbxMREVmVu4sC44cGY4CPB069OR1bXpist/+Oq8JFqoykguGFiIisRmakL0znzUvuSbBPMSRZDC9ERETkVBheiIjIbOYONuLYJLImhhciIrIaH3fzl8z79Y/X2qASkjKGFyIiMlt/b3e913+5dRRuHBWK26403Bl3wbSR8PFwwbNpw7vsGxHqY5MaSbq4qjQREZnt/6ZfgdqLLbgvOQoAMHNcDGaOizF6/OD+3shbeAPk8rYHSCNDfXC0rE63f+38CZjx/u82rZmkg+GFiIjMFuztjo/nmDejeXtwAYBVj4/D62sP4/ZLLTWjI/ysWh9JG8MLERHZnV8/Vw6ZJouxzwsRERE5FYYXIiIicioML0RE5BDmTRyk+/4Pk4bA39NVxGrIkTG8EBGRQ3hleqzu+5GhPtj36vUiVkOOjOGFiIgcUufRSa4KztFLHRheiIjI4cUEeYldAjkQhhciIiJyKgwvRETkcEaF+eq9Fowcd3dihO2LIYfDSeqIiMhhbH9xMspVTRgW0vN6RzeNDsX0MQOxOuesHSojR+JwLS/FxcVITU1FbGwsxowZg9WrV4tdEhER2UlkoCfGxgT2eNyfbx2Ffz+YCIWcHXn7IodreXFxccHSpUuRkJCAsrIyJCYm4qabboKXFztrERH1VYKg/+BILmNo6cscLrwMHDgQAwcOBACEhoYiODgY1dXVDC9ERH2Yl7vDfVyRiMx+bLRt2zbMmDEDYWFhkMlkWLNmTZdjlEolYmJi4OHhgZSUFGRlZVlUXE5ODjQaDSIjIy16PxERObf3H7gSI0N9sOSeeKz/48Ruj/39pcl2qorEZnaUbWhoQHx8PB5++GHccccdXfavWrUKGRkZWLZsGVJSUrB06VJMnToVx44dw4ABAwAACQkJaG1t7fLeDRs2ICwsDABQXV2NWbNmYfny5eaWSEREEnHzmDDcPCbM6H4ZOh4fRQR42qMkcgBmh5dp06Zh2rRpRvcvWbIE8+bNw9y5cwEAy5Ytw7p16/DJJ59gwYIFAIDc3Nxur6FWq3HbbbdhwYIFuOaaa3o8Vq1W616rVCoTfxIiIiJyRlYdbdTc3IycnBykpaV1XEAuR1paGnbt2mXSOQRBwJw5czBlyhTMnDmzx+MXL14MPz8/3RcfMREREUmbVcNLZWUlNBoNQkJC9LaHhISgrKzMpHPs2LEDq1atwpo1a5CQkICEhAQcPHjQ6PEvv/wyamtrdV/FxcW9+hmIiMj5xUf6i10C2ZDDdd+eMGECtFqtyce7u7vD3d3dhhUREZGj6Wmk9H1Jkbgqyh+f7jhll3rIvqza8hIcHAyFQoHy8nK97eXl5QgNDbXmpYiIqA/zcmv73bu7ENO5My9Ji1XDi5ubGxITE5GZmanbptVqkZmZiXHjxlnzUl0olUrExsYiKSnJptchIiLxvHZzLKaOCsH0MQPFLoVEZHZ4qa+vR25urm7EUFFREXJzc3HmzBkAQEZGBpYvX47PPvsMR44cwRNPPIGGhgbd6CNbSU9PR35+Pvbu3WvT6xARkXgemTAIH84cC1dF28dXVKDx4dGmTMK74dlrrVUa2ZHZfV6ys7MxeXLHREAZGRkAgNmzZ2PlypW49957UVFRgYULF6KsrAwJCQlYv359l068REREvRUZ6IkvHklGgKdbl30ThgXj49+Lumx/9/4r8fTX+wF0H37IcZkdXlJTU7usMXG5+fPnY/78+RYXRUREZKqJw/p32RYd5Ilxg4MMHh8f4WfrksjGHG5VaUuxzwsREX37h3FYfMdoXDMkGDKZDD8/PRFpVwzg6tMSI5nwwj4vREQ0NiYQ9ydH6V7HhvlixewkjAjxEbEqsjbJhBciIiJjOnfe9e/X0T9GIZfB39NVhIqoNxxukjoiIiJb8vN0xZePpsBVIYerQo5ALzfUNLaIXRaZgeGFiIgkz0Wh/6Bh/NBg3fdD+3vjZEUDAMDDVY6mFtNneSdxSCa8KJVKKJVKaDQasUshIiIH89adYzDrkz145rrhXfb97Y7RCPB0w/0pUYgO9MTp6kbcptwhQpVkKpnQ07hnJ6NSqeDn54fa2lr4+vqKXQ4RETkIQRAgM2XmOgAxC9bZuBrnd+rN6VY9nzmf3+ywS0REfYKpwYUcH8MLERERORWGFyIiInIqDC9ERETkVCQTXrg8ABEROZJ+rgqxS5AsyYQXLg9ARETWsuyhqzAqzBfPpnUdWm2qnNfSrFgRdSaZeV6IiIis5ca4gbgxbiDW5pVa9P6XbhwJTzd+xNqKZFpeiIiIrK19Jt7BwV5mvS/tigG2KIcuYXghIiIyItDLDYden4oNz15r1vs4pYxtMbwQERF1w9vdBS4KOf6YNkzsUugShhciIiITuCrM+chk04stSSa8cKg0ERHZy0s3jux2Px8b2ZZkwguHShMRkb08kToE7i76H6GPThgkUjV9j2TCCxERkS0JgqD3ev/C65H1ynV48cYRGDbAG49NGqzb11PDS1Sgpw0q7DsYXoiIiCzg6eaCAT4eeDJ1KH7LmIQATzfdvp5WsJ4WF6r7fmSoj81qlCqGFyIiIitQdAoswd5tQWbVY1cbPNat0yOn+VOG2rYwCeL0f0RERCboqTVFLpdh8/OpaNFo4ePhCgBIGRzU43mDvNytUl9fwvBCRERkgsv7vBgyyMyZeMkyfGxEREQkIjmHVZtNMuGF87wQEZGzGB6i30n3wJ9uEKkS5ySZ8MJ5XoiIyNGtnT8Bf751FG4eM1Bvu++lPjJkGsmEFyIiIluaNLxtpWgPV8s/OkdH+GHWuBiDnX+vjw2x+Lx9DcMLERGRCUZH+OG3Z6/Fnv9LM+t9Pzx5DUaG+iD71e7fFx/hZ3Tfq9OvMOuaUsfwQkREZKJhIT7w62feI54rowKw/o/XItjb8JDo9jFM18eGGtwPACNDfc26ptQxvBARETmAEaE+2P7iZIP7uNCjPoYXIiIiBxFpZM0jZhd9DC9EREQiCvRy6/GYIQO87VCJ8+AMu0RERCL494NXoVzV1GXOF0NCfD3wyzMT4ePhggl/32yH6hwbwwsREZEIbho9sOeDOrliIDvttuNjIyIiIgm548pwsUuwOcmEFy4PQEREBCy4aaTYJdicZMILlwcgIiIC3BSS+Wg3Svo/IRERUR/i7+mGp6cMxZSRA8QuxWbYYZeIiMgBhfl5oLS2yaL3ZtwwAgCwo7ASD67YY82yAAC+HuLGB7a8EBEROZANz16LT+cmYWQPo4tuGm18OQFbCw8wPJmevTC8EBEROZDhIT6YPML4I59P5ozFbQlh+PudY3o8lyD0eIjOMDMmwhN7xl8+NiIiInJAgpHkMWVkCKaMDLHadfr7uOOLR5KhkMlw/T+3We28tsSWFyIiIid151URvT6HDG2rVsvlYrenmI7hhYiIyAGZ8sTnjdvi8O8Hr8KQ/l49Hjs8RDrrIzG8EBEROal+bgrcNHogvD1cDe7369exffzQYPhYaZSQTORGGoYXIiIiiRod4Ye0Kwagv487nrs0fLpdYnQAAOCesZFilNYr7LBLRETkgMwZKeThYrwtYsVsw8vmrJybhJzTFzB+aLC5pbHlhYiIiHpn8R2jMTjYC/+4q/vh0w8kRwEAxg8Ngo+HK1JHDICrBcsJyEQeLM2WFyIiIgcUGdjP5GMH9/fGpudTezzu+akjMH5osO6RUWcBnm7mlCcqhhciIiIH9MLUkVC3aHH7leFWO6erQo5rh/c3uC/Qi+HF7pRKJZRKJTQajdilEBER9ZpfP1f84+54u14zMrAfiqsv2vWalpAJxqbwc1IqlQp+fn6ora2Fr2/360IQERFRh+qGZuw9VY3Hv8jp9rgxEX743/wJVr22OZ/f7LBLREREANoeHU0dJd6Cj6ZieCEiIiKziL2QAMMLERERORWGFyIiIjKPyLPUMbwQERGRnu+fvAYPpkSJXYZRDC9ERESk56qoADxz3TCxyzCK4YWIiIi6GODrgYgA02f5tSeGFyIiIjLo+ctWom4n9mgjycywS0RERNZ1S3wYKuvVuCo6AB9vL8K6g+cAiN5fly0vREREZJhcLsOjEwfjqqgAKB+8SuxydBheiIiIyKkwvBAREZFZxO7zwvBCREREToXhhYiIiJwKwwsRERGZRcblAYiIiIhMx/BCREREZmGHXSIiIiIzMLwQERGRWTjD7mVqamowduxYJCQkIC4uDsuXLxe7JCIiInIgDre2kY+PD7Zt2wZPT080NDQgLi4Od9xxB4KCgsQujYiIiByAw7W8KBQKeHp6AgDUajUEQYAgCCJXRURERI7C7PCybds2zJgxA2FhYZDJZFizZk2XY5RKJWJiYuDh4YGUlBRkZWWZdY2amhrEx8cjIiICL7zwAoKDg80tk4iIiGxEJvJ4I7PDS0NDA+Lj46FUKg3uX7VqFTIyMrBo0SLs27cP8fHxmDp1Ks6fP687pr0/y+VfpaWlAAB/f3/k5eWhqKgIX331FcrLyy388YiIiMjqRO6wa3afl2nTpmHatGlG9y9ZsgTz5s3D3LlzAQDLli3DunXr8Mknn2DBggUAgNzcXJOuFRISgvj4eGzfvh133XWXwWPUajXUarXutUqlMvEnISIiImdk1T4vzc3NyMnJQVpaWscF5HKkpaVh165dJp2jvLwcdXV1AIDa2lps27YNI0aMMHr84sWL4efnp/uKjIzs3Q9BREREDs2q4aWyshIajQYhISF620NCQlBWVmbSOU6fPo2JEyciPj4eEydOxFNPPYXRo0cbPf7ll19GbW2t7qu4uLhXPwMRERF1T+wZdh1uqHRycrLJj5UAwN3dHe7u7rYriIiIiByKVVtegoODoVAounSwLS8vR2hoqDUvRURERH2UVcOLm5sbEhMTkZmZqdum1WqRmZmJcePGWfNSXSiVSsTGxiIpKcmm1yEiIurrxF4ewOzHRvX19SgsLNS9LioqQm5uLgIDAxEVFYWMjAzMnj0bY8eORXJyMpYuXYqGhgbd6CNbSU9PR3p6OlQqFfz8/Gx6LSIior5M7HlezA4v2dnZmDx5su51RkYGAGD27NlYuXIl7r33XlRUVGDhwoUoKytDQkIC1q9f36UTLxEREZElzA4vqampPU7XP3/+fMyfP9/iooiIiIiMcbi1jSzFPi9ERET2IXafF8mEl/T0dOTn52Pv3r1il0JEREQ2JJnwQkRERH0DwwsRERGZhY+NiIiIyKmIPVRaMuGFHXaJiIj6BsmEF3bYJSIi6hskE16IiIiob2B4ISIiIrOwwy4RERGRGRheiIiIyCyebgpRry+Z8MLRRkRERLb1j7vGIC7cF4tmjBK1DpnQ0yqLTkalUsHPzw+1tbXw9fUVuxwiIiIygTmf35JpeSEiIqK+geGFiIiInArDCxERETkVhhciIiJyKgwvRERE5FQkE144VJqIiKhv4FBpIiIiEh2HShMREZFkMbwQERGRU2F4ISIiIqfC8EJEREROheGFiIiInArDCxERETkVyYQXzvNCRETUN0hunpfa2lr4+/ujuLiY87wQERE5CZVKhcjISNTU1MDPz6/bY13sVJPd1NXVAQAiIyNFroSIiIjMVVdX12N4kVzLi1arRWlpKXx8fCCTyXTbk5KSsHfv3i7Hm7O9PRU6QquOsbrFOKc57zPl2J6O6W6/qfeT97L37+O9NMza99Me99LU4y25X93t4720zXtt+W/TlvdSEATU1dUhLCwMcnn3vVok1/Iil8sRERHRZbtCoTD4B2judgDw9fUV/R9Wd/XZ+5zmvM+UY3s6prv95t5P3kvL38d7aZi176c97qWpx1tyv7rbx3tpm/fa8t+mre9lTy0u7STTYbcn6enpVtnuKGxRn6XnNOd9phzb0zHd7XfG+8l7ad4+R76XgPXrs8e9NPV4S+5Xd/t4L23zXlv+23SUeym5x0a2xEUfpYP3Ujp4L6WD91I6bH0v+0zLizW4u7tj0aJFcHd3F7sU6iXeS+ngvZQO3kvpsPW9ZMsLERERORW2vBAREZFTYXghIiIip8LwQkRERE6F4YWIiIicCsMLERERORWGFyv56aefMGLECAwbNgwrVqwQuxzqhdtvvx0BAQG46667xC6Feqm4uBipqamIjY3FmDFjsHr1arFLIgvV1NRg7NixSEhIQFxcHJYvXy52SdRLjY2NiI6OxvPPP2/2ezlU2gpaW1sRGxuLzZs3w8/PD4mJidi5cyeCgoLELo0ssGXLFtTV1eGzzz7Dt99+K3Y51Avnzp1DeXk5EhISUFZWhsTERBQUFMDLy0vs0shMGo0GarUanp6eaGhoQFxcHLKzs/n/rBN75ZVXUFhYiMjISLz99ttmvZctL1aQlZWFUaNGITw8HN7e3pg2bRo2bNggdllkodTUVPj4+IhdBlnBwIEDkZCQAAAIDQ1FcHAwqqurxS2KLKJQKODp6QkAUKvVEAQB/N3beR0/fhxHjx7FtGnTLHo/wwuAbdu2YcaMGQgLC4NMJsOaNWu6HKNUKhETEwMPDw+kpKQgKytLt6+0tBTh4eG61+Hh4SgpKbFH6XSZ3t5LcizWvJ85OTnQaDSIjIy0cdVkiDXuZU1NDeLj4xEREYEXXngBwcHBdqqeOrPGvXz++eexePFii2tgeAHQ0NCA+Ph4KJVKg/tXrVqFjIwMLFq0CPv27UN8fDymTp2K8+fP27lS6gnvpbRY635WV1dj1qxZ+Oijj+xRNhlgjXvp7++PvLw8FBUV4auvvkJ5ebm9yqdOensvf/zxRwwfPhzDhw+3vAiB9AAQfvjhB71tycnJQnp6uu61RqMRwsLChMWLFwuCIAg7duwQbrvtNt3+Z555Rvjyyy/tUi8ZZ8m9bLd582bhzjvvtEeZZCJL72dTU5MwceJE4fPPP7dXqdSD3vzbbPfEE08Iq1evtmWZZAJL7uWCBQuEiIgIITo6WggKChJ8fX2F119/3azrsuWlB83NzcjJyUFaWppum1wuR1paGnbt2gUASE5OxqFDh1BSUoL6+nr88ssvmDp1qlglkxGm3EtyHqbcT0EQMGfOHEyZMgUzZ84Uq1TqgSn3sry8HHV1dQCA2tpabNu2DSNGjBClXjLOlHu5ePFiFBcX49SpU3j77bcxb948LFy40KzruFi1agmqrKyERqNBSEiI3vaQkBAcPXoUAODi4oJ33nkHkydPhlarxYsvvsge8A7IlHsJAGlpacjLy0NDQwMiIiKwevVqjBs3zt7lUg9MuZ87duzAqlWrMGbMGN1z+S+++AKjR4+2d7nUDVPu5enTp/HYY4/pOuo+9dRTvI8OyNT/Z3uL4cVKbrnlFtxyyy1il0FWsHHjRrFLICuZMGECtFqt2GWQFSQnJyM3N1fsMsjK5syZY9H7+NioB8HBwVAoFF06hpWXlyM0NFSkqsgSvJfSwvspHbyX0mGve8nw0gM3NzckJiYiMzNTt02r1SIzM5OPEpwM76W08H5KB++ldNjrXvKxEYD6+noUFhbqXhcVFSE3NxeBgYGIiopCRkYGZs+ejbFjxyI5ORlLly5FQ0MD5s6dK2LVZAjvpbTwfkoH76V0OMS97N0gKWnYvHmzAKDL1+zZs3XHvPfee0JUVJTg5uYmJCcnC7t37xavYDKK91JaeD+lg/dSOhzhXnJtIyIiInIq7PNCREREToXhhYiIiJwKwwsRERE5FYYXIiIicioML0RERORUGF6IiIjIqTC8EBERkVNheCEiIiKnwvBCREREToXhhYiIiJwKwwsRERE5FYYXIiIicir/DxrpaYCzSwssAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "plt.plot(losses)\n",
    "plt.yscale(\"log\")\n",
    "plt.xscale(\"log\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390619 0.00390619 0.00390619 0.00390619]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390619 0.00390619 0.00390619 0.00390619]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n",
      "[0.00390625 0.00390625 0.00390625 0.00390625]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(Array(0.84375, dtype=float32, weak_type=True),\n",
       " Array([0.        , 0.        , 0.        , 0.        , 0.07407407,\n",
       "        0.9259259 ], dtype=float32, weak_type=True))"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import random\n",
    "import jax\n",
    "closenesses = jax.vmap(lambda key: (diffusion.sample(lambda x, t: run_net(ema_params, x, t), key, 256, (seq_len,)) == data))(jax.random.split(jax.random.key(0), 32))\n",
    "# p = closenesses.mean(2)\n",
    "# p = p / p.sum(-1, keepdims=True)\n",
    "# closenesses.min(2).max(1).mean(), p.mean(0)\n",
    "eq = 0\n",
    "corr = 0\n",
    "for sample in closenesses:\n",
    "    correct = sample.all(-1)\n",
    "    corr = corr + correct.any()\n",
    "    if correct.any():\n",
    "        eq = eq + correct\n",
    "corr / len(closenesses), eq / corr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "key = jax.random.key(0)\n",
    "score_fn = partial(run_net, ema_params)\n",
    "noise_key, transition_key = jax.random.split(key, 2)\n",
    "x = data.repeat(128, 0)\n",
    "t = jax.random.uniform(noise_key, x.shape[:-1] + (1,))\n",
    "total_noise, rate_noise = diffusion.noise_schedule(t)\n",
    "data_perturbed = diffusion.sample_transition(transition_key, x, total_noise)\n",
    "logits = score_fn(data_perturbed, total_noise)\n",
    "# logits[..., -1]\n",
    "xs = []\n",
    "for i in range(len(t)):\n",
    "    for j, (l, u) in enumerate(zip(logits[i], data_perturbed[i] == n_classes)):\n",
    "        # if not u:\n",
    "            # continue\n",
    "        probs = diffusion.staggered_score(jnp.exp(l), rate_noise[i, 0]) * diffusion.transp_transition(x[i], rate_noise[i])[j]\n",
    "        l_last = (probs / (1e-10 + probs.sum()))[..., -1]\n",
    "        xs.append((float(t[i, 0]), float(l_last)))\n",
    "plt.scatter(*np.array(xs).T)\n",
    "plt.ylim(-200, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}