dqn_train.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Learn to play Pong with a Deep Q-Network (DQN)\n",
    "\n",
    "Train an agent that can play the game Pong for Atari 2600 using the REINFORCE algorithm, which is a Policy Gradient Method.\n",
    "\n",
    "### 1. Import Packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from dqn.pong_environment import PongFromFeatures\n",
    "from dqn.dqn_agent import Agent\n",
    "\n",
    "import torch\n",
    "import numpy as np\n",
    "from collections import deque\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Instantiate the Environment and Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "A.L.E: Arcade Learning Environment (version 0.8.1+53f58b7)\n",
      "[Powered by Stella]\n",
      "/home/ale/dev/pong/DLMAIRIL01/iu_rl/lib/python3.10/site-packages/gymnasium/core.py:311: UserWarning: \u001b[33mWARN: env.seed to get variables from other wrappers is deprecated and will be removed in v1.0, to get this variable you can do `env.unwrapped.seed` for environment variables or `env.get_wrapper_attr('seed')` that will search the reminding wrappers.\u001b[0m\n",
      "  logger.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "State shape:  (12,)\n",
      "Number of actions:  3\n"
     ]
    }
   ],
   "source": [
    "env = PongFromFeatures(False)\n",
    "env.seed(0)\n",
    "\n",
    "print('State shape: ', env.observation_space.shape)\n",
    "print('Number of actions: ', env.action_space.n)\n",
    "\n",
    "agent = Agent(state_size=env.observation_space.shape[0], action_size=env.action_space.n, seed=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Train the Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 100\tAverage Score: -20.11\n",
      "Episode 200\tAverage Score: -19.36\n",
      "Episode 300\tAverage Score: -18.75\n",
      "Episode 400\tAverage Score: -17.92\n",
      "Episode 500\tAverage Score: -17.28\n",
      "Episode 600\tAverage Score: -16.64\n",
      "Episode 700\tAverage Score: -15.99\n",
      "Episode 800\tAverage Score: -15.36\n",
      "Episode 900\tAverage Score: -14.74\n",
      "Episode 1000\tAverage Score: -13.93\n",
      "Episode 1100\tAverage Score: -13.65\n",
      "Episode 1200\tAverage Score: -12.72\n",
      "Episode 1300\tAverage Score: -11.13\n",
      "Episode 1400\tAverage Score: -11.77\n",
      "Episode 1500\tAverage Score: -8.948\n",
      "Episode 1600\tAverage Score: -7.43\n",
      "Episode 1700\tAverage Score: -7.53\n",
      "Episode 1800\tAverage Score: -6.25\n",
      "Episode 1900\tAverage Score: -3.89\n",
      "Episode 2000\tAverage Score: -1.70\n",
      "Episode 2100\tAverage Score: 3.220\n",
      "Episode 2200\tAverage Score: 6.83\n",
      "Episode 2300\tAverage Score: 7.43\n",
      "Episode 2400\tAverage Score: 9.39\n",
      "Episode 2500\tAverage Score: 9.221\n",
      "Episode 2600\tAverage Score: 10.48\n",
      "Episode 2700\tAverage Score: 10.13\n",
      "Episode 2800\tAverage Score: 10.98\n",
      "Episode 2900\tAverage Score: 10.32\n",
      "Episode 3000\tAverage Score: 11.19\n",
      "Episode 3100\tAverage Score: 12.18\n",
      "Episode 3200\tAverage Score: 12.51\n",
      "Episode 3300\tAverage Score: 13.29\n",
      "Episode 3400\tAverage Score: 12.93\n",
      "Episode 3500\tAverage Score: 13.53\n",
      "Episode 3600\tAverage Score: 13.92\n",
      "Episode 3700\tAverage Score: 13.81\n",
      "Episode 3800\tAverage Score: 13.37\n",
      "Episode 3900\tAverage Score: 13.76\n",
      "Episode 4000\tAverage Score: 13.80\n",
      "Episode 4100\tAverage Score: 13.34\n",
      "Episode 4200\tAverage Score: 14.03\n",
      "Episode 4300\tAverage Score: 12.91\n",
      "Episode 4400\tAverage Score: 13.13\n",
      "Episode 4500\tAverage Score: 13.58\n",
      "Episode 4600\tAverage Score: 13.80\n",
      "Episode 4700\tAverage Score: 13.61\n",
      "Episode 4800\tAverage Score: 12.12\n",
      "Episode 4900\tAverage Score: 13.77\n",
      "Episode 5000\tAverage Score: 12.95\n",
      "Episode 5100\tAverage Score: 13.69\n",
      "Episode 5200\tAverage Score: 13.66\n",
      "Episode 5300\tAverage Score: 14.22\n",
      "Episode 5400\tAverage Score: 13.68\n",
      "Episode 5500\tAverage Score: 13.37\n",
      "Episode 5600\tAverage Score: 13.00\n",
      "Episode 5700\tAverage Score: 13.53\n",
      "Episode 5800\tAverage Score: 14.39\n",
      "Episode 5900\tAverage Score: 13.22\n",
      "Episode 6000\tAverage Score: 14.92\n",
      "Episode 6100\tAverage Score: 13.90\n",
      "Episode 6200\tAverage Score: 13.72\n",
      "Episode 6300\tAverage Score: 13.35\n",
      "Episode 6400\tAverage Score: 14.68\n",
      "Episode 6500\tAverage Score: 14.50\n",
      "Episode 6600\tAverage Score: 13.84\n",
      "Episode 6700\tAverage Score: 15.11\n",
      "Episode 6800\tAverage Score: 14.77\n",
      "Episode 6900\tAverage Score: 15.36\n",
      "Episode 7000\tAverage Score: 14.90\n",
      "Episode 7100\tAverage Score: 14.79\n",
      "Episode 7200\tAverage Score: 14.74\n",
      "Episode 7300\tAverage Score: 15.10\n",
      "Episode 7400\tAverage Score: 15.24\n",
      "Episode 7500\tAverage Score: 14.61\n",
      "Episode 7600\tAverage Score: 14.74\n",
      "Episode 7700\tAverage Score: 15.92\n",
      "Episode 7800\tAverage Score: 14.95\n",
      "Episode 7900\tAverage Score: 15.65\n",
      "Episode 8000\tAverage Score: 15.55\n",
      "Episode 8100\tAverage Score: 16.11\n",
      "Episode 8200\tAverage Score: 17.19\n",
      "Episode 8300\tAverage Score: 16.60\n",
      "Episode 8400\tAverage Score: 17.24\n",
      "Episode 8500\tAverage Score: 17.71\n",
      "Episode 8600\tAverage Score: 17.79\n",
      "Episode 8700\tAverage Score: 17.59\n",
      "Episode 8800\tAverage Score: 17.15\n",
      "Episode 8900\tAverage Score: 17.09\n",
      "Episode 9000\tAverage Score: 16.38\n",
      "Episode 9100\tAverage Score: 17.14\n",
      "Episode 9200\tAverage Score: 16.91\n",
      "Episode 9300\tAverage Score: 17.02\n",
      "Episode 9400\tAverage Score: 16.71\n",
      "Episode 9500\tAverage Score: 16.73\n",
      "Episode 9600\tAverage Score: 17.19\n",
      "Episode 9700\tAverage Score: 17.25\n",
      "Episode 9800\tAverage Score: 16.92\n",
      "Episode 9900\tAverage Score: 16.98\n",
      "Episode 10000\tAverage Score: 16.91\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAj4AAAGwCAYAAACpYG+ZAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAA9hAAAPYQGoP6dpAABSvElEQVR4nO3deXwTZf4H8E+OJr3ofQFtoaVAC5QCLUe5kcohi6KsJyIogiKoHMJSBVQ8iqyLrv4QVkXQ9cBjFVdEFDnlEATlRhQBYYGCCLQFeibP74/S0LRJm6RJZibzeb9eVTIzmfnOkzTz6TMzTzRCCAEiIiIiFdBKXQARERGRtzD4EBERkWow+BAREZFqMPgQERGRajD4EBERkWow+BAREZFqMPgQERGRauilLkBuzGYzTp06hUaNGkGj0UhdDhERETlACIGioiI0adIEWq39fh0GnxpOnTqFhIQEqcsgIiIiF5w4cQLx8fF25zP41NCoUSMAlQ0XEhIicTVERETkiMLCQiQkJFiO4/Yw+NRQdXorJCSEwYeIiEhh6rtMhRc3ExERkWow+BAREZFqMPgQERGRajD4EBERkWow+BAREZFqMPgQERGRajD4EBERkWow+BAREZFqMPgQERGRajD4EBERkWow+BAREZFqMPgQERGRavBLSknVSitM0Gu10GmvfaldcZkJAQadzeXLKszQaAA/nWt/M1wpq4BWo4G/n86yPq0G0Ou0KCk3wajXWn3BXl211FRYUo5APx30Oi1KK0y4VFKBiCADSsrNlnUUl5mg0QBCAEa9FlqtBhculyE8yGBZT4XJDLMADPpr+2g2C/x5uQzhgX7Q67RWdQkhLNuoml5WYYZZCJSbzGjk7wchBC6XmWAWAiH+figpN1naoOZ+FpeZAAABBh2KSsrh76eztLcQAqUVZrvPdVT1172q3UsrzDBc3U6ZyQwAlveFBoBZACazqLWt4jITtNrKNq0wCwT66VBUUoHQQD8Ul5lQUm6yat8qFy6Xwd9PB38/LS5eKbcsU1JuQmm5GX56DQINeqvtGPRaVJjNMOorayi/WqefTguTWeByWQX8tFoEGHRW++XvV/na6HUalJvMEAIIMuot6/X3q3zf2XoP1mw3rUYDsxAw6nU4U1gCg06L0AA/aLUalFWYUVxeuT6jXmd5b5RWVL7eVe1d9Z4oraisv/rrWVJusuxT1fJmIVBhFtBqNAj006HMVLlPl0srYBLCqkatRgOdRgN/Py0qzMKyrpJyE66UmaDVAGGBtV8PoPJ9fvJiMSKCDBBXawk26mEyC1SYBPS6yt/dP4pKER7kh7IKM8oqzNBrtSg3mxFo0KGk3AwhBC6VVqCwuAJNwwNQWmGCBhpcKq1AbIgRJrNAcXnlNK0GKCguR7C/HoF+epSZKp9vFoBep0FhcTniwwNRYTbjUkkFGvn7Wd6DZ4tKoNdqUWEyo8IsYBYCTUID8OflMhh0WpSZzPDTaXDuUhmaRQbij6JS6HUay+ufX1CMpKhgnLpYjJAAP0AAFWYzTELAT6uFWQjodVqYzQJlJjP02sr35LlLpdBogOhGRpSbBMorzBAALpdWIMioR3G5CSaTQFyoP7Qa4GxRKYx6LYx+lZ8NETZ+H7yJwYdUq7TChE5zViMmxB/rHusLAHhh1c9YuP43vD+2K7q3iLJavsJkRtfnv4VBr8XWGf2h1db9DcA1vbXpKOasOAAA2P3kAAT46ZD57GqE+Pvhk/HZyM5bi+tSY/DW6M4AgA+2H0fup3sx76/tcVtWQp3rXn/oLEYv+QEAcHDOIKTNXmU1f9m4blh94AwWbzpqmZbVLBxmIfDj8YvIahaOT8Z3hxACveetw+UyE3bMzLEEji7Pf4tzl8oAAPP+2h7TP9mDvFvScWeXREz6cBc+33UKU65vhfmrf8ETN6ThtfWHceFKOQDgu+n98PzKg/hqXz4AoGdKFDYdPof/jM9GZrMIvPndETz75UG8NqITfjh2Hks2HwMA3NklER9sPw4AOJp3AzQaDR749058c+AMNv2tH+LDA/HRDycw/T97MG94e9zWue42qlJSbkLmM5Wv+4cPdEOX59YgvWko9p4sQOfm4dBrtdh65E/L8lHBRui0wJnCUgDAlOtb4ZH+LQEA+04W4C+vbrK5nap1AkB8eAA2/e06y7xbXtuMH49ftFr+ketScF/PJHSYs9oy7Zlh7TCyWzPM/nwf3tn6OwAgwE+Hn2ZfDz+dFtl5a6DRaPB9bn90emY1Coor2/yG9Dis3JtvWc/MIWl49suDVtv7dkofmMwCA1/eiL9mxiN3cCoyn/0W3VtE4v2x3Wrtz5WyCnSYsxplFWYY9FrENDLifxeKLfMPPzcYrWZ+ZXn88zODMGnZLqzan19rXZNzWmH7sT+x+XBlOz9xQxrG9k7GpdIKtHvyawBAi+gg/Gd8d6v2qO7h61Lw6trDNucBwJD2jbHj2HmUmwS+mdwbWc9+a5l3W1Y85v01o9Zzkh9faXd9cvLYgFZYsec0fs4vkroUl2TEh+LziT0l2z5PdZFq/ZJ/CZfLTDh67rJl2sL1vwEAnl1xsNby+YUluHClHGcKS3Hl6l+lzqgKPQCw8Zc/cOzPyygqqcDJi8X4ZMf/AABrfz5rWSb3070AgOmf7Kl33VWhB4DlYFvd8ysPWoUeANjx+wXLwXfH7xcAVPZ0nCooQUFxOU5dvHZQqwo91eupqu/zXacAAPNX/wIAeG7lQUvoAYBX1vxqCT0AsOnwOQDAC6sOAYDlgDz1o92W0APAEnoAWHoGvjlwBgDw4Q8nKmv5zx6r/zviUH6R5XVfsfs0gGtt9sOxC1ahp3LfSy2hp/p+Vu2bPdVfh+oBAUCt0AMAr6w9jG8PnrWaNmv5PgCwhB4AKC43Yf+pQvx5qRTnLpXhj6JSFBaXW0IPAKvQA6BW6AGAJZuPYtGGyvf7Jzv/h5VXX6Mtv/1Za1kA2Pn7BZRdfR3KKsy19im/sMTq8c/5RTZDDwC89O0vltADVL5nAOD7atv+7Y/LVr8PNdUVegDgyz2ncaawFOcvl+GDbcet5n109fdNqV785hfFhh4A2P2/2p9R3sTgQ0SyICDqX8gNtHZO4yiLB9pK1L3OembbWN47ryeRsxh8iEgaNY6LzhwnG3JMrZ571H5o9mQ4Mau9cUm2GHyIJKCETgd3HRMd3VdvHSfd2fY8tteFrUPypJjgk5eXh86dO6NRo0aIiYnBsGHDcOjQIatlSkpKMGHCBERGRiI4OBjDhw/HmTNnJKqYyHucvM7aLg28l8i8dWqrpur7KLfTMd7Ow/bu3rKlvpaq2ZQya1oiC8UEnw0bNmDChAn4/vvvsXr1apSXl2PAgAG4fPnahamTJ0/GF198gY8//hgbNmzAqVOncMstt0hYNZFtNQNGQ48R1a9bkd/BXF5dPlanuuTVVE5xR+mefK8ouGnJxynmdvZVq6xvz126dCliYmKwc+dO9O7dGwUFBVi8eDHef/99XHdd5W2jS5YsQVpaGr7//nt061b79kwie1w9HeLomDJmISzjldhSc171cW9sbcNc7QBmasDFFdV7YcwOHBSrxtxxVfX9rBo7x57SCpPNf1cpKilHcZkJfjqtzXFzqpRX244j+2hLSbnJpYt9q8a0saeopLzWtItXympNKyk3W7VdhQuveUl55Zg7tlSN9wRUjoFTUFyOSyUVda6v5n6ZXaip5u/d5Qa+v6oU2mjX/124Ap1WgxB/PxSVVCAkQDGHQ58ghHCqx9GdFPtKFxRU3g4XEREBANi5cyfKy8uRk5NjWSY1NRWJiYnYunWr3eBTWlqK0tJrt6oWFhZ6sGpSCleOh//eegyzPt+Pl2/vgGEdm9a57MMf/GT1uPot0nv+dxE3/t9mq/mps1Zh58wcHMovwl1vbsOEfi0wbWCqZX71Y8xdb26rtb09Ltw+OmzBZux5aiAWrLN/23DN8YJssffZdraoFKmz6n8+APxw7DxGLt5uefzGd0fx8NWxdKqkP/WN5d93d0vEs8PSa63ncmmFVdvmffWzQ9uvydG6q+v/jw04Um3oBFue+uJArWm2xrEZUeM17vnCWqfr+c+P1rd0Vw0zADj2utaUM3+j1WNnc8+kZT9h+dWhEapU3c7fUG98d7TWtJ4vrHPLusk15SYBg16a4KOYU13Vmc1mTJo0CT169EC7du0AAPn5+TAYDAgLC7NaNjY2Fvn5tseSACqvHQoNDbX8JCQ4NggaUU2zPt8PAJj04a4GreelaiGoulX78/H01QPjgnW/NWgb9lQ/LVV49S/8v399yN7iDfL7n1ccXnbM0h21pm05bHu8GQB49/vjNqdXjVckhfpCT0NUjXPUEF/vl/Z6yJqhh8hTFBl8JkyYgH379mHZsmUNXldubi4KCgosPydOnHBDhUSuq6v7V6oLgqXmrv2W2/VPRGol5Z2tijvVNXHiRKxYsQIbN25EfHy8ZXpcXBzKyspw8eJFq16fM2fOIC4uzu76jEYjjEajJ0smHyTF76xSj9nu+IBT6r6rmVpDOjlGyhE9FNPjI4TAxIkT8dlnn2Ht2rVISkqymp+ZmQk/Pz+sWbPGMu3QoUM4fvw4srOzvV0uKUBDPpg9+ZFe1weCMgNAwz/iFLnbRGSXVBc2Awrq8ZkwYQLef/99fP7552jUqJHlup3Q0FAEBAQgNDQUY8aMwZQpUxAREYGQkBA8/PDDyM7O5h1d5DQpu2Htbdvewd+dp288s9+MLURkTcoeH8UEn4ULFwIA+vbtazV9yZIlGD16NADgpZdeglarxfDhw1FaWoqBAwfitdde83KlRA0l3UeCMnuUSJb4XqI68BofBzjyV62/vz8WLFiABQsWeKEi8jXVx6RxNADYG7en4Eo5DHotDHotistNCPCrf2yfKrbGHAGAwuJynLhgfSeU2SzqHf/GUX9eKkVJjbuDXBmLpaayioavw5WxiYpKylFhEjDotTAJAb1WU+fYSeRel0rrHveHSCqKCT5E7lb91u1FG37DXCfHdNny2znc9cY2jO/bAn8blGo1L2PON1aP48MDHF7v9qPnbU63dVv5jQs24eDpIofXXZfMZ7+tNa37XOfHh6mp5ngx7vLguzvrnF99XB/yvnH/rvv1IXWT8hofxVzcTORJzoYeAJhzdUydhevrH1PnfxeKnV6/I/adLGzQSM31yS8s8di6iYikwOBDREREqsHgQ2SDlBfeERGR5zD4ELmAuYiISJkYfIgcVPNOLykvziMiItcw+BDZwPFsiIh8E29nJ7Lh4pUyCCFQWmGGySxQYRJW1/2UlJusxoSRYnyYMjd8IzcRkdow+JBq1fVdXacKSpCUu9Jq2p1dEi3/rjnmTeqsVe4tzgGtZn7l9W0SESkdT3UROeiD7celLoGIiBqIwYeIiIhUg8GHiIiIVIPBh4iIiFSDwYeIiIhUg8GHiIiIVIO3s5MqVI3JAwBGvRYajQYafvEEEZHqMPiQKjzw75345sAZAEBOWgzeHNUZr679VeKqiIjI23iqi1ShKvQAwLcHz9aaRkRE6sDgQ0RERKrB4ENERESqweBDREREqsHgQ0RERKrB4ENERESqwdvZyacJIVBQXG5zOhERqQ+DD/m0pNyVNqffvXiblyshIiI54KkuUqXNh/+UugQiIpIAgw8RERGpBoMPERERqQaDDxEREakGgw8RERGpBoMPERERqQaDDxEREakGgw8RERGpBoMPERERqQaDDxEREakGgw8RERGpBoMPERERqQaDDxEREakGgw8RERGpBoMPERERqQaDDxEREakGgw8RERGpBoMPERERqQaDDxEREakGgw8RERGpBoMPERERqQaDDxEREamGXuoCiBqq3GSGEIDJLBBg0AEASspNKCqpkLgyIiKSGwYfUjSzWaDXC+uQX1gCAFg1qReig43IfPZbiSsjIiI5YvAhRbtSbrKEHgD414YjyGoeLmFFREQkZ7zGh3yKEELqEoiISMYYfIiIiEg1GHzIp7C/h4iI6sLgQ0RERKrB4EM+RQhAA43UZRARkUwx+JBiFZeZakUcAaC43CRFOUREpAC8nZ0UafPhcxjx5jbc26O51fTf/7yML3afkqYoIiKSPfb4kCI99d/9AIAlm49ZTd/zvwIJqiEiX9Y8MtDuvPjwAC9WQu7A4ENERFSH9dP62Z236W/XubxehiZpMPgQERFJgOOtSoPBh4iIiFSDwYcUiX8oEZHSaTjyhiQYfIiIiCTAU13SYPAhyZRwvB0iIvIyBh+SxP5TBUidtQq5n+6RuhQiIlIRBh+SxIJ1hwEAH2w/IXElRESu+XBcNwDA4HZxEldCzmDwISIictKxuUPQNTkSALDw7kwcmzvEresfmtHELesZ1zvZLevxJQw+REREMiPcdOWzu9bjSxh8iIiIJMBQIg1FBZ+NGzdi6NChaNKkCTQaDZYvX241XwiB2bNno3HjxggICEBOTg5+/fVXaYolIiJykbsikYaDBdWiqOBz+fJlZGRkYMGCBTbnz5s3D6+88goWLVqEbdu2ISgoCAMHDkRJSYmXKyVP419KRKR03vgU42dlbXqpC3DG4MGDMXjwYJvzhBB4+eWXMXPmTNx0000AgHfeeQexsbFYvnw57rjjDm+WSh5UXMbxf4iIyDWK6vGpy9GjR5Gfn4+cnBzLtNDQUHTt2hVbt261+7zS0lIUFhZa/ZB8rfv5LNJmr8Jvf1yWuhQiogbx99PZn8mOGo/xmeCTn58PAIiNjbWaHhsba5lnS15eHkJDQy0/CQkJHq2TGmYGBzwkIplrEurv0HL/d1dHu/MEk4/H+EzwcVVubi4KCgosPydOcEA9IiIlubdHc9yaGS91GRaP9G/p0HJtm4SiV8som/PkcGnON5N7W/49KruZhJW4l88En7i4ypEzz5w5YzX9zJkzlnm2GI1GhISEWP0QERFJSQ7Bx1f5TPBJSkpCXFwc1qxZY5lWWFiIbdu2ITs7W8LKiIjI0+R017YztdgLOGYmH49R1F1dly5dwuHDhy2Pjx49il27diEiIgKJiYmYNGkSnn32WbRs2RJJSUmYNWsWmjRpgmHDhklXNBEReZQGGp/rIfGx3ZEVRQWfHTt2oF+/fpbHU6ZMAQCMGjUKS5cuxfTp03H58mWMGzcOFy9eRM+ePbFq1Sr4+zt2oRnJn699uBER2cLxdzxHUcGnb9++db4ZNBoN5syZgzlz5nixKnKFM7/TJeUmGPVaaDQadv8SUS1yOs3lLHu186POc3zmGh/yTcfOXUbqrFWY+MFPAIBzl8okroiI5EZuIcEd9chhl7TVQpmfznfigu/sCfmkd7b+DgD4cs9piSshInumDWwtdQkNlhwd5PFtOHfRszTRJyrYiJSYYEwf1BotooPRq2UUbsxogof6pSA2xAgAeLTG7fqtYxvBoNMiJSbYanp8eIDX6nYGgw8RETVIzxTbY9G44tjcIW5bl6MMOi3WTu3r8PJGvfOHzudvTkd4oMHh5c0C6JES6fR2GioiyA/fTumDh/qmQKPR4N9juuKVOzsiIsiAbY/n4NjcIZh8fSur53w9uTd+eW4wvp3Sx2r64zekebN0hzH4EBFRgyj5GhsAgJvrd0d7CFTerdbg9TjZceSObcodgw8REZHMCCH4tRUewuBDklD8X4hEJBsN/jyRYb5w1yU+zraNOz+b5foxz+BDssaARCR/Up8eEUKW2cWKs59l7O3xHAYfkq2SchOulJmkLoOIfJ0XcpuzPThmszTbVQNFDWBI6nG2qARdnltT/4JEpHoCQlbjzNgLG/5O3A1m0GtR4a704wRX7lizR66ZSz7vFKJqVuzmuD1EcvLXzHi781w5Jd0/NQZJUdZj50zo18L5FV11f6+kOucnRgTan+nAEbrO5ztAowFevyfLoWW1GmDOTW3d0ltT87W5MaMJMpuFY9HdmTaXf/HWDKfWnxrXyOmavDFmUl0YfIhU7tU7O0pdgurk3ZKO5ChpP/yd9eywdlaPnxzapkHrWzy6M9Y91tcqUE0bmOrSujTQICzAz2pah4Qwq8dZzcNdWneVh69LcbweG0FQCKBd01B0dqCOdY/1RbNI97w/qoenxIhAvHJnR/xnfHcMaheHY3OHWI2btHhUFlrGOhdk2jQJcbqmqddLO+Algw8REdVL7deKqGH31XIzCYMPkcqp4QOd3E9Ox0iNxoH3cV0LyGlnVEDqgMXgQ0RE9fLUwUoWmcML6d+V9pM6IPgqBh+ShNq7zYnIs+T2EePMZ17Vst7+nFTL5zKDD3lVsY1xeWxNIyLlYM+EMnBQxEoMPuQ1H+04gbTZq7Bs+3Gr6WmzV2HW8n0SVUUkEYYFAO7pmWlwT4UDr4Vee20hfz9dnctqbSRBqcKhXkbjG8kFW4S8ZvonewAAMz7dW2vev7//3dvl0FU8/nqfo23eKTHMk2U4pebBXuOFI/lDfR0b18dWT0at6uoq9+rTl4zubHe8nhvSG1v+vfTezjaXGdsrCe2ahmBoRpPamxBVZdTfbo427d8G2b/9f9rA1mgZE4wHeic7tjIntluX+3tWjqd0Z5fEhq/MQzhyM5GKtWns/BgcStY8MhDH/rwidRkOG9e7BR58d2eD1xNs1ONSaYUbKrKt+sFcr9Wgwux4F0xdx9rMZg0be8dZ/VJj0C81Bs1nfFlrnr+fzmrMG1ueGFL/2EbOnG6qrydrfN8WeGHVzzbnTeiXggn9HB97yF1m/qUNZv6lsh1W7rU9EK3U1xKxx4dIxXhtBrmbO99Tjh4gNdDUWlZuV7Pwd00+GHyIiGTKXQdLHnPlQ+pvsq+Lu3tipO7ZsYfBh0jlZPrZRG7kiddYVDuqKaI3g290uorBh0jF5PoXGZEzNBqFhC+J1ff7rpY2ZPAhIpIpdwVTTxzP3HVXl1oOts7w9T9IpH7NGXyISDW8cQu2O2ndVK6nj6PVm7W+MW5qcsdB3tY6jE6MX2P08/yhUCeD955R7/79NNTRzjLYZZsYfEgScv2FUBuNRl0XvooGHGWTo4PcWIljOiSEeX2bdQk21j8Cyv/d1anO+c0iA7F4VJa7SrKpkb8eecPTrSfWeKNXfQY1DQvA2/d1cXsN747pisSIQLSPD0XbJiG4sUPtsX3sceTz8aXbMxxe3yt3dkSzyMB6XxtnzByShhbRQZhyfSunnyt1jxbH8SFSkb6to7H+0B/1Lndnl0R8UGOEbTnokBCG5RN62BxnxZNGdmuGZ4a18+p2+6fGQOtAl8+xuUPqrcvVcDu6e3Ms3XLs2nocWFFa40Z1zt8wrZ+L1VyTHBWEI+cuW02rfjBd/1hfRAYb61xH3s3puMODg+z1bBmFjdMbvq+2jOmZhJs7xju8/I0ZTXCjjUEVa3ImkNzfKxn396p7cESpA4497PEhIqqHVN9x5K7eOFerr9lDZq8er9+ibaf3RhGUVKuPYvAhScj1LwEib/CFU4xuG2NI6Q3hLGe+pd3LgVstrwWDDxEpRkM/mOWSt70d/H39eMY/pMgZDD5EKqekY4ZUB3ClH1hdPtVV47G9u+KqT5btyMRyeQ1l2jxqwuBDslNcZkJphVnqMlTBfg+KXI4S1pR2O7o9juyGPF8BkgPf+C2QDu/qIll5/LO9eH+b7buJFm34zcvV+D6l92Q4Sy4HDK2DAc5dL48n9lvnpkGG/OoYB0ZrZ1bN9lPSyM2OjKVT1Wtm1Ds3JlJDufsPC3vvESeGWPII9viQrNgLPQAw96ufvViJetg5eeHlKhzjzqqqPuP9nRi8rlNimFu2PdSBW4ud2deJ/VLqnG8rQL0/tmudz3nx1oxaNVQ/Lt7SMR5tGofggT7JtZZ7+LoUpMbVfVt7lUdzWqJFdBCeuCHNMu3ubonomBiGXi2jbT6ndVwj9EiJtDxuHhmIyCADctJikZMWi4ggAwBgyejO1Yq3vy81vXGP58YZenZYOyRFBWHuLem15g1Jb4w+raKREBEAAJhzU1skRQUhr8ay7v57ZUTXRHRKDEOvlCi3rve61BhkJIRhVHazGtNj3bodZ7HHh4h8UkSQASXlJlwpM9mcP75PC0wflIqPd5zAtE/2OLTObsmR+PH4RZdrOjZ3iFPLVz8292oZhUP5RThbVFpruZs6NMH/rTtsNW1kt2b49/e/21139xZRODZ3CO58/XtsPfKn1bzR3Zvjr5nx2PO/i3afH2DQYeWjvQAAv54pspo3dUBrTB3Q2qFxj2Ia+WPN1L5W054dVjsUVKcB8N793a6tX6OBRqPBmzUGRuyXGlPv9m25vk0smkcG4tifV1x6fl2aRQZh3WN9bc5bMKKT3WVzP91b53q/mNgTizb8hi/3nna6pudurru9XWXQa/H5hB4AgLe3Vr4XG4f6w+CBEaSdwR4fIlKN6n8pK+XUiKusLzh2D7mcGvWVa73cjs3iEAYfIhWpeeBS2vHDE/U6chAVNf6vNN78ri6lvafskXO4km9lysDgQ0SKIfWt0lL3eDizeU98+adcskBDvnPt2jrcUAgpEoMPkcop6vNfJgdeqbh6sFZ5s1Vysu3cEa5Inhh8yCtKyq0vMOVnCilJ1ftVqu/saqi6qnbPPjFakXIw+JDHnbxYjNRZq6ymrdqfL1E16uboqQqDTp4HMmeq8rdx50j1MVT09gaJqYPBjQOQ1HcNie3xbWyHFFurqmt8HEfVXIc83xWATi7n4LxEb+O1VUoTODKOkadJXwH5vA9/OCF1CYqXFBVUa9rb93VBYkRgg9dt6/Py4f4tG7xeeww6LVJigm3Ou7lj0zqf6+iHe7PIQPxrZFatfVt6bxdMub4VWsUG474eSY6trJr7eyajVWyww2PUALC7r/Z0SAhDclQQZv4lrVY4qtlT+rdBqbWenxEfijs6J2BCvxaWaTXbITbE6FAtE/q1QEpMsM3t2OPN4+8DfZLRpnFIve8bAMr+RncAt1zdxxbRQXigd7LV9OzkSLRpHCJVaQ75v7s6ollkYK1b9qXAcXyIFOBvg1IxqF0cnllxAIs3HQUA9GkVjY3T+9kdK+XnZwZBq9Gg1cyv7K7X3sXCUcGOHRhd8WCfZEyxM8bLS7d3wEu3d3Bo/Je6bJjWr9a0pKggNAkLwCP9W+IRp4NdZeIIDfTDN5P7AACSc7+EuZ6zRJ2bh6OguNypLY3s1gzDM+MBAH9eqj1mTxV7YwJ99GB2rRF/a5Y5vk8LOCIy2Ihvp/RxaFkp5A5OQ+5gqavwjvm3d8D82zvYnK4Ef2nfBH9pX//And7AHh8iFVPqNStK4s7r2eT6aimt94TUjcGHyIf52gHJE7ezK6mJXL3TqK59dKZN7V2XxJsVSEkYfIh8mK8dkHwtyDlDo9E41OMj9VhH3uBjb2vyMgYfIlI1Vw+iPPheo+ZASsrD4EOyUXOsH/IObx/AG7I9NR9g6z7NJZ+Gke1XPTCp0lUMPuR5Dp5vqTnWDzVcfccgpZ0WqXm3kjs40gK23sKOtpy/n3M1W3+R6rWtGHRal09dGp2sQe7cMZ6SViYBzRPvaaobgw+Rwj1/c7rN6TUPuPaWc+bjPyUmGFOvb4WM+FDc1TUR6U1DcX/PJHRJinBiLa57amhbAMC4auOYyN0/bstA88hAvHR7ht1lujrQflMGtHL64uZFd2eiWWQgFo/Kcup5zvJmhIhpZMT0Qa2df6IG+HBcN8vDoRnyuLX6yaFtkBwVhOdubid1KarBcXyIFMH+Ae+urol4/LO9VtMGtY2rtVzX5Ah8c6BhI2ZXjelib4DDho6/U5/EyMoBGx+/IQ2vbzzi0W0dmzvE4f1JjWuEn/OLak3XQINWsY2w3sa4QtV9+EB2vduKDfG3O89e58WgdnEY1K72e8HdvHUWyd7YRY7qmhzZ4HW4W0JEINY+1lfqMlSFPT5EPszX7upylruuN5GqHWv28Ngrw1v1yePkkHfI9lolajAGH/I8foC4gbIP4NVJ9W6Q27vQ2Xo0gENdK1L8usmtbd2B387uuxh8iHxQ1cGPmdM9vDnCtSsvmZxeZxmVQmQTgw+Rgjh6UOEfq7W5u0m8fSpEQPo7suUUsIhcxeBDnsejsNuwJdWl9rez1/8OkOLXjdfDkJI0KPiUlZXh0KFDqKiocFc9RGRTw49mcjk0KTG86W2MG+Ovb9jfjc6O7wMAAQbbz3H2tdW5OA6OvTFneD2MPBjdML6RGrjUSleuXMGYMWMQGBiItm3b4vjx4wCAhx9+GHPnznVrgUTkOj+dFn9p3xi9W0UjKSqo3uXrG2+mPu/d3xWJEYEuP7/KlOtbNXgdjrLVWVE13svMIWloER2ESTm1b99/toHjrjgyvk9Ni0d1RrPIQPxrZKbVdGdjx/BOTZ18RqV/jawcF+i1EZ1cej551rRBrZEcHYQnh7aRuhRZcyn45ObmYvfu3Vi/fj38/a+NLZGTk4MPP/zQbcURUcP9312d8M59XRw6HbF+Wj/c3DHe5W31SInCxunXxqwZ0CYWXz3ay+n1PGJnnCBbOjcPd3r91dXsrHjkuhR0TY4EANzfKxlrpvZFTKPaY+gkRlwLktXXodM61v9SNb6PM+2dkRCGDdP6YaCNcZocNbxTPAINrg3h1q5pKDZM64cb0htbTeepLnloHBqAtVP74t4eSVKXImsuvfuXL1+ODz/8EN26dbN6w7dt2xa//fab24ojoiqaav9twFpUfHzy1q43dDuunDRqyDbdfceamt9jpAwu9fj88ccfiImJqTX98uXLTP5ECuILv65y+L4xu+GhgaXx2hki93Mp+GRlZeHLL68Nr14Vdt58801kZ2e7pzIiqoWHQec4kjtqhj9X2lhO+aS+Hpya++uO4Ch99CRynEunup5//nkMHjwYBw4cQEVFBf75z3/iwIED2LJlCzZs2ODuGomIkUcx3PlKOdqDLnVPO9+dpCQu9fj07NkTu3fvRkVFBdLT0/HNN98gJiYGW7duRWZmZv0rICLyITzwEymH0z0+5eXleOCBBzBr1iy88cYbnqiJiOxw9O96PwfHmDHotNA7eBeSK2zV4ehdT46qa33+Bh2KSivHGTPYaRN311OdsYFj/TjKmV3wc2CsFz+dc22isfq3PE98GTjGDV3l9DvBz88P//nPfzxRCxG5QXJ0EHIHp9a7XPPIQMwd3h7902KRkRCG0d2bu62GOTe1RXJUEB6/Ic1qerPIQMuttvfVuOX27m6JNteVHB2EpKgg/P2v7W3Ov6F9Y5vTAeDNe7Is//7nHR1tLjOoXRzax4faXYcjal6E/MLwdCRFBeGZmxo21o+jEiMC0adVNIakN7Y7yGDVmESTbYxJVGX6oNZIiQnG+L4pHqnz5o6ujR8EANMGtnbpeTOHpCE5OsirY0ORvLl0jc+wYcOwfPlyTJ482d31EJFNlX9FO3JKZe3Uvg6tcf20a+PtfD6hhws12XdPdnPck90cAFBYXG6Zvm5qX2ivdk/MHtoGb20+CgB4+74u6NMq2ua6kqOC8eaoLJvzgMrRao/NHYJx7+zANwfOWM3LSAjDsblD6qzVqNfhvxN7ovmML+tcri41X5fbOyfi9s62g1x9XPuSUg3evq9Lncvc3ysZ9/dKrnOZh/qm4CEPhB4/nQa/PndDg9YRHx7g0vMc2W9SF5eCT8uWLTFnzhxs3rwZmZmZCAqyHhH2kUcecUtxRFRFvVeR+MIt92on19NfpE4uBZ/FixcjLCwMO3fuxM6dO63maTQaBh8isnAkuNS1iJxuFZcS24HIPVwKPkePHnV3HW61YMEC/P3vf0d+fj4yMjLw6quvokuXuruBiZSgwaMC++LBU2adCe4sR2a7RuQTGnyZuxBCVqOLfvjhh5gyZQqefPJJ/Pjjj8jIyMDAgQNx9uxZqUtTLfm8O0gKMvp4cCurniw37qOw828501jf1kUkay4Hn3feeQfp6ekICAhAQEAA2rdvj3//+9/urM0l8+fPx9ixY3HvvfeiTZs2WLRoEQIDA/HWW29JXRqR6vnS9TrVA503AoovtR2RlFw61TV//nzMmjULEydORI8elXeDbNq0CQ8++CDOnTsn2d1eZWVl2LlzJ3Jzcy3TtFotcnJysHXrVpvPKS0tRWlpqeVxYWGhx+skcpVSegCIiOTKpeDz6quvYuHChbjnnnss02688Ua0bdsWTz31lGTB59y5czCZTIiNjbWaHhsbi59//tnmc/Ly8vD00097ozzV4h+q1BDe7ulwZXNyOt1PRHVz6VTX6dOn0b1791rTu3fvjtOnTze4KG/Kzc1FQUGB5efEiRNSl0RkF0MkEVHDuBR8UlJS8NFHH9Wa/uGHH6JlS/ujgnpaVFQUdDodzpyxHsTszJkziIuLs/kco9GIkJAQqx8iko96O1Pc3Nni0rezN/D5Dm2DnUpEbuHSqa6nn34at99+OzZu3Gi5xmfz5s1Ys2aNzUDkLQaDAZmZmVizZg2GDRsGADCbzVizZg0mTpwoWV1E5HlSXvzrqVCilB6+6gMU8iJskjuXgs/w4cOxbds2vPTSS1i+fDkAIC0tDdu3b0fHjra/D8dbpkyZglGjRiErKwtdunTByy+/jMuXL+Pee++VtC414x+q7qPEtuSBUN74+pDauBR8ACAzMxPvvvuuO2txi9tvvx1//PEHZs+ejfz8fHTo0AGrVq2qdcEzkRop8SBXb80y2CehyEhaiafQSG1cCj4rV66ETqfDwIEDraZ//fXXMJvNGDx4sFuKc9XEiRN5aksiJeUmGHRayxdRknuxVeWvoa+Rh8ZFJKKrXLq4ecaMGTCZTLWmCyEwY8aMBhdFylRUUo62T36Nm1/bLHUppDDu6ImSsufCG9tWYm+dkvn76aQugTzEpeDz66+/ok2bNrWmp6am4vDhww0uipRp8+E/YTIL7P5fgdSleEVyVJDb1xngp0Or2GC3r9cVL92egeaRgVhwVyc0jwzES7dnuLQeNZxK8cY+KqUdfSWfvXxHBzSPDMQ/7+ggdSnkZi6d6goNDcWRI0fQvHlzq+mHDx9GUJD7DwZEcrT2sb5oPuNLt67zH7dloGtSBDKf/dat63XFzR3jcXPHeADAkPaN3bJODbst6lU93yiltXzxZU2NC8H6af2kLoM8wKUen5tuugmTJk3Cb7/9Zpl2+PBhTJ06FTfeeKPbiiMiIiJyJ5eCz7x58xAUFITU1FQkJSUhKSkJqampiIyMxIsvvujuGomI7FPIKSAikgeXT3Vt2bIFq1evxu7duxEQEICMjAz06tXL3fUREZRzfQcRkdw51eOzdetWrFixAkDlufoBAwYgJiYGL774IoYPH45x48ZZfdM5qYsvnuf3Nk8HHLkGKI1irmaxjV9SSqQcTgWfOXPmYP/+/ZbHe/fuxdixY3H99ddjxowZ+OKLL5CXl+f2IkkZ+NnvOUoOlUqunZzHC9hJ7pwKPrt27UL//v0tj5ctW4YuXbrgjTfewJQpU/DKK69I+l1dRL7OT+/SZXm+TQbH2eqZ353jv/hKhjD6Nfx9q+OgqOQmTr0bL1y4YPXVDxs2bLAapblz5844ceKE+6ojUpn6DnTjeiWjZUwwpg1s7Z2CZMDbh7uGbu/t+7q4pQ4ACA3ww4A2schJi0FUsMFt6/WWN+/JQrPIwAa1yejuzZERH4rr2/Brh8g9nLq4OTY2FkePHkVCQgLKysrw448/4umnn7bMLyoqgp+fn9uLJPIF84a3x7NfHkBhSYXL6wgPMmD1lD5urEoe6vquK2+fQXVle9VP82Y2C3dbLRqNBq/fk+W29XlbTptY5DQwsDx1Y1s3VUNUyakenxtuuAEzZszAd999h9zcXAQGBlrdybVnzx60aNHC7UWSMtjrreC1P1c1oCuBbShvSv6SUiK1carH55lnnsEtt9yCPn36IDg4GG+//TYMhmvdr2+99RYGDBjg9iKJfAKPjXYp/a4ud2JLEHmWU8EnKioKGzduREFBAYKDg6HTWV/E9/HHHyM4WB7fM0RE8qCG3ip37qPSm4vBjeTO5QEMbYmIiGhQMUS+ztVbfX3l7h5X1LvrSk8KEnPHe0vN709SHt4bS25RUm6yO8+khj/53YRNpUx82YiUg8GHGuyp/+5H6qxV+HzXScu07379A0DliLYL1/9m76mkAt7qDfB2r0P1cWUC3Dh2DxF5FoMPNdjSLccAACv35lumPfXfyhG+SyvMUpTUIAkRAfUuM7xTvBcqucZdPUFKPCVR765LtE/tm4aiW3IEhneKx2sjOqFZZCAW3NVJmmKIyGEuXeND5MuaRQThxPniOpf5x20ZTq+Xtzzbp8RAptVqsGxctuXxhmn9JKzGdTy9SmrDHh8i8gk8gBORIxh8yCP4RYW1aaBRZM+G1Nhk8sffd1ISBh8iL2nIqS4eV4iI3IPBhzyCx2l54ukg8jS+xUjueHEzuay4rI6xe8yVH38VZn4MugMDCxGRezD4kEsOn72EnPkb7M4/cu4yhBDoM2+dF6uSN61GgwA/HS6i3O4yOh/sg9U6cJ6uriX89PU0iptDYfXxeaSg9PcAe3tJ7hT+K0ZSWbzpaL3LlFaY8eflMi9UI53eraLrXWZsryS0axqCoRlN8PrILDSLDMQN6XFIjAi0LNM1KQIZCWG4LjXWk+VKomVMMHqmROGmDk3sLmMru8y5qS2So4Lw+A1pniuumsk5rdAqNhj39kjyyvZqGtMzCelNQzG4XWNJtu8u7JwkuWOPD5EDwgP9cOFK7Z6aaQNaY+Mvf9T53CeGtLH8Oz0+1DLei9kskPz4SgDAUze2RVrjEDdWLB8ajQbv3t/V6efdk90c92Q3d2ADztdky6M5LfFoTkv3rMwFs/7Spv6FZIq9PKQk7PEhIiIi1WDwIVIA3s5OSiF4JT7JHIMPeQwP1u7DYwkRkXsw+BCR5JiRfQdHcSa548XN5JRrY/ewC4LIF7g7p/BUF8kdgw857KfjF3Dza1ukLoOIZIadPKQkPNVFDvvHN79IXQJRbexgICInMPiQqo3rnSx1CQ7hX9SkFMyhJHcMPkQK4POXTTDYEZGXMPiQxyjhYC3lhZhS9OL4ZM/R1X3yyX1TIL4MJHcMPkQNwIMtkTUF/L1DKsfgQ0Q+QQk9jHLkjnbTsJ+HFITBRwHKTWaUm8we3861MXpsc7Z3o6S87vWR9zEcEJHaMfjInMkskJ23Btl5a2Eye+6ote9kAdJmr0Lup3vtLvPdr+ecWmeHOasbWhZdpdX69l/UWp4zVDS+fKQkDD4yd/5yGc5dKsO5S6W4eKXMY9v5v7WHAQAfbD/usW3IRZNQf8u/NRoNZgxOrfc5Ug3DP6R9Y2QkhKFf65g6l3t3TFckRgTi/bFdvVSZe9zROQGdm4ejc/MIqUuhBohpZET/1BgMbBuLEH8/qcshqhNHbibV2ZLbH81nfGl5/GCfFpj71c8urcvTp46evrEtooKN9S7Xs2UUNk7v59liPGDu8PZSl0BuoNFosHh0Z6nLIHIIe3yISNl43RIROYHBh0jGeDEyEZF7MfgQkbLxwloicgKDDwHgXRlERKQODD5ERCrGP3pIbRh8iGow6Gv/WvjbmEZERMrDT3PyaYkRgXh3jHNj2zw1tK3V46dvbIvX78lyZ1lERCQRjuNDAHy3u7u+sW1qfjv7tIGtkRgZaHn89I1tMap7cwBAl6QIbD963u01VtbhkdUSEVEN7PEhkohUo0ETEakZgw8RERGpBoMPqVp9vS7slFEAniYkIicw+BAREZFqMPioRHGZqc75mmrD3xaVlFuWN5kFCorLPVqbkimtR0hp9TrEF/eJiDyGd3WpwEc7TmD6J3sw95Z03NElsd7l05/6BgAwuntzLN1yDADw+shMT5YoG3qt/aOo0QfG8uHdY0Skdsr/JPdx1f9Cd/UuoOmf7AEAzPh0r1PPqwo9ADDu3ztd2rZSPHJdClLjGmFEt2ZW06u3+LPD2iEpKghzb0m3TKseJAa3iwMAJEUF4cmhbZzavk/2xHgZ29A1DMOkNuzxkbnqH0o1x5whx+Xdko7cOoLflAGtMWVA6zrX0SwyCOse62t3/sK7r/WKrTl4xukaqWH460FEjmCPD1XiX8tERKQCDD5ERESkGgw+RHVx8cIRXm9CRCRPDD6kCrz+g4iIAAYf1TGbBUrKr43pU1phgsnMVEBEROrAu7pU5uaFW7DvZAF+mn099FoNOjy9GsnRQfg5v0jq0iRR17g9AKBz8ZyVtgHnuvjlpeRNfLuR2jD4yJw7xvGpbveJiwCALYf/RJBRhzKTWRWhp2bTPTagFT7fdQrjeifbXP6BPsn47pdzuLljU5e21zMlCp0Sw5DWOMTp54YH+mFAm1iYhUBUsMGl7RMRkW0MPqRKE69riYnXtbQ7P3dwGnIHu75+vU6LTx/q4fDy1a9B0mg0eP2eLNc3TkREdvEaH5njAIZERETuw+CjUjyvT0REaqSY4PPcc8+he/fuCAwMRFhYmM1ljh8/jiFDhiAwMBAxMTGYNm0aKioqvFsoERERyZZirvEpKyvDrbfeiuzsbCxevLjWfJPJhCFDhiAuLg5btmzB6dOncc8998DPzw/PP/+8BBXLH8+cERGR2iimx+fpp5/G5MmTkZ6ebnP+N998gwMHDuDdd99Fhw4dMHjwYDzzzDNYsGABysrKvFyt9wkhUFxmqjW9rjF6zEw+RESkMooJPvXZunUr0tPTERsba5k2cOBAFBYWYv/+/XafV1paisLCQqsfJXpk2S6kzV6Fo+cuW01v8fhKm8trAIxe8oMXKpMHXT3j9bhKaddKGfQ+8ytv4epYS0SkTj7zKZifn28VegBYHufn59t9Xl5eHkJDQy0/CQkJHq3TWY6O4/PF7lMAgLe3HPNwRcp0Y0YTtGsagrG9kty63jaNQ9C9RaTL4/142+y/tEFyVBCeHdZO6lIa7P6eSWjXNARD2jeWuhQiUhBJr/GZMWMGXnjhhTqXOXjwIFJTUz1WQ25uLqZMmWJ5XFhYKLvw4wlqGx3Y30+HFQ/3cvt6tVoN3h/bze3r9ZSEiECsfayv1GW4xcy/tJG6BCJSIEmDz9SpUzF69Og6l0lOtj2ybk1xcXHYvn271bQzZ85Y5tljNBphNBod2oYUOI4PERGR+0gafKKjoxEdHe2WdWVnZ+O5557D2bNnERMTAwBYvXo1QkJC0KYN/zIkIiIiBd3Ofvz4cZw/fx7Hjx+HyWTCrl27AAApKSkIDg7GgAED0KZNG4wcORLz5s1Dfn4+Zs6ciQkTJsi6R4cIUN5F0uQ72JFMaqOY4DN79my8/fbblscdO3YEAKxbtw59+/aFTqfDihUrMH78eGRnZyMoKAijRo3CnDlzpCpZ1nicJSIiNVJM8Fm6dCmWLl1a5zLNmjXDypW2b99Wk5JyE4QAjHXculxSUXvMHyIiIl+nmOBDjikoLkfqrFUAgC7NI+wuN/H9n7xVEhERkWz4zDg+vsrRcXyqfHvwjOXf24+d90RJ5AG8zoKkwuvLSG0YfHwND6BERER2Mfj4GH7/FhERkX0MPjLHAQyJiIjch8GHiIiIVIPBx8ewT4iIiMg+Bh8fU2Fi9CEiIrKHwccH5H661/LvMpNZwkqIiIjkjcFH5hwZx+eD7ce9VA0REZGyMfiQIk0b2FrqEoiISIEYfIiIiEg1GHyIiIhINRh8ZI4DGBIREbkPgw8RERGpBoOPgpRWmGE2CxSXmSzT2AtERETkOL3UBZDjus9da/n3v8d0Qa+W0Rj7zg4JKyJ3sTNSATnBoOffca4wst1IZfiOlzl7B8Sn/rsfAPDtwbNerIbcbViHJuiREonWsY2kLkXxHr8hDcnRQXj6xrZSl6IoT9/YDklRQci7JV3qUoi8gj0+CqW2E1wh/noUllRIXYbbvXxHR6lL8BlNwgKwdmpfqctQnMTIQKx7rK/UZRB5DXt8SBHUFvSIiMgzGHxIkXhNDBERuYLBh4iIiFSDwUfm7N6tznM/RERETmPwkanqY/XYcqm0AmUVZi9VQ0RE5BsYfGRox7HzSJu9Cs+uOGB3mbNFpWg18ysvViWtAD+d1CUQEZEPYPCRoRdW/QwAeHPTUdVfxNuvdTSubxOLd8Z0cfq5wzvFe6AiIiJSMo7jQ7I1KrsZnr6pneXxsblD0HzGlw49NyM+FP+4LcNTpRERkUKxx4d8k9q7yoiIyCYGHyIiIlINBh9SJA3Yo0NERM5j8JE5u+P4EBERkdMYfGSuoLhM6hKIiIh8BoOPzOXM3yh1CZLR6+y/PfXauk91Get4LhERqRePDiQrfrprgeahvi1qzX/4uhSkxjXCnV0TcWtmPLokRWDFwz3RLDIQLwxPtyyXV+3fREREVTRC8CqS6goLCxEaGoqCggKEhIRIUsOti7bgh2MXJNm21PJuScedXRKlLoOIiBTG0eM3e3yIiIhINRh8iIiISDUYfEhWeOKViIg8icGHZEWAyYeIiDyHX1IqAyXlJgCVXy+l5XdMEREReQyDj8TOXSpF1rPfWh5HBRtw7pJ6By3kqS4iIvIknuqS2Ff78q0eqzH05A5OlboEIiJSCQYfqamoi2PxqCwcmzsEXZMiLNMym4XjgT61ByokIiLyBAYfiakn9hAREUmPwYe8xpHOLQZBIiLyJAYfianoTJdNtb4xRe0NQkREHsXgIzF+VRoREZH3MPhIqOBKOQqKK6Quw2uqIh6HKiIiIqlwHB+JfL7rJB5dtkvqMiRh1OukLoGIiFSKPT4SUWvoAYA5N7W1O48n/oiIyJMYfMhrqq5nahYZJHElRESkVgw+JCu81puIiDyJwYe8xpFMw7vciIjIkxh8iIiISDUYfEhS7N8hIiJvYvAhr+FZLCIikhqDD3mNTlt75EKDzvotyGxERESexOBDbtOuaQjCA/2spt3XIwkjuzVDh4Qw9G0dbZk+76/t0TwyEHm3pFstz14hIiLyJI7crDJdkiKw/eh5m/MyEsKw+8TFOp8fFWzEjpk5lsfZeWtwuqAEALDi4V4AgCkf7cKnP54EAMwe2sbmem7LSsBtWQnOlk9ERNQg7PGhBmEPDRERKQmDDzlJ1HjE5ENERMrB4KM2Ms8pMi+PiIgUjsGHGoSnuoiISEl4cbOXmM0CJy5cQbBRj8KSCukKqX1HeYO4O/fwKyuIiMiTGHy8JPnxlVKXUC+jrv4OQKNeZ/U4wE9nZ0nX6G2M9UNEROQuPNWlNjY6VG7u2BRJUUHIG249pk5iRCAa+V/Lxs0iA/GvkZlWy7w2ohOaRQZiwV2dGlTWQ31bIK1xCG7rzFvciYjIc9jjo3LH5g6xOX3mkDTc3ysZ497ZgW8OnAEAbJjWr9Zy7ZqG1pquceF82vRBqZg+KNXp5xERETmDPT5kEy+1ISIiX8TgozKOjrvD8XmIiMgXKSL4HDt2DGPGjEFSUhICAgLQokULPPnkkygrK7Nabs+ePejVqxf8/f2RkJCAefPmSVSxfDl6Goo9PkRE5IsUcY3Pzz//DLPZjH/9619ISUnBvn37MHbsWFy+fBkvvvgiAKCwsBADBgxATk4OFi1ahL179+K+++5DWFgYxo0bJ/EeKA9zDxER+SJFBJ9BgwZh0KBBlsfJyck4dOgQFi5caAk+7733HsrKyvDWW2/BYDCgbdu22LVrF+bPn8/gU43Dp7qYfIiIyAcp4lSXLQUFBYiIiLA83rp1K3r37g2DwWCZNnDgQBw6dAgXLlywu57S0lIUFhZa/fgyrcaxU11V4+kY9M6/RVx5DhERkTco8gh1+PBhvPrqq3jggQcs0/Lz8xEbG2u1XNXj/Px8u+vKy8tDaGio5SchQVnjyLw7pqtTy2ckhFn+fXe3xFrzH74uBalxjXBn18p5uTekITk6CHNuauvwNiZf3xItooMwc0iaU7URERF5mqSnumbMmIEXXnihzmUOHjyI1NRr47ucPHkSgwYNwq233oqxY8c2uIbc3FxMmTLF8riwsFD24ef9sV3RvUVUremNjHrsfXogAOCVNb9i/upfAFSO1dN8xpcAKr+xwt7YPQAwdUBrTB3Q2vK4aVgA1k7t61R9MY38scbJ5xAREXmDpMFn6tSpGD16dJ3LJCcnW/596tQp9OvXD927d8frr79utVxcXBzOnDljNa3qcVxcnN31G41GGI1GJyuXlisDBBIREZHEwSc6OhrR0dEOLXvy5En069cPmZmZWLJkCbRa67N02dnZeOKJJ1BeXg4/Pz8AwOrVq9G6dWuEh4e7vXYpOXiZjp0nu60MIiIixVHENT4nT55E3759kZiYiBdffBF//PEH8vPzra7dueuuu2AwGDBmzBjs378fH374If75z39ancYi8D51IiJSNUXczr569WocPnwYhw8fRnx8vNU8cfW+69DQUHzzzTeYMGECMjMzERUVhdmzZ/vkrezstCEiInKNIoLP6NGj670WCADat2+P7777zvMFSUzTkHNdTE1ERKRiijjVRdZ0dl41o5/O8m+9znbC8dPyJSciIvXiUVDGOiSE4Z7sZlg8KgvNIgORHB2ErkkR6JBgfbH2v0ZmollkIBaPyrJMG9mtGVLjGuGR/i0BAFOvb4WWMcEY2ysZREREaqURgl9OUF1hYSFCQ0NRUFCAkJAQqcshIiIiBzh6/GaPDxEREakGgw8RERGpBoMPERERqQaDDxEREakGgw8RERGpBoMPERERqQaDDxEREakGgw8RERGpBoMPERERqQaDDxEREakGgw8RERGpBoMPERERqQaDDxEREakGgw8RERGphl7qAuRGCAGg8uvtiYiISBmqjttVx3F7GHxqKCoqAgAkJCRIXAkRERE5q6ioCKGhoXbna0R90UhlzGYzTp06hUaNGkGj0bhtvYWFhUhISMCJEycQEhLitvWSNbazd7CdvYdt7R1sZ+/wZDsLIVBUVIQmTZpAq7V/JQ97fGrQarWIj4/32PpDQkL4S+UFbGfvYDt7D9vaO9jO3uGpdq6rp6cKL24mIiIi1WDwISIiItVg8PESo9GIJ598EkajUepSfBrb2TvYzt7DtvYOtrN3yKGdeXEzERERqQZ7fIiIiEg1GHyIiIhINRh8iIiISDUYfIiIiEg1GHy8ZMGCBWjevDn8/f3RtWtXbN++XeqSZCsvLw+dO3dGo0aNEBMTg2HDhuHQoUNWy5SUlGDChAmIjIxEcHAwhg8fjjNnzlgtc/z4cQwZMgSBgYGIiYnBtGnTUFFRYbXM+vXr0alTJxiNRqSkpGDp0qWe3j3Zmjt3LjQaDSZNmmSZxnZ2j5MnT+Luu+9GZGQkAgICkJ6ejh07dljmCyEwe/ZsNG7cGAEBAcjJycGvv/5qtY7z589jxIgRCAkJQVhYGMaMGYNLly5ZLbNnzx706tUL/v7+SEhIwLx587yyf3JgMpkwa9YsJCUlISAgAC1atMAzzzxj9b1NbGfXbNy4EUOHDkWTJk2g0WiwfPlyq/nebNePP/4Yqamp8Pf3R3p6OlauXOn8DgnyuGXLlgmDwSDeeustsX//fjF27FgRFhYmzpw5I3VpsjRw4ECxZMkSsW/fPrFr1y5xww03iMTERHHp0iXLMg8++KBISEgQa9asETt27BDdunUT3bt3t8yvqKgQ7dq1Ezk5OeKnn34SK1euFFFRUSI3N9eyzJEjR0RgYKCYMmWKOHDggHj11VeFTqcTq1at8ur+ysH27dtF8+bNRfv27cWjjz5qmc52brjz58+LZs2aidGjR4tt27aJI0eOiK+//locPnzYsszcuXNFaGioWL58udi9e7e48cYbRVJSkiguLrYsM2jQIJGRkSG+//578d1334mUlBRx5513WuYXFBSI2NhYMWLECLFv3z7xwQcfiICAAPGvf/3Lq/srleeee05ERkaKFStWiKNHj4qPP/5YBAcHi3/+85+WZdjOrlm5cqV44oknxKeffioAiM8++8xqvrfadfPmzUKn04l58+aJAwcOiJkzZwo/Pz+xd+9ep/aHwccLunTpIiZMmGB5bDKZRJMmTUReXp6EVSnH2bNnBQCxYcMGIYQQFy9eFH5+fuLjjz+2LHPw4EEBQGzdulUIUfmLqtVqRX5+vmWZhQsXipCQEFFaWiqEEGL69Omibdu2Vtu6/fbbxcCBAz29S7JSVFQkWrZsKVavXi369OljCT5sZ/f429/+Jnr27Gl3vtlsFnFxceLvf/+7ZdrFixeF0WgUH3zwgRBCiAMHDggA4ocffrAs89VXXwmNRiNOnjwphBDitddeE+Hh4ZZ2r9p269at3b1LsjRkyBBx3333WU275ZZbxIgRI4QQbGd3qRl8vNmut912mxgyZIhVPV27dhUPPPCAU/vAU10eVlZWhp07dyInJ8cyTavVIicnB1u3bpWwMuUoKCgAAERERAAAdu7cifLycqs2TU1NRWJioqVNt27divT0dMTGxlqWGThwIAoLC7F//37LMtXXUbWM2l6XCRMmYMiQIbXagu3sHv/973+RlZWFW2+9FTExMejYsSPeeOMNy/yjR48iPz/fqo1CQ0PRtWtXq3YOCwtDVlaWZZmcnBxotVps27bNskzv3r1hMBgsywwcOBCHDh3ChQsXPL2bkuvevTvWrFmDX375BQCwe/dubNq0CYMHDwbAdvYUb7aruz5LGHw87Ny5czCZTFYHBgCIjY1Ffn6+RFUph9lsxqRJk9CjRw+0a9cOAJCfnw+DwYCwsDCrZau3aX5+vs02r5pX1zKFhYUoLi72xO7IzrJly/Djjz8iLy+v1jy2s3scOXIECxcuRMuWLfH1119j/PjxeOSRR/D2228DuNZOdX1G5OfnIyYmxmq+Xq9HRESEU6+FL5sxYwbuuOMOpKamws/PDx07dsSkSZMwYsQIAGxnT/Fmu9pbxtl257ezk6xNmDAB+/btw6ZNm6QuxeecOHECjz76KFavXg1/f3+py/FZZrMZWVlZeP755wEAHTt2xL59+7Bo0SKMGjVK4up8x0cffYT33nsP77//Ptq2bYtdu3Zh0qRJaNKkCduZrLDHx8OioqKg0+lq3Qlz5swZxMXFSVSVMkycOBErVqzAunXrEB8fb5keFxeHsrIyXLx40Wr56m0aFxdns82r5tW1TEhICAICAty9O7Kzc+dOnD17Fp06dYJer4der8eGDRvwyiuvQK/XIzY2lu3sBo0bN0abNm2spqWlpeH48eMArrVTXZ8RcXFxOHv2rNX8iooKnD9/3qnXwpdNmzbN0uuTnp6OkSNHYvLkyZbeTLazZ3izXe0t42y7M/h4mMFgQGZmJtasWWOZZjabsWbNGmRnZ0tYmXwJITBx4kR89tlnWLt2LZKSkqzmZ2Zmws/Pz6pNDx06hOPHj1vaNDs7G3v37rX6ZVu9ejVCQkIsB6Hs7GyrdVQto5bXpX///ti7dy927dpl+cnKysKIESMs/2Y7N1yPHj1qDcfwyy+/oFmzZgCApKQkxMXFWbVRYWEhtm3bZtXOFy9exM6dOy3LrF27FmazGV27drUss3HjRpSXl1uWWb16NVq3bo3w8HCP7Z9cXLlyBVqt9SFNp9PBbDYDYDt7ijfb1W2fJU5dCk0uWbZsmTAajWLp0qXiwIEDYty4cSIsLMzqThi6Zvz48SI0NFSsX79enD592vJz5coVyzIPPvigSExMFGvXrhU7duwQ2dnZIjs72zK/6jbrAQMGiF27dolVq1aJ6Ohom7dZT5s2TRw8eFAsWLBAVbdZ21L9ri4h2M7usH37dqHX68Vzzz0nfv31V/Hee++JwMBA8e6771qWmTt3rggLCxOff/652LNnj7jpppts3g7csWNHsW3bNrFp0ybRsmVLq9uBL168KGJjY8XIkSPFvn37xLJly0RgYKBP32Zd3ahRo0TTpk0tt7N/+umnIioqSkyfPt2yDNvZNUVFReKnn34SP/30kwAg5s+fL3766Sfx+++/CyG8166bN28Wer1evPjii+LgwYPiySef5O3scvbqq6+KxMREYTAYRJcuXcT3338vdUmyBcDmz5IlSyzLFBcXi4ceekiEh4eLwMBAcfPNN4vTp09brefYsWNi8ODBIiAgQERFRYmpU6eK8vJyq2XWrVsnOnToIAwGg0hOTrbahhrVDD5sZ/f44osvRLt27YTRaBSpqani9ddft5pvNpvFrFmzRGxsrDAajaJ///7i0KFDVsv8+eef4s477xTBwcEiJCRE3HvvvaKoqMhqmd27d4uePXsKo9EomjZtKubOnevxfZOLwsJC8eijj4rExETh7+8vkpOTxRNPPGF1ezTb2TXr1q2z+Zk8atQoIYR32/Wjjz4SrVq1EgaDQbRt21Z8+eWXTu+PRohqw1oSERER+TBe40NERESqweBDREREqsHgQ0RERKrB4ENERESqweBDREREqsHgQ0RERKrB4ENERESqweBDREREqsHgQ0SKcOzYMWg0Guzatctj2xg9ejSGDRvmsfUTkfQYfIjI40aPHg2NRlPrZ9CgQQ6vIyEhAadPn0a7du08WKl7/fDDD2jSpAkA4NSpUwgICEBZWZnEVRGpm17qAohIHQYNGoQlS5ZYTTMajQ4/X6fTIS4uzt1ledTWrVvRo0cPAMB3332HrKwsGAwGiasiUjf2+BCRVxiNRsTFxVn9hIeHW+ZrNBosXLgQgwcPRkBAAJKTk/HJJ59Y5tc81XXhwgWMGDEC0dHRCAgIQMuWLa2C1d69e3HdddchICAAkZGRGDduHC5dumSZbzKZMGXKFISFhSEyMhLTp09Hza8uNJvNyMvLQ1JSEgICApCRkWFVU322bNliCT6bNm2y/JuIpMPgQ0SyMWvWLAwfPhy7d+/GiBEjcMcdd+DgwYN2lz1w4AC++uorHDx4EAsXLkRUVBQA4PLlyxg4cCDCw8Pxww8/4OOPP8a3336LiRMnWp7/j3/8A0uXLsVbb72FTZs24fz58/jss8+stpGXl4d33nkHixYtwv79+zF58mTcfffd2LBhg9192LRpE8LCwhAWFoZPPvkETzzxBMLCwrBo0SK88sorCAsLw9y5c93QWkTkEqe/z52IyEmjRo0SOp1OBAUFWf0899xzlmUAiAcffNDqeV27dhXjx48XQghx9OhRAUD89NNPQgghhg4dKu69916b23v99ddFeHi4uHTpkmXal19+KbRarcjPzxdCCNG4cWMxb948y/zy8nIRHx8vbrrpJiGEECUlJSIwMFBs2bLFat1jxowRd955p919LS4uFkePHhVfffWVCA8PF0eOHBE7duwQBoNBHDx4UBw9elRcuHCh7gYjIo/hNT5E5BX9+vXDwoULraZFRERYPc7Ozq712N5dXOPHj8fw4cPx448/YsCAARg2bBi6d+8OADh48CAyMjIQFBRkWb5Hjx4wm804dOgQ/P39cfr0aXTt2tUyX6/XIysry3K66/Dhw7hy5Qquv/56q+2WlZWhY8eOdvfT398fzZs3x0cffYTBgwcjKSkJW7ZsQa9evZCammr3eUTkHQw+ROQVQUFBSElJcdv6Bg8ejN9//x0rV67E6tWr0b9/f0yYMAEvvviiW9ZfdT3Ql19+iaZNm1rNq+ui7ODgYABAaWkptFotPv/8c5SVlUEIgeDgYPTq1QtfffWVW2okIufxGh8iko3vv/++1uO0tDS7y0dHR2PUqFF499138fLLL+P1118HAKSlpWH37t24fPmyZdnNmzdDq9WidevWCA0NRePGjbFt2zbL/IqKCuzcudPyuE2bNjAajTh+/DhSUlKsfhISEuzWtGvXLuzYsQM6nQ5r1qzBrl27EBkZiY8++gi7du3Cm2++6XS7EJH7sMeHiLyitLQU+fn5VtP0er3lgmQA+Pjjj5GVlYWePXvivffew/bt27F48WKb65s9ezYyMzPRtm1blJaWYsWKFZaQNGLECDz55JMYNWoUnnrqKfzxxx94+OGHMXLkSMTGxgIAHn30UcydOxctW7ZEamoq5s+fj4sXL1rW36hRIzz22GOYPHkyzGYzevbsiYKCAmzevBkhISEYNWqUzbpSUlLw/fffIzY2Fj179sTx48dRVFSEoUOHQq/nRy6R1PhbSEResWrVKjRu3NhqWuvWrfHzzz9bHj/99NNYtmwZHnroITRu3BgffPAB2rRpY3N9BoMBubm5OHbsGAICAtCrVy8sW7YMABAYGIivv/4ajz76KDp37ozAwEAMHz4c8+fPtzx/6tSpOH36NEaNGgWtVov77rsPN998MwoKCizLPPPMM4iOjkZeXh6OHDmCsLAwdOrUCY8//nid+7p+/Xr07t0bALBhwwZkZ2cz9BDJhEaIGgNXEBFJQKPR4LPPPuNXRhCRR/EaHyIiIlINBh8iIiJSDZ50JiJZ4Fl3IvIG9vgQERGRajD4EBERkWow+BAREZFqMPgQERGRajD4EBERkWow+BAREZFqMPgQERGRajD4EBERkWr8P174vCqYcfwZAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def dqn(n_episodes=10000, max_t=5000, eps_start=1.0, eps_end=0.0075, eps_decay=0.9987):\n",
    "    \"\"\"\n",
    "    Trains the DQN agent using explerience replay.\n",
    "\n",
    "    @param n_episodes Number of episodes simulated for the training.\n",
    "    @param max_t Maximum number of stemps for each episode.\n",
    "    @param eps_start Initial probability to select a random action.\n",
    "    @þaram eps_end Minimum of the probability to select a random action.\n",
    "    @param eps_decay Decay factor of the probability to select a random action.\n",
    "\n",
    "    @return List of rewards.\n",
    "\n",
    "    \"\"\"\n",
    "    scores = []                        # list containing scores from each episode\n",
    "    scores_window = deque(maxlen=100)  # last 100 scores\n",
    "    eps = eps_start                    # initialize epsilon\n",
    "    for i_episode in range(1, n_episodes+1):\n",
    "        state = env.reset()\n",
    "        score = 0\n",
    "        for t in range(max_t):\n",
    "            action = agent.act(state, eps)\n",
    "            next_state, reward, done, _ = env.step(action)\n",
    "            agent.step(state, action, reward, next_state, done)\n",
    "            state = next_state\n",
    "            score += reward\n",
    "            if done:\n",
    "                break \n",
    "        scores_window.append(score)       # save most recent score\n",
    "        scores.append(score)              # save most recent score\n",
    "        eps = max(eps_end, eps_decay*eps) # decrease epsilon\n",
    "        print('\\rEpisode {}\\tAverage Score: {:.2f}'.format(i_episode, np.mean(scores_window)), end=\"\")\n",
    "        if i_episode % 100 == 0:\n",
    "            print('\\rEpisode {}\\tAverage Score: {:.2f}'.format(i_episode, np.mean(scores_window)))\n",
    "            torch.save(agent.qnetwork_local.state_dict(), \"./saved_models/checkpoint_\" + str(i_episode) + \".pt\")\n",
    "            np.save(\"./saved_models/scores_\" +  str(i_episode) +  \".npy\", np.array(scores))\n",
    "    return scores\n",
    "\n",
    "scores = dqn()\n",
    "\n",
    "# plot the scores\n",
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(111)\n",
    "plt.plot(np.arange(len(scores)), scores)\n",
    "plt.ylabel('Score')\n",
    "plt.xlabel('Episode #')\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}