V11_Goku.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/abdulazeezb/1_SparkFundsInvestmentAnalysis/blob/master/V11_Goku.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Checking GPU Version"
      ],
      "metadata": {
        "id": "NAjscLW7Sk9V"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "gpu_info = !nvidia-smi\n",
        "gpu_info = '\\n'.join(gpu_info)\n",
        "if gpu_info.find('failed') >= 0:\n",
        "  print('Not connected to a GPU')\n",
        "else:\n",
        "  print(gpu_info)"
      ],
      "metadata": {
        "id": "t8nxH9ZvSaY6",
        "outputId": "1162ccc5-0c10-40f5-d68f-475746f06e1b",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Tue May  2 17:52:37 2023       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 525.85.12    Driver Version: 525.85.12    CUDA Version: 12.0     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla T4            Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   45C    P8    10W /  70W |      0MiB / 15360MiB |      0%      Default |\n",
            "|                               |                      |                  N/A |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from psutil import virtual_memory\n",
        "ram_gb = virtual_memory().total / 1e9\n",
        "print('Your runtime has {:.1f} gigabytes of available RAM\\n'.format(ram_gb))\n",
        "\n",
        "if ram_gb < 20:\n",
        "  print('Not using a high-RAM runtime')\n",
        "else:\n",
        "  print('You are using a high-RAM runtime!')"
      ],
      "metadata": {
        "id": "KEPzY7qCS2Lc",
        "outputId": "ea032459-745d-4668-eb37-925562d4705e",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Your runtime has 27.3 gigabytes of available RAM\n",
            "\n",
            "You are using a high-RAM runtime!\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QMMqmdiYMkvi"
      },
      "source": [
        "## Part 1: Getting Started - Install Python Packages"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "23TOba33L4qf",
        "outputId": "01c6a326-18f5-442e-e4e6-433881f55f87"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting wrds\n",
            "  Downloading wrds-3.1.6-py3-none-any.whl (12 kB)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from wrds) (1.22.4)\n",
            "Collecting sqlalchemy<2\n",
            "  Downloading SQLAlchemy-1.4.48-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m21.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting psycopg2-binary\n",
            "  Downloading psycopg2_binary-2.9.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.0/3.0 MB\u001b[0m \u001b[31m85.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from wrds) (1.10.1)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from wrds) (1.5.3)\n",
            "Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.10/dist-packages (from sqlalchemy<2->wrds) (2.0.2)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->wrds) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->wrds) (2022.7.1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas->wrds) (1.16.0)\n",
            "Installing collected packages: sqlalchemy, psycopg2-binary, wrds\n",
            "  Attempting uninstall: sqlalchemy\n",
            "    Found existing installation: SQLAlchemy 2.0.10\n",
            "    Uninstalling SQLAlchemy-2.0.10:\n",
            "      Successfully uninstalled SQLAlchemy-2.0.10\n",
            "Successfully installed psycopg2-binary-2.9.6 sqlalchemy-1.4.48 wrds-3.1.6\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting swig\n",
            "  Downloading swig-4.1.1-py2.py3-none-manylinux_2_5_x86_64.manylinux1_x86_64.whl (1.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.8/1.8 MB\u001b[0m \u001b[31m21.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: swig\n",
            "Successfully installed swig-4.1.1\n",
            "⏬ Downloading https://github.com/conda-forge/miniforge/releases/download/23.1.0-1/Mambaforge-23.1.0-1-Linux-x86_64.sh...\n",
            "📦 Installing...\n",
            "📌 Adjusting configuration...\n",
            "🩹 Patching environment...\n",
            "⏲ Done in 0:00:10\n",
            "🔁 Restarting kernel...\n",
            "Selecting previously unselected package libgl1-mesa-glx:amd64.\n",
            "(Reading database ... 122518 files and directories currently installed.)\n",
            "Preparing to unpack .../libgl1-mesa-glx_21.2.6-0ubuntu0.1~20.04.2_amd64.deb ...\n",
            "Unpacking libgl1-mesa-glx:amd64 (21.2.6-0ubuntu0.1~20.04.2) ...\n",
            "Selecting previously unselected package swig4.0.\n",
            "Preparing to unpack .../swig4.0_4.0.1-5build1_amd64.deb ...\n",
            "Unpacking swig4.0 (4.0.1-5build1) ...\n",
            "Selecting previously unselected package swig.\n",
            "Preparing to unpack .../swig_4.0.1-5build1_all.deb ...\n",
            "Unpacking swig (4.0.1-5build1) ...\n",
            "Setting up libgl1-mesa-glx:amd64 (21.2.6-0ubuntu0.1~20.04.2) ...\n",
            "Setting up swig4.0 (4.0.1-5build1) ...\n",
            "Setting up swig (4.0.1-5build1) ...\n",
            "Processing triggers for man-db (2.9.1-1) ...\n"
          ]
        }
      ],
      "source": [
        "## install finrl library\n",
        "!pip install wrds\n",
        "!pip install swig\n",
        "!pip install -q condacolab\n",
        "import condacolab\n",
        "condacolab.install()\n",
        "!apt-get update -y -qq && apt-get install -y -qq cmake libopenmpi-dev python3-dev zlib1g-dev libgl1-mesa-glx swig"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install git+https://github.com/AI4Finance-Foundation/FinRL.git\n"
      ],
      "metadata": {
        "id": "haJsmd15TCdH",
        "outputId": "92def19c-d2a9-40d7-e294-9b2bd75fa95c",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting git+https://github.com/AI4Finance-Foundation/FinRL.git\n",
            "  Cloning https://github.com/AI4Finance-Foundation/FinRL.git to /tmp/pip-req-build-q2va2shv\n",
            "  Running command git clone --filter=blob:none --quiet https://github.com/AI4Finance-Foundation/FinRL.git /tmp/pip-req-build-q2va2shv\n",
            "  Resolved https://github.com/AI4Finance-Foundation/FinRL.git to commit a3598dae504bcd834d12b17110b5aa91c1c5305d\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting elegantrl@ git+https://github.com/AI4Finance-Foundation/ElegantRL.git#egg=elegantrl\n",
            "  Cloning https://github.com/AI4Finance-Foundation/ElegantRL.git to /tmp/pip-install-os9ju90j/elegantrl_90aa1bf2d4db4118a06efe929c011c29\n",
            "  Running command git clone --filter=blob:none --quiet https://github.com/AI4Finance-Foundation/ElegantRL.git /tmp/pip-install-os9ju90j/elegantrl_90aa1bf2d4db4118a06efe929c011c29\n",
            "  Resolved https://github.com/AI4Finance-Foundation/ElegantRL.git to commit b974a806e6235f59055c954418e54640fa549331\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2\n",
            "  Cloning https://github.com/quantopian/pyfolio.git to /tmp/pip-install-os9ju90j/pyfolio_f9c321886fb644439a8b2964cd1e48ad\n",
            "  Running command git clone --filter=blob:none --quiet https://github.com/quantopian/pyfolio.git /tmp/pip-install-os9ju90j/pyfolio_f9c321886fb644439a8b2964cd1e48ad\n",
            "  Resolved https://github.com/quantopian/pyfolio.git to commit 4b901f6d73aa02ceb6d04b7d83502e5c6f2e81aa\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: gputil in /usr/local/lib/python3.10/site-packages (from finrl==0.3.5) (1.4.0)\n",
            "Requirement already satisfied: numpy>=1.17.3 in /usr/local/lib/python3.10/site-packages (from finrl==0.3.5) (1.24.3)\n",
            "Requirement already satisfied: stable-baselines3<2.0.0,>=1.6.2 in /usr/local/lib/python3.10/site-packages (from finrl==0.3.5) (1.8.0)\n",
            "Requirement already satisfied: exchange_calendars==3.6.3 in /usr/local/lib/python3.10/site-packages (from finrl==0.3.5) (3.6.3)\n",
            "Requirement already satisfied: lz4 in /usr/local/lib/python3.10/site-packages (from finrl==0.3.5) (4.3.2)\n",
            "Requirement already satisfied: tensorboardX in /usr/local/lib/python3.10/site-packages (from finrl==0.3.5) (2.6)\n",
            "Requirement already satisfied: scikit-learn>=0.21.0 in /usr/local/lib/python3.10/site-packages (from finrl==0.3.5) (1.2.2)\n",
            "Requirement already satisfied: ccxt>=1.66.32 in /usr/local/lib/python3.10/site-packages (from finrl==0.3.5) (3.0.89)\n",
            "Requirement already satisfied: jqdatasdk in /usr/local/lib/python3.10/site-packages (from finrl==0.3.5) (1.8.11)\n",
            "Requirement already satisfied: alpaca_trade_api>=2.1.0 in /usr/local/lib/python3.10/site-packages (from finrl==0.3.5) (3.0.0)\n",
            "Requirement already satisfied: pandas>=1.1.5 in /usr/local/lib/python3.10/site-packages (from finrl==0.3.5) (2.0.1)\n",
            "Requirement already satisfied: stockstats>=0.4.0 in /usr/local/lib/python3.10/site-packages (from finrl==0.3.5) (0.5.2)\n",
            "Requirement already satisfied: wrds>=3.1.6 in /usr/local/lib/python3.10/site-packages (from finrl==0.3.5) (3.1.6)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.10/site-packages (from finrl==0.3.5) (3.7.1)\n",
            "Requirement already satisfied: gym>=0.17 in /usr/local/lib/python3.10/site-packages (from finrl==0.3.5) (0.21.0)\n",
            "Requirement already satisfied: yfinance in /usr/local/lib/python3.10/site-packages (from finrl==0.3.5) (0.2.18)\n",
            "Requirement already satisfied: importlib-metadata==4.13.0 in /usr/local/lib/python3.10/site-packages (from finrl==0.3.5) (4.13.0)\n",
            "Requirement already satisfied: ray[default,tune]>=2.0.0 in /usr/local/lib/python3.10/site-packages (from finrl==0.3.5) (2.4.0)\n",
            "Requirement already satisfied: korean-lunar-calendar in /usr/local/lib/python3.10/site-packages (from exchange_calendars==3.6.3->finrl==0.3.5) (0.3.1)\n",
            "Requirement already satisfied: toolz in /usr/local/lib/python3.10/site-packages (from exchange_calendars==3.6.3->finrl==0.3.5) (0.12.0)\n",
            "Requirement already satisfied: pytz in /usr/local/lib/python3.10/site-packages (from exchange_calendars==3.6.3->finrl==0.3.5) (2023.3)\n",
            "Requirement already satisfied: pyluach in /usr/local/lib/python3.10/site-packages (from exchange_calendars==3.6.3->finrl==0.3.5) (2.2.0)\n",
            "Requirement already satisfied: python-dateutil in /usr/local/lib/python3.10/site-packages (from exchange_calendars==3.6.3->finrl==0.3.5) (2.8.2)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.10/site-packages (from importlib-metadata==4.13.0->finrl==0.3.5) (3.15.0)\n",
            "Requirement already satisfied: requests<3,>2 in /usr/local/lib/python3.10/site-packages (from alpaca_trade_api>=2.1.0->finrl==0.3.5) (2.28.2)\n",
            "Requirement already satisfied: websocket-client<2,>=0.56.0 in /usr/local/lib/python3.10/site-packages (from alpaca_trade_api>=2.1.0->finrl==0.3.5) (1.5.1)\n",
            "Requirement already satisfied: deprecation==2.1.0 in /usr/local/lib/python3.10/site-packages (from alpaca_trade_api>=2.1.0->finrl==0.3.5) (2.1.0)\n",
            "Requirement already satisfied: msgpack==1.0.3 in /usr/local/lib/python3.10/site-packages (from alpaca_trade_api>=2.1.0->finrl==0.3.5) (1.0.3)\n",
            "Requirement already satisfied: PyYAML==6.0 in /usr/local/lib/python3.10/site-packages (from alpaca_trade_api>=2.1.0->finrl==0.3.5) (6.0)\n",
            "Requirement already satisfied: aiohttp==3.8.1 in /usr/local/lib/python3.10/site-packages (from alpaca_trade_api>=2.1.0->finrl==0.3.5) (3.8.1)\n",
            "Requirement already satisfied: urllib3<2,>1.24 in /usr/local/lib/python3.10/site-packages (from alpaca_trade_api>=2.1.0->finrl==0.3.5) (1.26.15)\n",
            "Requirement already satisfied: websockets<11,>=9.0 in /usr/local/lib/python3.10/site-packages (from alpaca_trade_api>=2.1.0->finrl==0.3.5) (10.4)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/site-packages (from aiohttp==3.8.1->alpaca_trade_api>=2.1.0->finrl==0.3.5) (1.3.3)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/site-packages (from aiohttp==3.8.1->alpaca_trade_api>=2.1.0->finrl==0.3.5) (1.9.2)\n",
            "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /usr/local/lib/python3.10/site-packages (from aiohttp==3.8.1->alpaca_trade_api>=2.1.0->finrl==0.3.5) (2.1.1)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/site-packages (from aiohttp==3.8.1->alpaca_trade_api>=2.1.0->finrl==0.3.5) (1.3.1)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/site-packages (from aiohttp==3.8.1->alpaca_trade_api>=2.1.0->finrl==0.3.5) (6.0.4)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/site-packages (from aiohttp==3.8.1->alpaca_trade_api>=2.1.0->finrl==0.3.5) (4.0.2)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/site-packages (from aiohttp==3.8.1->alpaca_trade_api>=2.1.0->finrl==0.3.5) (23.1.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/site-packages (from deprecation==2.1.0->alpaca_trade_api>=2.1.0->finrl==0.3.5) (23.1)\n",
            "Requirement already satisfied: cryptography>=2.6.1 in /usr/local/lib/python3.10/site-packages (from ccxt>=1.66.32->finrl==0.3.5) (40.0.1)\n",
            "Requirement already satisfied: setuptools>=60.9.0 in /usr/local/lib/python3.10/site-packages (from ccxt>=1.66.32->finrl==0.3.5) (65.6.3)\n",
            "Requirement already satisfied: certifi>=2018.1.18 in /usr/local/lib/python3.10/site-packages (from ccxt>=1.66.32->finrl==0.3.5) (2022.12.7)\n",
            "Requirement already satisfied: aiodns>=1.1.1 in /usr/local/lib/python3.10/site-packages (from ccxt>=1.66.32->finrl==0.3.5) (3.0.0)\n",
            "Requirement already satisfied: cloudpickle>=1.2.0 in /usr/local/lib/python3.10/site-packages (from gym>=0.17->finrl==0.3.5) (2.2.1)\n",
            "Requirement already satisfied: tzdata>=2022.1 in /usr/local/lib/python3.10/site-packages (from pandas>=1.1.5->finrl==0.3.5) (2023.3)\n",
            "Requirement already satisfied: protobuf!=3.19.5,>=3.15.3 in /usr/local/lib/python3.10/site-packages (from ray[default,tune]>=2.0.0->finrl==0.3.5) (3.20.3)\n",
            "Requirement already satisfied: virtualenv<20.21.1,>=20.0.24 in /usr/local/lib/python3.10/site-packages (from ray[default,tune]>=2.0.0->finrl==0.3.5) (20.21.0)\n",
            "Requirement already satisfied: jsonschema in /usr/local/lib/python3.10/site-packages (from ray[default,tune]>=2.0.0->finrl==0.3.5) (4.17.3)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/site-packages (from ray[default,tune]>=2.0.0->finrl==0.3.5) (3.12.0)\n",
            "Requirement already satisfied: grpcio<=1.51.3,>=1.42.0 in /usr/local/lib/python3.10/site-packages (from ray[default,tune]>=2.0.0->finrl==0.3.5) (1.51.3)\n",
            "Requirement already satisfied: click>=7.0 in /usr/local/lib/python3.10/site-packages (from ray[default,tune]>=2.0.0->finrl==0.3.5) (8.1.3)\n",
            "Requirement already satisfied: colorful in /usr/local/lib/python3.10/site-packages (from ray[default,tune]>=2.0.0->finrl==0.3.5) (0.5.5)\n",
            "Requirement already satisfied: smart-open in /usr/local/lib/python3.10/site-packages (from ray[default,tune]>=2.0.0->finrl==0.3.5) (6.3.0)\n",
            "Requirement already satisfied: pydantic in /usr/local/lib/python3.10/site-packages (from ray[default,tune]>=2.0.0->finrl==0.3.5) (1.10.7)\n",
            "Requirement already satisfied: gpustat>=1.0.0 in /usr/local/lib/python3.10/site-packages (from ray[default,tune]>=2.0.0->finrl==0.3.5) (1.1)\n",
            "Requirement already satisfied: prometheus-client>=0.7.1 in /usr/local/lib/python3.10/site-packages (from ray[default,tune]>=2.0.0->finrl==0.3.5) (0.16.0)\n",
            "Requirement already satisfied: py-spy>=0.2.0 in /usr/local/lib/python3.10/site-packages (from ray[default,tune]>=2.0.0->finrl==0.3.5) (0.3.14)\n",
            "Requirement already satisfied: aiohttp-cors in /usr/local/lib/python3.10/site-packages (from ray[default,tune]>=2.0.0->finrl==0.3.5) (0.7.0)\n",
            "Requirement already satisfied: opencensus in /usr/local/lib/python3.10/site-packages (from ray[default,tune]>=2.0.0->finrl==0.3.5) (0.11.2)\n",
            "Requirement already satisfied: tabulate in /usr/local/lib/python3.10/site-packages (from ray[default,tune]>=2.0.0->finrl==0.3.5) (0.9.0)\n",
            "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/site-packages (from scikit-learn>=0.21.0->finrl==0.3.5) (1.2.0)\n",
            "Requirement already satisfied: scipy>=1.3.2 in /usr/local/lib/python3.10/site-packages (from scikit-learn>=0.21.0->finrl==0.3.5) (1.10.1)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/site-packages (from scikit-learn>=0.21.0->finrl==0.3.5) (3.1.0)\n",
            "Requirement already satisfied: torch>=1.11 in /usr/local/lib/python3.10/site-packages (from stable-baselines3<2.0.0,>=1.6.2->finrl==0.3.5) (2.0.0)\n",
            "Requirement already satisfied: sqlalchemy<2 in /usr/local/lib/python3.10/site-packages (from wrds>=3.1.6->finrl==0.3.5) (1.4.48)\n",
            "Requirement already satisfied: psycopg2-binary in /usr/local/lib/python3.10/site-packages (from wrds>=3.1.6->finrl==0.3.5) (2.9.6)\n",
            "Requirement already satisfied: thriftpy2>=0.3.9 in /usr/local/lib/python3.10/site-packages (from jqdatasdk->finrl==0.3.5) (0.4.16)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.10/site-packages (from jqdatasdk->finrl==0.3.5) (1.16.0)\n",
            "Requirement already satisfied: pymysql>=0.7.6 in /usr/local/lib/python3.10/site-packages (from jqdatasdk->finrl==0.3.5) (1.0.3)\n",
            "Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.10/site-packages (from matplotlib->finrl==0.3.5) (9.5.0)\n",
            "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/site-packages (from matplotlib->finrl==0.3.5) (4.39.3)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/site-packages (from matplotlib->finrl==0.3.5) (0.11.0)\n",
            "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/site-packages (from matplotlib->finrl==0.3.5) (3.0.9)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/site-packages (from matplotlib->finrl==0.3.5) (1.4.4)\n",
            "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/site-packages (from matplotlib->finrl==0.3.5) (1.0.7)\n",
            "Requirement already satisfied: ipython>=3.2.3 in /usr/local/lib/python3.10/site-packages (from pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.5) (8.13.1)\n",
            "Requirement already satisfied: seaborn>=0.7.1 in /usr/local/lib/python3.10/site-packages (from pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.5) (0.12.2)\n",
            "Requirement already satisfied: empyrical>=0.5.0 in /usr/local/lib/python3.10/site-packages (from pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.5) (0.5.5)\n",
            "Requirement already satisfied: beautifulsoup4>=4.11.1 in /usr/local/lib/python3.10/site-packages (from yfinance->finrl==0.3.5) (4.12.2)\n",
            "Requirement already satisfied: frozendict>=2.3.4 in /usr/local/lib/python3.10/site-packages (from yfinance->finrl==0.3.5) (2.3.8)\n",
            "Requirement already satisfied: appdirs>=1.4.4 in /usr/local/lib/python3.10/site-packages (from yfinance->finrl==0.3.5) (1.4.4)\n",
            "Requirement already satisfied: html5lib>=1.1 in /usr/local/lib/python3.10/site-packages (from yfinance->finrl==0.3.5) (1.1)\n",
            "Requirement already satisfied: lxml>=4.9.1 in /usr/local/lib/python3.10/site-packages (from yfinance->finrl==0.3.5) (4.9.2)\n",
            "Requirement already satisfied: multitasking>=0.0.7 in /usr/local/lib/python3.10/site-packages (from yfinance->finrl==0.3.5) (0.0.11)\n",
            "Requirement already satisfied: pycares>=4.0.0 in /usr/local/lib/python3.10/site-packages (from aiodns>=1.1.1->ccxt>=1.66.32->finrl==0.3.5) (4.3.0)\n",
            "Requirement already satisfied: soupsieve>1.2 in /usr/local/lib/python3.10/site-packages (from beautifulsoup4>=4.11.1->yfinance->finrl==0.3.5) (2.4.1)\n",
            "Requirement already satisfied: cffi>=1.12 in /usr/local/lib/python3.10/site-packages (from cryptography>=2.6.1->ccxt>=1.66.32->finrl==0.3.5) (1.15.1)\n",
            "Requirement already satisfied: pandas-datareader>=0.2 in /usr/local/lib/python3.10/site-packages (from empyrical>=0.5.0->pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.5) (0.10.0)\n",
            "Requirement already satisfied: psutil>=5.6.0 in /usr/local/lib/python3.10/site-packages (from gpustat>=1.0.0->ray[default,tune]>=2.0.0->finrl==0.3.5) (5.9.5)\n",
            "Requirement already satisfied: blessed>=1.17.1 in /usr/local/lib/python3.10/site-packages (from gpustat>=1.0.0->ray[default,tune]>=2.0.0->finrl==0.3.5) (1.20.0)\n",
            "Requirement already satisfied: nvidia-ml-py>=11.450.129 in /usr/local/lib/python3.10/site-packages (from gpustat>=1.0.0->ray[default,tune]>=2.0.0->finrl==0.3.5) (11.525.112)\n",
            "Requirement already satisfied: webencodings in /usr/local/lib/python3.10/site-packages (from html5lib>=1.1->yfinance->finrl==0.3.5) (0.5.1)\n",
            "Requirement already satisfied: pygments>=2.4.0 in /usr/local/lib/python3.10/site-packages (from ipython>=3.2.3->pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.5) (2.15.1)\n",
            "Requirement already satisfied: jedi>=0.16 in /usr/local/lib/python3.10/site-packages (from ipython>=3.2.3->pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.5) (0.18.2)\n",
            "Requirement already satisfied: matplotlib-inline in /usr/local/lib/python3.10/site-packages (from ipython>=3.2.3->pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.5) (0.1.6)\n",
            "Requirement already satisfied: decorator in /usr/local/lib/python3.10/site-packages (from ipython>=3.2.3->pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.5) (5.1.1)\n",
            "Requirement already satisfied: pickleshare in /usr/local/lib/python3.10/site-packages (from ipython>=3.2.3->pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.5) (0.7.5)\n",
            "Requirement already satisfied: prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30 in /usr/local/lib/python3.10/site-packages (from ipython>=3.2.3->pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.5) (3.0.38)\n",
            "Requirement already satisfied: stack-data in /usr/local/lib/python3.10/site-packages (from ipython>=3.2.3->pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.5) (0.6.2)\n",
            "Requirement already satisfied: traitlets>=5 in /usr/local/lib/python3.10/site-packages (from ipython>=3.2.3->pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.5) (5.9.0)\n",
            "Requirement already satisfied: backcall in /usr/local/lib/python3.10/site-packages (from ipython>=3.2.3->pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.5) (0.2.0)\n",
            "Requirement already satisfied: pexpect>4.3 in /usr/local/lib/python3.10/site-packages (from ipython>=3.2.3->pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.5) (4.8.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/site-packages (from requests<3,>2->alpaca_trade_api>=2.1.0->finrl==0.3.5) (3.4)\n",
            "Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.10/site-packages (from sqlalchemy<2->wrds>=3.1.6->finrl==0.3.5) (2.0.2)\n",
            "Requirement already satisfied: ply<4.0,>=3.4 in /usr/local/lib/python3.10/site-packages (from thriftpy2>=0.3.9->jqdatasdk->finrl==0.3.5) (3.11)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/site-packages (from torch>=1.11->stable-baselines3<2.0.0,>=1.6.2->finrl==0.3.5) (4.5.0)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/site-packages (from torch>=1.11->stable-baselines3<2.0.0,>=1.6.2->finrl==0.3.5) (1.11.1)\n",
            "Requirement already satisfied: nvidia-curand-cu11==10.2.10.91 in /usr/local/lib/python3.10/site-packages (from torch>=1.11->stable-baselines3<2.0.0,>=1.6.2->finrl==0.3.5) (10.2.10.91)\n",
            "Requirement already satisfied: nvidia-cudnn-cu11==8.5.0.96 in /usr/local/lib/python3.10/site-packages (from torch>=1.11->stable-baselines3<2.0.0,>=1.6.2->finrl==0.3.5) (8.5.0.96)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/site-packages (from torch>=1.11->stable-baselines3<2.0.0,>=1.6.2->finrl==0.3.5) (3.1.2)\n",
            "Requirement already satisfied: nvidia-cublas-cu11==11.10.3.66 in /usr/local/lib/python3.10/site-packages (from torch>=1.11->stable-baselines3<2.0.0,>=1.6.2->finrl==0.3.5) (11.10.3.66)\n",
            "Requirement already satisfied: nvidia-nvtx-cu11==11.7.91 in /usr/local/lib/python3.10/site-packages (from torch>=1.11->stable-baselines3<2.0.0,>=1.6.2->finrl==0.3.5) (11.7.91)\n",
            "Requirement already satisfied: nvidia-cuda-cupti-cu11==11.7.101 in /usr/local/lib/python3.10/site-packages (from torch>=1.11->stable-baselines3<2.0.0,>=1.6.2->finrl==0.3.5) (11.7.101)\n",
            "Requirement already satisfied: nvidia-cusolver-cu11==11.4.0.1 in /usr/local/lib/python3.10/site-packages (from torch>=1.11->stable-baselines3<2.0.0,>=1.6.2->finrl==0.3.5) (11.4.0.1)\n",
            "Requirement already satisfied: nvidia-cusparse-cu11==11.7.4.91 in /usr/local/lib/python3.10/site-packages (from torch>=1.11->stable-baselines3<2.0.0,>=1.6.2->finrl==0.3.5) (11.7.4.91)\n",
            "Requirement already satisfied: nvidia-cuda-runtime-cu11==11.7.99 in /usr/local/lib/python3.10/site-packages (from torch>=1.11->stable-baselines3<2.0.0,>=1.6.2->finrl==0.3.5) (11.7.99)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/site-packages (from torch>=1.11->stable-baselines3<2.0.0,>=1.6.2->finrl==0.3.5) (3.1)\n",
            "Requirement already satisfied: nvidia-nccl-cu11==2.14.3 in /usr/local/lib/python3.10/site-packages (from torch>=1.11->stable-baselines3<2.0.0,>=1.6.2->finrl==0.3.5) (2.14.3)\n",
            "Requirement already satisfied: nvidia-cufft-cu11==10.9.0.58 in /usr/local/lib/python3.10/site-packages (from torch>=1.11->stable-baselines3<2.0.0,>=1.6.2->finrl==0.3.5) (10.9.0.58)\n",
            "Requirement already satisfied: nvidia-cuda-nvrtc-cu11==11.7.99 in /usr/local/lib/python3.10/site-packages (from torch>=1.11->stable-baselines3<2.0.0,>=1.6.2->finrl==0.3.5) (11.7.99)\n",
            "Requirement already satisfied: triton==2.0.0 in /usr/local/lib/python3.10/site-packages (from torch>=1.11->stable-baselines3<2.0.0,>=1.6.2->finrl==0.3.5) (2.0.0)\n",
            "Requirement already satisfied: wheel in /usr/local/lib/python3.10/site-packages (from nvidia-cublas-cu11==11.10.3.66->torch>=1.11->stable-baselines3<2.0.0,>=1.6.2->finrl==0.3.5) (0.40.0)\n",
            "Requirement already satisfied: cmake in /usr/local/lib/python3.10/site-packages (from triton==2.0.0->torch>=1.11->stable-baselines3<2.0.0,>=1.6.2->finrl==0.3.5) (3.26.3)\n",
            "Requirement already satisfied: lit in /usr/local/lib/python3.10/site-packages (from triton==2.0.0->torch>=1.11->stable-baselines3<2.0.0,>=1.6.2->finrl==0.3.5) (16.0.2)\n",
            "Requirement already satisfied: platformdirs<4,>=2.4 in /usr/local/lib/python3.10/site-packages (from virtualenv<20.21.1,>=20.0.24->ray[default,tune]>=2.0.0->finrl==0.3.5) (3.5.0)\n",
            "Requirement already satisfied: distlib<1,>=0.3.6 in /usr/local/lib/python3.10/site-packages (from virtualenv<20.21.1,>=20.0.24->ray[default,tune]>=2.0.0->finrl==0.3.5) (0.3.6)\n",
            "Requirement already satisfied: box2d-py==2.3.5 in /usr/local/lib/python3.10/site-packages (from gym>=0.17->finrl==0.3.5) (2.3.5)\n",
            "Requirement already satisfied: pyglet>=1.4.0 in /usr/local/lib/python3.10/site-packages (from gym>=0.17->finrl==0.3.5) (2.0.6)\n",
            "Requirement already satisfied: pyrsistent!=0.17.0,!=0.17.1,!=0.17.2,>=0.14.0 in /usr/local/lib/python3.10/site-packages (from jsonschema->ray[default,tune]>=2.0.0->finrl==0.3.5) (0.19.3)\n",
            "Requirement already satisfied: google-api-core<3.0.0,>=1.0.0 in /usr/local/lib/python3.10/site-packages (from opencensus->ray[default,tune]>=2.0.0->finrl==0.3.5) (2.11.0)\n",
            "Requirement already satisfied: opencensus-context>=0.1.3 in /usr/local/lib/python3.10/site-packages (from opencensus->ray[default,tune]>=2.0.0->finrl==0.3.5) (0.1.3)\n",
            "Requirement already satisfied: wcwidth>=0.1.4 in /usr/local/lib/python3.10/site-packages (from blessed>=1.17.1->gpustat>=1.0.0->ray[default,tune]>=2.0.0->finrl==0.3.5) (0.2.6)\n",
            "Requirement already satisfied: pycparser in /usr/local/lib/python3.10/site-packages (from cffi>=1.12->cryptography>=2.6.1->ccxt>=1.66.32->finrl==0.3.5) (2.21)\n",
            "Requirement already satisfied: googleapis-common-protos<2.0dev,>=1.56.2 in /usr/local/lib/python3.10/site-packages (from google-api-core<3.0.0,>=1.0.0->opencensus->ray[default,tune]>=2.0.0->finrl==0.3.5) (1.59.0)\n",
            "Requirement already satisfied: google-auth<3.0dev,>=2.14.1 in /usr/local/lib/python3.10/site-packages (from google-api-core<3.0.0,>=1.0.0->opencensus->ray[default,tune]>=2.0.0->finrl==0.3.5) (2.17.3)\n",
            "Requirement already satisfied: parso<0.9.0,>=0.8.0 in /usr/local/lib/python3.10/site-packages (from jedi>=0.16->ipython>=3.2.3->pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.5) (0.8.3)\n",
            "Requirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.10/site-packages (from pexpect>4.3->ipython>=3.2.3->pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.5) (0.7.0)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/site-packages (from jinja2->torch>=1.11->stable-baselines3<2.0.0,>=1.6.2->finrl==0.3.5) (2.1.2)\n",
            "Requirement already satisfied: executing>=1.2.0 in /usr/local/lib/python3.10/site-packages (from stack-data->ipython>=3.2.3->pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.5) (1.2.0)\n",
            "Requirement already satisfied: pure-eval in /usr/local/lib/python3.10/site-packages (from stack-data->ipython>=3.2.3->pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.5) (0.2.2)\n",
            "Requirement already satisfied: asttokens>=2.1.0 in /usr/local/lib/python3.10/site-packages (from stack-data->ipython>=3.2.3->pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2->finrl==0.3.5) (2.2.1)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/site-packages (from sympy->torch>=1.11->stable-baselines3<2.0.0,>=1.6.2->finrl==0.3.5) (1.3.0)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.10/site-packages (from google-auth<3.0dev,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default,tune]>=2.0.0->finrl==0.3.5) (4.9)\n",
            "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.10/site-packages (from google-auth<3.0dev,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default,tune]>=2.0.0->finrl==0.3.5) (5.3.0)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.10/site-packages (from google-auth<3.0dev,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default,tune]>=2.0.0->finrl==0.3.5) (0.3.0)\n",
            "Requirement already satisfied: pyasn1<0.6.0,>=0.4.6 in /usr/local/lib/python3.10/site-packages (from pyasn1-modules>=0.2.1->google-auth<3.0dev,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default,tune]>=2.0.0->finrl==0.3.5) (0.5.0)\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
            "\u001b[0m"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 1.2 Check if the additional packages needed are present, if not install them"
      ],
      "metadata": {
        "id": "DMaIlOcVTJfp"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# !pip install trading_calendars\n",
        "# !pip install alpaca_trade_api\n",
        "# !pip install ccxt\n",
        "# !pip install jqdatasdk\n",
        "# !pip install wrds\n",
        "\n",
        "# !pip install lz4\n",
        "# !pip install ray[tune]\n",
        "# !pip install tensorboardX\n",
        "# !pip install gputil\n",
        "\n",
        "#%%capture\n",
        "if True:\n",
        "    # installing packages\n",
        "    !pip install pyfolio-reloaded  #original pyfolio no longer maintained\n",
        "    !pip install optuna\n",
        "    !pip install -U \"ray[rllib]\"\n",
        "    !pip install plotly\n",
        "    !pip install ipywidgets\n",
        "    !pip install -U kaleido   # enables saving plots to file\n",
        "    \n",
        "    "
      ],
      "metadata": {
        "id": "UfmXkH1rTFM7",
        "outputId": "3267870a-561c-456e-d300-619d31d66159",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting pyfolio-reloaded\n",
            "  Downloading pyfolio_reloaded-0.9.5-py3-none-any.whl (6.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.3/6.3 MB\u001b[0m \u001b[31m49.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.11.1 in /usr/local/lib/python3.10/site-packages (from pyfolio-reloaded) (1.24.3)\n",
            "Requirement already satisfied: matplotlib>=1.4.0 in /usr/local/lib/python3.10/site-packages (from pyfolio-reloaded) (3.7.1)\n",
            "Requirement already satisfied: ipython>=3.2.3 in /usr/local/lib/python3.10/site-packages (from pyfolio-reloaded) (8.13.1)\n",
            "Requirement already satisfied: scipy>=0.14.0 in /usr/local/lib/python3.10/site-packages (from pyfolio-reloaded) (1.10.1)\n",
            "Requirement already satisfied: scikit-learn>=0.16.1 in /usr/local/lib/python3.10/site-packages (from pyfolio-reloaded) (1.2.2)\n",
            "Requirement already satisfied: seaborn>=0.7.1 in /usr/local/lib/python3.10/site-packages (from pyfolio-reloaded) (0.12.2)\n",
            "Requirement already satisfied: pytz>=2014.10 in /usr/local/lib/python3.10/site-packages (from pyfolio-reloaded) (2023.3)\n",
            "Collecting empyrical-reloaded>=0.5.8\n",
            "  Downloading empyrical_reloaded-0.5.9-py3-none-any.whl (32 kB)\n",
            "Requirement already satisfied: pandas>=0.18.1 in /usr/local/lib/python3.10/site-packages (from pyfolio-reloaded) (2.0.1)\n",
            "Requirement already satisfied: yfinance>=0.1.63 in /usr/local/lib/python3.10/site-packages (from empyrical-reloaded>=0.5.8->pyfolio-reloaded) (0.2.18)\n",
            "Collecting bottleneck>=1.3.0\n",
            "  Downloading Bottleneck-1.3.7-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (354 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m354.0/354.0 kB\u001b[0m \u001b[31m30.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pandas-datareader>=0.4 in /usr/local/lib/python3.10/site-packages (from empyrical-reloaded>=0.5.8->pyfolio-reloaded) (0.10.0)\n",
            "Requirement already satisfied: pickleshare in /usr/local/lib/python3.10/site-packages (from ipython>=3.2.3->pyfolio-reloaded) (0.7.5)\n",
            "Requirement already satisfied: backcall in /usr/local/lib/python3.10/site-packages (from ipython>=3.2.3->pyfolio-reloaded) (0.2.0)\n",
            "Requirement already satisfied: pygments>=2.4.0 in /usr/local/lib/python3.10/site-packages (from ipython>=3.2.3->pyfolio-reloaded) (2.15.1)\n",
            "Requirement already satisfied: stack-data in /usr/local/lib/python3.10/site-packages (from ipython>=3.2.3->pyfolio-reloaded) (0.6.2)\n",
            "Requirement already satisfied: jedi>=0.16 in /usr/local/lib/python3.10/site-packages (from ipython>=3.2.3->pyfolio-reloaded) (0.18.2)\n",
            "Requirement already satisfied: pexpect>4.3 in /usr/local/lib/python3.10/site-packages (from ipython>=3.2.3->pyfolio-reloaded) (4.8.0)\n",
            "Requirement already satisfied: traitlets>=5 in /usr/local/lib/python3.10/site-packages (from ipython>=3.2.3->pyfolio-reloaded) (5.9.0)\n",
            "Requirement already satisfied: decorator in /usr/local/lib/python3.10/site-packages (from ipython>=3.2.3->pyfolio-reloaded) (5.1.1)\n",
            "Requirement already satisfied: prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30 in /usr/local/lib/python3.10/site-packages (from ipython>=3.2.3->pyfolio-reloaded) (3.0.38)\n",
            "Requirement already satisfied: matplotlib-inline in /usr/local/lib/python3.10/site-packages (from ipython>=3.2.3->pyfolio-reloaded) (0.1.6)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/site-packages (from matplotlib>=1.4.0->pyfolio-reloaded) (0.11.0)\n",
            "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/site-packages (from matplotlib>=1.4.0->pyfolio-reloaded) (3.0.9)\n",
            "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.10/site-packages (from matplotlib>=1.4.0->pyfolio-reloaded) (2.8.2)\n",
            "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/site-packages (from matplotlib>=1.4.0->pyfolio-reloaded) (1.0.7)\n",
            "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/site-packages (from matplotlib>=1.4.0->pyfolio-reloaded) (4.39.3)\n",
            "Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.10/site-packages (from matplotlib>=1.4.0->pyfolio-reloaded) (9.5.0)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/site-packages (from matplotlib>=1.4.0->pyfolio-reloaded) (23.1)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/site-packages (from matplotlib>=1.4.0->pyfolio-reloaded) (1.4.4)\n",
            "Requirement already satisfied: tzdata>=2022.1 in /usr/local/lib/python3.10/site-packages (from pandas>=0.18.1->pyfolio-reloaded) (2023.3)\n",
            "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/site-packages (from scikit-learn>=0.16.1->pyfolio-reloaded) (1.2.0)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/site-packages (from scikit-learn>=0.16.1->pyfolio-reloaded) (3.1.0)\n",
            "Requirement already satisfied: parso<0.9.0,>=0.8.0 in /usr/local/lib/python3.10/site-packages (from jedi>=0.16->ipython>=3.2.3->pyfolio-reloaded) (0.8.3)\n",
            "Requirement already satisfied: lxml in /usr/local/lib/python3.10/site-packages (from pandas-datareader>=0.4->empyrical-reloaded>=0.5.8->pyfolio-reloaded) (4.9.2)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.10/site-packages (from pandas-datareader>=0.4->empyrical-reloaded>=0.5.8->pyfolio-reloaded) (2.28.2)\n",
            "Requirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.10/site-packages (from pexpect>4.3->ipython>=3.2.3->pyfolio-reloaded) (0.7.0)\n",
            "Requirement already satisfied: wcwidth in /usr/local/lib/python3.10/site-packages (from prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30->ipython>=3.2.3->pyfolio-reloaded) (0.2.6)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/site-packages (from python-dateutil>=2.7->matplotlib>=1.4.0->pyfolio-reloaded) (1.16.0)\n",
            "Requirement already satisfied: frozendict>=2.3.4 in /usr/local/lib/python3.10/site-packages (from yfinance>=0.1.63->empyrical-reloaded>=0.5.8->pyfolio-reloaded) (2.3.8)\n",
            "Requirement already satisfied: multitasking>=0.0.7 in /usr/local/lib/python3.10/site-packages (from yfinance>=0.1.63->empyrical-reloaded>=0.5.8->pyfolio-reloaded) (0.0.11)\n",
            "Requirement already satisfied: cryptography>=3.3.2 in /usr/local/lib/python3.10/site-packages (from yfinance>=0.1.63->empyrical-reloaded>=0.5.8->pyfolio-reloaded) (40.0.1)\n",
            "Requirement already satisfied: beautifulsoup4>=4.11.1 in /usr/local/lib/python3.10/site-packages (from yfinance>=0.1.63->empyrical-reloaded>=0.5.8->pyfolio-reloaded) (4.12.2)\n",
            "Requirement already satisfied: appdirs>=1.4.4 in /usr/local/lib/python3.10/site-packages (from yfinance>=0.1.63->empyrical-reloaded>=0.5.8->pyfolio-reloaded) (1.4.4)\n",
            "Requirement already satisfied: html5lib>=1.1 in /usr/local/lib/python3.10/site-packages (from yfinance>=0.1.63->empyrical-reloaded>=0.5.8->pyfolio-reloaded) (1.1)\n",
            "Requirement already satisfied: executing>=1.2.0 in /usr/local/lib/python3.10/site-packages (from stack-data->ipython>=3.2.3->pyfolio-reloaded) (1.2.0)\n",
            "Requirement already satisfied: asttokens>=2.1.0 in /usr/local/lib/python3.10/site-packages (from stack-data->ipython>=3.2.3->pyfolio-reloaded) (2.2.1)\n",
            "Requirement already satisfied: pure-eval in /usr/local/lib/python3.10/site-packages (from stack-data->ipython>=3.2.3->pyfolio-reloaded) (0.2.2)\n",
            "Requirement already satisfied: soupsieve>1.2 in /usr/local/lib/python3.10/site-packages (from beautifulsoup4>=4.11.1->yfinance>=0.1.63->empyrical-reloaded>=0.5.8->pyfolio-reloaded) (2.4.1)\n",
            "Requirement already satisfied: cffi>=1.12 in /usr/local/lib/python3.10/site-packages (from cryptography>=3.3.2->yfinance>=0.1.63->empyrical-reloaded>=0.5.8->pyfolio-reloaded) (1.15.1)\n",
            "Requirement already satisfied: webencodings in /usr/local/lib/python3.10/site-packages (from html5lib>=1.1->yfinance>=0.1.63->empyrical-reloaded>=0.5.8->pyfolio-reloaded) (0.5.1)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/site-packages (from requests>=2.19.0->pandas-datareader>=0.4->empyrical-reloaded>=0.5.8->pyfolio-reloaded) (2022.12.7)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/site-packages (from requests>=2.19.0->pandas-datareader>=0.4->empyrical-reloaded>=0.5.8->pyfolio-reloaded) (3.4)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/site-packages (from requests>=2.19.0->pandas-datareader>=0.4->empyrical-reloaded>=0.5.8->pyfolio-reloaded) (2.1.1)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/site-packages (from requests>=2.19.0->pandas-datareader>=0.4->empyrical-reloaded>=0.5.8->pyfolio-reloaded) (1.26.15)\n",
            "Requirement already satisfied: pycparser in /usr/local/lib/python3.10/site-packages (from cffi>=1.12->cryptography>=3.3.2->yfinance>=0.1.63->empyrical-reloaded>=0.5.8->pyfolio-reloaded) (2.21)\n",
            "Installing collected packages: bottleneck, empyrical-reloaded, pyfolio-reloaded\n",
            "Successfully installed bottleneck-1.3.7 empyrical-reloaded-0.5.9 pyfolio-reloaded-0.9.5\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
            "\u001b[0mLooking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting optuna\n",
            "  Downloading optuna-3.1.1-py3-none-any.whl (365 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m365.7/365.7 kB\u001b[0m \u001b[31m7.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: PyYAML in /usr/local/lib/python3.10/site-packages (from optuna) (6.0)\n",
            "Requirement already satisfied: sqlalchemy>=1.3.0 in /usr/local/lib/python3.10/site-packages (from optuna) (1.4.48)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/site-packages (from optuna) (4.65.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/site-packages (from optuna) (1.24.3)\n",
            "Collecting cmaes>=0.9.1\n",
            "  Downloading cmaes-0.9.1-py3-none-any.whl (21 kB)\n",
            "Collecting colorlog\n",
            "  Downloading colorlog-6.7.0-py2.py3-none-any.whl (11 kB)\n",
            "Collecting alembic>=1.5.0\n",
            "  Downloading alembic-1.10.4-py3-none-any.whl (212 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m212.9/212.9 kB\u001b[0m \u001b[31m21.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/site-packages (from optuna) (23.1)\n",
            "Collecting Mako\n",
            "  Downloading Mako-1.2.4-py3-none-any.whl (78 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m78.7/78.7 kB\u001b[0m \u001b[31m9.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: typing-extensions>=4 in /usr/local/lib/python3.10/site-packages (from alembic>=1.5.0->optuna) (4.5.0)\n",
            "Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.10/site-packages (from sqlalchemy>=1.3.0->optuna) (2.0.2)\n",
            "Requirement already satisfied: MarkupSafe>=0.9.2 in /usr/local/lib/python3.10/site-packages (from Mako->alembic>=1.5.0->optuna) (2.1.2)\n",
            "Installing collected packages: Mako, colorlog, cmaes, alembic, optuna\n",
            "Successfully installed Mako-1.2.4 alembic-1.10.4 cmaes-0.9.1 colorlog-6.7.0 optuna-3.1.1\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
            "\u001b[0mLooking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: ray[rllib] in /usr/local/lib/python3.10/site-packages (2.4.0)\n",
            "Requirement already satisfied: grpcio<=1.51.3,>=1.42.0 in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (1.51.3)\n",
            "Requirement already satisfied: protobuf!=3.19.5,>=3.15.3 in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (3.20.3)\n",
            "Requirement already satisfied: numpy>=1.19.3 in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (1.24.3)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (23.1)\n",
            "Requirement already satisfied: jsonschema in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (4.17.3)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (6.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (3.12.0)\n",
            "Requirement already satisfied: aiosignal in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (1.3.1)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (2.28.2)\n",
            "Requirement already satisfied: virtualenv<20.21.1,>=20.0.24 in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (20.21.0)\n",
            "Requirement already satisfied: frozenlist in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (1.3.3)\n",
            "Requirement already satisfied: click>=7.0 in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (8.1.3)\n",
            "Requirement already satisfied: msgpack<2.0.0,>=1.0.0 in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (1.0.3)\n",
            "Requirement already satisfied: attrs in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (23.1.0)\n",
            "Requirement already satisfied: tensorboardX>=1.9 in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (2.6)\n",
            "Collecting rich\n",
            "  Downloading rich-13.3.5-py3-none-any.whl (238 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m238.7/238.7 kB\u001b[0m \u001b[31m5.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: lz4 in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (4.3.2)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (2.0.1)\n",
            "Collecting scikit-image\n",
            "  Downloading scikit_image-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (13.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m13.2/13.2 MB\u001b[0m \u001b[31m68.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: scipy in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (1.10.1)\n",
            "Collecting dm-tree\n",
            "  Downloading dm_tree-0.1.8-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (152 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m152.8/152.8 kB\u001b[0m \u001b[31m17.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting typer\n",
            "  Downloading typer-0.9.0-py3-none-any.whl (45 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m45.9/45.9 kB\u001b[0m \u001b[31m5.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting gymnasium==0.26.3\n",
            "  Downloading Gymnasium-0.26.3-py3-none-any.whl (836 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m836.9/836.9 kB\u001b[0m \u001b[31m44.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: tabulate in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (0.9.0)\n",
            "Collecting gymnasium-notices>=0.0.1\n",
            "  Downloading gymnasium_notices-0.0.1-py3-none-any.whl (2.8 kB)\n",
            "Requirement already satisfied: cloudpickle>=1.2.0 in /usr/local/lib/python3.10/site-packages (from gymnasium==0.26.3->ray[rllib]) (2.2.1)\n",
            "Requirement already satisfied: distlib<1,>=0.3.6 in /usr/local/lib/python3.10/site-packages (from virtualenv<20.21.1,>=20.0.24->ray[rllib]) (0.3.6)\n",
            "Requirement already satisfied: platformdirs<4,>=2.4 in /usr/local/lib/python3.10/site-packages (from virtualenv<20.21.1,>=20.0.24->ray[rllib]) (3.5.0)\n",
            "Requirement already satisfied: pyrsistent!=0.17.0,!=0.17.1,!=0.17.2,>=0.14.0 in /usr/local/lib/python3.10/site-packages (from jsonschema->ray[rllib]) (0.19.3)\n",
            "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/site-packages (from pandas->ray[rllib]) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/site-packages (from pandas->ray[rllib]) (2023.3)\n",
            "Requirement already satisfied: tzdata>=2022.1 in /usr/local/lib/python3.10/site-packages (from pandas->ray[rllib]) (2023.3)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/site-packages (from requests->ray[rllib]) (1.26.15)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/site-packages (from requests->ray[rllib]) (2022.12.7)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/site-packages (from requests->ray[rllib]) (3.4)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/site-packages (from requests->ray[rllib]) (2.1.1)\n",
            "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/site-packages (from rich->ray[rllib]) (2.15.1)\n",
            "Collecting markdown-it-py<3.0.0,>=2.2.0\n",
            "  Downloading markdown_it_py-2.2.0-py3-none-any.whl (84 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m84.5/84.5 kB\u001b[0m \u001b[31m9.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting lazy_loader>=0.1\n",
            "  Downloading lazy_loader-0.2-py3-none-any.whl (8.6 kB)\n",
            "Collecting tifffile>=2019.7.26\n",
            "  Downloading tifffile-2023.4.12-py3-none-any.whl (219 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m219.4/219.4 kB\u001b[0m \u001b[31m22.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting PyWavelets>=1.1.1\n",
            "  Downloading PyWavelets-1.4.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (6.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.8/6.8 MB\u001b[0m \u001b[31m70.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: networkx>=2.8 in /usr/local/lib/python3.10/site-packages (from scikit-image->ray[rllib]) (3.1)\n",
            "Requirement already satisfied: pillow>=9.0.1 in /usr/local/lib/python3.10/site-packages (from scikit-image->ray[rllib]) (9.5.0)\n",
            "Collecting imageio>=2.4.1\n",
            "  Downloading imageio-2.28.1-py3-none-any.whl (3.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.4/3.4 MB\u001b[0m \u001b[31m76.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/site-packages (from typer->ray[rllib]) (4.5.0)\n",
            "Collecting mdurl~=0.1\n",
            "  Downloading mdurl-0.1.2-py3-none-any.whl (10.0 kB)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/site-packages (from python-dateutil>=2.8.2->pandas->ray[rllib]) (1.16.0)\n",
            "Installing collected packages: gymnasium-notices, dm-tree, typer, tifffile, PyWavelets, mdurl, lazy_loader, imageio, gymnasium, scikit-image, markdown-it-py, rich\n",
            "Successfully installed PyWavelets-1.4.1 dm-tree-0.1.8 gymnasium-0.26.3 gymnasium-notices-0.0.1 imageio-2.28.1 lazy_loader-0.2 markdown-it-py-2.2.0 mdurl-0.1.2 rich-13.3.5 scikit-image-0.20.0 tifffile-2023.4.12 typer-0.9.0\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
            "\u001b[0mLooking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting plotly\n",
            "  Downloading plotly-5.14.1-py2.py3-none-any.whl (15.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m15.3/15.3 MB\u001b[0m \u001b[31m76.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.10/site-packages (from plotly) (23.1)\n",
            "Collecting tenacity>=6.2.0\n",
            "  Downloading tenacity-8.2.2-py3-none-any.whl (24 kB)\n",
            "Installing collected packages: tenacity, plotly\n",
            "Successfully installed plotly-5.14.1 tenacity-8.2.2\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
            "\u001b[0mLooking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting ipywidgets\n",
            "  Downloading ipywidgets-8.0.6-py3-none-any.whl (138 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m138.3/138.3 kB\u001b[0m \u001b[31m4.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting widgetsnbextension~=4.0.7\n",
            "  Downloading widgetsnbextension-4.0.7-py3-none-any.whl (2.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.1/2.1 MB\u001b[0m \u001b[31m35.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting ipykernel>=4.5.1\n",
            "  Downloading ipykernel-6.22.0-py3-none-any.whl (149 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m150.0/150.0 kB\u001b[0m \u001b[31m18.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting jupyterlab-widgets~=3.0.7\n",
            "  Downloading jupyterlab_widgets-3.0.7-py3-none-any.whl (198 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m198.2/198.2 kB\u001b[0m \u001b[31m22.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: traitlets>=4.3.1 in /usr/local/lib/python3.10/site-packages (from ipywidgets) (5.9.0)\n",
            "Requirement already satisfied: ipython>=6.1.0 in /usr/local/lib/python3.10/site-packages (from ipywidgets) (8.13.1)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/site-packages (from ipykernel>=4.5.1->ipywidgets) (23.1)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.10/site-packages (from ipykernel>=4.5.1->ipywidgets) (5.9.5)\n",
            "Collecting debugpy>=1.6.5\n",
            "  Downloading debugpy-1.6.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.0/3.0 MB\u001b[0m \u001b[31m82.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting tornado>=6.1\n",
            "  Downloading tornado-6.3.1-cp38-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (426 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m426.8/426.8 kB\u001b[0m \u001b[31m41.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pyzmq>=20\n",
            "  Downloading pyzmq-25.0.2-cp310-cp310-manylinux_2_28_x86_64.whl (1.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.1 MB\u001b[0m \u001b[31m58.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting comm>=0.1.1\n",
            "  Downloading comm-0.1.3-py3-none-any.whl (6.6 kB)\n",
            "Collecting jupyter-core!=5.0.*,>=4.12\n",
            "  Downloading jupyter_core-5.3.0-py3-none-any.whl (93 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m93.2/93.2 kB\u001b[0m \u001b[31m12.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nest-asyncio\n",
            "  Downloading nest_asyncio-1.5.6-py3-none-any.whl (5.2 kB)\n",
            "Collecting jupyter-client>=6.1.12\n",
            "  Downloading jupyter_client-8.2.0-py3-none-any.whl (103 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m103.2/103.2 kB\u001b[0m \u001b[31m12.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: matplotlib-inline>=0.1 in /usr/local/lib/python3.10/site-packages (from ipykernel>=4.5.1->ipywidgets) (0.1.6)\n",
            "Requirement already satisfied: pexpect>4.3 in /usr/local/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets) (4.8.0)\n",
            "Requirement already satisfied: jedi>=0.16 in /usr/local/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets) (0.18.2)\n",
            "Requirement already satisfied: pygments>=2.4.0 in /usr/local/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets) (2.15.1)\n",
            "Requirement already satisfied: backcall in /usr/local/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets) (0.2.0)\n",
            "Requirement already satisfied: prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30 in /usr/local/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets) (3.0.38)\n",
            "Requirement already satisfied: pickleshare in /usr/local/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets) (0.7.5)\n",
            "Requirement already satisfied: stack-data in /usr/local/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets) (0.6.2)\n",
            "Requirement already satisfied: decorator in /usr/local/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets) (5.1.1)\n",
            "Requirement already satisfied: parso<0.9.0,>=0.8.0 in /usr/local/lib/python3.10/site-packages (from jedi>=0.16->ipython>=6.1.0->ipywidgets) (0.8.3)\n",
            "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/site-packages (from jupyter-client>=6.1.12->ipykernel>=4.5.1->ipywidgets) (2.8.2)\n",
            "Requirement already satisfied: platformdirs>=2.5 in /usr/local/lib/python3.10/site-packages (from jupyter-core!=5.0.*,>=4.12->ipykernel>=4.5.1->ipywidgets) (3.5.0)\n",
            "Requirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.10/site-packages (from pexpect>4.3->ipython>=6.1.0->ipywidgets) (0.7.0)\n",
            "Requirement already satisfied: wcwidth in /usr/local/lib/python3.10/site-packages (from prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30->ipython>=6.1.0->ipywidgets) (0.2.6)\n",
            "Requirement already satisfied: executing>=1.2.0 in /usr/local/lib/python3.10/site-packages (from stack-data->ipython>=6.1.0->ipywidgets) (1.2.0)\n",
            "Requirement already satisfied: asttokens>=2.1.0 in /usr/local/lib/python3.10/site-packages (from stack-data->ipython>=6.1.0->ipywidgets) (2.2.1)\n",
            "Requirement already satisfied: pure-eval in /usr/local/lib/python3.10/site-packages (from stack-data->ipython>=6.1.0->ipywidgets) (0.2.2)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.10/site-packages (from asttokens>=2.1.0->stack-data->ipython>=6.1.0->ipywidgets) (1.16.0)\n",
            "Installing collected packages: widgetsnbextension, tornado, pyzmq, nest-asyncio, jupyterlab-widgets, jupyter-core, debugpy, comm, jupyter-client, ipykernel, ipywidgets\n",
            "Successfully installed comm-0.1.3 debugpy-1.6.7 ipykernel-6.22.0 ipywidgets-8.0.6 jupyter-client-8.2.0 jupyter-core-5.3.0 jupyterlab-widgets-3.0.7 nest-asyncio-1.5.6 pyzmq-25.0.2 tornado-6.3.1 widgetsnbextension-4.0.7\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
            "\u001b[0mLooking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting kaleido\n",
            "  Downloading kaleido-0.2.1-py2.py3-none-manylinux1_x86_64.whl (79.9 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m79.9/79.9 MB\u001b[0m \u001b[31m6.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: kaleido\n",
            "Successfully installed kaleido-0.2.1\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
            "\u001b[0m"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install pandas==1.5.3\n",
        "!pip install gymnasium"
      ],
      "metadata": {
        "id": "9Zkmtvr-TFn_",
        "outputId": "10770b25-4151-41cf-b05a-6eba7c860ad3",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting pandas==1.5.3\n",
            "  Downloading pandas-1.5.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m12.1/12.1 MB\u001b[0m \u001b[31m74.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/site-packages (from pandas==1.5.3) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/site-packages (from pandas==1.5.3) (2023.3)\n",
            "Requirement already satisfied: numpy>=1.21.0 in /usr/local/lib/python3.10/site-packages (from pandas==1.5.3) (1.24.3)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/site-packages (from python-dateutil>=2.8.1->pandas==1.5.3) (1.16.0)\n",
            "Installing collected packages: pandas\n",
            "  Attempting uninstall: pandas\n",
            "    Found existing installation: pandas 2.0.1\n",
            "    Uninstalling pandas-2.0.1:\n",
            "      Successfully uninstalled pandas-2.0.1\n",
            "Successfully installed pandas-1.5.3\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
            "\u001b[0mLooking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: gymnasium in /usr/local/lib/python3.10/site-packages (0.26.3)\n",
            "Requirement already satisfied: cloudpickle>=1.2.0 in /usr/local/lib/python3.10/site-packages (from gymnasium) (2.2.1)\n",
            "Requirement already satisfied: numpy>=1.18.0 in /usr/local/lib/python3.10/site-packages (from gymnasium) (1.24.3)\n",
            "Requirement already satisfied: gymnasium-notices>=0.0.1 in /usr/local/lib/python3.10/site-packages (from gymnasium) (0.0.1)\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
            "\u001b[0m"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import sys\n",
        "sys.path.append(\"../FinRL-Library\")"
      ],
      "metadata": {
        "id": "cMn1VNKCR0ld"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 1.3 Import packages"
      ],
      "metadata": {
        "id": "BbOMJnGdTRYX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#Importing the libraries\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import matplotlib\n",
        "import matplotlib.pyplot as plt\n",
        "# matplotlib.use('Agg')\n",
        "import datetime\n",
        "import optuna\n",
        "from pathlib import Path\n",
        "from google.colab import files\n",
        "%matplotlib inline\n",
        "from finrl import config\n",
        "from finrl import config_tickers\n",
        "from optuna.integration import PyTorchLightningPruningCallback\n",
        "from finrl.meta.preprocessor.yahoodownloader import YahooDownloader\n",
        "from finrl.meta.preprocessor.preprocessors import FeatureEngineer, data_split\n",
        "from finrl.meta.env_stock_trading.env_stocktrading import StockTradingEnv\n",
        "from finrl.meta.env_stock_trading.env_stocktrading_np import StockTradingEnv as StockTradingEnv_numpy\n",
        "from finrl.agents.stablebaselines3.models import DRLAgent\n",
        "from finrl.agents.rllib.models import DRLAgent as DRLAgent_rllib\n",
        "from finrl.meta.data_processor import DataProcessor\n",
        "import joblib\n",
        "from finrl.plot import backtest_stats, backtest_plot, get_daily_return, get_baseline\n",
        "import ray\n",
        "from pprint import pprint\n",
        "import kaleido\n",
        "\n",
        "\n",
        "\n",
        "import itertools\n",
        "\n",
        "import torch\n",
        "if torch.cuda.is_available():\n",
        "  device = torch.device(\"cuda\")\n",
        "else:\n",
        "  device = torch.device(\"cpu\")\n",
        "print(f'Torch device: {device}')"
      ],
      "metadata": {
        "id": "tG5qIkVHTNv8",
        "outputId": "5f3f9d83-8350-4599-ea7c-c281a753b1f9",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Torch device: cuda\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/site-packages/pyfolio/pos.py:25: UserWarning: Module \"zipline.assets\" not found; multipliers will not be applied to position notionals.\n",
            "  warnings.warn(\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "config_tickers.DOW_30_TICKER"
      ],
      "metadata": {
        "id": "DB11ygQLVLUD",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8286e8f5-ff5f-4efd-dffd-131033c1a048"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['AXP',\n",
              " 'AMGN',\n",
              " 'AAPL',\n",
              " 'BA',\n",
              " 'CAT',\n",
              " 'CSCO',\n",
              " 'CVX',\n",
              " 'GS',\n",
              " 'HD',\n",
              " 'HON',\n",
              " 'IBM',\n",
              " 'INTC',\n",
              " 'JNJ',\n",
              " 'KO',\n",
              " 'JPM',\n",
              " 'MCD',\n",
              " 'MMM',\n",
              " 'MRK',\n",
              " 'MSFT',\n",
              " 'NKE',\n",
              " 'PG',\n",
              " 'TRV',\n",
              " 'UNH',\n",
              " 'CRM',\n",
              " 'VZ',\n",
              " 'V',\n",
              " 'WBA',\n",
              " 'WMT',\n",
              " 'DIS',\n",
              " 'DOW']"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "fMaPm8MsUo0R"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "if not os.path.exists(\"./\" + config.DATA_SAVE_DIR):\n",
        "    os.makedirs(\"./\" + config.DATA_SAVE_DIR)\n",
        "if not os.path.exists(\"./\" + config.TRAINED_MODEL_DIR):\n",
        "    os.makedirs(\"./\" + config.TRAINED_MODEL_DIR)\n",
        "if not os.path.exists(\"./\" + config.TENSORBOARD_LOG_DIR):\n",
        "    os.makedirs(\"./\" + config.TENSORBOARD_LOG_DIR)\n",
        "if not os.path.exists(\"./\" + config.RESULTS_DIR):\n",
        "    os.makedirs(\"./\" + config.RESULTS_DIR)"
      ],
      "metadata": {
        "id": "FAZpwTH3VVi-"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Collecting data and preprocessing"
      ],
      "metadata": {
        "id": "cqJ8ngFWVcNG"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#config_tickers.DOW_30_TICKER = [\"PYPL\"]"
      ],
      "metadata": {
        "id": "4M4UwcP4Ap_o"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#Custom ticker list dataframe download\n",
        "#TODO save df to avoid download\n",
        "path_pf = '/content/ticker_data.csv'\n",
        "if Path(path_pf).is_file():\n",
        "  print('Reading ticker data')\n",
        "  df = pd.read_csv(path_pf)\n",
        "  \n",
        "else:\n",
        "  print('Downloading ticker data')\n",
        "  ticker_list = config_tickers.DOW_30_TICKER\n",
        "  df = YahooDownloader(start_date = '2009-01-01',\n",
        "                     end_date = '2023-04-30',\n",
        "                     ticker_list = ticker_list).fetch_data()\n",
        "  df.to_csv('ticker_data.csv')"
      ],
      "metadata": {
        "id": "7BJNlsAuVZ-o",
        "outputId": "c5c049c9-e1f9-4cb1-9ea6-e91c7d44a7cc",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading ticker data\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "Shape of DataFrame:  (105581, 8)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def create_processed_full(processed):\n",
        "  list_ticker = processed[\"tic\"].unique().tolist()\n",
        "  list_date = list(pd.date_range(processed['date'].min(),processed['date'].max()).astype(str))\n",
        "  combination = list(itertools.product(list_date,list_ticker))\n",
        "\n",
        "  processed_full = pd.DataFrame(combination,columns=[\"date\",\"tic\"]).merge(processed,on=[\"date\",\"tic\"],how=\"left\")\n",
        "  processed_full = processed_full[processed_full['date'].isin(processed['date'])]\n",
        "  processed_full = processed_full.sort_values(['date','tic'])\n",
        "\n",
        "  processed_full = processed_full.fillna(0)\n",
        "  processed_full.sort_values(['date','tic'],ignore_index=True).head(5)\n",
        "\n",
        "  processed_full.to_csv('processed_full.csv')\n",
        "  return processed_full"
      ],
      "metadata": {
        "id": "RD6wwtGSVZVU"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#You can add technical indicators and turbulence factor to dataframe\n",
        "#Just set the use_technical_indicator=True, use_vix=True and use_turbulence=True\n",
        "def create_techind():\n",
        "  fe = FeatureEngineer(\n",
        "                    use_technical_indicator=True,\n",
        "                    tech_indicator_list = config.INDICATORS,\n",
        "                    use_vix=True,\n",
        "                    use_turbulence=True,\n",
        "                    user_defined_feature = False)\n",
        "\n",
        "  processed = fe.preprocess_data(df)\n",
        "  return processed"
      ],
      "metadata": {
        "id": "UNqEALmbVoJo"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#Load price and technical indicator data from file if available\n",
        "path_pf = '/content/processed_full.csv'\n",
        "if Path(path_pf).is_file():\n",
        "  print('Reading processed_full data')\n",
        "  processed_full = pd.read_csv(path_pf)\n",
        "\n",
        "else:\n",
        "  print('Creating processed_full file')\n",
        "  processed=create_techind()\n",
        "  processed_full=create_processed_full(processed)"
      ],
      "metadata": {
        "id": "Qr1Qd-lMVpzg",
        "outputId": "a2f98172-0812-40b1-96c8-6cbabaa1a96f",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Creating processed_full file\n",
            "Successfully added technical indicators\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "Shape of DataFrame:  (3604, 8)\n",
            "Successfully added vix\n",
            "Successfully added turbulence index\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "date_col = \"date\"\n",
        "tic_col = \"tic\"\n",
        "\n",
        "init_train_trade_data = processed_full.sort_values([date_col, tic_col])\n",
        "\n",
        "init_train_trade_data = processed_full.fillna(0)\n",
        "\n",
        "init_train_data = data_split(\n",
        "    init_train_trade_data, '2020-01-01', '2020-05-01')\n",
        "init_trade_data = data_split(\n",
        "    init_train_trade_data, '2021-05-01','2021-10-01')\n",
        "\n",
        "print(f'Number of training samples: {len(init_train_data)}')\n",
        "print(f'Number of testing samples: {len(init_train_trade_data)}')"
      ],
      "metadata": {
        "id": "KFs4AjWIVr2s",
        "outputId": "fb984121-bd90-477a-9abe-87c316c3da2b",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Number of training samples: 2407\n",
            "Number of testing samples: 104516\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "init_trade_data.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 357
        },
        "id": "GI2qgPGFTerU",
        "outputId": "bd545047-9350-4b6e-9970-1ed1f6b09b1a"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "         date   tic        open        high         low       close  \\\n",
              "0  2021-05-03  AAPL  132.039993  134.070007  131.830002  130.963562   \n",
              "0  2021-05-03  AMGN  240.669998  247.020004  240.550003  230.412506   \n",
              "0  2021-05-03   AXP  154.589996  156.050003  154.009995  150.957855   \n",
              "0  2021-05-03    BA  234.110001  237.100006  233.809998  235.190002   \n",
              "0  2021-05-03   CAT  230.000000  230.929993  227.210007  218.290527   \n",
              "\n",
              "       volume  day      macd     boll_ub     boll_lb     rsi_30      cci_30  \\\n",
              "0  75135100.0  0.0  1.805509  135.539556  126.477482  53.856523   50.478498   \n",
              "0   3587700.0  0.0 -0.410711  248.223104  221.790972  50.842041  -81.617817   \n",
              "0   2726200.0  0.0  2.754006  151.027340  139.179363  62.209688  175.796174   \n",
              "0   9887800.0  0.0 -2.593339  260.290298  227.831702  49.445883 -102.331978   \n",
              "0   3182900.0  0.0  1.167006  223.005770  216.688284  56.081928  -17.678266   \n",
              "\n",
              "       dx_30  close_30_sma  close_60_sma        vix  turbulence  \n",
              "0  20.583396    127.527258    126.185904  18.309999   20.080178  \n",
              "0   5.785497    234.734413    226.585524  18.309999   20.080178  \n",
              "0  22.874057    142.745665    138.440773  18.309999   20.080178  \n",
              "0  17.827725    245.819000    237.101333  18.309999   20.080178  \n",
              "0   2.780977    218.974405    211.601994  18.309999   20.080178  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-d4f44b9d-8436-4aca-8ce9-01c7a9b62faf\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>date</th>\n",
              "      <th>tic</th>\n",
              "      <th>open</th>\n",
              "      <th>high</th>\n",
              "      <th>low</th>\n",
              "      <th>close</th>\n",
              "      <th>volume</th>\n",
              "      <th>day</th>\n",
              "      <th>macd</th>\n",
              "      <th>boll_ub</th>\n",
              "      <th>boll_lb</th>\n",
              "      <th>rsi_30</th>\n",
              "      <th>cci_30</th>\n",
              "      <th>dx_30</th>\n",
              "      <th>close_30_sma</th>\n",
              "      <th>close_60_sma</th>\n",
              "      <th>vix</th>\n",
              "      <th>turbulence</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2021-05-03</td>\n",
              "      <td>AAPL</td>\n",
              "      <td>132.039993</td>\n",
              "      <td>134.070007</td>\n",
              "      <td>131.830002</td>\n",
              "      <td>130.963562</td>\n",
              "      <td>75135100.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.805509</td>\n",
              "      <td>135.539556</td>\n",
              "      <td>126.477482</td>\n",
              "      <td>53.856523</td>\n",
              "      <td>50.478498</td>\n",
              "      <td>20.583396</td>\n",
              "      <td>127.527258</td>\n",
              "      <td>126.185904</td>\n",
              "      <td>18.309999</td>\n",
              "      <td>20.080178</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2021-05-03</td>\n",
              "      <td>AMGN</td>\n",
              "      <td>240.669998</td>\n",
              "      <td>247.020004</td>\n",
              "      <td>240.550003</td>\n",
              "      <td>230.412506</td>\n",
              "      <td>3587700.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>-0.410711</td>\n",
              "      <td>248.223104</td>\n",
              "      <td>221.790972</td>\n",
              "      <td>50.842041</td>\n",
              "      <td>-81.617817</td>\n",
              "      <td>5.785497</td>\n",
              "      <td>234.734413</td>\n",
              "      <td>226.585524</td>\n",
              "      <td>18.309999</td>\n",
              "      <td>20.080178</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2021-05-03</td>\n",
              "      <td>AXP</td>\n",
              "      <td>154.589996</td>\n",
              "      <td>156.050003</td>\n",
              "      <td>154.009995</td>\n",
              "      <td>150.957855</td>\n",
              "      <td>2726200.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>2.754006</td>\n",
              "      <td>151.027340</td>\n",
              "      <td>139.179363</td>\n",
              "      <td>62.209688</td>\n",
              "      <td>175.796174</td>\n",
              "      <td>22.874057</td>\n",
              "      <td>142.745665</td>\n",
              "      <td>138.440773</td>\n",
              "      <td>18.309999</td>\n",
              "      <td>20.080178</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2021-05-03</td>\n",
              "      <td>BA</td>\n",
              "      <td>234.110001</td>\n",
              "      <td>237.100006</td>\n",
              "      <td>233.809998</td>\n",
              "      <td>235.190002</td>\n",
              "      <td>9887800.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>-2.593339</td>\n",
              "      <td>260.290298</td>\n",
              "      <td>227.831702</td>\n",
              "      <td>49.445883</td>\n",
              "      <td>-102.331978</td>\n",
              "      <td>17.827725</td>\n",
              "      <td>245.819000</td>\n",
              "      <td>237.101333</td>\n",
              "      <td>18.309999</td>\n",
              "      <td>20.080178</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2021-05-03</td>\n",
              "      <td>CAT</td>\n",
              "      <td>230.000000</td>\n",
              "      <td>230.929993</td>\n",
              "      <td>227.210007</td>\n",
              "      <td>218.290527</td>\n",
              "      <td>3182900.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.167006</td>\n",
              "      <td>223.005770</td>\n",
              "      <td>216.688284</td>\n",
              "      <td>56.081928</td>\n",
              "      <td>-17.678266</td>\n",
              "      <td>2.780977</td>\n",
              "      <td>218.974405</td>\n",
              "      <td>211.601994</td>\n",
              "      <td>18.309999</td>\n",
              "      <td>20.080178</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-d4f44b9d-8436-4aca-8ce9-01c7a9b62faf')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-d4f44b9d-8436-4aca-8ce9-01c7a9b62faf button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-d4f44b9d-8436-4aca-8ce9-01c7a9b62faf');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Creating Environment"
      ],
      "metadata": {
        "id": "L3VlkfBMW8e7"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from __future__ import annotations\n",
        "\n",
        "from typing import List\n",
        "\n",
        "import gym\n",
        "import matplotlib\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "np.set_printoptions(linewidth=np.inf)\n",
        "import pandas as pd\n",
        "from gym import spaces\n",
        "from gym.utils import seeding\n",
        "from stable_baselines3.common.vec_env import DummyVecEnv\n",
        "\n",
        "matplotlib.use(\"Agg\")\n",
        "\n",
        "# from stable_baselines3.common.logger import Logger, KVWriter, CSVOutputFormat\n",
        "\n",
        "\n",
        "class GokuEnv(gym.Env):\n",
        "    \"\"\"A stock trading environment for OpenAI gym\"\"\"\n",
        "\n",
        "    metadata = {\"render.modes\": [\"human\"]}\n",
        "\n",
        "    def __init__(\n",
        "        self,\n",
        "        df: pd.DataFrame,\n",
        "        stock_dim: int,\n",
        "        hmax: int,\n",
        "        initial_amount: int,\n",
        "        num_stock_shares: list[int],\n",
        "        buy_cost_pct: list[float],\n",
        "        sell_cost_pct: list[float],\n",
        "        reward_scaling: float,\n",
        "        state_space: int,\n",
        "        action_space: int,\n",
        "        tech_indicator_list: list[str],\n",
        "        turbulence_threshold=None,\n",
        "        risk_indicator_col=\"turbulence\",\n",
        "        make_plots: bool = False,\n",
        "        print_verbosity=10,\n",
        "        day=0,\n",
        "        initial=True,\n",
        "        previous_state=[],\n",
        "        model_name=\"\",\n",
        "        mode=\"\",\n",
        "        iteration=\"\",\n",
        "    ):\n",
        "        self.day = day\n",
        "        self.df = df\n",
        "        self.stock_dim = stock_dim\n",
        "        self.hmax = hmax\n",
        "        self.num_stock_shares = num_stock_shares\n",
        "        self.initial_amount = initial_amount  # get the initial cash\n",
        "        self.buy_cost_pct = buy_cost_pct\n",
        "        self.sell_cost_pct = sell_cost_pct\n",
        "        self.reward_scaling = reward_scaling\n",
        "        self.state_space = state_space\n",
        "        self.action_space = action_space\n",
        "        self.tech_indicator_list = tech_indicator_list\n",
        "        self.action_space = spaces.Box(low=-1, high=1, shape=(self.action_space,))\n",
        "        self.observation_space = spaces.Box(\n",
        "            low=-np.inf, high=np.inf, shape=(self.state_space,)\n",
        "        )\n",
        "        self.data = self.df.loc[self.day, :]\n",
        "        self.terminal = False\n",
        "        self.make_plots = make_plots\n",
        "        self.print_verbosity = print_verbosity\n",
        "        self.turbulence_threshold = turbulence_threshold\n",
        "        self.risk_indicator_col = risk_indicator_col\n",
        "        self.initial = initial\n",
        "        self.previous_state = previous_state\n",
        "        self.model_name = model_name\n",
        "        self.mode = mode\n",
        "        self.iteration = iteration\n",
        "        # initalize state\n",
        "        self.state = self._initiate_state()\n",
        "\n",
        "        # initialize reward\n",
        "        self.reward = 0\n",
        "        self.turbulence = 0\n",
        "        self.cost = 0\n",
        "        self.trades = 0\n",
        "        self.episode = 0\n",
        "        # memorize all the total balance change\n",
        "        self.asset_memory = [\n",
        "            self.initial_amount\n",
        "            + np.sum(\n",
        "                np.array(self.num_stock_shares)\n",
        "                * np.array(self.state[1 : 1 + self.stock_dim])\n",
        "            )\n",
        "        ]  # the initial total asset is calculated by cash + sum (num_share_stock_i * price_stock_i)\n",
        "        self.rewards_memory = []\n",
        "        self.actions_memory = []\n",
        "        self.state_memory = (\n",
        "            []\n",
        "        )  # we need sometimes to preserve the state in the middle of trading process\n",
        "        self.date_memory = [self._get_date()]\n",
        "        #         self.logger = Logger('results',[CSVOutputFormat])\n",
        "        # self.reset()\n",
        "        self._seed()\n",
        "        self.total_price = np.array([0.0] * 29)\n",
        "        self.avg_price = np.array([0.0] * 29)\n",
        "        self.total_stockss = np.array([0.0] * 29)\n",
        "        self.frame  = pd.DataFrame(columns=list(range(1,30)))\n",
        "\n",
        "    def _sell_stock(self, index, action):\n",
        "        def _do_sell_normal():\n",
        "            if (\n",
        "                self.state[index + 2 * self.stock_dim + 1] != True\n",
        "            ):  # check if the stock is able to sell, for simlicity we just add it in techical index\n",
        "                # if self.state[index + 1] > 0: # if we use price<0 to denote a stock is unable to trade in that day, the total asset calculation may be wrong for the price is unreasonable\n",
        "                # Sell only if the price is > 0 (no missing data in this particular date)\n",
        "                # perform sell action based on the sign of the action\n",
        "                if self.state[index + self.stock_dim + 1] > 0:\n",
        "                    # Sell only if current asset is > 0\n",
        "                    sell_num_shares = min(\n",
        "                        abs(action), self.state[index + self.stock_dim + 1]\n",
        "                    )\n",
        "                    sell_amount = (\n",
        "                        self.state[index + 1]\n",
        "                        * sell_num_shares\n",
        "                        * (1 - self.sell_cost_pct[index])\n",
        "                    )\n",
        "                    # update balance\n",
        "                    self.state[0] += sell_amount\n",
        "\n",
        "                    self.state[index + self.stock_dim + 1] -= sell_num_shares\n",
        "                    self.cost += (\n",
        "                        self.state[index + 1]\n",
        "                        * sell_num_shares\n",
        "                        * self.sell_cost_pct[index]\n",
        "                    )\n",
        "                    self.trades += 1\n",
        "                    #if sell_num_shares >0:\n",
        "                    #  print(\"stocks Sold sell_num_shares\", sell_num_shares)\n",
        "                else:\n",
        "                    sell_num_shares = 0\n",
        "            else:\n",
        "                sell_num_shares = 0\n",
        "            return sell_num_shares\n",
        "\n",
        "        # perform sell action based on the sign of the action\n",
        "        if self.turbulence_threshold is not None:\n",
        "            if self.turbulence >= self.turbulence_threshold:\n",
        "                if self.state[index + 1] > 0:\n",
        "                    # Sell only if the price is > 0 (no missing data in this particular date)\n",
        "                    # if turbulence goes over threshold, just clear out all positions\n",
        "                    if self.state[index + self.stock_dim + 1] > 0:\n",
        "                        # Sell only if current asset is > 0\n",
        "                        sell_num_shares = self.state[index + self.stock_dim + 1]\n",
        "                        sell_amount = (\n",
        "                            self.state[index + 1]\n",
        "                            * sell_num_shares\n",
        "                            * (1 - self.sell_cost_pct[index])\n",
        "                        )\n",
        "                        # update balance\n",
        "                        self.state[0] += sell_amount\n",
        "                        self.state[index + self.stock_dim + 1] = 0\n",
        "                        self.cost += (\n",
        "                            self.state[index + 1]\n",
        "                            * sell_num_shares\n",
        "                            * self.sell_cost_pct[index]\n",
        "                        )\n",
        "                        self.trades += 1\n",
        "                    else:\n",
        "                        sell_num_shares = 0\n",
        "                else:\n",
        "                    sell_num_shares = 0\n",
        "            else:\n",
        "                sell_num_shares = _do_sell_normal()\n",
        "        else:\n",
        "            sell_num_shares = _do_sell_normal()\n",
        "        #print(\"stocks Sold sell_num_shares\", sell_num_shares)\n",
        "        return sell_num_shares\n",
        "\n",
        "    def _buy_stock(self, index, action):\n",
        "        def _do_buy():\n",
        "            if (self.state[index + 2 * self.stock_dim + 1] != True):  # check if the stock is able to buy\n",
        "                # if self.state[index + 1] >0:\n",
        "                # Buy only if the price is > 0 (no missing data in this particular date)\n",
        "                available_amount = self.state[0] // (self.state[index + 1] * (1 + self.buy_cost_pct[index]))\n",
        "                # when buying stocks, we should consider the cost of trading when calculating available_amount, or we may be have cash<0\n",
        "                # print('available_amount:{}'.format(available_amount))\n",
        "                # update balance\n",
        "                buy_num_shares = min(available_amount, action)\n",
        "                buy_amount = (self.state[index + 1] * buy_num_shares * (1 + self.buy_cost_pct[index]))\n",
        "                self.state[0] -= buy_amount\n",
        "                self.state[index + self.stock_dim + 1] += buy_num_shares\n",
        "                self.cost += (self.state[index + 1] * buy_num_shares * self.buy_cost_pct[index])\n",
        "                self.trades += 1\n",
        "            else:\n",
        "                buy_num_shares = 0\n",
        "\n",
        "            return buy_num_shares\n",
        "\n",
        "        # perform buy action based on the sign of the action\n",
        "        if self.turbulence_threshold is None:\n",
        "            buy_num_shares = _do_buy()\n",
        "        else:\n",
        "            if self.turbulence < self.turbulence_threshold:\n",
        "                buy_num_shares = _do_buy()\n",
        "            else:\n",
        "                buy_num_shares = 0\n",
        "                pass\n",
        "\n",
        "        return buy_num_shares\n",
        "\n",
        "    def _make_plot(self):\n",
        "        plt.plot(self.asset_memory, \"r\")\n",
        "        plt.savefig(f\"results/account_value_trade_{self.episode}.png\")\n",
        "        plt.close()\n",
        "\n",
        "    def step(self, actions):\n",
        "        self.terminal = self.day >= len(self.df.index.unique()) - 1\n",
        "        if self.terminal:\n",
        "            # print(f\"Episode: {self.episode}\")\n",
        "            if self.make_plots:\n",
        "                self._make_plot()\n",
        "            end_total_asset = self.state[0] + sum(np.array(self.state[1 : (self.stock_dim + 1)]) * np.array(self.state[(self.stock_dim + 1) : (self.stock_dim * 2 + 1)]))\n",
        "            df_total_value = pd.DataFrame(self.asset_memory)\n",
        "            tot_reward = (self.state[0] + \n",
        "                          sum(np.array(self.state[1 : (self.stock_dim + 1)]) \n",
        "                          * np.array(self.state[(self.stock_dim + 1) : (self.stock_dim * 2 + 1)])) \n",
        "                          - self.asset_memory[0])  # initial_amount is only cash part of our initial asset\n",
        "            df_total_value.columns = [\"account_value\"]\n",
        "            df_total_value[\"date\"] = self.date_memory\n",
        "            df_total_value[\"daily_return\"] = df_total_value[\"account_value\"].pct_change(1)\n",
        "            if df_total_value[\"daily_return\"].std() != 0:\n",
        "                sharpe = ((252**0.5) \n",
        "                * df_total_value[\"daily_return\"].mean() \n",
        "                / df_total_value[\"daily_return\"].std())\n",
        "            df_rewards = pd.DataFrame(self.rewards_memory)\n",
        "            df_rewards.columns = [\"account_rewards\"]\n",
        "            df_rewards[\"date\"] = self.date_memory[:-1]\n",
        "            \n",
        "            if self.episode % self.print_verbosity == 0:\n",
        "                print(f\"day: {self.day}, episode: {self.episode}\")\n",
        "                print(f\"begin_total_asset: {self.asset_memory[0]:0.2f}\")\n",
        "                print(f\"end_total_asset: {end_total_asset:0.2f}\")\n",
        "                print(f\"total_reward: {tot_reward:0.2f}\")\n",
        "                print(f\"total_cost: {self.cost:0.2f}\")\n",
        "                print(f\"total_trades: {self.trades}\")\n",
        "                if df_total_value[\"daily_return\"].std() != 0:\n",
        "                    print(f\"Sharpe: {sharpe:0.3f}\")\n",
        "                print(\"=================================\")\n",
        "\n",
        "            if (self.model_name != \"\") and (self.mode != \"\"):\n",
        "                df_actions = self.save_action_memory()\n",
        "                df_actions.to_csv(\"results/actions_{}_{}_{}.csv\".format(self.mode, self.model_name, self.iteration))\n",
        "                df_total_value.to_csv(\"results/account_value_{}_{}_{}.csv\".format(self.mode, self.model_name, self.iteration),index=False,)\n",
        "                df_rewards.to_csv(\"results/account_rewards_{}_{}_{}.csv\".format(self.mode, self.model_name, self.iteration),index=False,)\n",
        "                plt.plot(self.asset_memory, \"r\")\n",
        "                plt.savefig(\"results/account_value_{}_{}_{}.png\".format(self.mode, self.model_name, self.iteration))\n",
        "                plt.close()\n",
        "            return self.state, self.reward, self.terminal, {}\n",
        "\n",
        "        else:\n",
        "            actions = actions * self.hmax  # actions initially is scaled between 0 to 1\n",
        "            actions = actions.astype(int)  # convert into integer because we can't by fraction of shares\n",
        "\n",
        "\n",
        "            if self.turbulence_threshold is not None:\n",
        "                if self.turbulence >= self.turbulence_threshold:\n",
        "                    actions = np.array([-self.hmax] * self.stock_dim)\n",
        "            \n",
        "            current_price = np.array(self.state[1 : (self.stock_dim + 1)])\n",
        "            #actions = np.where(((current_price > ( self.avg_price * 0.6  + self.avg_price))& (self.avg_price >0.0)), self.total_stockss*-1,actions  )\n",
        "            #actions = np.where(((current_price < (self.avg_price - self.avg_price * .2))& (self.avg_price >0)), self.total_stockss*-1,actions  )\n",
        "            \n",
        "            \n",
        "            # # Mandatory Selling Profits and Losses\n",
        "            for i in range(0,len(actions)):\n",
        "              # self.total_price\n",
        "              # self.total_stockss\n",
        "              # self.avg_price\n",
        "              # print(\"Current Price\")\n",
        "              # print(np.array(self.state[1 : (self.stock_dim + 1)])[i])\n",
        "              current_price  = np.array(self.state[1 : (self.stock_dim + 1)])[i]\n",
        "              avg_price = self.avg_price[i]\n",
        "              #print(avg_price)\n",
        "              if (current_price > ( avg_price * 0.6  + avg_price))  and (avg_price > 0.0):\n",
        "                actions[i] = self.total_stockss[i] * -1\n",
        "                #print(\"Updated actions\")\n",
        "                #self.total_price[i] = 0.0\n",
        "                print(round(current_price) , \"|\", \" avg_price\", avg_price, \"Profit\", current_price -  avg_price)\n",
        "              elif current_price < (avg_price - avg_price * 0.2)  and avg_price > 0.0:\n",
        "                actions[i] = self.total_stockss[i] * -1\n",
        "                print(round(current_price) , \"|\", \" avg_price\", avg_price,  \"loss\", current_price -  avg_price )\n",
        "                #self.total_price[i] = 0.0\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "            begin_total_asset = self.state[0] + sum(np.array(self.state[1 : (self.stock_dim + 1)])* np.array(self.state[(self.stock_dim + 1) : (self.stock_dim * 2 + 1)]))\n",
        "            # print(\"begin_total_asset:{}\".format(begin_total_asset))\n",
        "            #print(\"*\" * 20 )\n",
        "            #print(np.array(self.state[1 : (self.stock_dim + 1)]))\n",
        "            \n",
        "            #print(np.array(self.state[(self.stock_dim + 1) : (self.stock_dim * 2 + 1)]))\n",
        "            #print(\"*\" * 20 )\n",
        "            \n",
        "            #print(\"AVG PRices\")  \n",
        "            #print(self.avg_price)\n",
        "            #print(np.where(np.array(self.state[1 : (self.stock_dim + 1)]) > (self.avg_price + self.avg_price*.15))) \n",
        "            #print(self.avg_price)\n",
        "            #print(self.total_stockss)\n",
        "            #print(\"*\"*20)\n",
        "\n",
        "            # if sum(self.total_stockss) !=0:\n",
        "            #   for i in np.where(np.array(self.state[1 : (self.stock_dim + 1)]) > (self.avg_price + self.avg_price*.15)):\n",
        "            #     for a in i:\n",
        "            #       actions[a] = actions[a]*-1\n",
        "\n",
        "            # for a in actions:\n",
        "            #   print(a)\n",
        "\n",
        "            argsort_actions = np.argsort(actions)\n",
        "            sell_index = argsort_actions[: np.where(actions < 0)[0].shape[0]]\n",
        "            buy_index = argsort_actions[::-1][: np.where(actions > 0)[0].shape[0]]\n",
        "\n",
        "            for index in sell_index:\n",
        "              actions[index] = self._sell_stock(index, actions[index]) * (-1)\n",
        "\n",
        "            for index in buy_index:\n",
        "                actions[index] = self._buy_stock(index, actions[index])\n",
        "\n",
        "\n",
        "            self.actions_memory.append(actions)\n",
        "\n",
        "            # state: s -> s+1\n",
        "            self.day += 1\n",
        "            self.data = self.df.loc[self.day, :]\n",
        "            if self.turbulence_threshold is not None:\n",
        "                if len(self.df.tic.unique()) == 1:\n",
        "                    self.turbulence = self.data[self.risk_indicator_col]\n",
        "                elif len(self.df.tic.unique()) > 1:\n",
        "                    self.turbulence = self.data[self.risk_indicator_col].values[0]\n",
        "\n",
        "\n",
        "\n",
        "            \n",
        "            #self.frame.loc[self.frame.shape[0]+1] = recent_price\n",
        "            recent_price = self.state[1 : (self.stock_dim + 1)]\n",
        "            recent_buy_sell = actions\n",
        "            recent_buy_sell_price = np.where(recent_buy_sell != 0.0, recent_buy_sell * recent_price, 0.0)\n",
        "            #print(recent_price)\n",
        "            #print(recent_buy_sell)\n",
        "            #print(recent_buy_sell_price)\n",
        "            self.frame.loc[self.frame.shape[0]+1] = recent_price\n",
        "            self.frame.loc[self.frame.shape[0]+1] = recent_buy_sell\n",
        "            self.frame.loc[self.frame.shape[0]+1] = recent_buy_sell_price\n",
        "\n",
        "            \n",
        "            self.total_stockss = actions + self.total_stockss\n",
        "            self.total_price = recent_buy_sell_price + self.total_price\n",
        "            self.avg_price = np.divide(self.total_price,\n",
        "                                       self.total_stockss,\n",
        "                                       out=np.zeros_like(self.total_price),\n",
        "                                       #where=((recent_buy_sell!=0.0)) &(self.total_stockss >0.0) )\n",
        "                                       where=recent_buy_sell!=0.0)\n",
        "\n",
        "            self.avg_price = np.where(~np.isfinite(self.avg_price), 0.0, self.avg_price)\n",
        "            self.avg_price = np.where(self.avg_price<0.0, 0.0, self.avg_price)\n",
        "\n",
        "            self.frame.loc[self.frame.shape[0]+1] = recent_price\n",
        "            self.frame.loc[self.frame.shape[0]+2] = recent_buy_sell\n",
        "            self.frame.loc[self.frame.shape[0]+3] = recent_buy_sell_price\n",
        "            self.frame.loc[self.frame.shape[0]+4] = self.total_stockss\n",
        "            self.frame.loc[self.frame.shape[0]+5] = self.total_price\n",
        "            self.frame.loc[self.frame.shape[0]+6] = self.avg_price\n",
        "            self.frame.to_csv(\"test.csv\")\n",
        "            # dx = np.where(((self.avg_price < 0.00)))\n",
        "            # print(dx)\n",
        "            #if len(dx) > 0:\n",
        "            #print(self.total_stockss)\n",
        "            #print(self.total_price )\n",
        "            #print(self.avg_price)\n",
        "\n",
        "\n",
        "            \n",
        "\n",
        "\n",
        "\n",
        "\n",
        "            # self.total_price = np.add(recent_buy_sell_price,\n",
        "            #                           self.total_price,\n",
        "            #                           out=np.zeros_like(self.total_price),\n",
        "            #                                where=(recent_buy_sell!=0.0  )\n",
        "            \n",
        "            # #self.total_price = np.where(self.total_price<0.0, 0.0, self.total_price)\n",
        "            # self.total_stockss = actions + self.total_stockss\n",
        "            \n",
        "            # self.avg_price = np.divide(self.total_price,\n",
        "            #                            self.total_stockss,\n",
        "            #                            out=np.zeros_like(self.total_price),\n",
        "            #                            where=recent_buy_sell!=0.0)\n",
        "          \n",
        "            # print(\"Total Stocks\")\n",
        "            # print(self.total_stockss)\n",
        "            # print(\"Price\")\n",
        "            # print(recent_price)\n",
        "\n",
        "            # print(\"Avg Price\")\n",
        "            # print(self.avg_price)\n",
        "            # print(\"Total Price\")\n",
        "            # print(self.total_price)\n",
        "\n",
        "\n",
        "            self.state = self._update_state()\n",
        "\n",
        "            end_total_asset = self.state[0] + sum(\n",
        "                np.array(self.state[1 : (self.stock_dim + 1)])\n",
        "                * np.array(self.state[(self.stock_dim + 1) : (self.stock_dim * 2 + 1)])\n",
        "            )\n",
        "            self.asset_memory.append(end_total_asset)\n",
        "            self.date_memory.append(self._get_date())\n",
        "            self.reward = end_total_asset - begin_total_asset\n",
        "            self.rewards_memory.append(self.reward)\n",
        "            self.reward = self.reward * self.reward_scaling\n",
        "            self.state_memory.append(\n",
        "                self.state\n",
        "            )  # add current state in state_recorder for each step\n",
        "\n",
        "        return self.state, self.reward, self.terminal, {}\n",
        "\n",
        "    def reset(self):\n",
        "        # initiate state\n",
        "        self.state = self._initiate_state()\n",
        "\n",
        "        if self.initial:\n",
        "            self.asset_memory = [\n",
        "                self.initial_amount\n",
        "                + np.sum(\n",
        "                    np.array(self.num_stock_shares)\n",
        "                    * np.array(self.state[1 : 1 + self.stock_dim])\n",
        "                )\n",
        "            ]\n",
        "        else:\n",
        "            previous_total_asset = self.previous_state[0] + sum(\n",
        "                np.array(self.state[1 : (self.stock_dim + 1)])\n",
        "                * np.array(\n",
        "                    self.previous_state[(self.stock_dim + 1) : (self.stock_dim * 2 + 1)]\n",
        "                )\n",
        "            )\n",
        "            self.asset_memory = [previous_total_asset]\n",
        "\n",
        "        self.day = 0\n",
        "        self.data = self.df.loc[self.day, :]\n",
        "        self.turbulence = 0\n",
        "        self.cost = 0\n",
        "        self.trades = 0\n",
        "        self.terminal = False\n",
        "        # self.iteration=self.iteration\n",
        "        self.rewards_memory = []\n",
        "        self.actions_memory = []\n",
        "        self.date_memory = [self._get_date()]\n",
        "        if self.total_price[5] > 0.0:\n",
        "          dd = pd.DataFrame(data = self.avg_price)\n",
        "          dd.to_csv(\"avg\")\n",
        "          dd = pd.DataFrame(data = self.total_price)\n",
        "          dd.to_csv(\"tp\")\n",
        "        \n",
        "        self.episode += 1\n",
        "        self.total_price = np.array([0.0] * 29)\n",
        "        self.avg_price = np.array([0.0] * 29)\n",
        "        self.total_stockss = np.array([0.0] * 29)\n",
        "        #print(\"Resteting Account\")\n",
        "        return self.state\n",
        "\n",
        "    def render(self, mode=\"human\", close=False):\n",
        "        return self.state\n",
        "\n",
        "    def _initiate_state(self):\n",
        "        if self.initial:\n",
        "            # For Initial State\n",
        "            if len(self.df.tic.unique()) > 1:\n",
        "                # for multiple stock\n",
        "                state = (\n",
        "                    [self.initial_amount]\n",
        "                    + self.data.close.values.tolist()\n",
        "                    + self.num_stock_shares\n",
        "                    + sum(\n",
        "                        (\n",
        "                            self.data[tech].values.tolist()\n",
        "                            for tech in self.tech_indicator_list\n",
        "                        ),\n",
        "                        [],\n",
        "                    )\n",
        "                )  # append initial stocks_share to initial state, instead of all zero\n",
        "            else:\n",
        "                # for single stock\n",
        "                state = (\n",
        "                    [self.initial_amount]\n",
        "                    + [self.data.close]\n",
        "                    + [0] * self.stock_dim\n",
        "                    + sum(([self.data[tech]] for tech in self.tech_indicator_list), [])\n",
        "                )\n",
        "        else:\n",
        "            # Using Previous State\n",
        "            if len(self.df.tic.unique()) > 1:\n",
        "                # for multiple stock\n",
        "                state = (\n",
        "                    [self.previous_state[0]]\n",
        "                    + self.data.close.values.tolist()\n",
        "                    + self.previous_state[\n",
        "                        (self.stock_dim + 1) : (self.stock_dim * 2 + 1)\n",
        "                    ]\n",
        "                    + sum(\n",
        "                        (\n",
        "                            self.data[tech].values.tolist()\n",
        "                            for tech in self.tech_indicator_list\n",
        "                        ),\n",
        "                        [],\n",
        "                    )\n",
        "                )\n",
        "            else:\n",
        "                # for single stock\n",
        "                state = (\n",
        "                    [self.previous_state[0]]\n",
        "                    + [self.data.close]\n",
        "                    + self.previous_state[\n",
        "                        (self.stock_dim + 1) : (self.stock_dim * 2 + 1)\n",
        "                    ]\n",
        "                    + sum(([self.data[tech]] for tech in self.tech_indicator_list), [])\n",
        "                )\n",
        "        return state\n",
        "\n",
        "    def _update_state(self):\n",
        "        if len(self.df.tic.unique()) > 1:\n",
        "            # for multiple stock\n",
        "            state = (\n",
        "                [self.state[0]]\n",
        "                + self.data.close.values.tolist()\n",
        "                + list(self.state[(self.stock_dim + 1) : (self.stock_dim * 2 + 1)])\n",
        "                + sum((self.data[tech].values.tolist() for tech in self.tech_indicator_list),[],))\n",
        "\n",
        "        else:\n",
        "            # for single stock\n",
        "            state = ([self.state[0]] + [self.data.close] \n",
        "                     + list(self.state[(self.stock_dim + 1) : (self.stock_dim * 2 + 1)])\n",
        "                     + sum(([self.data[tech]] for tech in self.tech_indicator_list), []))\n",
        "\n",
        "        return state\n",
        "\n",
        "    def _get_date(self):\n",
        "        if len(self.df.tic.unique()) > 1:\n",
        "            date = self.data.date.unique()[0]\n",
        "        else:\n",
        "            date = self.data.date\n",
        "        return date\n",
        "\n",
        "\n",
        "    def save_asset_memory(self):\n",
        "        date_list = self.date_memory\n",
        "        asset_list = self.asset_memory\n",
        "        # print(len(date_list))\n",
        "        # print(len(asset_list))\n",
        "        df_account_value = pd.DataFrame({\"date\": date_list, \"account_value\": asset_list})\n",
        "        return df_account_value\n",
        "\n",
        "    def save_action_memory(self):\n",
        "        if len(self.df.tic.unique()) > 1:\n",
        "            # date and close price length must match actions length\n",
        "            date_list = self.date_memory[:-1]\n",
        "            df_date = pd.DataFrame(date_list)\n",
        "            df_date.columns = [\"date\"]\n",
        "\n",
        "            action_list = self.actions_memory\n",
        "            df_actions = pd.DataFrame(action_list)\n",
        "            df_actions.columns = self.data.tic.values\n",
        "            df_actions.index = df_date.date\n",
        "            # df_actions = pd.DataFrame({'date':date_list,'actions':action_list})\n",
        "        else:\n",
        "            date_list = self.date_memory[:-1]\n",
        "            action_list = self.actions_memory\n",
        "            df_actions = pd.DataFrame({\"date\": date_list, \"actions\": action_list})\n",
        "        return df_actions\n",
        "\n",
        "    def _seed(self, seed=None):\n",
        "        self.np_random, seed = seeding.np_random(seed)\n",
        "        return [seed]\n",
        "\n",
        "    def get_sb_env(self):\n",
        "        e = DummyVecEnv([lambda: self])\n",
        "        obs = e.reset()\n",
        "        return e, obs\n"
      ],
      "metadata": {
        "id": "4p02b2cx_dj5"
      },
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "k5e9VO3mBLhm"
      },
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "stock_dimension = len(init_train_data.tic.unique())\n",
        "state_space = 1 + 2 * stock_dimension + len(config.INDICATORS) * stock_dimension\n",
        "print(f\"Stock Dimension: {stock_dimension}, State Space: {state_space}\")\n",
        "buy_cost_list = sell_cost_list = [0.001] * stock_dimension\n",
        "num_stock_shares = [0] * stock_dimension"
      ],
      "metadata": {
        "id": "rg7DTZbVV10n",
        "outputId": "c5fed6b2-c54f-423e-a530-a0f6066406b1",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Stock Dimension: 29, State Space: 291\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Defining the environment kwargs\n",
        "\n",
        "initial_amount = 500000\n",
        "env_kwargs = {\n",
        "    \"hmax\": 500,\n",
        "    \"initial_amount\": initial_amount,\n",
        "    \"num_stock_shares\": num_stock_shares,\n",
        "    \"buy_cost_pct\": buy_cost_list,\n",
        "    \"sell_cost_pct\": sell_cost_list,\n",
        "    \"state_space\": state_space,\n",
        "    \"stock_dim\": stock_dimension,\n",
        "    \"tech_indicator_list\": config.INDICATORS,\n",
        "    \"action_space\": stock_dimension,\n",
        "    \"reward_scaling\": 1e-4\n",
        "}"
      ],
      "metadata": {
        "id": "zODPmc5hV42J"
      },
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#Instantiate the training gym compatible environment\n",
        "e_train_gym = GokuEnv(df = init_train_data, **env_kwargs)\n",
        "env_train, _ = e_train_gym.get_sb_env()\n",
        "agent = DRLAgent(env = env_train)"
      ],
      "metadata": {
        "id": "7n2QYGP5V58p"
      },
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#Instantiate the trading environment\n",
        "e_trade_gym = GokuEnv(df = init_trade_data, turbulence_threshold = None, **env_kwargs)"
      ],
      "metadata": {
        "id": "xoBM74XEX1oI"
      },
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Trade performance code\n",
        "The following code calculates trade performance metrics, which are then used as an objective for optimizing hyperparameter values.\n",
        "\n",
        "There are several available metrics. In this tutorial, the default choice is the ratio of average value of winning to losing trades."
      ],
      "metadata": {
        "id": "Dq6zJ14ZYsTK"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "#Main method\n",
        "# Calculates Trade Performance for Objective\n",
        "# Called from objective method\n",
        "# Returns selected trade perf metric(s)\n",
        "# Requires actions and associated prices\n",
        "\n",
        "def calc_trade_perf_metric(df_actions, \n",
        "                           df_prices_trade,\n",
        "                           tp_metric,\n",
        "                           dbg=False):\n",
        "  \n",
        "    df_actions_p, df_prices_p, tics = prep_data(df_actions.copy(),\n",
        "                                                df_prices_trade.copy())\n",
        "    # actions predicted by trained model on trade data\n",
        "    df_actions_p.to_csv('df_actions.csv') \n",
        "\n",
        "    \n",
        "    # Confirms that actions, prices and tics are consistent\n",
        "    df_actions_s, df_prices_s, tics_prtfl = \\\n",
        "        sync_tickers(df_actions_p.copy(),df_prices_p.copy(),tics)\n",
        "    \n",
        "    # copy to ensure that tics from portfolio remains unchanged\n",
        "    tics = tics_prtfl.copy()\n",
        "    \n",
        "    # Analysis is performed on each portfolio ticker\n",
        "    perf_data= collect_performance_data(df_actions_s, df_prices_s, tics)\n",
        "    # profit/loss for each ticker\n",
        "    pnl_all = calc_pnl_all(perf_data, tics)\n",
        "    # values for trade performance metrics\n",
        "    perf_results = calc_trade_perf(pnl_all)\n",
        "    df = pd.DataFrame.from_dict(perf_results, orient='index')\n",
        "    \n",
        "    # calculate and return trade metric value as objective\n",
        "    m = calc_trade_metric(df,tp_metric)\n",
        "    print(f'Ratio Avg Win/Avg Loss: {m}')\n",
        "    k = str(len(tpm_hist)+1)\n",
        "    # save metric value\n",
        "    tpm_hist[k] = m\n",
        "    return m\n",
        "\n",
        "\n",
        "# Supporting methods\n",
        "def calc_trade_metric(df,metric='avgwl'):\n",
        "    '''# trades', '# wins', '# losses', 'wins total value', 'wins avg value',\n",
        "       'losses total value', 'losses avg value'''\n",
        "    # For this tutorial, the only metric available is the ratio of \n",
        "    #  average values of winning to losing trades. Others are in development.\n",
        "    \n",
        "    # some test cases produce no losing trades.\n",
        "    # The code below assigns a value as a multiple of the highest value during\n",
        "    # previous hp optimization runs. If the first run experiences no losses,\n",
        "    # a fixed value is assigned for the ratio\n",
        "    tpm_mult = 1.0\n",
        "    avgwl_no_losses = 25\n",
        "    if metric == 'avgwl':\n",
        "        if sum(df['# losses']) == 0:\n",
        "          try:\n",
        "            return max(tpm_hist.values())*tpm_mult\n",
        "          except ValueError:\n",
        "            return avgwl_no_losses\n",
        "        avg_w = sum(df['wins total value'])/sum(df['# wins'])\n",
        "        avg_l = sum(df['losses total value'])/sum(df['# losses'])\n",
        "        m = abs(avg_w/avg_l)\n",
        "\n",
        "    return m\n",
        "\n",
        "\n",
        "def prep_data(df_actions,\n",
        "              df_prices_trade):\n",
        "    \n",
        "    df=df_prices_trade[['date','close','tic']]\n",
        "    df['Date'] = pd.to_datetime(df['date'])\n",
        "    df = df.set_index('Date')\n",
        "    # set indices on both df to datetime\n",
        "    idx = pd.to_datetime(df_actions.index, infer_datetime_format=True)\n",
        "    df_actions.index=idx\n",
        "    tics = np.unique(df.tic)\n",
        "    n_tics = len(tics)\n",
        "    print(f'Number of tickers: {n_tics}')\n",
        "    print(f'Tickers: {tics}')\n",
        "    dategr = df.groupby('tic')\n",
        "    p_d={t:dategr.get_group(t).loc[:,'close'] for t in tics}\n",
        "    df_prices = pd.DataFrame.from_dict(p_d)\n",
        "    df_prices.index = df_prices.index.normalize()\n",
        "    return df_actions, df_prices, tics\n",
        "\n",
        "\n",
        "# prepares for integrating action and price files\n",
        "def link_prices_actions(df_a,\n",
        "                        df_p):\n",
        "    cols_a = [t + '_a' for t in df_a.columns]\n",
        "    df_a.columns = cols_a\n",
        "    cols_p = [t + '_p' for t in df_p.columns]\n",
        "    df_p.columns = cols_p\n",
        "    return df_a, df_p\n",
        "\n",
        "\n",
        "def sync_tickers(df_actions,df_tickers_p,tickers):\n",
        "    # Some DOW30 components may not be included in portfolio\n",
        "    # passed tickers includes all DOW30 components\n",
        "    # actions and ticker files may have different length indices\n",
        "    if len(df_actions) != len(df_tickers_p):\n",
        "      msng_dates = set(df_actions.index)^set(df_tickers_p.index)\n",
        "      try:\n",
        "        #assumption is prices has one additional timestamp (row)\n",
        "        df_tickers_p.drop(msng_dates,inplace=True)\n",
        "      except:\n",
        "        df_actions.drop(msng_dates,inplace=True)\n",
        "    df_actions, df_tickers_p = link_prices_actions(df_actions,df_tickers_p)\n",
        "    # identify any DOW components not in portfolio\n",
        "    t_not_in_a = [t for t in tickers if t + '_a' not in list(df_actions.columns)]\n",
        "  \n",
        "    # remove t_not_in_a from df_tickers_p\n",
        "    drop_cols = [t + '_p' for t in t_not_in_a]\n",
        "    df_tickers_p.drop(columns=drop_cols,inplace=True)\n",
        "    \n",
        "    # Tickers in portfolio\n",
        "    tickers_prtfl = [c.split('_')[0] for c in df_actions.columns]\n",
        "    return df_actions,df_tickers_p, tickers_prtfl\n",
        "\n",
        "def collect_performance_data(dfa,dfp,tics, dbg=False):\n",
        "    \n",
        "    perf_data = {}\n",
        "    # In current version, files columns include secondary identifier\n",
        "    for t in tics:\n",
        "        # actions: purchase/sale of DOW equities\n",
        "        acts = dfa['_'.join([t,'a'])].values\n",
        "        # ticker prices\n",
        "        prices = dfp['_'.join([t,'p'])].values\n",
        "        # market value of purchases/sales\n",
        "        tvals_init = np.multiply(acts,prices)\n",
        "        d={'actions':acts, 'prices':prices,'init_values':tvals_init}\n",
        "        perf_data[t]=d\n",
        "\n",
        "    return perf_data\n",
        "\n",
        "\n",
        "def calc_pnl_all(perf_dict, tics_all):\n",
        "    # calculate profit/loss for each ticker\n",
        "    print(f'Calculating profit/loss for each ticker')\n",
        "    pnl_all = {}\n",
        "    for tic in tics_all:\n",
        "        pnl_t = []\n",
        "        tic_data = perf_dict[tic]\n",
        "        init_values = tic_data['init_values']\n",
        "        acts = tic_data['actions']\n",
        "        prices = tic_data['prices']\n",
        "        cs = np.cumsum(acts)\n",
        "        args_s = [i + 1 for i in range(len(cs) - 1) if cs[i + 1] < cs[i]]\n",
        "        # tic actions with no sales\n",
        "        if not args_s:\n",
        "            pnl = complete_calc_buyonly(acts, prices, init_values)\n",
        "            pnl_all[tic] = pnl\n",
        "            continue\n",
        "        # copy acts: acts_rev will be revised based on closing/reducing init positions\n",
        "        pnl_all = execute_position_sales(tic,acts,prices,args_s,pnl_all)\n",
        "\n",
        "    return pnl_all\n",
        "\n",
        "\n",
        "def complete_calc_buyonly(actions, prices, init_values):\n",
        "    # calculate final pnl for each ticker assuming no sales\n",
        "    fnl_price = prices[-1]\n",
        "    final_values = np.multiply(fnl_price, actions)\n",
        "    pnl = np.subtract(final_values, init_values)\n",
        "    return pnl\n",
        "\n",
        "\n",
        "def execute_position_sales(tic,acts,prices,args_s,pnl_all):\n",
        "  # calculate final pnl for each ticker with sales\n",
        "    pnl_t = []\n",
        "    acts_rev = acts.copy()\n",
        "    # location of sales transactions\n",
        "    for s in args_s:  # s is scaler\n",
        "        # price_s = [prices[s]]\n",
        "        act_s = [acts_rev[s]]\n",
        "        args_b = [i for i in range(s) if acts_rev[i] > 0]\n",
        "        prcs_init_trades = prices[args_b]\n",
        "        acts_init_trades = acts_rev[args_b]\n",
        "  \n",
        "        # update actions for sales\n",
        "        # reduce/eliminate init values through trades\n",
        "        # always start with earliest purchase that has not been closed through sale\n",
        "        # selectors for purchase and sales trades\n",
        "        # find earliest remaining purchase\n",
        "        arg_sel = min(args_b)\n",
        "        # sel_s = len(acts_trades) - 1\n",
        "\n",
        "        # closing part/all of earliest init trade not yet closed\n",
        "        # sales actions are negative\n",
        "        # in this test case, abs_val of init and sales share counts are same\n",
        "        # zero-out sales actions\n",
        "        # market value of sale\n",
        "        # max number of shares to be closed: may be less than # originally purchased\n",
        "        acts_shares = min(abs(act_s.pop()), acts_rev[arg_sel])\n",
        "\n",
        "        # mv of shares when purchased\n",
        "        mv_p = abs(acts_shares * prices[arg_sel])\n",
        "        # mv of sold shares\n",
        "        mv_s = abs(acts_shares * prices[s])\n",
        "\n",
        "        # calc pnl\n",
        "        pnl = mv_s - mv_p\n",
        "        # reduce init share count\n",
        "        # close all/part of init purchase\n",
        "        acts_rev[arg_sel] -= acts_shares\n",
        "        acts_rev[s] += acts_shares\n",
        "        # calculate pnl for trade\n",
        "        # value of associated purchase\n",
        "        \n",
        "        # find earliest non-zero positive act in acts_revs\n",
        "        pnl_t.append(pnl)\n",
        "    \n",
        "    pnl_op = calc_pnl_for_open_positions(acts_rev, prices)\n",
        "    #pnl_op is list\n",
        "    # add pnl_op results (if any) to pnl_t (both lists)\n",
        "    pnl_t.extend(pnl_op)\n",
        "    #print(f'Total pnl for {tic}: {np.sum(pnl_t)}')\n",
        "    pnl_all[tic] = np.array(pnl_t)\n",
        "    return pnl_all\n",
        "\n",
        "\n",
        "def calc_pnl_for_open_positions(acts,prices):\n",
        "    # identify any positive share values after accounting for sales\n",
        "    pnl = []\n",
        "    fp = prices[-1] # last price\n",
        "    open_pos_arg = np.argwhere(acts>0)\n",
        "    if len(open_pos_arg)==0:return pnl # no open positions\n",
        "\n",
        "    mkt_vals_open = np.multiply(acts[open_pos_arg], prices[open_pos_arg])\n",
        "    # mkt val at end of testing period\n",
        "    # treat as trades for purposes of calculating pnl at end of testing period\n",
        "    mkt_vals_final = np.multiply(fp, acts[open_pos_arg])\n",
        "    pnl_a = np.subtract(mkt_vals_final, mkt_vals_open)\n",
        "    #convert to list\n",
        "    pnl = [i[0] for i in pnl_a.tolist()]\n",
        "    #print(f'Market value of open positions at end of testing {pnl}')\n",
        "    return pnl\n",
        "\n",
        "\n",
        "def calc_trade_perf(pnl_d):\n",
        "    # calculate trade performance metrics\n",
        "    perf_results = {}\n",
        "    for t,pnl in pnl_d.items():\n",
        "        wins = pnl[pnl>0]  # total val\n",
        "        losses = pnl[pnl<0]\n",
        "        n_wins = len(wins)\n",
        "        n_losses = len(losses)\n",
        "        n_trades = n_wins + n_losses\n",
        "        wins_val = np.sum(wins)\n",
        "        losses_val = np.sum(losses)\n",
        "        wins_avg = 0 if n_wins==0 else np.mean(wins)\n",
        "        #print(f'{t} n_wins: {n_wins} n_losses: {n_losses}')\n",
        "        losses_avg = 0 if n_losses==0 else np.mean(losses)\n",
        "        d = {'# trades':n_trades,'# wins':n_wins,'# losses':n_losses,\n",
        "             'wins total value':wins_val, 'wins avg value':wins_avg,\n",
        "             'losses total value':losses_val, 'losses avg value':losses_avg,}\n",
        "        perf_results[t] = d\n",
        "    return perf_results"
      ],
      "metadata": {
        "id": "QydaFexDX5BQ"
      },
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Tuning hyperparameters using Optuna"
      ],
      "metadata": {
        "id": "c_AT1dsuZApj"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def sample_ddpg_params(trial:optuna.Trial):\n",
        "  # Size of the replay buffer\n",
        "  buffer_size = trial.suggest_categorical(\"buffer_size\", [int(1e4), int(1e5), int(1e6)])\n",
        "  learning_rate = trial.suggest_loguniform(\"learning_rate\", 1e-5, 1)\n",
        "  batch_size = trial.suggest_categorical(\"batch_size\", [32, 64, 128, 256, 512])\n",
        "  \n",
        "  return {\"buffer_size\": buffer_size,\n",
        "          \"learning_rate\":learning_rate,\n",
        "          \"batch_size\":batch_size}"
      ],
      "metadata": {
        "id": "fKGkKHV7Y9AA"
      },
      "execution_count": 34,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Set Variables\n",
        "## Fixed\n",
        "tpm_hist = {}  # record tp metric values for trials\n",
        "tp_metric = 'avgwl'  # specified trade_param_metric: ratio avg value win/loss\n",
        "## Settable by User\n",
        "n_trials = 100  # number of HP optimization runs\n",
        "total_timesteps = 2000 # per HP optimization run\n",
        "## Logging callback params\n",
        "lc_threshold=1e-5\n",
        "lc_patience=15\n",
        "lc_trial_number=5"
      ],
      "metadata": {
        "id": "N6blMWpz-pR0"
      },
      "execution_count": 35,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "OPTIONAL CODE FOR SAMPLING HYPERPARAMETERS\n",
        "\n",
        "Replace current call in function objective with\n",
        "\n",
        "hyperparameters = sample_ddpg_params_all(trial)"
      ],
      "metadata": {
        "id": "fHET-odKZShg"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def sample_ddpg_params_all(trial:optuna.Trial,\n",
        "                           # fixed values from previous study\n",
        "                           learning_rate=0.0103,\n",
        "                           batch_size=128,\n",
        "                           buffer_size=int(1e6)):\n",
        "\n",
        "    gamma = trial.suggest_categorical(\"gamma\", [0.94, 0.96, 0.98])\n",
        "    # Polyak coeff\n",
        "    tau = trial.suggest_categorical(\"tau\", [0.08, 0.1, 0.12])\n",
        "\n",
        "    train_freq = trial.suggest_categorical(\"train_freq\", [512,768,1024])\n",
        "    gradient_steps = train_freq\n",
        "    \n",
        "    noise_type = trial.suggest_categorical(\"noise_type\", [\"ornstein-uhlenbeck\", \"normal\", None])\n",
        "    noise_std = trial.suggest_categorical(\"noise_std\", [.1,.2,.3] )\n",
        "\n",
        "    # NOTE: Add \"verybig\" to net_arch when tuning HER (see TD3)\n",
        "    net_arch = trial.suggest_categorical(\"net_arch\", [\"small\", \"big\"])\n",
        "    # activation_fn = trial.suggest_categorical('activation_fn', [nn.Tanh, nn.ReLU, nn.ELU, nn.LeakyReLU])\n",
        "\n",
        "    net_arch = {\n",
        "        \"small\": [64, 64],\n",
        "        \"medium\": [256, 256],\n",
        "        \"big\": [512, 512],\n",
        "    }[net_arch]\n",
        "  \n",
        "    hyperparams = {\n",
        "        \"batch_size\": batch_size,\n",
        "        \"buffer_size\": buffer_size,\n",
        "        \"gamma\": gamma,\n",
        "        \"gradient_steps\": gradient_steps,\n",
        "        \"learning_rate\": learning_rate,\n",
        "        \"tau\": tau,\n",
        "        \"train_freq\": train_freq,\n",
        "        #\"noise_std\": noise_std,\n",
        "        #\"noise_type\": noise_type,\n",
        "        \n",
        "        \"policy_kwargs\": dict(net_arch=net_arch)\n",
        "    }\n",
        "    return hyperparams"
      ],
      "metadata": {
        "id": "htLdZHKTZKPJ"
      },
      "execution_count": 36,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def sample_ddpg_params_all(trial:optuna.Trial,\n",
        "                           # fixed values from previous study\n",
        "                           learning_rate=0.0103,\n",
        "                           batch_size=128,\n",
        "                           buffer_size=int(1e6)):\n",
        "\n",
        "    gamma = trial.suggest_categorical(\"gamma\", [0.90, 0.92, 0.94, 0.96, 0.98])\n",
        "    # Polyak coeff\n",
        "    tau = trial.suggest_categorical(\"tau\", [0.02,0.04,0.06,0.08, 0.1, 0.12])\n",
        "\n",
        "    train_freq = trial.suggest_categorical(\"train_freq\", [512,768,1024])\n",
        "    gradient_steps = train_freq\n",
        "    \n",
        "    noise_type = trial.suggest_categorical(\"noise_type\", [\"ornstein-uhlenbeck\", \"normal\", None])\n",
        "    noise_std = trial.suggest_categorical(\"noise_std\", [.1,.2,.3,.4,.5] )\n",
        "\n",
        "    # NOTE: Add \"verybig\" to net_arch when tuning HER (see TD3)\n",
        "    net_arch = trial.suggest_categorical(\"net_arch\", [\"small\", \"big\"])\n",
        "    # activation_fn = trial.suggest_categorical('activation_fn', [nn.Tanh, nn.ReLU, nn.ELU, nn.LeakyReLU])\n",
        "\n",
        "    net_arch = {\n",
        "        \"small\": [32, 32],\n",
        "        \"medium\": [64, 64],\n",
        "        \"big\": [256, 256],\n",
        "    }[net_arch]\n",
        "  \n",
        "    hyperparams = {\n",
        "        \"batch_size\": batch_size,\n",
        "        \"buffer_size\": buffer_size,\n",
        "        \"gamma\": gamma,\n",
        "        \"gradient_steps\": gradient_steps,\n",
        "        \"learning_rate\": learning_rate,\n",
        "        \"tau\": tau,\n",
        "        \"train_freq\": train_freq,\n",
        "        #\"noise_std\": noise_std,\n",
        "        #\"noise_type\": noise_type,\n",
        "        \n",
        "        \"policy_kwargs\": dict(net_arch=net_arch)\n",
        "    }\n",
        "    return hyperparams"
      ],
      "metadata": {
        "id": "dJYdh-E4ZQKP"
      },
      "execution_count": 37,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Callbacks\n",
        "\n",
        "\n",
        "1. The callback will terminate if the improvement margin is below certain point\n",
        "2. It will terminate after certain number of trial_number are reached, not before that\n",
        "3. It will hold its patience to reach the threshold"
      ],
      "metadata": {
        "id": "mWEoJMPLZY4o"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class LoggingCallback:\n",
        "    def __init__(self,threshold,trial_number,patience):\n",
        "      '''\n",
        "      threshold:int tolerance for increase in objective\n",
        "      trial_number: int Prune after minimum number of trials\n",
        "      patience: int patience for the threshold\n",
        "      '''\n",
        "      self.threshold = threshold\n",
        "      self.trial_number  = trial_number\n",
        "      self.patience = patience\n",
        "      print(f'Callback threshold {self.threshold}, \\\n",
        "            trial_number {self.trial_number}, \\\n",
        "            patience {self.patience}')\n",
        "      self.cb_list = [] #Trials list for which threshold is reached\n",
        "    def __call__(self,study:optuna.study, frozen_trial:optuna.Trial):\n",
        "      #Setting the best value in the current trial\n",
        "      study.set_user_attr(\"previous_best_value\", study.best_value)\n",
        "      \n",
        "      #Checking if the minimum number of trials have pass\n",
        "      if frozen_trial.number >self.trial_number:\n",
        "          previous_best_value = study.user_attrs.get(\"previous_best_value\",None)\n",
        "          #Checking if the previous and current objective values have the same sign\n",
        "          if previous_best_value * study.best_value >=0:\n",
        "              #Checking for the threshold condition\n",
        "              if abs(previous_best_value-study.best_value) < self.threshold: \n",
        "                  self.cb_list.append(frozen_trial.number)\n",
        "                  #If threshold is achieved for the patience amount of time\n",
        "                  if len(self.cb_list)>self.patience:\n",
        "                      print('The study stops now...')\n",
        "                      print('With number',frozen_trial.number ,'and value ',frozen_trial.value)\n",
        "                      print('The previous and current best values are {} and {} respectively'\n",
        "                              .format(previous_best_value, study.best_value))\n",
        "                      study.stop()"
      ],
      "metadata": {
        "id": "He7GTZZUZWP8"
      },
      "execution_count": 38,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#Calculate the Sharpe ratio\n",
        "#This is our objective for tuning\n",
        "def calculate_sharpe(df):\n",
        "  df['daily_return'] = df['account_value'].pct_change(1)\n",
        "  if df['daily_return'].std() !=0:\n",
        "    sharpe = (252**0.5)*df['daily_return'].mean()/ \\\n",
        "          df['daily_return'].std()\n",
        "    return sharpe\n",
        "  else:\n",
        "    return 0"
      ],
      "metadata": {
        "id": "s9orU-WplQ70"
      },
      "execution_count": 39,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from IPython.display import clear_output\n",
        "import sys   \n",
        "\n",
        "os.makedirs(\"models\",exist_ok=True)\n",
        "\n",
        "def objective(trial:optuna.Trial):\n",
        "  #Trial will suggest a set of hyperparamters from the specified range\n",
        "\n",
        "  # Optional to optimize larger set of parameters\n",
        "  # hyperparameters = sample_ddpg_params_all(trial)\n",
        "  \n",
        "  # Optimize buffer size, batch size, learning rate\n",
        "  hyperparameters = sample_ddpg_params_all(trial)\n",
        "  print(f'Hyperparameters from objective: {hyperparameters.keys()}')\n",
        "  policy_kwargs = None  # default\n",
        "  if 'policy_kwargs' in hyperparameters.keys():\n",
        "    policy_kwargs = hyperparameters['policy_kwargs']\n",
        "    del hyperparameters['policy_kwargs']\n",
        "    print(f'Policy keyword arguments {policy_kwargs}')\n",
        "  model_ddpg = agent.get_model(\"ddpg\",\n",
        "                               policy_kwargs = policy_kwargs,\n",
        "                               model_kwargs = hyperparameters )\n",
        "  \n",
        "  #You can increase it for better comparison\n",
        "  trained_ddpg = agent.train_model(model=model_ddpg,\n",
        "                                   tb_log_name=\"ddpg\",\n",
        "                                   total_timesteps=total_timesteps)\n",
        "  trained_ddpg.save('models/ddpg_{}.pth'.format(trial.number))\n",
        "  clear_output(wait=True)\n",
        "  \n",
        "  #For the given hyperparamters, determine the account value in the trading period\n",
        "  df_account_value, df_actions = DRLAgent.DRL_prediction(\n",
        "    model=trained_ddpg, \n",
        "    environment = e_trade_gym)\n",
        " \n",
        "  # Calculate trade performance metric\n",
        "  # Currently ratio of average win and loss market values\n",
        "  #tpm = calc_trade_perf_metric(df_actions,init_trade_data,tp_metric)\n",
        "  tpm = calculate_sharpe(df_account_value)\n",
        "  return tpm\n",
        "\n",
        "#Create a study object and specify the direction as 'maximize'\n",
        "#As you want to maximize sharpe\n",
        "#Pruner stops not promising iterations\n",
        "#Use a pruner, else you will get error related to divergence of model\n",
        "#You can also use Multivariate samplere\n",
        "#sampler = optuna.samplers.TPESampler(multivarite=True,seed=42)\n",
        "sampler = optuna.samplers.TPESampler()\n",
        "\n",
        "study = optuna.create_study(study_name=\"ddpg_study\",direction='maximize',\n",
        "                            sampler = sampler, pruner=optuna.pruners.HyperbandPruner())\n",
        "\n",
        "logging_callback = LoggingCallback(threshold=lc_threshold,\n",
        "                                   patience=lc_patience,\n",
        "                                   trial_number=lc_trial_number)\n",
        "#You can increase the n_trials for a better search space scanning\n",
        "study.optimize(objective, n_trials=n_trials,catch=(ValueError,),callbacks=[logging_callback])"
      ],
      "metadata": {
        "id": "b6EiwiiIZpRU",
        "outputId": "52d510ee-2107-4b85-c6c2-0615bf36b8fc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\u001b[32m[I 2023-05-02 18:32:36,822]\u001b[0m A new study created in memory with name: ddpg_study\u001b[0m\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Callback threshold 1e-05,             trial_number 5,             patience 15\n",
            "Hyperparameters from objective: dict_keys(['batch_size', 'buffer_size', 'gamma', 'gradient_steps', 'learning_rate', 'tau', 'train_freq', 'policy_kwargs'])\n",
            "Policy keyword arguments {'net_arch': [32, 32]}\n",
            "{'batch_size': 128, 'buffer_size': 1000000, 'gamma': 0.9, 'gradient_steps': 768, 'learning_rate': 0.0103, 'tau': 0.08, 'train_freq': 768}\n",
            "Using cuda device\n",
            "166 |  avg_price 496.71120834350586 loss -330.26467514038086\n",
            "47 |  avg_price 317.5266876220703 loss -270.17238998413086\n",
            "113 |  avg_price 160.61541748046875 loss -48.03044891357422\n",
            "130 |  avg_price 40.24806431361607 Profit 90.20109340122768\n",
            "47 |  avg_price 317.5179901123047 loss -270.64228439331055\n",
            "97 |  avg_price 800.5945358276367 loss -703.458610534668\n",
            "162 |  avg_price 81.49164095791903 Profit 80.18568142977628\n",
            "115 |  avg_price 505.2396011352539 loss -389.89788818359375\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "320 |  avg_price 882.4515869140625 loss -562.3361999511719\n",
            "200 |  avg_price 78.36731583032852 Profit 122.08446639623398\n",
            "138 |  avg_price 219.77936717442103 loss -81.96936961582728\n",
            "316 |  avg_price 672.8479423522949 loss -356.4603691101074\n",
            "122 |  avg_price 167.98552139945653 loss -46.34196671195653\n",
            "58 |  avg_price 1746.1953239440918 loss -1687.975284576416\n",
            "204 |  avg_price 104.99450807313661 Profit 98.6662127520587\n",
            "291 |  avg_price 175.40569444136187 Profit 116.012976212935\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "127 |  avg_price 527.574331665039 loss -400.7631591796875\n",
            "59 |  avg_price 78.94202041625977 loss -19.49980926513672\n",
            "54 |  avg_price 1440.5773849487305 loss -1386.33012008667\n",
            "82 |  avg_price 1790.6979751586914 loss -1708.764663696289\n",
            "154 |  avg_price 198.17116222642872 loss -44.356556513538095\n",
            "50 |  avg_price 1437.0353736877441 loss -1386.6116676330566\n",
            "115 |  avg_price 147.1398416037088 loss -31.951220082712695\n",
            "160 |  avg_price 13120.13591003418 loss -12960.166000366211\n",
            "74 |  avg_price 24.039565577651516 Profit 50.1699833725438\n",
            "114 |  avg_price 213.16352667039442 loss -99.18352331346082\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "231 |  avg_price 433.73415801104375 loss -202.7241635042078\n",
            "156 |  avg_price 516.7536529541015 loss -360.27467956542966\n",
            "161 |  avg_price 209.41112743166576 loss -48.866342885767324\n",
            "68 |  avg_price 85.70125191552299 loss -17.65230934960502\n",
            "92 |  avg_price 143.37563295787191 loss -51.565635399278165\n",
            "95 |  avg_price 18.17575720584754 Profit 76.95884057247278\n",
            "124 |  avg_price 978.6967163085938 loss -854.5400009155273\n",
            "30 |  avg_price 40.07291681036529 loss -9.623076264710996\n",
            "81 |  avg_price 106.15533107715649 loss -24.661587180672115\n",
            "41 |  avg_price 51.08956515509437 loss -10.574409362858042\n",
            "132 |  avg_price 258.7754581996373 loss -127.09273420061385\n",
            "66 |  avg_price 409.7481689453125 loss -343.5696258544922\n",
            "145 |  avg_price 68.14726229580965 Profit 77.34811429110441\n",
            "34 |  avg_price 105.32575244529575 loss -71.26275271995395\n",
            "129 |  avg_price 220.0663868209063 loss -90.79182017051568\n",
            "106 |  avg_price 134.2273217118494 loss -28.317180720638476\n",
            "95 |  avg_price 154.00075717088652 loss -59.47757144090605\n",
            "44 |  avg_price 216.55086517333984 loss -172.501953125\n",
            "66 |  avg_price 39.44335720862871 Profit 26.424921294789257\n",
            "105 |  avg_price 134.28390978787348 loss -29.057957639435983\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "58 |  avg_price 1755.9592323303223 loss -1698.032283782959\n",
            "98 |  avg_price 134.44755555973498 loss -36.32755281315295\n",
            "146 |  avg_price 73.3458898016747 Profit 72.6541101983253\n",
            "162 |  avg_price 81.86286220265858 Profit 80.37900486277111\n",
            "174 |  avg_price 86.86655680338542 Profit 87.32704671223958\n",
            "102 |  avg_price 127.05488461997888 loss -25.54604978111169\n",
            "95 |  avg_price 137.49226996536655 loss -42.572271796421234\n",
            "86 |  avg_price 182.84062132543448 loss -96.36248717016105\n",
            "45 |  avg_price 13.01555061340332 Profit 31.98577308654785\n",
            "123 |  avg_price 161.35948894670616 loss -38.08949230363976\n",
            "36 |  avg_price 3669.023431777954 loss -3632.7317028045654\n",
            "137 |  avg_price 425.72581960042316 loss -288.4935808308919\n",
            "49 |  avg_price 67.60953150397519 loss -18.44823465949277\n",
            "77 |  avg_price 25.44707489013672 Profit 51.3697509765625\n",
            "238 |  avg_price 140.87568685743543 Profit 97.19187906053332\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "54 |  avg_price 158.06376072145858 loss -104.4976520178453\n",
            "172 |  avg_price 262.5252091311337 loss -90.426896753204\n",
            "104 |  avg_price 137.3537531843089 loss -33.853753184308914\n",
            "166 |  avg_price 1386.7466393673058 loss -1220.2803002559776\n",
            "132 |  avg_price 165.50671564737956 loss -33.15199457804363\n",
            "38 |  avg_price 7304.472454071045 loss -7266.612339019775\n",
            "98 |  avg_price 128.5180097553092 loss -30.894939992125614\n",
            "216 |  avg_price 110.15167092377285 Profit 105.9683852285709\n",
            "38 |  avg_price 262.5909175872803 loss -224.97846031188965\n",
            "96 |  avg_price 126.25293404174502 loss -30.217312398678615\n",
            "135 |  avg_price 181.84123534344613 loss -46.871234122743004\n",
            "38 |  avg_price 149.83835887908936 loss -111.82212734222412\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "101 |  avg_price 134.64981491211427 loss -33.459812470708016\n",
            "103 |  avg_price 12277.387825012207 loss -12174.805046081543\n",
            "54 |  avg_price 82.32337837818406 loss -28.312250906260232\n",
            "203 |  avg_price 6.593419161709872 Profit 196.44745913418856\n",
            "50 |  avg_price 7.23345596559586 Profit 42.47203033201156\n",
            "39 |  avg_price 486.287841796875 loss -447.095401763916\n",
            "40 |  avg_price 3672.0507793426514 loss -3632.5310916900635\n",
            "158 |  avg_price 51.781157027843385 Profit 106.24702595555505\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "50 |  avg_price 252.75654983520508 loss -202.45745468139648\n",
            "43 |  avg_price 18.08901042322959 Profit 25.2778566422001\n",
            "183 |  avg_price 99.28684140558113 Profit 83.40316103582512\n",
            "78 |  avg_price 40.69497267405192 Profit 37.41749986012777\n",
            "215 |  avg_price 75.01034249199762 Profit 139.99869071112738\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "51 |  avg_price 253.71746063232422 loss -202.7720947265625\n",
            "138 |  avg_price 510.20196533203125 loss -371.83197021484375\n",
            "127 |  avg_price 192.73025602935462 loss -65.98387236236243\n",
            "131 |  avg_price 209.212458788864 loss -78.11522978495776\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "227 |  avg_price 363.1976925229845 loss -136.02769435403917\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "331 |  avg_price 141.02000427246094 Profit 189.7718963623047\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-28-fb55fa656c81>:358: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "\u001b[33m[W 2023-05-02 18:32:41,965]\u001b[0m Trial 0 failed with parameters: {'gamma': 0.9, 'tau': 0.08, 'train_freq': 768, 'noise_type': 'ornstein-uhlenbeck', 'noise_std': 0.5, 'net_arch': 'small'} because of the following error: KeyboardInterrupt().\u001b[0m\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/site-packages/optuna/study/_optimize.py\", line 200, in _run_trial\n",
            "    value_or_values = func(trial)\n",
            "  File \"<ipython-input-40-26bf33718e81>\", line 25, in objective\n",
            "    trained_ddpg = agent.train_model(model=model_ddpg,\n",
            "  File \"/usr/local/lib/python3.10/site-packages/finrl/agents/stablebaselines3/models.py\", line 103, in train_model\n",
            "    model = model.learn(\n",
            "  File \"/usr/local/lib/python3.10/site-packages/stable_baselines3/ddpg/ddpg.py\", line 123, in learn\n",
            "    return super().learn(\n",
            "  File \"/usr/local/lib/python3.10/site-packages/stable_baselines3/td3/td3.py\", line 216, in learn\n",
            "    return super().learn(\n",
            "  File \"/usr/local/lib/python3.10/site-packages/stable_baselines3/common/off_policy_algorithm.py\", line 311, in learn\n",
            "    rollout = self.collect_rollouts(\n",
            "  File \"/usr/local/lib/python3.10/site-packages/stable_baselines3/common/off_policy_algorithm.py\", line 543, in collect_rollouts\n",
            "    new_obs, rewards, dones, infos = env.step(actions)\n",
            "  File \"/usr/local/lib/python3.10/site-packages/stable_baselines3/common/vec_env/base_vec_env.py\", line 163, in step\n",
            "    return self.step_wait()\n",
            "  File \"/usr/local/lib/python3.10/site-packages/stable_baselines3/common/vec_env/dummy_vec_env.py\", line 54, in step_wait\n",
            "    obs, self.buf_rews[env_idx], self.buf_dones[env_idx], self.buf_infos[env_idx] = self.envs[env_idx].step(\n",
            "  File \"<ipython-input-28-fb55fa656c81>\", line 373, in step\n",
            "    self.frame.to_csv(\"test.csv\")\n",
            "  File \"/usr/local/lib/python3.10/site-packages/pandas/util/_decorators.py\", line 211, in wrapper\n",
            "    return func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/site-packages/pandas/core/generic.py\", line 3720, in to_csv\n",
            "    return DataFrameRenderer(formatter).to_csv(\n",
            "  File \"/usr/local/lib/python3.10/site-packages/pandas/util/_decorators.py\", line 211, in wrapper\n",
            "    return func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/site-packages/pandas/io/formats/format.py\", line 1189, in to_csv\n",
            "    csv_formatter.save()\n",
            "  File \"/usr/local/lib/python3.10/site-packages/pandas/io/formats/csvs.py\", line 261, in save\n",
            "    self._save()\n",
            "  File \"/usr/local/lib/python3.10/site-packages/pandas/io/formats/csvs.py\", line 266, in _save\n",
            "    self._save_body()\n",
            "  File \"/usr/local/lib/python3.10/site-packages/pandas/io/formats/csvs.py\", line 304, in _save_body\n",
            "    self._save_chunk(start_i, end_i)\n",
            "  File \"/usr/local/lib/python3.10/site-packages/pandas/io/formats/csvs.py\", line 315, in _save_chunk\n",
            "    libwriters.write_csv_rows(\n",
            "  File \"pandas/_libs/writers.pyx\", line 55, in pandas._libs.writers.write_csv_rows\n",
            "KeyboardInterrupt\n",
            "\u001b[33m[W 2023-05-02 18:32:41,969]\u001b[0m Trial 0 failed with value None.\u001b[0m\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-40-26bf33718e81>\u001b[0m in \u001b[0;36m<cell line: 57>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     55\u001b[0m                                    trial_number=lc_trial_number)\n\u001b[1;32m     56\u001b[0m \u001b[0;31m#You can increase the n_trials for a better search space scanning\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 57\u001b[0;31m \u001b[0mstudy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobjective\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_trials\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mn_trials\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mcatch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mValueError\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mlogging_callback\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/optuna/study/study.py\u001b[0m in \u001b[0;36moptimize\u001b[0;34m(self, func, n_trials, timeout, n_jobs, catch, callbacks, gc_after_trial, show_progress_bar)\u001b[0m\n\u001b[1;32m    423\u001b[0m         \"\"\"\n\u001b[1;32m    424\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 425\u001b[0;31m         _optimize(\n\u001b[0m\u001b[1;32m    426\u001b[0m             \u001b[0mstudy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    427\u001b[0m             \u001b[0mfunc\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/optuna/study/_optimize.py\u001b[0m in \u001b[0;36m_optimize\u001b[0;34m(study, func, n_trials, timeout, n_jobs, catch, callbacks, gc_after_trial, show_progress_bar)\u001b[0m\n\u001b[1;32m     64\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     65\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mn_jobs\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 66\u001b[0;31m             _optimize_sequential(\n\u001b[0m\u001b[1;32m     67\u001b[0m                 \u001b[0mstudy\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     68\u001b[0m                 \u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/optuna/study/_optimize.py\u001b[0m in \u001b[0;36m_optimize_sequential\u001b[0;34m(study, func, n_trials, timeout, catch, callbacks, gc_after_trial, reseed_sampler_rng, time_start, progress_bar)\u001b[0m\n\u001b[1;32m    161\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    162\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 163\u001b[0;31m             \u001b[0mfrozen_trial\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_run_trial\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstudy\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcatch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    164\u001b[0m         \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    165\u001b[0m             \u001b[0;31m# The following line mitigates memory problems that can be occurred in some\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/optuna/study/_optimize.py\u001b[0m in \u001b[0;36m_run_trial\u001b[0;34m(study, func, catch)\u001b[0m\n\u001b[1;32m    249\u001b[0m         \u001b[0;32mand\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc_err\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcatch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    250\u001b[0m     ):\n\u001b[0;32m--> 251\u001b[0;31m         \u001b[0;32mraise\u001b[0m \u001b[0mfunc_err\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    252\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mfrozen_trial\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    253\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/optuna/study/_optimize.py\u001b[0m in \u001b[0;36m_run_trial\u001b[0;34m(study, func, catch)\u001b[0m\n\u001b[1;32m    198\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mget_heartbeat_thread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrial\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_trial_id\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstudy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_storage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    199\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 200\u001b[0;31m             \u001b[0mvalue_or_values\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrial\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    201\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mexceptions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTrialPruned\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    202\u001b[0m             \u001b[0;31m# TODO(mamu): Handle multi-objective cases.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-40-26bf33718e81>\u001b[0m in \u001b[0;36mobjective\u001b[0;34m(trial)\u001b[0m\n\u001b[1;32m     23\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m   \u001b[0;31m#You can increase it for better comparison\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 25\u001b[0;31m   trained_ddpg = agent.train_model(model=model_ddpg,\n\u001b[0m\u001b[1;32m     26\u001b[0m                                    \u001b[0mtb_log_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"ddpg\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     27\u001b[0m                                    total_timesteps=total_timesteps)\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/finrl/agents/stablebaselines3/models.py\u001b[0m in \u001b[0;36mtrain_model\u001b[0;34m(self, model, tb_log_name, total_timesteps)\u001b[0m\n\u001b[1;32m    101\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    102\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mtrain_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtb_log_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m5000\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 103\u001b[0;31m         model = model.learn(\n\u001b[0m\u001b[1;32m    104\u001b[0m             \u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    105\u001b[0m             \u001b[0mtb_log_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtb_log_name\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/stable_baselines3/ddpg/ddpg.py\u001b[0m in \u001b[0;36mlearn\u001b[0;34m(self, total_timesteps, callback, log_interval, tb_log_name, reset_num_timesteps, progress_bar)\u001b[0m\n\u001b[1;32m    121\u001b[0m         \u001b[0mprogress_bar\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mbool\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    122\u001b[0m     ) -> SelfDDPG:\n\u001b[0;32m--> 123\u001b[0;31m         return super().learn(\n\u001b[0m\u001b[1;32m    124\u001b[0m             \u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    125\u001b[0m             \u001b[0mcallback\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallback\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/stable_baselines3/td3/td3.py\u001b[0m in \u001b[0;36mlearn\u001b[0;34m(self, total_timesteps, callback, log_interval, tb_log_name, reset_num_timesteps, progress_bar)\u001b[0m\n\u001b[1;32m    214\u001b[0m         \u001b[0mprogress_bar\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mbool\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    215\u001b[0m     ) -> SelfTD3:\n\u001b[0;32m--> 216\u001b[0;31m         return super().learn(\n\u001b[0m\u001b[1;32m    217\u001b[0m             \u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    218\u001b[0m             \u001b[0mcallback\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallback\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/stable_baselines3/common/off_policy_algorithm.py\u001b[0m in \u001b[0;36mlearn\u001b[0;34m(self, total_timesteps, callback, log_interval, tb_log_name, reset_num_timesteps, progress_bar)\u001b[0m\n\u001b[1;32m    309\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    310\u001b[0m         \u001b[0;32mwhile\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnum_timesteps\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 311\u001b[0;31m             rollout = self.collect_rollouts(\n\u001b[0m\u001b[1;32m    312\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menv\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    313\u001b[0m                 \u001b[0mtrain_freq\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_freq\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/stable_baselines3/common/off_policy_algorithm.py\u001b[0m in \u001b[0;36mcollect_rollouts\u001b[0;34m(self, env, callback, train_freq, replay_buffer, action_noise, learning_starts, log_interval)\u001b[0m\n\u001b[1;32m    541\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    542\u001b[0m             \u001b[0;31m# Rescale and perform action\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 543\u001b[0;31m             \u001b[0mnew_obs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrewards\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdones\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minfos\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mactions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    544\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    545\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnum_timesteps\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnum_envs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/stable_baselines3/common/vec_env/base_vec_env.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, actions)\u001b[0m\n\u001b[1;32m    161\u001b[0m         \"\"\"\n\u001b[1;32m    162\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep_async\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mactions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 163\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep_wait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    164\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    165\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mget_images\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mSequence\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndarray\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/stable_baselines3/common/vec_env/dummy_vec_env.py\u001b[0m in \u001b[0;36mstep_wait\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     52\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mstep_wait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mVecEnvStepReturn\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     53\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0menv_idx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnum_envs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 54\u001b[0;31m             obs, self.buf_rews[env_idx], self.buf_dones[env_idx], self.buf_infos[env_idx] = self.envs[env_idx].step(\n\u001b[0m\u001b[1;32m     55\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mactions\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0menv_idx\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     56\u001b[0m             )\n",
            "\u001b[0;32m<ipython-input-28-fb55fa656c81>\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, actions)\u001b[0m\n\u001b[1;32m    371\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mframe\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mframe\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtotal_price\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    372\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mframe\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mframe\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m6\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mavg_price\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 373\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mframe\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"test.csv\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    374\u001b[0m             \u001b[0;31m# dx = np.where(((self.avg_price < 0.00)))\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    375\u001b[0m             \u001b[0;31m# print(dx)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/pandas/util/_decorators.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    209\u001b[0m                 \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    210\u001b[0m                     \u001b[0mkwargs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnew_arg_name\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnew_arg_value\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 211\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    212\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    213\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mF\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36mto_csv\u001b[0;34m(self, path_or_buf, sep, na_rep, float_format, columns, header, index, index_label, mode, encoding, compression, quoting, quotechar, lineterminator, chunksize, date_format, doublequote, escapechar, decimal, errors, storage_options)\u001b[0m\n\u001b[1;32m   3718\u001b[0m         )\n\u001b[1;32m   3719\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3720\u001b[0;31m         return DataFrameRenderer(formatter).to_csv(\n\u001b[0m\u001b[1;32m   3721\u001b[0m             \u001b[0mpath_or_buf\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3722\u001b[0m             \u001b[0mlineterminator\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlineterminator\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/pandas/util/_decorators.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    209\u001b[0m                 \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    210\u001b[0m                     \u001b[0mkwargs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnew_arg_name\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnew_arg_value\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 211\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    212\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    213\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mF\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/pandas/io/formats/format.py\u001b[0m in \u001b[0;36mto_csv\u001b[0;34m(self, path_or_buf, encoding, sep, columns, index_label, mode, compression, quoting, quotechar, lineterminator, chunksize, date_format, doublequote, escapechar, errors, storage_options)\u001b[0m\n\u001b[1;32m   1187\u001b[0m             \u001b[0mformatter\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfmt\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1188\u001b[0m         )\n\u001b[0;32m-> 1189\u001b[0;31m         \u001b[0mcsv_formatter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1190\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1191\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mcreated_buffer\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/pandas/io/formats/csvs.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    259\u001b[0m             )\n\u001b[1;32m    260\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 261\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_save\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    262\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    263\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_save\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/pandas/io/formats/csvs.py\u001b[0m in \u001b[0;36m_save\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    264\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_need_to_save_header\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    265\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_save_header\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 266\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_save_body\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    267\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    268\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_save_header\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/pandas/io/formats/csvs.py\u001b[0m in \u001b[0;36m_save_body\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    302\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mstart_i\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0mend_i\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    303\u001b[0m                 \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 304\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_save_chunk\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstart_i\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mend_i\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    305\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    306\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_save_chunk\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstart_i\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mend_i\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/pandas/io/formats/csvs.py\u001b[0m in \u001b[0;36m_save_chunk\u001b[0;34m(self, start_i, end_i)\u001b[0m\n\u001b[1;32m    313\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    314\u001b[0m         \u001b[0mix\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata_index\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mslicer\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_format_native_types\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_number_format\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 315\u001b[0;31m         libwriters.write_csv_rows(\n\u001b[0m\u001b[1;32m    316\u001b[0m             \u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    317\u001b[0m             \u001b[0mix\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/pandas/_libs/writers.pyx\u001b[0m in \u001b[0;36mpandas._libs.writers.write_csv_rows\u001b[0;34m()\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "joblib.dump(study, \"final_ddpg_study__.pkl\")"
      ],
      "metadata": {
        "id": "1yIXC6W2ZqPP"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#Get the best hyperparamters\n",
        "print('Hyperparameters after tuning',study.best_params)\n",
        "print('Hyperparameters before tuning',config.DDPG_PARAMS)"
      ],
      "metadata": {
        "id": "gXzfenQTdqq_",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ad76d6e8-c0ff-4d10-d0f2-efd5b0972e3e"
      },
      "execution_count": 166,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Hyperparameters after tuning {'gamma': 0.98, 'tau': 0.04, 'train_freq': 1024, 'noise_type': 'normal', 'noise_std': 0.4, 'net_arch': 'big'}\n",
            "Hyperparameters before tuning {'batch_size': 128, 'buffer_size': 50000, 'learning_rate': 0.001}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "study.best_trial"
      ],
      "metadata": {
        "id": "VQcuj-yidu0O",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0496257d-abd8-4566-b571-8cecf333c4c7"
      },
      "execution_count": 167,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "FrozenTrial(number=4, state=TrialState.COMPLETE, values=[1.3926463459776786], datetime_start=datetime.datetime(2023, 5, 2, 14, 40, 26, 156017), datetime_complete=datetime.datetime(2023, 5, 2, 14, 40, 42, 543469), params={'gamma': 0.98, 'tau': 0.04, 'train_freq': 1024, 'noise_type': 'normal', 'noise_std': 0.4, 'net_arch': 'big'}, user_attrs={}, system_attrs={}, intermediate_values={}, distributions={'gamma': CategoricalDistribution(choices=(0.9, 0.92, 0.94, 0.96, 0.98)), 'tau': CategoricalDistribution(choices=(0.02, 0.04, 0.06, 0.08, 0.1, 0.12)), 'train_freq': CategoricalDistribution(choices=(512, 768, 1024)), 'noise_type': CategoricalDistribution(choices=('ornstein-uhlenbeck', 'normal', None)), 'noise_std': CategoricalDistribution(choices=(0.1, 0.2, 0.3, 0.4, 0.5)), 'net_arch': CategoricalDistribution(choices=('small', 'big'))}, trial_id=4, value=None)"
            ]
          },
          "metadata": {},
          "execution_count": 167
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from stable_baselines3 import DDPG\n",
        "tuned_model_ddpg = DDPG.load('models/ddpg_{}.pth'.format(study.best_trial.number),env=env_train)"
      ],
      "metadata": {
        "id": "ZjYe9lOIdw3i"
      },
      "execution_count": 168,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#Trading period account value with tuned model\n",
        "df_account_value_tuned, df_actions_tuned = DRLAgent.DRL_prediction(\n",
        "    model=tuned_model_ddpg, \n",
        "    environment = e_trade_gym)"
      ],
      "metadata": {
        "id": "3bnbx87vdzHZ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a84cd21e-3154-43e3-ba2e-db3271bb3de4"
      },
      "execution_count": 178,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "hit end!\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_account_value_tuned[105:106][\"account_value\"]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "G1Y1jjMLf0qF",
        "outputId": "6e205e9b-9567-445f-d6d9-63f680806e02"
      },
      "execution_count": 179,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "105    533043.809441\n",
              "Name: account_value, dtype: float64"
            ]
          },
          "metadata": {},
          "execution_count": 179
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Backtesting with our pruned model\n",
        "print(\"==============Get Backtest Results===========\")\n",
        "now = datetime.datetime.now().strftime('%Y%m%d-%Hh%M')\n",
        "\n",
        "perf_stats_all_tuned = backtest_stats(account_value=df_account_value_tuned)\n",
        "perf_stats_all_tuned = pd.DataFrame(perf_stats_all_tuned)\n",
        "perf_stats_all_tuned.to_csv(\"./\"+config.RESULTS_DIR+\"/perf_stats_all_tuned_\"+now+'.csv')"
      ],
      "metadata": {
        "id": "FJrv144od3G_",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5fd8aee3-2ce7-4267-ca58-96a66364eba5"
      },
      "execution_count": 180,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==============Get Backtest Results===========\n",
            "Annual return          0.164324\n",
            "Cumulative returns     0.066088\n",
            "Annual volatility      0.115015\n",
            "Sharpe ratio           1.392646\n",
            "Calmar ratio           5.177199\n",
            "Stability              0.778060\n",
            "Max drawdown          -0.031740\n",
            "Omega ratio            1.269228\n",
            "Sortino ratio          2.187981\n",
            "Skew                        NaN\n",
            "Kurtosis                    NaN\n",
            "Tail ratio             1.171742\n",
            "Daily value at risk   -0.013855\n",
            "dtype: float64\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Now train with not tuned hyperaparameters\n",
        "#Default config.ddpg_PARAMS\n",
        "non_tuned_model_ddpg = agent.get_model(\"ddpg\",model_kwargs = config.DDPG_PARAMS )\n",
        "trained_ddpg = agent.train_model(model=non_tuned_model_ddpg, \n",
        "                             tb_log_name='ddpg',\n",
        "                             total_timesteps=50000)"
      ],
      "metadata": {
        "id": "asWsWzemd5dk",
        "outputId": "78a3262d-e665-4d79-a47d-a1ccfd892d70",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "execution_count": 88,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'batch_size': 128, 'buffer_size': 50000, 'learning_rate': 0.001}\n",
            "Using cuda device\n",
            "day: 82, episode: 570\n",
            "begin_total_asset: 500000.00\n",
            "end_total_asset: 434938.17\n",
            "total_reward: -65061.83\n",
            "total_cost: 25569.22\n",
            "total_trades: 1611\n",
            "Sharpe: -0.514\n",
            "=================================\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-64-5018ead22eec>:354: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-64-5018ead22eec>:268: RuntimeWarning: invalid value encountered in subtract\n",
            "  actions = np.where(((current_price < (self.avg_price - self.avg_price * .1))& (self.avg_price >0)), self.total_stockss*-1,actions  )\n",
            "<ipython-input-64-5018ead22eec>:354: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-64-5018ead22eec>:268: RuntimeWarning: invalid value encountered in subtract\n",
            "  actions = np.where(((current_price < (self.avg_price - self.avg_price * .1))& (self.avg_price >0)), self.total_stockss*-1,actions  )\n",
            "<ipython-input-64-5018ead22eec>:354: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-64-5018ead22eec>:268: RuntimeWarning: invalid value encountered in subtract\n",
            "  actions = np.where(((current_price < (self.avg_price - self.avg_price * .1))& (self.avg_price >0)), self.total_stockss*-1,actions  )\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "-----------------------------------\n",
            "| time/              |            |\n",
            "|    episodes        | 4          |\n",
            "|    fps             | 233        |\n",
            "|    time_elapsed    | 1          |\n",
            "|    total_timesteps | 332        |\n",
            "| train/             |            |\n",
            "|    actor_loss      | -21.1      |\n",
            "|    critic_loss     | 2.07e+03   |\n",
            "|    learning_rate   | 0.001      |\n",
            "|    n_updates       | 166        |\n",
            "|    reward          | -0.8715689 |\n",
            "-----------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-64-5018ead22eec>:354: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-64-5018ead22eec>:268: RuntimeWarning: invalid value encountered in subtract\n",
            "  actions = np.where(((current_price < (self.avg_price - self.avg_price * .1))& (self.avg_price >0)), self.total_stockss*-1,actions  )\n",
            "<ipython-input-64-5018ead22eec>:354: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-64-5018ead22eec>:268: RuntimeWarning: invalid value encountered in subtract\n",
            "  actions = np.where(((current_price < (self.avg_price - self.avg_price * .1))& (self.avg_price >0)), self.total_stockss*-1,actions  )\n",
            "<ipython-input-64-5018ead22eec>:354: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-64-5018ead22eec>:268: RuntimeWarning: invalid value encountered in subtract\n",
            "  actions = np.where(((current_price < (self.avg_price - self.avg_price * .1))& (self.avg_price >0)), self.total_stockss*-1,actions  )\n",
            "<ipython-input-64-5018ead22eec>:354: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-64-5018ead22eec>:268: RuntimeWarning: invalid value encountered in subtract\n",
            "  actions = np.where(((current_price < (self.avg_price - self.avg_price * .1))& (self.avg_price >0)), self.total_stockss*-1,actions  )\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "-----------------------------------\n",
            "| time/              |            |\n",
            "|    episodes        | 8          |\n",
            "|    fps             | 176        |\n",
            "|    time_elapsed    | 3          |\n",
            "|    total_timesteps | 664        |\n",
            "| train/             |            |\n",
            "|    actor_loss      | -15.6      |\n",
            "|    critic_loss     | 3.43e+03   |\n",
            "|    learning_rate   | 0.001      |\n",
            "|    n_updates       | 498        |\n",
            "|    reward          | -0.8715689 |\n",
            "-----------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-64-5018ead22eec>:354: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-64-5018ead22eec>:268: RuntimeWarning: invalid value encountered in subtract\n",
            "  actions = np.where(((current_price < (self.avg_price - self.avg_price * .1))& (self.avg_price >0)), self.total_stockss*-1,actions  )\n",
            "<ipython-input-64-5018ead22eec>:354: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-64-5018ead22eec>:268: RuntimeWarning: invalid value encountered in subtract\n",
            "  actions = np.where(((current_price < (self.avg_price - self.avg_price * .1))& (self.avg_price >0)), self.total_stockss*-1,actions  )\n",
            "<ipython-input-64-5018ead22eec>:354: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-64-5018ead22eec>:268: RuntimeWarning: invalid value encountered in subtract\n",
            "  actions = np.where(((current_price < (self.avg_price - self.avg_price * .1))& (self.avg_price >0)), self.total_stockss*-1,actions  )\n",
            "<ipython-input-64-5018ead22eec>:354: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-64-5018ead22eec>:268: RuntimeWarning: invalid value encountered in subtract\n",
            "  actions = np.where(((current_price < (self.avg_price - self.avg_price * .1))& (self.avg_price >0)), self.total_stockss*-1,actions  )\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "day: 82, episode: 580\n",
            "begin_total_asset: 500000.00\n",
            "end_total_asset: 498030.54\n",
            "total_reward: -1969.46\n",
            "total_cost: 703.88\n",
            "total_trades: 1231\n",
            "Sharpe: 0.281\n",
            "=================================\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-88-3b1b0264cc27>\u001b[0m in \u001b[0;36m<cell line: 4>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;31m#Default config.ddpg_PARAMS\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mnon_tuned_model_ddpg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0magent\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"ddpg\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mmodel_kwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDDPG_PARAMS\u001b[0m \u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m trained_ddpg = agent.train_model(model=non_tuned_model_ddpg, \n\u001b[0m\u001b[1;32m      5\u001b[0m                              \u001b[0mtb_log_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'ddpg'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m                              total_timesteps=50000)\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/finrl/agents/stablebaselines3/models.py\u001b[0m in \u001b[0;36mtrain_model\u001b[0;34m(self, model, tb_log_name, total_timesteps)\u001b[0m\n\u001b[1;32m    101\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    102\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mtrain_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtb_log_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m5000\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 103\u001b[0;31m         model = model.learn(\n\u001b[0m\u001b[1;32m    104\u001b[0m             \u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    105\u001b[0m             \u001b[0mtb_log_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtb_log_name\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/stable_baselines3/ddpg/ddpg.py\u001b[0m in \u001b[0;36mlearn\u001b[0;34m(self, total_timesteps, callback, log_interval, tb_log_name, reset_num_timesteps, progress_bar)\u001b[0m\n\u001b[1;32m    121\u001b[0m         \u001b[0mprogress_bar\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mbool\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    122\u001b[0m     ) -> SelfDDPG:\n\u001b[0;32m--> 123\u001b[0;31m         return super().learn(\n\u001b[0m\u001b[1;32m    124\u001b[0m             \u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    125\u001b[0m             \u001b[0mcallback\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallback\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/stable_baselines3/td3/td3.py\u001b[0m in \u001b[0;36mlearn\u001b[0;34m(self, total_timesteps, callback, log_interval, tb_log_name, reset_num_timesteps, progress_bar)\u001b[0m\n\u001b[1;32m    214\u001b[0m         \u001b[0mprogress_bar\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mbool\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    215\u001b[0m     ) -> SelfTD3:\n\u001b[0;32m--> 216\u001b[0;31m         return super().learn(\n\u001b[0m\u001b[1;32m    217\u001b[0m             \u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    218\u001b[0m             \u001b[0mcallback\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallback\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/stable_baselines3/common/off_policy_algorithm.py\u001b[0m in \u001b[0;36mlearn\u001b[0;34m(self, total_timesteps, callback, log_interval, tb_log_name, reset_num_timesteps, progress_bar)\u001b[0m\n\u001b[1;32m    328\u001b[0m                 \u001b[0;31m# Special case when the user passes `gradient_steps=0`\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    329\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mgradient_steps\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 330\u001b[0;31m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient_steps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mgradient_steps\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    331\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    332\u001b[0m         \u001b[0mcallback\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_training_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/stable_baselines3/td3/td3.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self, gradient_steps, batch_size)\u001b[0m\n\u001b[1;32m    195\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    196\u001b[0m                 \u001b[0mpolyak_update\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcritic\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcritic_target\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtau\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 197\u001b[0;31m                 \u001b[0mpolyak_update\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mactor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mactor_target\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtau\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    198\u001b[0m                 \u001b[0;31m# Copy running stats, see GH issue #996\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    199\u001b[0m                 \u001b[0mpolyak_update\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcritic_batch_norm_stats\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcritic_batch_norm_stats_target\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1.0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/stable_baselines3/common/utils.py\u001b[0m in \u001b[0;36mpolyak_update\u001b[0;34m(params, target_params, tau)\u001b[0m\n\u001b[1;32m    466\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mth\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mno_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    467\u001b[0m         \u001b[0;31m# zip does not raise an exception if length of parameters does not match.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 468\u001b[0;31m         \u001b[0;32mfor\u001b[0m \u001b[0mparam\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_param\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip_strict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    469\u001b[0m             \u001b[0mtarget_param\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmul_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mtau\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    470\u001b[0m             \u001b[0mth\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtarget_param\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparam\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0malpha\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtau\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtarget_param\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/stable_baselines3/common/utils.py\u001b[0m in \u001b[0;36mzip_strict\u001b[0;34m(*iterables)\u001b[0m\n\u001b[1;32m    439\u001b[0m     \u001b[0msentinel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mobject\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    440\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mcombo\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip_longest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0miterables\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfillvalue\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msentinel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 441\u001b[0;31m         \u001b[0;32mif\u001b[0m \u001b[0msentinel\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mcombo\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    442\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Iterables have different lengths\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    443\u001b[0m         \u001b[0;32myield\u001b[0m \u001b[0mcombo\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_account_value, df_actions = DRLAgent.DRL_prediction(\n",
        "    model=trained_ddpg, \n",
        "    environment = e_trade_gym)"
      ],
      "metadata": {
        "id": "mv1zgwOgd9FY",
        "outputId": "3fd9a94a-98d4-41c0-8870-74b8271b6494",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-35-38368194789e>\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m df_account_value, df_actions = DRLAgent.DRL_prediction(\n\u001b[0;32m----> 2\u001b[0;31m     \u001b[0mmodel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrained_ddpg\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m     environment = e_trade_gym)\n",
            "\u001b[0;31mNameError\u001b[0m: name 'trained_ddpg' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Backtesting for not tuned hyperparamters\n",
        "print(\"==============Get Backtest Results===========\")\n",
        "now = datetime.datetime.now().strftime('%Y%m%d-%Hh%M')\n",
        "\n",
        "perf_stats_all = backtest_stats(account_value=df_account_value)\n",
        "perf_stats_all = pd.DataFrame(perf_stats_all)\n",
        "# perf_stats_all.to_csv(\"./\"+config.RESULTS_DIR+\"/perf_stats_all_\"+now+'.csv')"
      ],
      "metadata": {
        "id": "DuZL52qDeEz6",
        "outputId": "f4e9cd19-27cd-4dba-956e-353df0dabffa",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==============Get Backtest Results===========\n",
            "Annual return          0.247826\n",
            "Cumulative returns     0.368416\n",
            "Annual volatility      0.167226\n",
            "Sharpe ratio           1.412093\n",
            "Calmar ratio           2.531845\n",
            "Stability              0.897149\n",
            "Max drawdown          -0.097883\n",
            "Omega ratio            1.280406\n",
            "Sortino ratio          2.010344\n",
            "Skew                        NaN\n",
            "Kurtosis                    NaN\n",
            "Tail ratio             1.033938\n",
            "Daily value at risk   -0.020131\n",
            "dtype: float64\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#You can see with trial, our sharpe ratio is increasing\n",
        "#Certainly you can afford more number of trials for further optimization\n",
        "from optuna.visualization import plot_optimization_history\n",
        "plot_optimization_history(study)"
      ],
      "metadata": {
        "id": "5aSsEmpZeI5C",
        "outputId": "cba8ec8c-7654-44d7-88cb-bd6e1bad4a5b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 542
        }
      },
      "execution_count": 172,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<html>\n",
              "<head><meta charset=\"utf-8\" /></head>\n",
              "<body>\n",
              "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
              "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.20.0.min.js\"></script>                <div id=\"565b1505-44ed-4588-989d-e42e52b774ca\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"565b1505-44ed-4588-989d-e42e52b774ca\")) {                    Plotly.newPlot(                        \"565b1505-44ed-4588-989d-e42e52b774ca\",                        [{\"mode\":\"markers\",\"name\":\"Objective Value\",\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21],\"y\":[-0.49080299778360714,-0.3950000677893763,1.3011854021414007,-0.8267805987990207,1.3926463459776786,0.26111723817257965,-0.9640129324129495,0.2819918104525999,1.0018988775173625,-0.2701595920840011,1.1243951293107357,-0.40659941768702645,0.25085630521807395,0.2654094534716406,1.3315265826000702,-0.7288319825005803,0.25514408327043286,-1.0674379403572838,-0.6986206079122625,0.3810367303898103,-0.471019724578347,0.748178567824751],\"type\":\"scatter\"},{\"name\":\"Best Value\",\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21],\"y\":[-0.49080299778360714,-0.3950000677893763,1.3011854021414007,1.3011854021414007,1.3926463459776786,1.3926463459776786,1.3926463459776786,1.3926463459776786,1.3926463459776786,1.3926463459776786,1.3926463459776786,1.3926463459776786,1.3926463459776786,1.3926463459776786,1.3926463459776786,1.3926463459776786,1.3926463459776786,1.3926463459776786,1.3926463459776786,1.3926463459776786,1.3926463459776786,1.3926463459776786],\"type\":\"scatter\"}],                        {\"title\":{\"text\":\"Optimization History Plot\"},\"xaxis\":{\"title\":{\"text\":\"Trial\"}},\"yaxis\":{\"title\":{\"text\":\"Objective Value\"}},\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}}},                        {\"responsive\": true}                    ).then(function(){\n",
              "                            \n",
              "var gd = document.getElementById('565b1505-44ed-4588-989d-e42e52b774ca');\n",
              "var x = new MutationObserver(function (mutations, observer) {{\n",
              "        var display = window.getComputedStyle(gd).display;\n",
              "        if (!display || display === 'none') {{\n",
              "            console.log([gd, 'removed!']);\n",
              "            Plotly.purge(gd);\n",
              "            observer.disconnect();\n",
              "        }}\n",
              "}});\n",
              "\n",
              "// Listen for the removal of the full notebook cells\n",
              "var notebookContainer = gd.closest('#notebook-container');\n",
              "if (notebookContainer) {{\n",
              "    x.observe(notebookContainer, {childList: true});\n",
              "}}\n",
              "\n",
              "// Listen for the clearing of the current output cell\n",
              "var outputEl = gd.closest('.output');\n",
              "if (outputEl) {{\n",
              "    x.observe(outputEl, {childList: true});\n",
              "}}\n",
              "\n",
              "                        })                };                            </script>        </div>\n",
              "</body>\n",
              "</html>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from optuna.visualization import plot_contour\n",
        "from optuna.visualization import plot_edf\n",
        "from optuna.visualization import plot_intermediate_values\n",
        "from optuna.visualization import plot_optimization_history\n",
        "from optuna.visualization import plot_parallel_coordinate\n",
        "from optuna.visualization import plot_param_importances\n",
        "from optuna.visualization import plot_slice"
      ],
      "metadata": {
        "id": "K9zcey_BeJMe"
      },
      "execution_count": 173,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#Hyperparamters importance\n",
        "#Ent_coef is the most important\n",
        "plot_param_importances(study)"
      ],
      "metadata": {
        "id": "J6WmB5DneOgh",
        "outputId": "2d916588-ae89-4970-f430-249ef6b5c81a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 542
        }
      },
      "execution_count": 174,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<html>\n",
              "<head><meta charset=\"utf-8\" /></head>\n",
              "<body>\n",
              "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
              "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.20.0.min.js\"></script>                <div id=\"65abb362-3b70-4093-bbd4-fd5b77123c9e\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"65abb362-3b70-4093-bbd4-fd5b77123c9e\")) {                    Plotly.newPlot(                        \"65abb362-3b70-4093-bbd4-fd5b77123c9e\",                        [{\"cliponaxis\":false,\"hovertemplate\":[\"net_arch (CategoricalDistribution): 0.029499975293024668<extra></extra>\",\"train_freq (CategoricalDistribution): 0.0539951123903899<extra></extra>\",\"noise_type (CategoricalDistribution): 0.1655824179588198<extra></extra>\",\"tau (CategoricalDistribution): 0.1773850893223912<extra></extra>\",\"noise_std (CategoricalDistribution): 0.19437196725757405<extra></extra>\",\"gamma (CategoricalDistribution): 0.37916543777780054<extra></extra>\"],\"marker\":{\"color\":\"rgb(66,146,198)\"},\"orientation\":\"h\",\"text\":[\"0.03\",\"0.05\",\"0.17\",\"0.18\",\"0.19\",\"0.38\"],\"textposition\":\"outside\",\"x\":[0.029499975293024668,0.0539951123903899,0.1655824179588198,0.1773850893223912,0.19437196725757405,0.37916543777780054],\"y\":[\"net_arch\",\"train_freq\",\"noise_type\",\"tau\",\"noise_std\",\"gamma\"],\"type\":\"bar\"}],                        {\"showlegend\":false,\"title\":{\"text\":\"Hyperparameter Importances\"},\"xaxis\":{\"title\":{\"text\":\"Importance for Objective Value\"}},\"yaxis\":{\"title\":{\"text\":\"Hyperparameter\"}},\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}}},                        {\"responsive\": true}                    ).then(function(){\n",
              "                            \n",
              "var gd = document.getElementById('65abb362-3b70-4093-bbd4-fd5b77123c9e');\n",
              "var x = new MutationObserver(function (mutations, observer) {{\n",
              "        var display = window.getComputedStyle(gd).display;\n",
              "        if (!display || display === 'none') {{\n",
              "            console.log([gd, 'removed!']);\n",
              "            Plotly.purge(gd);\n",
              "            observer.disconnect();\n",
              "        }}\n",
              "}});\n",
              "\n",
              "// Listen for the removal of the full notebook cells\n",
              "var notebookContainer = gd.closest('#notebook-container');\n",
              "if (notebookContainer) {{\n",
              "    x.observe(notebookContainer, {childList: true});\n",
              "}}\n",
              "\n",
              "// Listen for the clearing of the current output cell\n",
              "var outputEl = gd.closest('.output');\n",
              "if (outputEl) {{\n",
              "    x.observe(outputEl, {childList: true});\n",
              "}}\n",
              "\n",
              "                        })                };                            </script>        </div>\n",
              "</body>\n",
              "</html>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def main():\n",
        "    import warnings\n",
        "    warnings.filterwarnings(\"ignore\")\n",
        "    import pandas as pd\n",
        "    import numpy as np\n",
        "    import matplotlib\n",
        "    import matplotlib.pyplot as plt\n",
        "    # matplotlib.use('Agg')\n",
        "    import datetime\n",
        "\n",
        "\n",
        "    from finrl.config_tickers import DOW_30_TICKER\n",
        "    from finrl.meta.preprocessor.yahoodownloader import YahooDownloader\n",
        "    from finrl.meta.preprocessor.preprocessors import FeatureEngineer, data_split\n",
        "    from finrl.meta.env_stock_trading.env_stocktrading import StockTradingEnv\n",
        "    from finrl.agents.stablebaselines3.models import DRLAgent, DRLEnsembleAgent\n",
        "    from finrl.plot import backtest_stats, backtest_plot, get_daily_return, get_baseline\n",
        "\n",
        "    from pprint import pprint\n",
        "\n",
        "    import sys\n",
        "    sys.path.append(\"../FinRL-Library\")\n",
        "\n",
        "    import itertools\n",
        "\n",
        "    import os\n",
        "    from finrl.main import check_and_make_directories\n",
        "    from finrl.config import (\n",
        "        DATA_SAVE_DIR,\n",
        "        TRAINED_MODEL_DIR,\n",
        "        TENSORBOARD_LOG_DIR,\n",
        "        RESULTS_DIR,\n",
        "        INDICATORS,\n",
        "        TRAIN_START_DATE,\n",
        "        TRAIN_END_DATE,\n",
        "        TEST_START_DATE,\n",
        "        TEST_END_DATE,\n",
        "        TRADE_START_DATE,\n",
        "        TRADE_END_DATE,\n",
        "    )\n",
        "\n",
        "    check_and_make_directories([DATA_SAVE_DIR, TRAINED_MODEL_DIR, TENSORBOARD_LOG_DIR, RESULTS_DIR])\n",
        "    print(DOW_30_TICKER)\n",
        "    TRAIN_START_DATE = '2009-04-01'\n",
        "    TRAIN_END_DATE = '2021-01-01'\n",
        "    TEST_START_DATE = '2021-01-01'\n",
        "    TEST_END_DATE = '2022-06-01'\n",
        "\n",
        "    df = YahooDownloader(start_date=TRAIN_START_DATE,\n",
        "                         end_date=TEST_END_DATE,\n",
        "                         ticker_list=DOW_30_TICKER).fetch_data()\n",
        "\n",
        "    df.sort_values(['date', 'tic']).head()\n",
        "\n",
        "    fe = FeatureEngineer(use_technical_indicator=True,\n",
        "                         tech_indicator_list=INDICATORS,\n",
        "                         use_turbulence=True,\n",
        "                         user_defined_feature=False)\n",
        "\n",
        "    processed = fe.preprocess_data(df)\n",
        "    processed = processed.copy()\n",
        "    processed = processed.fillna(0)\n",
        "    processed = processed.replace(np.inf, 0)\n",
        "\n",
        "    stock_dimension = len(processed.tic.unique())\n",
        "    state_space = 1 + 2 * stock_dimension + len(INDICATORS) * stock_dimension\n",
        "    print(f\"Stock Dimension: {stock_dimension}, State Space: {state_space}\")\n",
        "\n",
        "    env_kwargs = {\n",
        "        \"hmax\": 100,\n",
        "        \"initial_amount\": 1000000,\n",
        "        \"buy_cost_pct\": 0.001,\n",
        "        \"sell_cost_pct\": 0.001,\n",
        "        \"state_space\": state_space,\n",
        "        \"stock_dim\": stock_dimension,\n",
        "        \"tech_indicator_list\": INDICATORS,\n",
        "        \"action_space\": stock_dimension,\n",
        "        \"reward_scaling\": 1e-4,\n",
        "        \"print_verbosity\": 5\n",
        "\n",
        "    }\n",
        "\n",
        "    rebalance_window = 63  # rebalance_window is the number of days to retrain the model\n",
        "    validation_window = 63  # validation_window is the number of days to do validation and trading (e.g. if validation_window=63, then both validation and trading period will be 63 days)\n",
        "\n",
        "    ensemble_agent = DRLEnsembleAgent(df=processed,\n",
        "                                      train_period=(TRAIN_START_DATE, TRAIN_END_DATE),\n",
        "                                      val_test_period=(TEST_START_DATE, TEST_END_DATE),\n",
        "                                      rebalance_window=rebalance_window,\n",
        "                                      validation_window=validation_window,\n",
        "                                      **env_kwargs)\n",
        "\n",
        "    A2C_model_kwargs = {\n",
        "        'n_steps': 5,\n",
        "        'ent_coef': 0.005,\n",
        "        'learning_rate': 0.0007\n",
        "    }\n",
        "\n",
        "    PPO_model_kwargs = {\n",
        "        \"ent_coef\": 0.01,\n",
        "        \"n_steps\": 2048,\n",
        "        \"learning_rate\": 0.00025,\n",
        "        \"batch_size\": 128\n",
        "    }\n",
        "\n",
        "    DDPG_model_kwargs = {\n",
        "        # \"action_noise\":\"ornstein_uhlenbeck\",\n",
        "        \"buffer_size\": 10_000,\n",
        "        \"learning_rate\": 0.0005,\n",
        "        \"batch_size\": 64\n",
        "    }\n",
        "\n",
        "    timesteps_dict = {'a2c': 10_000,\n",
        "                      'ppo': 10_000,\n",
        "                      'ddpg': 10_000\n",
        "                      }\n",
        "    df_summary = ensemble_agent.run_ensemble_strategy(A2C_model_kwargs,\n",
        "                                                      PPO_model_kwargs,\n",
        "                                                      DDPG_model_kwargs,\n",
        "                                                      timesteps_dict)\n",
        "\n",
        "    unique_trade_date = processed[(processed.date > TEST_START_DATE) & (processed.date <= TEST_END_DATE)].date.unique()\n",
        "\n",
        "    df_trade_date = pd.DataFrame({'datadate': unique_trade_date})\n",
        "\n",
        "    df_account_value = pd.DataFrame()\n",
        "    for i in range(rebalance_window + validation_window, len(unique_trade_date) + 1, rebalance_window):\n",
        "        temp = pd.read_csv('results/account_value_trade_{}_{}.csv'.format('ensemble', i))\n",
        "        df_account_value = df_account_value.append(temp, ignore_index=True)\n",
        "    sharpe = (252 ** 0.5) * df_account_value.account_value.pct_change(\n",
        "        1).mean() / df_account_value.account_value.pct_change(1).std()\n",
        "    print('Sharpe Ratio: ', sharpe)\n",
        "    df_account_value = df_account_value.join(df_trade_date[validation_window:].reset_index(drop=True))\n",
        "\n",
        "    df_account_value.account_value.plot()\n",
        "\n",
        "    print(\"==============Get Backtest Results===========\")\n",
        "    now = datetime.datetime.now().strftime('%Y%m%d-%Hh%M')\n",
        "\n",
        "    perf_stats_all = backtest_stats(account_value=df_account_value)\n",
        "    perf_stats_all = pd.DataFrame(perf_stats_all)\n",
        "\n",
        "    # baseline stats\n",
        "    print(\"==============Get Baseline Stats===========\")\n",
        "    baseline_df = get_baseline(\n",
        "        ticker=\"^DJI\",\n",
        "        start=df_account_value.loc[0, 'date'],\n",
        "        end=df_account_value.loc[len(df_account_value) - 1, 'date'])\n",
        "\n",
        "    stats = backtest_stats(baseline_df, value_col_name='close')\n",
        "\n",
        "    print(\"==============Compare to DJIA===========\")\n",
        "\n",
        "    # S&P 500: ^GSPC\n",
        "    # Dow Jones Index: ^DJI\n",
        "    # NASDAQ 100: ^NDX\n",
        "    backtest_plot(df_account_value,\n",
        "                  baseline_ticker='^DJI',\n",
        "                  baseline_start=df_account_value.loc[0, 'date'],\n",
        "                  baseline_end=df_account_value.loc[len(df_account_value) - 1, 'date'])\n",
        "\n",
        "\n",
        "if __name__ == '__main__':\n",
        "    main()"
      ],
      "metadata": {
        "id": "s_xcVpBSeOyE",
        "outputId": "52b3d82a-1f8d-4af4-a405-6b5b9778781d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['AXP', 'AMGN', 'AAPL', 'BA', 'CAT', 'CSCO', 'CVX', 'GS', 'HD', 'HON', 'IBM', 'INTC', 'JNJ', 'KO', 'JPM', 'MCD', 'MMM', 'MRK', 'MSFT', 'NKE', 'PG', 'TRV', 'UNH', 'CRM', 'VZ', 'V', 'WBA', 'WMT', 'DIS', 'DOW']\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "Shape of DataFrame:  (96942, 8)\n",
            "Successfully added technical indicators\n",
            "Successfully added turbulence index\n",
            "Stock Dimension: 29, State Space: 291\n",
            "============Start Ensemble Strategy============\n",
            "============================================\n",
            "turbulence_threshold:  203.40117769841157\n",
            "======Model training from:  2009-04-01 to  2021-01-04\n",
            "======A2C Training========\n",
            "{'n_steps': 5, 'ent_coef': 0.005, 'learning_rate': 0.0007}\n",
            "Using cuda device\n",
            "Logging to tensorboard_log/a2c/a2c_126_1\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 100        |\n",
            "|    iterations         | 100        |\n",
            "|    time_elapsed       | 4          |\n",
            "|    total_timesteps    | 500        |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.1      |\n",
            "|    explained_variance | 0.00295    |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 99         |\n",
            "|    policy_loss        | -71.2      |\n",
            "|    reward             | 0.78650117 |\n",
            "|    std                | 1          |\n",
            "|    value_loss         | 12.2       |\n",
            "--------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 101        |\n",
            "|    iterations         | 200        |\n",
            "|    time_elapsed       | 9          |\n",
            "|    total_timesteps    | 1000       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.1      |\n",
            "|    explained_variance | 0.00111    |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 199        |\n",
            "|    policy_loss        | 6.59       |\n",
            "|    reward             | 0.66569686 |\n",
            "|    std                | 0.998      |\n",
            "|    value_loss         | 1.03       |\n",
            "--------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 102       |\n",
            "|    iterations         | 300       |\n",
            "|    time_elapsed       | 14        |\n",
            "|    total_timesteps    | 1500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.1     |\n",
            "|    explained_variance | 0         |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 299       |\n",
            "|    policy_loss        | -123      |\n",
            "|    reward             | 2.9206326 |\n",
            "|    std                | 0.998     |\n",
            "|    value_loss         | 15.8      |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 102        |\n",
            "|    iterations         | 400        |\n",
            "|    time_elapsed       | 19         |\n",
            "|    total_timesteps    | 2000       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.1      |\n",
            "|    explained_variance | -1.19e-07  |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 399        |\n",
            "|    policy_loss        | -6.35      |\n",
            "|    reward             | -1.8242201 |\n",
            "|    std                | 0.999      |\n",
            "|    value_loss         | 3.27       |\n",
            "--------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                 |               |\n",
            "|    fps                | 102           |\n",
            "|    iterations         | 500           |\n",
            "|    time_elapsed       | 24            |\n",
            "|    total_timesteps    | 2500          |\n",
            "| train/                |               |\n",
            "|    entropy_loss       | -41.2         |\n",
            "|    explained_variance | -1.19e-07     |\n",
            "|    learning_rate      | 0.0007        |\n",
            "|    n_updates          | 499           |\n",
            "|    policy_loss        | 16            |\n",
            "|    reward             | -0.0061751665 |\n",
            "|    std                | 1             |\n",
            "|    value_loss         | 0.657         |\n",
            "-----------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 102       |\n",
            "|    iterations         | 600       |\n",
            "|    time_elapsed       | 29        |\n",
            "|    total_timesteps    | 3000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.2     |\n",
            "|    explained_variance | -1.19e-07 |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 599       |\n",
            "|    policy_loss        | -33.6     |\n",
            "|    reward             | 0.5337091 |\n",
            "|    std                | 1         |\n",
            "|    value_loss         | 0.837     |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 102        |\n",
            "|    iterations         | 700        |\n",
            "|    time_elapsed       | 34         |\n",
            "|    total_timesteps    | 3500       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.2      |\n",
            "|    explained_variance | -1.97      |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 699        |\n",
            "|    policy_loss        | -76.4      |\n",
            "|    reward             | -1.3246601 |\n",
            "|    std                | 1          |\n",
            "|    value_loss         | 3.85       |\n",
            "--------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 101        |\n",
            "|    iterations         | 800        |\n",
            "|    time_elapsed       | 39         |\n",
            "|    total_timesteps    | 4000       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.2      |\n",
            "|    explained_variance | 0.0632     |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 799        |\n",
            "|    policy_loss        | 44.6       |\n",
            "|    reward             | -1.6959082 |\n",
            "|    std                | 1          |\n",
            "|    value_loss         | 1.53       |\n",
            "--------------------------------------\n",
            "------------------------------------\n",
            "| time/                 |          |\n",
            "|    fps                | 100      |\n",
            "|    iterations         | 900      |\n",
            "|    time_elapsed       | 44       |\n",
            "|    total_timesteps    | 4500     |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -41.2    |\n",
            "|    explained_variance | 0.0211   |\n",
            "|    learning_rate      | 0.0007   |\n",
            "|    n_updates          | 899      |\n",
            "|    policy_loss        | -33.2    |\n",
            "|    reward             | 3.994608 |\n",
            "|    std                | 1        |\n",
            "|    value_loss         | 3.53     |\n",
            "------------------------------------\n",
            "---------------------------------------\n",
            "| time/                 |             |\n",
            "|    fps                | 101         |\n",
            "|    iterations         | 1000        |\n",
            "|    time_elapsed       | 49          |\n",
            "|    total_timesteps    | 5000        |\n",
            "| train/                |             |\n",
            "|    entropy_loss       | -41.2       |\n",
            "|    explained_variance | 1.79e-07    |\n",
            "|    learning_rate      | 0.0007      |\n",
            "|    n_updates          | 999         |\n",
            "|    policy_loss        | 36.3        |\n",
            "|    reward             | -0.20298508 |\n",
            "|    std                | 1           |\n",
            "|    value_loss         | 1.68        |\n",
            "---------------------------------------\n",
            "---------------------------------------\n",
            "| time/                 |             |\n",
            "|    fps                | 101         |\n",
            "|    iterations         | 1100        |\n",
            "|    time_elapsed       | 54          |\n",
            "|    total_timesteps    | 5500        |\n",
            "| train/                |             |\n",
            "|    entropy_loss       | -41.1       |\n",
            "|    explained_variance | 0.0114      |\n",
            "|    learning_rate      | 0.0007      |\n",
            "|    n_updates          | 1099        |\n",
            "|    policy_loss        | 393         |\n",
            "|    reward             | -0.92543656 |\n",
            "|    std                | 1           |\n",
            "|    value_loss         | 120         |\n",
            "---------------------------------------\n",
            "----------------------------------------\n",
            "| time/                 |              |\n",
            "|    fps                | 101          |\n",
            "|    iterations         | 1200         |\n",
            "|    time_elapsed       | 59           |\n",
            "|    total_timesteps    | 6000         |\n",
            "| train/                |              |\n",
            "|    entropy_loss       | -41.2        |\n",
            "|    explained_variance | 0.0236       |\n",
            "|    learning_rate      | 0.0007       |\n",
            "|    n_updates          | 1199         |\n",
            "|    policy_loss        | 151          |\n",
            "|    reward             | -0.099366836 |\n",
            "|    std                | 1            |\n",
            "|    value_loss         | 19.4         |\n",
            "----------------------------------------\n",
            "---------------------------------------\n",
            "| time/                 |             |\n",
            "|    fps                | 101         |\n",
            "|    iterations         | 1300        |\n",
            "|    time_elapsed       | 64          |\n",
            "|    total_timesteps    | 6500        |\n",
            "| train/                |             |\n",
            "|    entropy_loss       | -41.2       |\n",
            "|    explained_variance | -0.000106   |\n",
            "|    learning_rate      | 0.0007      |\n",
            "|    n_updates          | 1299        |\n",
            "|    policy_loss        | -47.6       |\n",
            "|    reward             | -0.39146426 |\n",
            "|    std                | 1           |\n",
            "|    value_loss         | 3.15        |\n",
            "---------------------------------------\n",
            "---------------------------------------\n",
            "| time/                 |             |\n",
            "|    fps                | 101         |\n",
            "|    iterations         | 1400        |\n",
            "|    time_elapsed       | 69          |\n",
            "|    total_timesteps    | 7000        |\n",
            "| train/                |             |\n",
            "|    entropy_loss       | -41.2       |\n",
            "|    explained_variance | 0           |\n",
            "|    learning_rate      | 0.0007      |\n",
            "|    n_updates          | 1399        |\n",
            "|    policy_loss        | 31.8        |\n",
            "|    reward             | -0.25985184 |\n",
            "|    std                | 1           |\n",
            "|    value_loss         | 1.84        |\n",
            "---------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 101       |\n",
            "|    iterations         | 1500      |\n",
            "|    time_elapsed       | 74        |\n",
            "|    total_timesteps    | 7500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.2     |\n",
            "|    explained_variance | 0         |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1499      |\n",
            "|    policy_loss        | -30.8     |\n",
            "|    reward             | 3.1766434 |\n",
            "|    std                | 1         |\n",
            "|    value_loss         | 4.93      |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 101        |\n",
            "|    iterations         | 1600       |\n",
            "|    time_elapsed       | 78         |\n",
            "|    total_timesteps    | 8000       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.2      |\n",
            "|    explained_variance | 0.00673    |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 1599       |\n",
            "|    policy_loss        | 133        |\n",
            "|    reward             | -2.0720153 |\n",
            "|    std                | 1          |\n",
            "|    value_loss         | 10.9       |\n",
            "--------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 101       |\n",
            "|    iterations         | 1700      |\n",
            "|    time_elapsed       | 83        |\n",
            "|    total_timesteps    | 8500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.2     |\n",
            "|    explained_variance | 5.96e-08  |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1699      |\n",
            "|    policy_loss        | -84.2     |\n",
            "|    reward             | 3.8356218 |\n",
            "|    std                | 1         |\n",
            "|    value_loss         | 7         |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 101        |\n",
            "|    iterations         | 1800       |\n",
            "|    time_elapsed       | 88         |\n",
            "|    total_timesteps    | 9000       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.2      |\n",
            "|    explained_variance | 0.063      |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 1799       |\n",
            "|    policy_loss        | 42         |\n",
            "|    reward             | 0.97460705 |\n",
            "|    std                | 1          |\n",
            "|    value_loss         | 2.23       |\n",
            "--------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 101       |\n",
            "|    iterations         | 1900      |\n",
            "|    time_elapsed       | 93        |\n",
            "|    total_timesteps    | 9500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.2     |\n",
            "|    explained_variance | 0.0122    |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1899      |\n",
            "|    policy_loss        | -84.7     |\n",
            "|    reward             | 2.2624621 |\n",
            "|    std                | 1         |\n",
            "|    value_loss         | 8.34      |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 101        |\n",
            "|    iterations         | 2000       |\n",
            "|    time_elapsed       | 98         |\n",
            "|    total_timesteps    | 10000      |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.2      |\n",
            "|    explained_variance | 0          |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 1999       |\n",
            "|    policy_loss        | 45.9       |\n",
            "|    reward             | -1.0189797 |\n",
            "|    std                | 1          |\n",
            "|    value_loss         | 1.77       |\n",
            "--------------------------------------\n",
            "======A2C Validation from:  2021-01-04 to  2021-04-06\n",
            "A2C Sharpe Ratio:  0.26168708794170054\n",
            "======PPO Training========\n",
            "{'ent_coef': 0.01, 'n_steps': 2048, 'learning_rate': 0.00025, 'batch_size': 128}\n",
            "Using cuda device\n",
            "Logging to tensorboard_log/ppo/ppo_126_1\n",
            "----------------------------------\n",
            "| time/              |           |\n",
            "|    fps             | 111       |\n",
            "|    iterations      | 1         |\n",
            "|    time_elapsed    | 18        |\n",
            "|    total_timesteps | 2048      |\n",
            "| train/             |           |\n",
            "|    reward          | 1.8877938 |\n",
            "----------------------------------\n",
            "day: 2959, episode: 5\n",
            "begin_total_asset: 1000000.00\n",
            "end_total_asset: 3881161.61\n",
            "total_reward: 2881161.61\n",
            "total_cost: 376867.37\n",
            "total_trades: 82873\n",
            "Sharpe: 0.776\n",
            "=================================\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 109         |\n",
            "|    iterations           | 2           |\n",
            "|    time_elapsed         | 37          |\n",
            "|    total_timesteps      | 4096        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.016203282 |\n",
            "|    clip_fraction        | 0.192       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.2       |\n",
            "|    explained_variance   | -0.0104     |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 4           |\n",
            "|    n_updates            | 10          |\n",
            "|    policy_gradient_loss | -0.0191     |\n",
            "|    reward               | 1.0358981   |\n",
            "|    std                  | 1           |\n",
            "|    value_loss           | 11.3        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 109         |\n",
            "|    iterations           | 3           |\n",
            "|    time_elapsed         | 56          |\n",
            "|    total_timesteps      | 6144        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.012528071 |\n",
            "|    clip_fraction        | 0.126       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.2       |\n",
            "|    explained_variance   | -0.00217    |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 57          |\n",
            "|    n_updates            | 20          |\n",
            "|    policy_gradient_loss | -0.0176     |\n",
            "|    reward               | 0.07821791  |\n",
            "|    std                  | 1           |\n",
            "|    value_loss           | 55.7        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 107         |\n",
            "|    iterations           | 4           |\n",
            "|    time_elapsed         | 76          |\n",
            "|    total_timesteps      | 8192        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.012139827 |\n",
            "|    clip_fraction        | 0.146       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.2       |\n",
            "|    explained_variance   | -0.00454    |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 13.7        |\n",
            "|    n_updates            | 30          |\n",
            "|    policy_gradient_loss | -0.018      |\n",
            "|    reward               | -1.5925834  |\n",
            "|    std                  | 1           |\n",
            "|    value_loss           | 43.4        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 107         |\n",
            "|    iterations           | 5           |\n",
            "|    time_elapsed         | 94          |\n",
            "|    total_timesteps      | 10240       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.019309906 |\n",
            "|    clip_fraction        | 0.17        |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.3       |\n",
            "|    explained_variance   | -0.0161     |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 5.07        |\n",
            "|    n_updates            | 40          |\n",
            "|    policy_gradient_loss | -0.0225     |\n",
            "|    reward               | 0.0615912   |\n",
            "|    std                  | 1.01        |\n",
            "|    value_loss           | 11.3        |\n",
            "-----------------------------------------\n",
            "======PPO Validation from:  2021-01-04 to  2021-04-06\n",
            "PPO Sharpe Ratio:  0.3016310407987196\n",
            "======DDPG Training========\n",
            "{'buffer_size': 10000, 'learning_rate': 0.0005, 'batch_size': 64}\n",
            "Using cuda device\n",
            "Logging to tensorboard_log/ddpg/ddpg_126_1\n",
            "day: 2959, episode: 10\n",
            "begin_total_asset: 1000000.00\n",
            "end_total_asset: 4424085.63\n",
            "total_reward: 3424085.63\n",
            "total_cost: 1037.34\n",
            "total_trades: 50241\n",
            "Sharpe: 0.797\n",
            "=================================\n",
            "----------------------------------\n",
            "| time/              |           |\n",
            "|    episodes        | 4         |\n",
            "|    fps             | 78        |\n",
            "|    time_elapsed    | 151       |\n",
            "|    total_timesteps | 11840     |\n",
            "| train/             |           |\n",
            "|    actor_loss      | -60.3     |\n",
            "|    critic_loss     | 441       |\n",
            "|    learning_rate   | 0.0005    |\n",
            "|    n_updates       | 8880      |\n",
            "|    reward          | 3.0541244 |\n",
            "----------------------------------\n",
            "======DDPG Validation from:  2021-01-04 to  2021-04-06\n",
            "======Best Model Retraining from:  2009-04-01 to  2021-04-06\n",
            "======Trading from:  2021-04-06 to  2021-07-06\n",
            "============================================\n",
            "turbulence_threshold:  203.40117769841157\n",
            "======Model training from:  2009-04-01 to  2021-04-06\n",
            "======A2C Training========\n",
            "{'n_steps': 5, 'ent_coef': 0.005, 'learning_rate': 0.0007}\n",
            "Using cuda device\n",
            "Logging to tensorboard_log/a2c/a2c_189_1\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 99         |\n",
            "|    iterations         | 100        |\n",
            "|    time_elapsed       | 5          |\n",
            "|    total_timesteps    | 500        |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41        |\n",
            "|    explained_variance | -0.446     |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 99         |\n",
            "|    policy_loss        | -85.9      |\n",
            "|    reward             | -0.6058027 |\n",
            "|    std                | 0.996      |\n",
            "|    value_loss         | 9.16       |\n",
            "--------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 100       |\n",
            "|    iterations         | 200       |\n",
            "|    time_elapsed       | 9         |\n",
            "|    total_timesteps    | 1000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41       |\n",
            "|    explained_variance | 0         |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 199       |\n",
            "|    policy_loss        | -19.4     |\n",
            "|    reward             | 1.8647146 |\n",
            "|    std                | 0.995     |\n",
            "|    value_loss         | 0.253     |\n",
            "-------------------------------------\n",
            "------------------------------------\n",
            "| time/                 |          |\n",
            "|    fps                | 100      |\n",
            "|    iterations         | 300      |\n",
            "|    time_elapsed       | 14       |\n",
            "|    total_timesteps    | 1500     |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -41      |\n",
            "|    explained_variance | 0.0122   |\n",
            "|    learning_rate      | 0.0007   |\n",
            "|    n_updates          | 299      |\n",
            "|    policy_loss        | -179     |\n",
            "|    reward             | 3.291392 |\n",
            "|    std                | 0.996    |\n",
            "|    value_loss         | 22.1     |\n",
            "------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 100       |\n",
            "|    iterations         | 400       |\n",
            "|    time_elapsed       | 19        |\n",
            "|    total_timesteps    | 2000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41       |\n",
            "|    explained_variance | 0.0507    |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 399       |\n",
            "|    policy_loss        | -11.1     |\n",
            "|    reward             | 0.3743614 |\n",
            "|    std                | 0.996     |\n",
            "|    value_loss         | 1.09      |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 100        |\n",
            "|    iterations         | 500        |\n",
            "|    time_elapsed       | 24         |\n",
            "|    total_timesteps    | 2500       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41        |\n",
            "|    explained_variance | 0          |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 499        |\n",
            "|    policy_loss        | 169        |\n",
            "|    reward             | -0.3291204 |\n",
            "|    std                | 0.995      |\n",
            "|    value_loss         | 16         |\n",
            "--------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 100        |\n",
            "|    iterations         | 600        |\n",
            "|    time_elapsed       | 29         |\n",
            "|    total_timesteps    | 3000       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41        |\n",
            "|    explained_variance | 0.054      |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 599        |\n",
            "|    policy_loss        | 905        |\n",
            "|    reward             | -1.0711567 |\n",
            "|    std                | 0.994      |\n",
            "|    value_loss         | 521        |\n",
            "--------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 100        |\n",
            "|    iterations         | 700        |\n",
            "|    time_elapsed       | 34         |\n",
            "|    total_timesteps    | 3500       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41        |\n",
            "|    explained_variance | -0.0433    |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 699        |\n",
            "|    policy_loss        | -32.9      |\n",
            "|    reward             | -2.9666545 |\n",
            "|    std                | 0.995      |\n",
            "|    value_loss         | 0.946      |\n",
            "--------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 100        |\n",
            "|    iterations         | 800        |\n",
            "|    time_elapsed       | 39         |\n",
            "|    total_timesteps    | 4000       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41        |\n",
            "|    explained_variance | -1.19e-07  |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 799        |\n",
            "|    policy_loss        | -3.25      |\n",
            "|    reward             | -0.5585839 |\n",
            "|    std                | 0.994      |\n",
            "|    value_loss         | 1.05       |\n",
            "--------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 100       |\n",
            "|    iterations         | 900       |\n",
            "|    time_elapsed       | 44        |\n",
            "|    total_timesteps    | 4500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -40.9     |\n",
            "|    explained_variance | -0.66     |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 899       |\n",
            "|    policy_loss        | 306       |\n",
            "|    reward             | 2.3080251 |\n",
            "|    std                | 0.992     |\n",
            "|    value_loss         | 70.3      |\n",
            "-------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 100       |\n",
            "|    iterations         | 1000      |\n",
            "|    time_elapsed       | 49        |\n",
            "|    total_timesteps    | 5000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41       |\n",
            "|    explained_variance | 0         |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 999       |\n",
            "|    policy_loss        | -108      |\n",
            "|    reward             | 1.1761196 |\n",
            "|    std                | 0.994     |\n",
            "|    value_loss         | 10.8      |\n",
            "-------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 100       |\n",
            "|    iterations         | 1100      |\n",
            "|    time_elapsed       | 54        |\n",
            "|    total_timesteps    | 5500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41       |\n",
            "|    explained_variance | 0         |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1099      |\n",
            "|    policy_loss        | -117      |\n",
            "|    reward             | 3.2501192 |\n",
            "|    std                | 0.995     |\n",
            "|    value_loss         | 10.7      |\n",
            "-------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 100       |\n",
            "|    iterations         | 1200      |\n",
            "|    time_elapsed       | 59        |\n",
            "|    total_timesteps    | 6000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41       |\n",
            "|    explained_variance | -1.19e-07 |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1199      |\n",
            "|    policy_loss        | 362       |\n",
            "|    reward             | 6.932239  |\n",
            "|    std                | 0.994     |\n",
            "|    value_loss         | 84.6      |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 100        |\n",
            "|    iterations         | 1300       |\n",
            "|    time_elapsed       | 64         |\n",
            "|    total_timesteps    | 6500       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41        |\n",
            "|    explained_variance | 0          |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 1299       |\n",
            "|    policy_loss        | 44.8       |\n",
            "|    reward             | -1.5383602 |\n",
            "|    std                | 0.994      |\n",
            "|    value_loss         | 2.08       |\n",
            "--------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 100       |\n",
            "|    iterations         | 1400      |\n",
            "|    time_elapsed       | 69        |\n",
            "|    total_timesteps    | 7000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41       |\n",
            "|    explained_variance | 0         |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1399      |\n",
            "|    policy_loss        | -143      |\n",
            "|    reward             | 0.7461217 |\n",
            "|    std                | 0.994     |\n",
            "|    value_loss         | 17        |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 100        |\n",
            "|    iterations         | 1500       |\n",
            "|    time_elapsed       | 74         |\n",
            "|    total_timesteps    | 7500       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41        |\n",
            "|    explained_variance | 0          |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 1499       |\n",
            "|    policy_loss        | -145       |\n",
            "|    reward             | -1.6722031 |\n",
            "|    std                | 0.995      |\n",
            "|    value_loss         | 14.8       |\n",
            "--------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 100       |\n",
            "|    iterations         | 1600      |\n",
            "|    time_elapsed       | 79        |\n",
            "|    total_timesteps    | 8000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41       |\n",
            "|    explained_variance | 1.19e-07  |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1599      |\n",
            "|    policy_loss        | 4.35      |\n",
            "|    reward             | 3.1992574 |\n",
            "|    std                | 0.995     |\n",
            "|    value_loss         | 0.916     |\n",
            "-------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 100       |\n",
            "|    iterations         | 1700      |\n",
            "|    time_elapsed       | 84        |\n",
            "|    total_timesteps    | 8500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -40.9     |\n",
            "|    explained_variance | -0.00566  |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1699      |\n",
            "|    policy_loss        | -1.16e+03 |\n",
            "|    reward             | 11.774156 |\n",
            "|    std                | 0.993     |\n",
            "|    value_loss         | 936       |\n",
            "-------------------------------------\n",
            "------------------------------------\n",
            "| time/                 |          |\n",
            "|    fps                | 100      |\n",
            "|    iterations         | 1800     |\n",
            "|    time_elapsed       | 89       |\n",
            "|    total_timesteps    | 9000     |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -40.9    |\n",
            "|    explained_variance | -0.00464 |\n",
            "|    learning_rate      | 0.0007   |\n",
            "|    n_updates          | 1799     |\n",
            "|    policy_loss        | -185     |\n",
            "|    reward             | 4.000574 |\n",
            "|    std                | 0.993    |\n",
            "|    value_loss         | 41.6     |\n",
            "------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 100        |\n",
            "|    iterations         | 1900       |\n",
            "|    time_elapsed       | 94         |\n",
            "|    total_timesteps    | 9500       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -40.9      |\n",
            "|    explained_variance | 0          |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 1899       |\n",
            "|    policy_loss        | 12.7       |\n",
            "|    reward             | -1.3208354 |\n",
            "|    std                | 0.991      |\n",
            "|    value_loss         | 0.306      |\n",
            "--------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 100       |\n",
            "|    iterations         | 2000      |\n",
            "|    time_elapsed       | 99        |\n",
            "|    total_timesteps    | 10000     |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -40.9     |\n",
            "|    explained_variance | 0         |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1999      |\n",
            "|    policy_loss        | -10.6     |\n",
            "|    reward             | 0.7043803 |\n",
            "|    std                | 0.992     |\n",
            "|    value_loss         | 0.431     |\n",
            "-------------------------------------\n",
            "======A2C Validation from:  2021-04-06 to  2021-07-06\n",
            "A2C Sharpe Ratio:  0.23121777505990648\n",
            "======PPO Training========\n",
            "{'ent_coef': 0.01, 'n_steps': 2048, 'learning_rate': 0.00025, 'batch_size': 128}\n",
            "Using cuda device\n",
            "Logging to tensorboard_log/ppo/ppo_189_1\n",
            "----------------------------------\n",
            "| time/              |           |\n",
            "|    fps             | 104       |\n",
            "|    iterations      | 1         |\n",
            "|    time_elapsed    | 19        |\n",
            "|    total_timesteps | 2048      |\n",
            "| train/             |           |\n",
            "|    reward          | 1.1126131 |\n",
            "----------------------------------\n",
            "day: 3022, episode: 5\n",
            "begin_total_asset: 1000000.00\n",
            "end_total_asset: 3636477.12\n",
            "total_reward: 2636477.12\n",
            "total_cost: 386575.15\n",
            "total_trades: 84625\n",
            "Sharpe: 0.731\n",
            "=================================\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 105         |\n",
            "|    iterations           | 2           |\n",
            "|    time_elapsed         | 38          |\n",
            "|    total_timesteps      | 4096        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.015871149 |\n",
            "|    clip_fraction        | 0.234       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.2       |\n",
            "|    explained_variance   | -0.0129     |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 3.2         |\n",
            "|    n_updates            | 10          |\n",
            "|    policy_gradient_loss | -0.0282     |\n",
            "|    reward               | 0.88015693  |\n",
            "|    std                  | 1           |\n",
            "|    value_loss           | 10.1        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 105         |\n",
            "|    iterations           | 3           |\n",
            "|    time_elapsed         | 58          |\n",
            "|    total_timesteps      | 6144        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.019049045 |\n",
            "|    clip_fraction        | 0.166       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.2       |\n",
            "|    explained_variance   | 0.00262     |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 32.1        |\n",
            "|    n_updates            | 20          |\n",
            "|    policy_gradient_loss | -0.023      |\n",
            "|    reward               | 0.49640504  |\n",
            "|    std                  | 1           |\n",
            "|    value_loss           | 53.7        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 105         |\n",
            "|    iterations           | 4           |\n",
            "|    time_elapsed         | 77          |\n",
            "|    total_timesteps      | 8192        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.015561214 |\n",
            "|    clip_fraction        | 0.161       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.2       |\n",
            "|    explained_variance   | -0.0133     |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 31.9        |\n",
            "|    n_updates            | 30          |\n",
            "|    policy_gradient_loss | -0.0201     |\n",
            "|    reward               | -0.7624933  |\n",
            "|    std                  | 1           |\n",
            "|    value_loss           | 45.6        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 105         |\n",
            "|    iterations           | 5           |\n",
            "|    time_elapsed         | 96          |\n",
            "|    total_timesteps      | 10240       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.014756277 |\n",
            "|    clip_fraction        | 0.141       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.2       |\n",
            "|    explained_variance   | -0.00517    |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 6.56        |\n",
            "|    n_updates            | 40          |\n",
            "|    policy_gradient_loss | -0.0204     |\n",
            "|    reward               | 1.2296445   |\n",
            "|    std                  | 1           |\n",
            "|    value_loss           | 14.8        |\n",
            "-----------------------------------------\n",
            "======PPO Validation from:  2021-04-06 to  2021-07-06\n",
            "PPO Sharpe Ratio:  0.010011405217654536\n",
            "======DDPG Training========\n",
            "{'buffer_size': 10000, 'learning_rate': 0.0005, 'batch_size': 64}\n",
            "Using cuda device\n",
            "Logging to tensorboard_log/ddpg/ddpg_189_1\n",
            "day: 3022, episode: 10\n",
            "begin_total_asset: 1000000.00\n",
            "end_total_asset: 5009064.34\n",
            "total_reward: 4009064.34\n",
            "total_cost: 1282.47\n",
            "total_trades: 51375\n",
            "Sharpe: 0.880\n",
            "=================================\n",
            "---------------------------------\n",
            "| time/              |          |\n",
            "|    episodes        | 4        |\n",
            "|    fps             | 76       |\n",
            "|    time_elapsed    | 157      |\n",
            "|    total_timesteps | 12092    |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -65.5    |\n",
            "|    critic_loss     | 3.14e+03 |\n",
            "|    learning_rate   | 0.0005   |\n",
            "|    n_updates       | 9069     |\n",
            "|    reward          | 8.301459 |\n",
            "---------------------------------\n",
            "======DDPG Validation from:  2021-04-06 to  2021-07-06\n",
            "======Best Model Retraining from:  2009-04-01 to  2021-07-06\n",
            "======Trading from:  2021-07-06 to  2021-10-04\n",
            "============================================\n",
            "turbulence_threshold:  203.40117769841157\n",
            "======Model training from:  2009-04-01 to  2021-07-06\n",
            "======A2C Training========\n",
            "{'n_steps': 5, 'ent_coef': 0.005, 'learning_rate': 0.0007}\n",
            "Using cuda device\n",
            "Logging to tensorboard_log/a2c/a2c_252_1\n",
            "---------------------------------------\n",
            "| time/                 |             |\n",
            "|    fps                | 98          |\n",
            "|    iterations         | 100         |\n",
            "|    time_elapsed       | 5           |\n",
            "|    total_timesteps    | 500         |\n",
            "| train/                |             |\n",
            "|    entropy_loss       | -41.1       |\n",
            "|    explained_variance | -0.0351     |\n",
            "|    learning_rate      | 0.0007      |\n",
            "|    n_updates          | 99          |\n",
            "|    policy_loss        | -28.6       |\n",
            "|    reward             | -0.15436654 |\n",
            "|    std                | 0.998       |\n",
            "|    value_loss         | 5.15        |\n",
            "---------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 97        |\n",
            "|    iterations         | 200       |\n",
            "|    time_elapsed       | 10        |\n",
            "|    total_timesteps    | 1000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41       |\n",
            "|    explained_variance | 0         |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 199       |\n",
            "|    policy_loss        | 50.5      |\n",
            "|    reward             | 1.3676119 |\n",
            "|    std                | 0.996     |\n",
            "|    value_loss         | 2.12      |\n",
            "-------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 98        |\n",
            "|    iterations         | 300       |\n",
            "|    time_elapsed       | 15        |\n",
            "|    total_timesteps    | 1500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41       |\n",
            "|    explained_variance | 0         |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 299       |\n",
            "|    policy_loss        | -179      |\n",
            "|    reward             | 5.1952586 |\n",
            "|    std                | 0.995     |\n",
            "|    value_loss         | 22.2      |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 400        |\n",
            "|    time_elapsed       | 20         |\n",
            "|    total_timesteps    | 2000       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41        |\n",
            "|    explained_variance | 0          |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 399        |\n",
            "|    policy_loss        | 79.1       |\n",
            "|    reward             | -0.3978011 |\n",
            "|    std                | 0.995      |\n",
            "|    value_loss         | 7.82       |\n",
            "--------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 500        |\n",
            "|    time_elapsed       | 25         |\n",
            "|    total_timesteps    | 2500       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -40.9      |\n",
            "|    explained_variance | 0.19       |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 499        |\n",
            "|    policy_loss        | -8.81      |\n",
            "|    reward             | 0.19138733 |\n",
            "|    std                | 0.993      |\n",
            "|    value_loss         | 6.38       |\n",
            "--------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 600        |\n",
            "|    time_elapsed       | 30         |\n",
            "|    total_timesteps    | 3000       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41        |\n",
            "|    explained_variance | -0.00925   |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 599        |\n",
            "|    policy_loss        | -498       |\n",
            "|    reward             | -11.956305 |\n",
            "|    std                | 0.996      |\n",
            "|    value_loss         | 222        |\n",
            "--------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 700        |\n",
            "|    time_elapsed       | 35         |\n",
            "|    total_timesteps    | 3500       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41        |\n",
            "|    explained_variance | -0.169     |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 699        |\n",
            "|    policy_loss        | -68.7      |\n",
            "|    reward             | 0.39965674 |\n",
            "|    std                | 0.996      |\n",
            "|    value_loss         | 3.86       |\n",
            "--------------------------------------\n",
            "---------------------------------------\n",
            "| time/                 |             |\n",
            "|    fps                | 98          |\n",
            "|    iterations         | 800         |\n",
            "|    time_elapsed       | 40          |\n",
            "|    total_timesteps    | 4000        |\n",
            "| train/                |             |\n",
            "|    entropy_loss       | -41         |\n",
            "|    explained_variance | -0.0865     |\n",
            "|    learning_rate      | 0.0007      |\n",
            "|    n_updates          | 799         |\n",
            "|    policy_loss        | -164        |\n",
            "|    reward             | -0.33571205 |\n",
            "|    std                | 0.996       |\n",
            "|    value_loss         | 16.1        |\n",
            "---------------------------------------\n",
            "---------------------------------------\n",
            "| time/                 |             |\n",
            "|    fps                | 98          |\n",
            "|    iterations         | 900         |\n",
            "|    time_elapsed       | 45          |\n",
            "|    total_timesteps    | 4500        |\n",
            "| train/                |             |\n",
            "|    entropy_loss       | -41         |\n",
            "|    explained_variance | 0.159       |\n",
            "|    learning_rate      | 0.0007      |\n",
            "|    n_updates          | 899         |\n",
            "|    policy_loss        | 46.5        |\n",
            "|    reward             | -0.19355251 |\n",
            "|    std                | 0.996       |\n",
            "|    value_loss         | 1.99        |\n",
            "---------------------------------------\n",
            "---------------------------------------\n",
            "| time/                 |             |\n",
            "|    fps                | 98          |\n",
            "|    iterations         | 1000        |\n",
            "|    time_elapsed       | 50          |\n",
            "|    total_timesteps    | 5000        |\n",
            "| train/                |             |\n",
            "|    entropy_loss       | -41.1       |\n",
            "|    explained_variance | 0.0618      |\n",
            "|    learning_rate      | 0.0007      |\n",
            "|    n_updates          | 999         |\n",
            "|    policy_loss        | -15.1       |\n",
            "|    reward             | -0.48996487 |\n",
            "|    std                | 0.997       |\n",
            "|    value_loss         | 0.677       |\n",
            "---------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 98        |\n",
            "|    iterations         | 1100      |\n",
            "|    time_elapsed       | 55        |\n",
            "|    total_timesteps    | 5500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41       |\n",
            "|    explained_variance | -0.192    |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1099      |\n",
            "|    policy_loss        | -233      |\n",
            "|    reward             | 1.1524396 |\n",
            "|    std                | 0.997     |\n",
            "|    value_loss         | 35.2      |\n",
            "-------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 98        |\n",
            "|    iterations         | 1200      |\n",
            "|    time_elapsed       | 60        |\n",
            "|    total_timesteps    | 6000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.1     |\n",
            "|    explained_variance | -0.214    |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1199      |\n",
            "|    policy_loss        | 14.9      |\n",
            "|    reward             | -2.838138 |\n",
            "|    std                | 0.998     |\n",
            "|    value_loss         | 3.95      |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 1300       |\n",
            "|    time_elapsed       | 65         |\n",
            "|    total_timesteps    | 6500       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.2      |\n",
            "|    explained_variance | 0.167      |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 1299       |\n",
            "|    policy_loss        | 43.7       |\n",
            "|    reward             | -1.0078373 |\n",
            "|    std                | 1          |\n",
            "|    value_loss         | 1.86       |\n",
            "--------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 98        |\n",
            "|    iterations         | 1400      |\n",
            "|    time_elapsed       | 70        |\n",
            "|    total_timesteps    | 7000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.2     |\n",
            "|    explained_variance | -0.0727   |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1399      |\n",
            "|    policy_loss        | -17.9     |\n",
            "|    reward             | 2.1590974 |\n",
            "|    std                | 1         |\n",
            "|    value_loss         | 1.71      |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 1500       |\n",
            "|    time_elapsed       | 75         |\n",
            "|    total_timesteps    | 7500       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.2      |\n",
            "|    explained_variance | 0          |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 1499       |\n",
            "|    policy_loss        | 42.5       |\n",
            "|    reward             | 0.47163484 |\n",
            "|    std                | 1          |\n",
            "|    value_loss         | 1.17       |\n",
            "--------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 98        |\n",
            "|    iterations         | 1600      |\n",
            "|    time_elapsed       | 80        |\n",
            "|    total_timesteps    | 8000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.2     |\n",
            "|    explained_variance | 0.000262  |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1599      |\n",
            "|    policy_loss        | -73.2     |\n",
            "|    reward             | 0.8027305 |\n",
            "|    std                | 1         |\n",
            "|    value_loss         | 19.5      |\n",
            "-------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 98        |\n",
            "|    iterations         | 1700      |\n",
            "|    time_elapsed       | 85        |\n",
            "|    total_timesteps    | 8500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.2     |\n",
            "|    explained_variance | 0         |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1699      |\n",
            "|    policy_loss        | -184      |\n",
            "|    reward             | 0.3686392 |\n",
            "|    std                | 1         |\n",
            "|    value_loss         | 29.3      |\n",
            "-------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 98        |\n",
            "|    iterations         | 1800      |\n",
            "|    time_elapsed       | 90        |\n",
            "|    total_timesteps    | 9000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.2     |\n",
            "|    explained_variance | -1.19e-07 |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1799      |\n",
            "|    policy_loss        | 116       |\n",
            "|    reward             | 1.6162095 |\n",
            "|    std                | 1         |\n",
            "|    value_loss         | 21.6      |\n",
            "-------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 98        |\n",
            "|    iterations         | 1900      |\n",
            "|    time_elapsed       | 96        |\n",
            "|    total_timesteps    | 9500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.2     |\n",
            "|    explained_variance | -0.00951  |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1899      |\n",
            "|    policy_loss        | -1.74     |\n",
            "|    reward             | 1.1433389 |\n",
            "|    std                | 1         |\n",
            "|    value_loss         | 0.195     |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 2000       |\n",
            "|    time_elapsed       | 101        |\n",
            "|    total_timesteps    | 10000      |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.2      |\n",
            "|    explained_variance | -0.0113    |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 1999       |\n",
            "|    policy_loss        | -38.9      |\n",
            "|    reward             | 0.97623837 |\n",
            "|    std                | 1          |\n",
            "|    value_loss         | 0.806      |\n",
            "--------------------------------------\n",
            "======A2C Validation from:  2021-07-06 to  2021-10-04\n",
            "A2C Sharpe Ratio:  -0.0528373614471615\n",
            "======PPO Training========\n",
            "{'ent_coef': 0.01, 'n_steps': 2048, 'learning_rate': 0.00025, 'batch_size': 128}\n",
            "Using cuda device\n",
            "Logging to tensorboard_log/ppo/ppo_252_1\n",
            "----------------------------------\n",
            "| time/              |           |\n",
            "|    fps             | 101       |\n",
            "|    iterations      | 1         |\n",
            "|    time_elapsed    | 20        |\n",
            "|    total_timesteps | 2048      |\n",
            "| train/             |           |\n",
            "|    reward          | 1.2801732 |\n",
            "----------------------------------\n",
            "day: 3085, episode: 5\n",
            "begin_total_asset: 1000000.00\n",
            "end_total_asset: 4550937.53\n",
            "total_reward: 3550937.53\n",
            "total_cost: 408098.28\n",
            "total_trades: 86397\n",
            "Sharpe: 0.873\n",
            "=================================\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 102         |\n",
            "|    iterations           | 2           |\n",
            "|    time_elapsed         | 39          |\n",
            "|    total_timesteps      | 4096        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.018978704 |\n",
            "|    clip_fraction        | 0.233       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.2       |\n",
            "|    explained_variance   | -0.00994    |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 5.48        |\n",
            "|    n_updates            | 10          |\n",
            "|    policy_gradient_loss | -0.0284     |\n",
            "|    reward               | 0.49718028  |\n",
            "|    std                  | 1           |\n",
            "|    value_loss           | 10.8        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 103         |\n",
            "|    iterations           | 3           |\n",
            "|    time_elapsed         | 59          |\n",
            "|    total_timesteps      | 6144        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.012346724 |\n",
            "|    clip_fraction        | 0.155       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.2       |\n",
            "|    explained_variance   | -0.00278    |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 19.6        |\n",
            "|    n_updates            | 20          |\n",
            "|    policy_gradient_loss | -0.0154     |\n",
            "|    reward               | -0.8982904  |\n",
            "|    std                  | 1           |\n",
            "|    value_loss           | 50.6        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 103         |\n",
            "|    iterations           | 4           |\n",
            "|    time_elapsed         | 78          |\n",
            "|    total_timesteps      | 8192        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.014507987 |\n",
            "|    clip_fraction        | 0.123       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.2       |\n",
            "|    explained_variance   | -0.012      |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 43.8        |\n",
            "|    n_updates            | 30          |\n",
            "|    policy_gradient_loss | -0.016      |\n",
            "|    reward               | -0.0212465  |\n",
            "|    std                  | 1           |\n",
            "|    value_loss           | 74.7        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 104         |\n",
            "|    iterations           | 5           |\n",
            "|    time_elapsed         | 98          |\n",
            "|    total_timesteps      | 10240       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.016703494 |\n",
            "|    clip_fraction        | 0.186       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.3       |\n",
            "|    explained_variance   | -0.0496     |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 4.09        |\n",
            "|    n_updates            | 40          |\n",
            "|    policy_gradient_loss | -0.0285     |\n",
            "|    reward               | 1.0109037   |\n",
            "|    std                  | 1.01        |\n",
            "|    value_loss           | 14.2        |\n",
            "-----------------------------------------\n",
            "======PPO Validation from:  2021-07-06 to  2021-10-04\n",
            "PPO Sharpe Ratio:  -0.05262682094979255\n",
            "======DDPG Training========\n",
            "{'buffer_size': 10000, 'learning_rate': 0.0005, 'batch_size': 64}\n",
            "Using cuda device\n",
            "Logging to tensorboard_log/ddpg/ddpg_252_1\n",
            "day: 3085, episode: 10\n",
            "begin_total_asset: 1000000.00\n",
            "end_total_asset: 4501430.61\n",
            "total_reward: 3501430.61\n",
            "total_cost: 1082.97\n",
            "total_trades: 61741\n",
            "Sharpe: 0.737\n",
            "=================================\n",
            "----------------------------------\n",
            "| time/              |           |\n",
            "|    episodes        | 4         |\n",
            "|    fps             | 76        |\n",
            "|    time_elapsed    | 161       |\n",
            "|    total_timesteps | 12344     |\n",
            "| train/             |           |\n",
            "|    actor_loss      | -3.66     |\n",
            "|    critic_loss     | 176       |\n",
            "|    learning_rate   | 0.0005    |\n",
            "|    n_updates       | 9258      |\n",
            "|    reward          | 2.1645079 |\n",
            "----------------------------------\n",
            "======DDPG Validation from:  2021-07-06 to  2021-10-04\n",
            "======Best Model Retraining from:  2009-04-01 to  2021-10-04\n",
            "======Trading from:  2021-10-04 to  2022-01-03\n",
            "============================================\n",
            "turbulence_threshold:  203.40117769841157\n",
            "======Model training from:  2009-04-01 to  2021-10-04\n",
            "======A2C Training========\n",
            "{'n_steps': 5, 'ent_coef': 0.005, 'learning_rate': 0.0007}\n",
            "Using cuda device\n",
            "Logging to tensorboard_log/a2c/a2c_315_1\n",
            "----------------------------------------\n",
            "| time/                 |              |\n",
            "|    fps                | 98           |\n",
            "|    iterations         | 100          |\n",
            "|    time_elapsed       | 5            |\n",
            "|    total_timesteps    | 500          |\n",
            "| train/                |              |\n",
            "|    entropy_loss       | -41.3        |\n",
            "|    explained_variance | 0            |\n",
            "|    learning_rate      | 0.0007       |\n",
            "|    n_updates          | 99           |\n",
            "|    policy_loss        | -51.2        |\n",
            "|    reward             | -0.096114956 |\n",
            "|    std                | 1.01         |\n",
            "|    value_loss         | 4.56         |\n",
            "----------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 98        |\n",
            "|    iterations         | 200       |\n",
            "|    time_elapsed       | 10        |\n",
            "|    total_timesteps    | 1000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.3     |\n",
            "|    explained_variance | 5.96e-08  |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 199       |\n",
            "|    policy_loss        | -36.9     |\n",
            "|    reward             | 2.9646184 |\n",
            "|    std                | 1         |\n",
            "|    value_loss         | 0.756     |\n",
            "-------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 97        |\n",
            "|    iterations         | 300       |\n",
            "|    time_elapsed       | 15        |\n",
            "|    total_timesteps    | 1500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.3     |\n",
            "|    explained_variance | 0         |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 299       |\n",
            "|    policy_loss        | -236      |\n",
            "|    reward             | 1.1201329 |\n",
            "|    std                | 1.01      |\n",
            "|    value_loss         | 36.4      |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 400        |\n",
            "|    time_elapsed       | 20         |\n",
            "|    total_timesteps    | 2000       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.4      |\n",
            "|    explained_variance | 0          |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 399        |\n",
            "|    policy_loss        | -53.9      |\n",
            "|    reward             | 0.03644575 |\n",
            "|    std                | 1.01       |\n",
            "|    value_loss         | 2.41       |\n",
            "--------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 98        |\n",
            "|    iterations         | 500       |\n",
            "|    time_elapsed       | 25        |\n",
            "|    total_timesteps    | 2500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.3     |\n",
            "|    explained_variance | 0         |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 499       |\n",
            "|    policy_loss        | 34.1      |\n",
            "|    reward             | 0.3948357 |\n",
            "|    std                | 1         |\n",
            "|    value_loss         | 1.08      |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 600        |\n",
            "|    time_elapsed       | 30         |\n",
            "|    total_timesteps    | 3000       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.3      |\n",
            "|    explained_variance | -1.19e-07  |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 599        |\n",
            "|    policy_loss        | -618       |\n",
            "|    reward             | -12.502208 |\n",
            "|    std                | 1          |\n",
            "|    value_loss         | 338        |\n",
            "--------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 700        |\n",
            "|    time_elapsed       | 35         |\n",
            "|    total_timesteps    | 3500       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.2      |\n",
            "|    explained_variance | -0.0164    |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 699        |\n",
            "|    policy_loss        | -105       |\n",
            "|    reward             | 0.30092302 |\n",
            "|    std                | 1          |\n",
            "|    value_loss         | 10.8       |\n",
            "--------------------------------------\n",
            "---------------------------------------\n",
            "| time/                 |             |\n",
            "|    fps                | 98          |\n",
            "|    iterations         | 800         |\n",
            "|    time_elapsed       | 40          |\n",
            "|    total_timesteps    | 4000        |\n",
            "| train/                |             |\n",
            "|    entropy_loss       | -41.3       |\n",
            "|    explained_variance | 1.19e-07    |\n",
            "|    learning_rate      | 0.0007      |\n",
            "|    n_updates          | 799         |\n",
            "|    policy_loss        | 74.6        |\n",
            "|    reward             | 0.009111182 |\n",
            "|    std                | 1           |\n",
            "|    value_loss         | 4.73        |\n",
            "---------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 98        |\n",
            "|    iterations         | 900       |\n",
            "|    time_elapsed       | 45        |\n",
            "|    total_timesteps    | 4500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.3     |\n",
            "|    explained_variance | -0.0124   |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 899       |\n",
            "|    policy_loss        | -131      |\n",
            "|    reward             | 2.0478728 |\n",
            "|    std                | 1         |\n",
            "|    value_loss         | 16.5      |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 1000       |\n",
            "|    time_elapsed       | 50         |\n",
            "|    total_timesteps    | 5000       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.3      |\n",
            "|    explained_variance | 0          |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 999        |\n",
            "|    policy_loss        | -34.9      |\n",
            "|    reward             | 0.04952446 |\n",
            "|    std                | 1.01       |\n",
            "|    value_loss         | 2.24       |\n",
            "--------------------------------------\n",
            "---------------------------------------\n",
            "| time/                 |             |\n",
            "|    fps                | 98          |\n",
            "|    iterations         | 1100        |\n",
            "|    time_elapsed       | 55          |\n",
            "|    total_timesteps    | 5500        |\n",
            "| train/                |             |\n",
            "|    entropy_loss       | -41.3       |\n",
            "|    explained_variance | -0.0188     |\n",
            "|    learning_rate      | 0.0007      |\n",
            "|    n_updates          | 1099        |\n",
            "|    policy_loss        | 344         |\n",
            "|    reward             | -0.11844914 |\n",
            "|    std                | 1.01        |\n",
            "|    value_loss         | 64.7        |\n",
            "---------------------------------------\n",
            "---------------------------------------\n",
            "| time/                 |             |\n",
            "|    fps                | 98          |\n",
            "|    iterations         | 1200        |\n",
            "|    time_elapsed       | 60          |\n",
            "|    total_timesteps    | 6000        |\n",
            "| train/                |             |\n",
            "|    entropy_loss       | -41.4       |\n",
            "|    explained_variance | 0           |\n",
            "|    learning_rate      | 0.0007      |\n",
            "|    n_updates          | 1199        |\n",
            "|    policy_loss        | 58.9        |\n",
            "|    reward             | -0.28185675 |\n",
            "|    std                | 1.01        |\n",
            "|    value_loss         | 6.99        |\n",
            "---------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 1300       |\n",
            "|    time_elapsed       | 66         |\n",
            "|    total_timesteps    | 6500       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.4      |\n",
            "|    explained_variance | 0.549      |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 1299       |\n",
            "|    policy_loss        | -11.7      |\n",
            "|    reward             | -0.7957832 |\n",
            "|    std                | 1.01       |\n",
            "|    value_loss         | 0.135      |\n",
            "--------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 1400       |\n",
            "|    time_elapsed       | 71         |\n",
            "|    total_timesteps    | 7000       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.4      |\n",
            "|    explained_variance | 1.19e-07   |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 1399       |\n",
            "|    policy_loss        | -29.7      |\n",
            "|    reward             | 0.16117185 |\n",
            "|    std                | 1.01       |\n",
            "|    value_loss         | 0.859      |\n",
            "--------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 1500       |\n",
            "|    time_elapsed       | 76         |\n",
            "|    total_timesteps    | 7500       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.4      |\n",
            "|    explained_variance | 1.79e-07   |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 1499       |\n",
            "|    policy_loss        | 15.7       |\n",
            "|    reward             | 0.17504984 |\n",
            "|    std                | 1.01       |\n",
            "|    value_loss         | 0.528      |\n",
            "--------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 98        |\n",
            "|    iterations         | 1600      |\n",
            "|    time_elapsed       | 81        |\n",
            "|    total_timesteps    | 8000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.4     |\n",
            "|    explained_variance | 0         |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1599      |\n",
            "|    policy_loss        | 48.7      |\n",
            "|    reward             | 3.1540215 |\n",
            "|    std                | 1.01      |\n",
            "|    value_loss         | 6.42      |\n",
            "-------------------------------------\n",
            "---------------------------------------\n",
            "| time/                 |             |\n",
            "|    fps                | 98          |\n",
            "|    iterations         | 1700        |\n",
            "|    time_elapsed       | 86          |\n",
            "|    total_timesteps    | 8500        |\n",
            "| train/                |             |\n",
            "|    entropy_loss       | -41.4       |\n",
            "|    explained_variance | 0           |\n",
            "|    learning_rate      | 0.0007      |\n",
            "|    n_updates          | 1699        |\n",
            "|    policy_loss        | 22.9        |\n",
            "|    reward             | -0.09754947 |\n",
            "|    std                | 1.01        |\n",
            "|    value_loss         | 0.677       |\n",
            "---------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 98        |\n",
            "|    iterations         | 1800      |\n",
            "|    time_elapsed       | 91        |\n",
            "|    total_timesteps    | 9000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.4     |\n",
            "|    explained_variance | -0.171    |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1799      |\n",
            "|    policy_loss        | 99        |\n",
            "|    reward             | -1.175583 |\n",
            "|    std                | 1.01      |\n",
            "|    value_loss         | 9.98      |\n",
            "-------------------------------------\n",
            "------------------------------------\n",
            "| time/                 |          |\n",
            "|    fps                | 98       |\n",
            "|    iterations         | 1900     |\n",
            "|    time_elapsed       | 96       |\n",
            "|    total_timesteps    | 9500     |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -41.4    |\n",
            "|    explained_variance | 0        |\n",
            "|    learning_rate      | 0.0007   |\n",
            "|    n_updates          | 1899     |\n",
            "|    policy_loss        | -24.6    |\n",
            "|    reward             | 0.362812 |\n",
            "|    std                | 1.01     |\n",
            "|    value_loss         | 0.367    |\n",
            "------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 2000       |\n",
            "|    time_elapsed       | 101        |\n",
            "|    total_timesteps    | 10000      |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.4      |\n",
            "|    explained_variance | 0          |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 1999       |\n",
            "|    policy_loss        | -146       |\n",
            "|    reward             | -1.7740997 |\n",
            "|    std                | 1.01       |\n",
            "|    value_loss         | 13.5       |\n",
            "--------------------------------------\n",
            "======A2C Validation from:  2021-10-04 to  2022-01-03\n",
            "A2C Sharpe Ratio:  0.3744625013790845\n",
            "======PPO Training========\n",
            "{'ent_coef': 0.01, 'n_steps': 2048, 'learning_rate': 0.00025, 'batch_size': 128}\n",
            "Using cuda device\n",
            "Logging to tensorboard_log/ppo/ppo_315_1\n",
            "----------------------------------\n",
            "| time/              |           |\n",
            "|    fps             | 106       |\n",
            "|    iterations      | 1         |\n",
            "|    time_elapsed    | 19        |\n",
            "|    total_timesteps | 2048      |\n",
            "| train/             |           |\n",
            "|    reward          | 0.7561091 |\n",
            "----------------------------------\n",
            "day: 3148, episode: 5\n",
            "begin_total_asset: 1000000.00\n",
            "end_total_asset: 4259003.47\n",
            "total_reward: 3259003.47\n",
            "total_cost: 426068.70\n",
            "total_trades: 87901\n",
            "Sharpe: 0.790\n",
            "=================================\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 101         |\n",
            "|    iterations           | 2           |\n",
            "|    time_elapsed         | 40          |\n",
            "|    total_timesteps      | 4096        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.016951367 |\n",
            "|    clip_fraction        | 0.229       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.2       |\n",
            "|    explained_variance   | -0.000697   |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 5.25        |\n",
            "|    n_updates            | 10          |\n",
            "|    policy_gradient_loss | -0.0307     |\n",
            "|    reward               | 0.9373541   |\n",
            "|    std                  | 1           |\n",
            "|    value_loss           | 9.91        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 102         |\n",
            "|    iterations           | 3           |\n",
            "|    time_elapsed         | 60          |\n",
            "|    total_timesteps      | 6144        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.017283382 |\n",
            "|    clip_fraction        | 0.176       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.3       |\n",
            "|    explained_variance   | 0.00633     |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 33.4        |\n",
            "|    n_updates            | 20          |\n",
            "|    policy_gradient_loss | -0.0207     |\n",
            "|    reward               | 7.712371    |\n",
            "|    std                  | 1           |\n",
            "|    value_loss           | 56.7        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 102         |\n",
            "|    iterations           | 4           |\n",
            "|    time_elapsed         | 79          |\n",
            "|    total_timesteps      | 8192        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.010827523 |\n",
            "|    clip_fraction        | 0.0962      |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.3       |\n",
            "|    explained_variance   | 0.00682     |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 73.4        |\n",
            "|    n_updates            | 30          |\n",
            "|    policy_gradient_loss | -0.018      |\n",
            "|    reward               | -0.83698237 |\n",
            "|    std                  | 1.01        |\n",
            "|    value_loss           | 134         |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 102         |\n",
            "|    iterations           | 5           |\n",
            "|    time_elapsed         | 99          |\n",
            "|    total_timesteps      | 10240       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.013603406 |\n",
            "|    clip_fraction        | 0.172       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.4       |\n",
            "|    explained_variance   | 0.0243      |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 4.64        |\n",
            "|    n_updates            | 40          |\n",
            "|    policy_gradient_loss | -0.0243     |\n",
            "|    reward               | 0.5904208   |\n",
            "|    std                  | 1.01        |\n",
            "|    value_loss           | 12.7        |\n",
            "-----------------------------------------\n",
            "======PPO Validation from:  2021-10-04 to  2022-01-03\n",
            "PPO Sharpe Ratio:  0.0593551480877188\n",
            "======DDPG Training========\n",
            "{'buffer_size': 10000, 'learning_rate': 0.0005, 'batch_size': 64}\n",
            "Using cuda device\n",
            "Logging to tensorboard_log/ddpg/ddpg_315_1\n",
            "day: 3148, episode: 10\n",
            "begin_total_asset: 1000000.00\n",
            "end_total_asset: 4617783.42\n",
            "total_reward: 3617783.42\n",
            "total_cost: 1110.92\n",
            "total_trades: 49019\n",
            "Sharpe: 0.745\n",
            "=================================\n",
            "----------------------------------\n",
            "| time/              |           |\n",
            "|    episodes        | 4         |\n",
            "|    fps             | 75        |\n",
            "|    time_elapsed    | 167       |\n",
            "|    total_timesteps | 12596     |\n",
            "| train/             |           |\n",
            "|    actor_loss      | 36.1      |\n",
            "|    critic_loss     | 656       |\n",
            "|    learning_rate   | 0.0005    |\n",
            "|    n_updates       | 9447      |\n",
            "|    reward          | 5.4972343 |\n",
            "----------------------------------\n",
            "======DDPG Validation from:  2021-10-04 to  2022-01-03\n",
            "======Best Model Retraining from:  2009-04-01 to  2022-01-03\n",
            "======Trading from:  2022-01-03 to  2022-04-04\n",
            "Ensemble Strategy took:  25.210217595100403  minutes\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "AttributeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-30-4153cda81a48>\u001b[0m in \u001b[0;36m<cell line: 163>\u001b[0;34m()\u001b[0m\n\u001b[1;32m    162\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    163\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0m__name__\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'__main__'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 164\u001b[0;31m     \u001b[0mmain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-30-4153cda81a48>\u001b[0m in \u001b[0;36mmain\u001b[0;34m()\u001b[0m\n\u001b[1;32m    127\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrebalance_window\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mvalidation_window\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0munique_trade_date\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrebalance_window\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    128\u001b[0m         \u001b[0mtemp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'results/account_value_trade_{}_{}.csv'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'ensemble'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 129\u001b[0;31m         \u001b[0mdf_account_value\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdf_account_value\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtemp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mignore_index\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    130\u001b[0m     sharpe = (252 ** 0.5) * df_account_value.account_value.pct_change(\n\u001b[1;32m    131\u001b[0m         1).mean() / df_account_value.account_value.pct_change(1).std()\n",
            "\u001b[0;32m/usr/local/lib/python3.9/site-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36m__getattr__\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m   5987\u001b[0m         ):\n\u001b[1;32m   5988\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 5989\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mobject\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__getattribute__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   5990\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   5991\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__setattr__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mAttributeError\u001b[0m: 'DataFrame' object has no attribute 'append'"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "current_price  = 110\n",
        "avg_price = 130\n",
        "print(( current_price * 0.3 ) + current_price)\n",
        "if current_price > ( avg_price * 0.3  + avg_price)  and avg_price > 0:\n",
        "  #actions[i] = self.total_stockss[i] * -1\n",
        "  #print(\"Updated actions\")\n",
        "  print(\"Current Price\", current_price , \" \", \" avg_price\", \" \", avg_price, \"Profit\", current_price -  avg_price)\n",
        "elif current_price < (avg_price - avg_price * 0.15)  and avg_price > 0:\n",
        "  #actions[i] = self.total_stockss[i] * -1\n",
        "  print(\"Current Price\", current_price , \" \", \" avg_price\", \" \", avg_price, \"loss\", current_price -  avg_price )"
      ],
      "metadata": {
        "id": "R6wjk6Qwg-ar",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "65204f2e-8f07-4ca7-8505-b1dfcad5f769"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "143.0\n",
            "Current Price 110    avg_price   130 loss -20\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "avg_price < ( current_price * 0.3  + current_price)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lZm_y4awCWO9",
        "outputId": "89222edb-bc50-4871-c67d-ee4300ed49db"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 153
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "current_price - current_price * 0.15"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wr3yRoNwCvV7",
        "outputId": "60d7ebbb-13fb-4b17-ece3-b008217d2987"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "93.5"
            ]
          },
          "metadata": {},
          "execution_count": 163
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "JtmhGYiRC07B",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "401210c0-edca-413f-c39b-716e9fedeee3"
      },
      "execution_count": 401,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "False"
            ]
          },
          "metadata": {},
          "execution_count": 401
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "S15p3oh8smm1"
      },
      "execution_count": 401,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "IJO0W8Uxsr2i"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "colab": {
      "name": "Making the most of your colab subscription",
      "provenance": [],
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "accelerator": "GPU",
    "gpuClass": "standard"
  },
  "nbformat": 4,
  "nbformat_minor": 0
}