V10_Goku.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/abdulazeezb/1_SparkFundsInvestmentAnalysis/blob/master/V10_Goku.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Checking GPU Version"
      ],
      "metadata": {
        "id": "NAjscLW7Sk9V"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "gpu_info = !nvidia-smi\n",
        "gpu_info = '\\n'.join(gpu_info)\n",
        "if gpu_info.find('failed') >= 0:\n",
        "  print('Not connected to a GPU')\n",
        "else:\n",
        "  print(gpu_info)"
      ],
      "metadata": {
        "id": "t8nxH9ZvSaY6",
        "outputId": "9b48576b-a448-48a1-fc6c-646e0713a4f1",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Tue May  2 11:18:21 2023       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 525.85.12    Driver Version: 525.85.12    CUDA Version: 12.0     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla T4            Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   36C    P8     9W /  70W |      0MiB / 15360MiB |      0%      Default |\n",
            "|                               |                      |                  N/A |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from psutil import virtual_memory\n",
        "ram_gb = virtual_memory().total / 1e9\n",
        "print('Your runtime has {:.1f} gigabytes of available RAM\\n'.format(ram_gb))\n",
        "\n",
        "if ram_gb < 20:\n",
        "  print('Not using a high-RAM runtime')\n",
        "else:\n",
        "  print('You are using a high-RAM runtime!')"
      ],
      "metadata": {
        "id": "KEPzY7qCS2Lc",
        "outputId": "a63d5b82-8246-43b8-d958-df19a3e3bf8e",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Your runtime has 27.3 gigabytes of available RAM\n",
            "\n",
            "You are using a high-RAM runtime!\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QMMqmdiYMkvi"
      },
      "source": [
        "## Part 1: Getting Started - Install Python Packages"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "23TOba33L4qf",
        "outputId": "71444c4a-16f8-42d8-887d-74e43b3ffa99"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting wrds\n",
            "  Downloading wrds-3.1.6-py3-none-any.whl (12 kB)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from wrds) (1.22.4)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from wrds) (1.5.3)\n",
            "Collecting psycopg2-binary (from wrds)\n",
            "  Downloading psycopg2_binary-2.9.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.0/3.0 MB\u001b[0m \u001b[31m52.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from wrds) (1.10.1)\n",
            "Collecting sqlalchemy<2 (from wrds)\n",
            "  Downloading SQLAlchemy-1.4.48-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m99.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.10/dist-packages (from sqlalchemy<2->wrds) (2.0.2)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->wrds) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->wrds) (2022.7.1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas->wrds) (1.16.0)\n",
            "Installing collected packages: sqlalchemy, psycopg2-binary, wrds\n",
            "  Attempting uninstall: sqlalchemy\n",
            "    Found existing installation: SQLAlchemy 2.0.10\n",
            "    Uninstalling SQLAlchemy-2.0.10:\n",
            "      Successfully uninstalled SQLAlchemy-2.0.10\n",
            "Successfully installed psycopg2-binary-2.9.6 sqlalchemy-1.4.48 wrds-3.1.6\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting swig\n",
            "  Downloading swig-4.1.1-py2.py3-none-manylinux_2_5_x86_64.manylinux1_x86_64.whl (1.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.8/1.8 MB\u001b[0m \u001b[31m78.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: swig\n",
            "Successfully installed swig-4.1.1\n",
            "⏬ Downloading https://github.com/conda-forge/miniforge/releases/download/23.1.0-1/Mambaforge-23.1.0-1-Linux-x86_64.sh...\n",
            "📦 Installing...\n",
            "📌 Adjusting configuration...\n",
            "🩹 Patching environment...\n",
            "⏲ Done in 0:00:14\n",
            "🔁 Restarting kernel...\n",
            "Selecting previously unselected package libgl1-mesa-glx:amd64.\n",
            "(Reading database ... 122518 files and directories currently installed.)\n",
            "Preparing to unpack .../libgl1-mesa-glx_21.2.6-0ubuntu0.1~20.04.2_amd64.deb ...\n",
            "Unpacking libgl1-mesa-glx:amd64 (21.2.6-0ubuntu0.1~20.04.2) ...\n",
            "Selecting previously unselected package swig4.0.\n",
            "Preparing to unpack .../swig4.0_4.0.1-5build1_amd64.deb ...\n",
            "Unpacking swig4.0 (4.0.1-5build1) ...\n",
            "Selecting previously unselected package swig.\n",
            "Preparing to unpack .../swig_4.0.1-5build1_all.deb ...\n",
            "Unpacking swig (4.0.1-5build1) ...\n",
            "Setting up libgl1-mesa-glx:amd64 (21.2.6-0ubuntu0.1~20.04.2) ...\n",
            "Setting up swig4.0 (4.0.1-5build1) ...\n",
            "Setting up swig (4.0.1-5build1) ...\n",
            "Processing triggers for man-db (2.9.1-1) ...\n"
          ]
        }
      ],
      "source": [
        "## install finrl library\n",
        "!pip install wrds\n",
        "!pip install swig\n",
        "!pip install -q condacolab\n",
        "import condacolab\n",
        "condacolab.install()\n",
        "!apt-get update -y -qq && apt-get install -y -qq cmake libopenmpi-dev python3-dev zlib1g-dev libgl1-mesa-glx swig"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install git+https://github.com/AI4Finance-Foundation/FinRL.git"
      ],
      "metadata": {
        "id": "haJsmd15TCdH",
        "outputId": "d324905b-602d-4235-ecdd-a218c1ae118c",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting git+https://github.com/AI4Finance-Foundation/FinRL.git\n",
            "  Cloning https://github.com/AI4Finance-Foundation/FinRL.git to /tmp/pip-req-build-6a2bel8j\n",
            "  Running command git clone --filter=blob:none --quiet https://github.com/AI4Finance-Foundation/FinRL.git /tmp/pip-req-build-6a2bel8j\n",
            "  Resolved https://github.com/AI4Finance-Foundation/FinRL.git to commit a3598dae504bcd834d12b17110b5aa91c1c5305d\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2\n",
            "  Cloning https://github.com/quantopian/pyfolio.git to /tmp/pip-install-biljph5i/pyfolio_230a2398b59c4e58bf30b3a6ec5cece9\n",
            "  Running command git clone --filter=blob:none --quiet https://github.com/quantopian/pyfolio.git /tmp/pip-install-biljph5i/pyfolio_230a2398b59c4e58bf30b3a6ec5cece9\n",
            "  Resolved https://github.com/quantopian/pyfolio.git to commit 4b901f6d73aa02ceb6d04b7d83502e5c6f2e81aa\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting elegantrl@ git+https://github.com/AI4Finance-Foundation/ElegantRL.git#egg=elegantrl\n",
            "  Cloning https://github.com/AI4Finance-Foundation/ElegantRL.git to /tmp/pip-install-biljph5i/elegantrl_a9156de8fe9b43948d81e1877e845f12\n",
            "  Running command git clone --filter=blob:none --quiet https://github.com/AI4Finance-Foundation/ElegantRL.git /tmp/pip-install-biljph5i/elegantrl_a9156de8fe9b43948d81e1877e845f12\n",
            "  Resolved https://github.com/AI4Finance-Foundation/ElegantRL.git to commit b974a806e6235f59055c954418e54640fa549331\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting jqdatasdk\n",
            "  Downloading jqdatasdk-1.8.11-py3-none-any.whl (158 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m158.2/158.2 kB\u001b[0m \u001b[31m12.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting matplotlib\n",
            "  Downloading matplotlib-3.7.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (11.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m11.6/11.6 MB\u001b[0m \u001b[31m99.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting importlib-metadata==4.13.0\n",
            "  Downloading importlib_metadata-4.13.0-py3-none-any.whl (23 kB)\n",
            "Collecting gym>=0.17\n",
            "  Downloading gym-0.26.2.tar.gz (721 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m721.7/721.7 kB\u001b[0m \u001b[31m61.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting yfinance\n",
            "  Downloading yfinance-0.2.18-py2.py3-none-any.whl (60 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m60.3/60.3 kB\u001b[0m \u001b[31m8.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting gputil\n",
            "  Downloading GPUtil-1.4.0.tar.gz (5.5 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting numpy>=1.17.3\n",
            "  Downloading numpy-1.24.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (17.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m17.3/17.3 MB\u001b[0m \u001b[31m93.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting alpaca_trade_api>=2.1.0\n",
            "  Downloading alpaca_trade_api-3.0.0-py3-none-any.whl (33 kB)\n",
            "Collecting wrds>=3.1.6\n",
            "  Using cached wrds-3.1.6-py3-none-any.whl (12 kB)\n",
            "Collecting exchange_calendars==3.6.3\n",
            "  Downloading exchange_calendars-3.6.3.tar.gz (152 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m152.8/152.8 kB\u001b[0m \u001b[31m23.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting ccxt>=1.66.32\n",
            "  Downloading ccxt-3.0.86-py2.py3-none-any.whl (3.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.6/3.6 MB\u001b[0m \u001b[31m101.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting ray[default,tune]>=2.0.0\n",
            "  Downloading ray-2.4.0-cp310-cp310-manylinux2014_x86_64.whl (58.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.6/58.6 MB\u001b[0m \u001b[31m15.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting stockstats>=0.4.0\n",
            "  Downloading stockstats-0.5.2-py2.py3-none-any.whl (20 kB)\n",
            "Collecting lz4\n",
            "  Downloading lz4-4.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m83.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pandas>=1.1.5\n",
            "  Downloading pandas-2.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m12.3/12.3 MB\u001b[0m \u001b[31m118.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting tensorboardX\n",
            "  Downloading tensorboardX-2.6-py2.py3-none-any.whl (114 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.5/114.5 kB\u001b[0m \u001b[31m17.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting stable-baselines3<2.0.0,>=1.6.2\n",
            "  Downloading stable_baselines3-1.8.0-py3-none-any.whl (174 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m174.5/174.5 kB\u001b[0m \u001b[31m24.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting scikit-learn>=0.21.0\n",
            "  Downloading scikit_learn-1.2.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (9.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m9.6/9.6 MB\u001b[0m \u001b[31m109.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pyluach\n",
            "  Downloading pyluach-2.2.0-py3-none-any.whl (25 kB)\n",
            "Collecting python-dateutil\n",
            "  Downloading python_dateutil-2.8.2-py2.py3-none-any.whl (247 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m247.7/247.7 kB\u001b[0m \u001b[31m34.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pytz\n",
            "  Downloading pytz-2023.3-py2.py3-none-any.whl (502 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m502.3/502.3 kB\u001b[0m \u001b[31m53.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: toolz in /usr/local/lib/python3.10/site-packages (from exchange_calendars==3.6.3->finrl==0.3.5) (0.12.0)\n",
            "Collecting korean_lunar_calendar\n",
            "  Downloading korean_lunar_calendar-0.3.1-py3-none-any.whl (9.0 kB)\n",
            "Collecting zipp>=0.5\n",
            "  Downloading zipp-3.15.0-py3-none-any.whl (6.8 kB)\n",
            "Collecting aiohttp==3.8.1\n",
            "  Downloading aiohttp-3.8.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (1.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m81.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting websockets<11,>=9.0\n",
            "  Downloading websockets-10.4-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (106 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m106.8/106.8 kB\u001b[0m \u001b[31m17.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting websocket-client<2,>=0.56.0\n",
            "  Downloading websocket_client-1.5.1-py3-none-any.whl (55 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m55.9/55.9 kB\u001b[0m \u001b[31m9.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: urllib3<2,>1.24 in /usr/local/lib/python3.10/site-packages (from alpaca_trade_api>=2.1.0->finrl==0.3.5) (1.26.15)\n",
            "Collecting PyYAML==6.0\n",
            "  Downloading PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (682 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m682.2/682.2 kB\u001b[0m \u001b[31m66.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: requests<3,>2 in /usr/local/lib/python3.10/site-packages (from alpaca_trade_api>=2.1.0->finrl==0.3.5) (2.28.2)\n",
            "Collecting msgpack==1.0.3\n",
            "  Downloading msgpack-1.0.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (323 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m323.7/323.7 kB\u001b[0m \u001b[31m43.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting deprecation==2.1.0\n",
            "  Downloading deprecation-2.1.0-py2.py3-none-any.whl (11 kB)\n",
            "Collecting yarl<2.0,>=1.0\n",
            "  Downloading yarl-1.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (268 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m268.8/268.8 kB\u001b[0m \u001b[31m37.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting frozenlist>=1.1.1\n",
            "  Downloading frozenlist-1.3.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (149 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m149.6/149.6 kB\u001b[0m \u001b[31m23.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting multidict<7.0,>=4.5\n",
            "  Downloading multidict-6.0.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (114 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.5/114.5 kB\u001b[0m \u001b[31m17.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting async-timeout<5.0,>=4.0.0a3\n",
            "  Downloading async_timeout-4.0.2-py3-none-any.whl (5.8 kB)\n",
            "Collecting attrs>=17.3.0\n",
            "  Downloading attrs-23.1.0-py3-none-any.whl (61 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m61.2/61.2 kB\u001b[0m \u001b[31m9.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting charset-normalizer<3.0,>=2.0\n",
            "  Downloading charset_normalizer-2.1.1-py3-none-any.whl (39 kB)\n",
            "Collecting aiosignal>=1.1.2\n",
            "  Downloading aiosignal-1.3.1-py3-none-any.whl (7.6 kB)\n",
            "Collecting packaging\n",
            "  Downloading packaging-23.1-py3-none-any.whl (48 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m48.9/48.9 kB\u001b[0m \u001b[31m8.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: setuptools>=60.9.0 in /usr/local/lib/python3.10/site-packages (from ccxt>=1.66.32->finrl==0.3.5) (65.6.3)\n",
            "Requirement already satisfied: cryptography>=2.6.1 in /usr/local/lib/python3.10/site-packages (from ccxt>=1.66.32->finrl==0.3.5) (40.0.1)\n",
            "Collecting aiodns>=1.1.1\n",
            "  Downloading aiodns-3.0.0-py3-none-any.whl (5.0 kB)\n",
            "Requirement already satisfied: certifi>=2018.1.18 in /usr/local/lib/python3.10/site-packages (from ccxt>=1.66.32->finrl==0.3.5) (2022.12.7)\n",
            "Collecting cloudpickle>=1.2.0\n",
            "  Downloading cloudpickle-2.2.1-py3-none-any.whl (25 kB)\n",
            "Collecting gym-notices>=0.0.4\n",
            "  Downloading gym_notices-0.0.8-py3-none-any.whl (3.0 kB)\n",
            "Collecting tzdata>=2022.1\n",
            "  Downloading tzdata-2023.3-py2.py3-none-any.whl (341 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m341.8/341.8 kB\u001b[0m \u001b[31m42.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting click>=7.0\n",
            "  Downloading click-8.1.3-py3-none-any.whl (96 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m96.6/96.6 kB\u001b[0m \u001b[31m13.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting jsonschema\n",
            "  Downloading jsonschema-4.17.3-py3-none-any.whl (90 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m90.4/90.4 kB\u001b[0m \u001b[31m14.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting protobuf!=3.19.5,>=3.15.3\n",
            "  Downloading protobuf-4.22.3-cp37-abi3-manylinux2014_x86_64.whl (302 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m302.4/302.4 kB\u001b[0m \u001b[31m40.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting filelock\n",
            "  Downloading filelock-3.12.0-py3-none-any.whl (10 kB)\n",
            "Collecting grpcio<=1.51.3,>=1.42.0\n",
            "  Downloading grpcio-1.51.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (4.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.8/4.8 MB\u001b[0m \u001b[31m100.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting virtualenv<20.21.1,>=20.0.24\n",
            "  Downloading virtualenv-20.21.0-py3-none-any.whl (8.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m8.7/8.7 MB\u001b[0m \u001b[31m86.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting tabulate\n",
            "  Downloading tabulate-0.9.0-py3-none-any.whl (35 kB)\n",
            "Collecting smart-open\n",
            "  Downloading smart_open-6.3.0-py3-none-any.whl (56 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.8/56.8 kB\u001b[0m \u001b[31m8.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting colorful\n",
            "  Downloading colorful-0.5.5-py2.py3-none-any.whl (201 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m201.4/201.4 kB\u001b[0m \u001b[31m27.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pydantic\n",
            "  Downloading pydantic-1.10.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.1/3.1 MB\u001b[0m \u001b[31m115.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting py-spy>=0.2.0\n",
            "  Downloading py_spy-0.3.14-py2.py3-none-manylinux_2_5_x86_64.manylinux1_x86_64.whl (3.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.0/3.0 MB\u001b[0m \u001b[31m102.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting opencensus\n",
            "  Downloading opencensus-0.11.2-py2.py3-none-any.whl (128 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m128.2/128.2 kB\u001b[0m \u001b[31m20.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting prometheus-client>=0.7.1\n",
            "  Downloading prometheus_client-0.16.0-py3-none-any.whl (122 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m122.5/122.5 kB\u001b[0m \u001b[31m20.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting gpustat>=1.0.0\n",
            "  Downloading gpustat-1.1.tar.gz (97 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m97.9/97.9 kB\u001b[0m \u001b[31m16.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting aiohttp-cors\n",
            "  Downloading aiohttp_cors-0.7.0-py3-none-any.whl (27 kB)\n",
            "Collecting scipy>=1.3.2\n",
            "  Downloading scipy-1.10.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (34.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m34.4/34.4 MB\u001b[0m \u001b[31m37.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting joblib>=1.1.1\n",
            "  Downloading joblib-1.2.0-py3-none-any.whl (297 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m298.0/298.0 kB\u001b[0m \u001b[31m38.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting threadpoolctl>=2.0.0\n",
            "  Downloading threadpoolctl-3.1.0-py3-none-any.whl (14 kB)\n",
            "Collecting gym>=0.17\n",
            "  Downloading gym-0.21.0.tar.gz (1.5 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.5/1.5 MB\u001b[0m \u001b[31m91.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting torch>=1.11\n",
            "  Downloading torch-2.0.0-cp310-cp310-manylinux1_x86_64.whl (619.9 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m619.9/619.9 MB\u001b[0m \u001b[31m2.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting protobuf!=3.19.5,>=3.15.3\n",
            "  Downloading protobuf-3.20.3-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (1.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.1 MB\u001b[0m \u001b[31m73.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting sqlalchemy<2\n",
            "  Using cached SQLAlchemy-1.4.48-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.6 MB)\n",
            "Collecting psycopg2-binary\n",
            "  Using cached psycopg2_binary-2.9.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.0 MB)\n",
            "Collecting six\n",
            "  Downloading six-1.16.0-py2.py3-none-any.whl (11 kB)\n",
            "Collecting thriftpy2>=0.3.9\n",
            "  Downloading thriftpy2-0.4.16.tar.gz (643 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m643.4/643.4 kB\u001b[0m \u001b[31m67.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting pymysql>=0.7.6\n",
            "  Downloading PyMySQL-1.0.3-py3-none-any.whl (43 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m43.7/43.7 kB\u001b[0m \u001b[31m6.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pillow>=6.2.0\n",
            "  Downloading Pillow-9.5.0-cp310-cp310-manylinux_2_28_x86_64.whl (3.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.4/3.4 MB\u001b[0m \u001b[31m112.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting contourpy>=1.0.1\n",
            "  Downloading contourpy-1.0.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (300 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m300.3/300.3 kB\u001b[0m \u001b[31m38.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting cycler>=0.10\n",
            "  Downloading cycler-0.11.0-py3-none-any.whl (6.4 kB)\n",
            "Collecting kiwisolver>=1.0.1\n",
            "  Downloading kiwisolver-1.4.4-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (1.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m92.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pyparsing>=2.3.1\n",
            "  Downloading pyparsing-3.0.9-py3-none-any.whl (98 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m98.3/98.3 kB\u001b[0m \u001b[31m16.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting fonttools>=4.22.0\n",
            "  Downloading fonttools-4.39.3-py3-none-any.whl (1.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.0/1.0 MB\u001b[0m \u001b[31m79.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting ipython>=3.2.3\n",
            "  Downloading ipython-8.13.1-py3-none-any.whl (797 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m797.6/797.6 kB\u001b[0m \u001b[31m68.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting seaborn>=0.7.1\n",
            "  Downloading seaborn-0.12.2-py3-none-any.whl (293 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m293.3/293.3 kB\u001b[0m \u001b[31m39.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting empyrical>=0.5.0\n",
            "  Downloading empyrical-0.5.5.tar.gz (52 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m52.8/52.8 kB\u001b[0m \u001b[31m8.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting multitasking>=0.0.7\n",
            "  Downloading multitasking-0.0.11-py3-none-any.whl (8.5 kB)\n",
            "Collecting frozendict>=2.3.4\n",
            "  Downloading frozendict-2.3.8-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (115 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m115.3/115.3 kB\u001b[0m \u001b[31m18.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting appdirs>=1.4.4\n",
            "  Downloading appdirs-1.4.4-py2.py3-none-any.whl (9.6 kB)\n",
            "Collecting lxml>=4.9.1\n",
            "  Downloading lxml-4.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl (7.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.1/7.1 MB\u001b[0m \u001b[31m135.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting html5lib>=1.1\n",
            "  Downloading html5lib-1.1-py2.py3-none-any.whl (112 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m112.2/112.2 kB\u001b[0m \u001b[31m18.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting beautifulsoup4>=4.11.1\n",
            "  Downloading beautifulsoup4-4.12.2-py3-none-any.whl (142 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m143.0/143.0 kB\u001b[0m \u001b[31m23.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pycares>=4.0.0\n",
            "  Downloading pycares-4.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (288 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m288.7/288.7 kB\u001b[0m \u001b[31m36.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting soupsieve>1.2\n",
            "  Downloading soupsieve-2.4.1-py3-none-any.whl (36 kB)\n",
            "Requirement already satisfied: cffi>=1.12 in /usr/local/lib/python3.10/site-packages (from cryptography>=2.6.1->ccxt>=1.66.32->finrl==0.3.5) (1.15.1)\n",
            "Collecting pandas-datareader>=0.2\n",
            "  Downloading pandas_datareader-0.10.0-py3-none-any.whl (109 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m109.5/109.5 kB\u001b[0m \u001b[31m16.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting blessed>=1.17.1\n",
            "  Downloading blessed-1.20.0-py2.py3-none-any.whl (58 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.4/58.4 kB\u001b[0m \u001b[31m10.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting psutil>=5.6.0\n",
            "  Downloading psutil-5.9.5-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (282 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m282.1/282.1 kB\u001b[0m \u001b[31m37.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-ml-py>=11.450.129\n",
            "  Downloading nvidia_ml_py-11.525.112-py3-none-any.whl (35 kB)\n",
            "Collecting webencodings\n",
            "  Downloading webencodings-0.5.1-py2.py3-none-any.whl (11 kB)\n",
            "Collecting decorator\n",
            "  Downloading decorator-5.1.1-py3-none-any.whl (9.1 kB)\n",
            "Collecting traitlets>=5\n",
            "  Downloading traitlets-5.9.0-py3-none-any.whl (117 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m117.4/117.4 kB\u001b[0m \u001b[31m18.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting jedi>=0.16\n",
            "  Downloading jedi-0.18.2-py2.py3-none-any.whl (1.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m90.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30\n",
            "  Downloading prompt_toolkit-3.0.38-py3-none-any.whl (385 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m385.8/385.8 kB\u001b[0m \u001b[31m46.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting matplotlib-inline\n",
            "  Downloading matplotlib_inline-0.1.6-py3-none-any.whl (9.4 kB)\n",
            "Collecting pygments>=2.4.0\n",
            "  Downloading Pygments-2.15.1-py3-none-any.whl (1.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.1 MB\u001b[0m \u001b[31m85.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pickleshare\n",
            "  Downloading pickleshare-0.7.5-py2.py3-none-any.whl (6.9 kB)\n",
            "Collecting stack-data\n",
            "  Downloading stack_data-0.6.2-py3-none-any.whl (24 kB)\n",
            "Collecting backcall\n",
            "  Downloading backcall-0.2.0-py2.py3-none-any.whl (11 kB)\n",
            "Collecting pexpect>4.3\n",
            "  Downloading pexpect-4.8.0-py2.py3-none-any.whl (59 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m59.0/59.0 kB\u001b[0m \u001b[31m9.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/site-packages (from requests<3,>2->alpaca_trade_api>=2.1.0->finrl==0.3.5) (3.4)\n",
            "Collecting greenlet!=0.4.17\n",
            "  Downloading greenlet-2.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (613 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m613.7/613.7 kB\u001b[0m \u001b[31m65.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting ply<4.0,>=3.4\n",
            "  Downloading ply-3.11-py2.py3-none-any.whl (49 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.6/49.6 kB\u001b[0m \u001b[31m8.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cusparse-cu11==11.7.4.91\n",
            "  Downloading nvidia_cusparse_cu11-11.7.4.91-py3-none-manylinux1_x86_64.whl (173.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m173.2/173.2 MB\u001b[0m \u001b[31m5.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cublas-cu11==11.10.3.66\n",
            "  Downloading nvidia_cublas_cu11-11.10.3.66-py3-none-manylinux1_x86_64.whl (317.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m317.1/317.1 MB\u001b[0m \u001b[31m3.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting jinja2\n",
            "  Downloading Jinja2-3.1.2-py3-none-any.whl (133 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m133.1/133.1 kB\u001b[0m \u001b[31m19.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-curand-cu11==10.2.10.91\n",
            "  Downloading nvidia_curand_cu11-10.2.10.91-py3-none-manylinux1_x86_64.whl (54.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m54.6/54.6 MB\u001b[0m \u001b[31m28.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cuda-runtime-cu11==11.7.99\n",
            "  Downloading nvidia_cuda_runtime_cu11-11.7.99-py3-none-manylinux1_x86_64.whl (849 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m849.3/849.3 kB\u001b[0m \u001b[31m69.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting typing-extensions\n",
            "  Using cached typing_extensions-4.5.0-py3-none-any.whl (27 kB)\n",
            "Collecting nvidia-cufft-cu11==10.9.0.58\n",
            "  Downloading nvidia_cufft_cu11-10.9.0.58-py3-none-manylinux1_x86_64.whl (168.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m168.4/168.4 MB\u001b[0m \u001b[31m5.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cusolver-cu11==11.4.0.1\n",
            "  Downloading nvidia_cusolver_cu11-11.4.0.1-2-py3-none-manylinux1_x86_64.whl (102.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m102.6/102.6 MB\u001b[0m \u001b[31m9.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting networkx\n",
            "  Downloading networkx-3.1-py3-none-any.whl (2.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.1/2.1 MB\u001b[0m \u001b[31m107.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-nccl-cu11==2.14.3\n",
            "  Downloading nvidia_nccl_cu11-2.14.3-py3-none-manylinux1_x86_64.whl (177.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m177.1/177.1 MB\u001b[0m \u001b[31m5.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-nvtx-cu11==11.7.91\n",
            "  Downloading nvidia_nvtx_cu11-11.7.91-py3-none-manylinux1_x86_64.whl (98 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m98.6/98.6 kB\u001b[0m \u001b[31m13.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting triton==2.0.0\n",
            "  Downloading triton-2.0.0-1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (63.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m63.3/63.3 MB\u001b[0m \u001b[31m20.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cuda-cupti-cu11==11.7.101\n",
            "  Downloading nvidia_cuda_cupti_cu11-11.7.101-py3-none-manylinux1_x86_64.whl (11.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m11.8/11.8 MB\u001b[0m \u001b[31m25.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cudnn-cu11==8.5.0.96\n",
            "  Downloading nvidia_cudnn_cu11-8.5.0.96-2-py3-none-manylinux1_x86_64.whl (557.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m557.1/557.1 MB\u001b[0m \u001b[31m2.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cuda-nvrtc-cu11==11.7.99\n",
            "  Downloading nvidia_cuda_nvrtc_cu11-11.7.99-2-py3-none-manylinux1_x86_64.whl (21.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m21.0/21.0 MB\u001b[0m \u001b[31m90.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting sympy\n",
            "  Downloading sympy-1.11.1-py3-none-any.whl (6.5 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.5/6.5 MB\u001b[0m \u001b[31m102.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: wheel in /usr/local/lib/python3.10/site-packages (from nvidia-cublas-cu11==11.10.3.66->torch>=1.11->stable-baselines3<2.0.0,>=1.6.2->finrl==0.3.5) (0.40.0)\n",
            "Collecting cmake\n",
            "  Downloading cmake-3.26.3-py2.py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (24.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m24.0/24.0 MB\u001b[0m \u001b[31m74.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting lit\n",
            "  Downloading lit-16.0.2.tar.gz (137 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m137.9/137.9 kB\u001b[0m \u001b[31m21.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting distlib<1,>=0.3.6\n",
            "  Downloading distlib-0.3.6-py2.py3-none-any.whl (468 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m468.5/468.5 kB\u001b[0m \u001b[31m51.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting platformdirs<4,>=2.4\n",
            "  Downloading platformdirs-3.5.0-py3-none-any.whl (15 kB)\n",
            "Collecting gym[box2d]\n",
            "  Downloading gym-0.26.1.tar.gz (719 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m719.9/719.9 kB\u001b[0m \u001b[31m53.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Downloading gym-0.26.0.tar.gz (710 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m710.3/710.3 kB\u001b[0m \u001b[31m58.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Downloading gym-0.25.2.tar.gz (734 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m734.5/734.5 kB\u001b[0m \u001b[31m63.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Downloading gym-0.25.1.tar.gz (732 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m732.2/732.2 kB\u001b[0m \u001b[31m66.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Downloading gym-0.25.0.tar.gz (720 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m720.4/720.4 kB\u001b[0m \u001b[31m66.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Downloading gym-0.24.1.tar.gz (696 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m696.4/696.4 kB\u001b[0m \u001b[31m53.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Downloading gym-0.24.0.tar.gz (694 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m694.4/694.4 kB\u001b[0m \u001b[31m53.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Downloading gym-0.23.1.tar.gz (626 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m626.2/626.2 kB\u001b[0m \u001b[31m17.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Downloading gym-0.23.0.tar.gz (624 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m624.4/624.4 kB\u001b[0m \u001b[31m44.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Downloading gym-0.22.0.tar.gz (631 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m631.1/631.1 kB\u001b[0m \u001b[31m60.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting box2d-py==2.3.5\n",
            "  Downloading box2d-py-2.3.5.tar.gz (374 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m374.4/374.4 kB\u001b[0m \u001b[31m41.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting pyglet>=1.4.0\n",
            "  Downloading pyglet-2.0.6-py3-none-any.whl (840 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m840.9/840.9 kB\u001b[0m \u001b[31m62.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pyrsistent!=0.17.0,!=0.17.1,!=0.17.2,>=0.14.0\n",
            "  Downloading pyrsistent-0.19.3-py3-none-any.whl (57 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m57.5/57.5 kB\u001b[0m \u001b[31m9.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting google-api-core<3.0.0,>=1.0.0\n",
            "  Downloading google_api_core-2.11.0-py3-none-any.whl (120 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m120.3/120.3 kB\u001b[0m \u001b[31m16.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting opencensus-context>=0.1.3\n",
            "  Downloading opencensus_context-0.1.3-py2.py3-none-any.whl (5.1 kB)\n",
            "Collecting wcwidth>=0.1.4\n",
            "  Downloading wcwidth-0.2.6-py2.py3-none-any.whl (29 kB)\n",
            "Requirement already satisfied: pycparser in /usr/local/lib/python3.10/site-packages (from cffi>=1.12->cryptography>=2.6.1->ccxt>=1.66.32->finrl==0.3.5) (2.21)\n",
            "Collecting google-auth<3.0dev,>=2.14.1\n",
            "  Downloading google_auth-2.17.3-py2.py3-none-any.whl (178 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m178.2/178.2 kB\u001b[0m \u001b[31m23.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting googleapis-common-protos<2.0dev,>=1.56.2\n",
            "  Downloading googleapis_common_protos-1.59.0-py2.py3-none-any.whl (223 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m223.6/223.6 kB\u001b[0m \u001b[31m30.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting parso<0.9.0,>=0.8.0\n",
            "  Downloading parso-0.8.3-py2.py3-none-any.whl (100 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m100.8/100.8 kB\u001b[0m \u001b[31m15.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting ptyprocess>=0.5\n",
            "  Downloading ptyprocess-0.7.0-py2.py3-none-any.whl (13 kB)\n",
            "Collecting MarkupSafe>=2.0\n",
            "  Downloading MarkupSafe-2.1.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (25 kB)\n",
            "Collecting executing>=1.2.0\n",
            "  Downloading executing-1.2.0-py2.py3-none-any.whl (24 kB)\n",
            "Collecting pure-eval\n",
            "  Downloading pure_eval-0.2.2-py3-none-any.whl (11 kB)\n",
            "Collecting asttokens>=2.1.0\n",
            "  Downloading asttokens-2.2.1-py2.py3-none-any.whl (26 kB)\n",
            "Collecting mpmath>=0.19\n",
            "  Downloading mpmath-1.3.0-py3-none-any.whl (536 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m536.2/536.2 kB\u001b[0m \u001b[31m54.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pyasn1-modules>=0.2.1\n",
            "  Downloading pyasn1_modules-0.3.0-py2.py3-none-any.whl (181 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m181.3/181.3 kB\u001b[0m \u001b[31m24.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting rsa<5,>=3.1.4\n",
            "  Downloading rsa-4.9-py3-none-any.whl (34 kB)\n",
            "Collecting cachetools<6.0,>=2.0.0\n",
            "  Downloading cachetools-5.3.0-py3-none-any.whl (9.3 kB)\n",
            "Collecting pyasn1<0.6.0,>=0.4.6\n",
            "  Downloading pyasn1-0.5.0-py2.py3-none-any.whl (83 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m83.9/83.9 kB\u001b[0m \u001b[31m12.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hBuilding wheels for collected packages: finrl, exchange_calendars, gym, elegantrl, gputil, pyfolio, empyrical, gpustat, thriftpy2, box2d-py, lit\n",
            "  Building wheel for finrl (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for finrl: filename=finrl-0.3.5-py3-none-any.whl size=4668722 sha256=b5d9990feddfce4dac1109677bd1616fcf78abb54bcf1b30c280638a7696b8d0\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-x5y5swzs/wheels/72/3b/1a/0fc805a8cc65ecd5bfe4f74a3c586b6075678b8ba53fd8f749\n",
            "  Building wheel for exchange_calendars (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for exchange_calendars: filename=exchange_calendars-3.6.3-py3-none-any.whl size=182636 sha256=4c9e03737bfe54b95bfd464559773ea9208e3438fa8f485b0cee85b3b510ca85\n",
            "  Stored in directory: /root/.cache/pip/wheels/93/ca/8f/1e1c90cc79fb3ca9b5413ff58e3fccf3baf2182c994c6dfd37\n",
            "  \u001b[1;31merror\u001b[0m: \u001b[1msubprocess-exited-with-error\u001b[0m\n",
            "  \n",
            "  \u001b[31m×\u001b[0m \u001b[32mpython setup.py bdist_wheel\u001b[0m did not run successfully.\n",
            "  \u001b[31m│\u001b[0m exit code: \u001b[1;36m1\u001b[0m\n",
            "  \u001b[31m╰─>\u001b[0m See above for output.\n",
            "  \n",
            "  \u001b[1;35mnote\u001b[0m: This error originates from a subprocess, and is likely not a problem with pip.\n",
            "  Building wheel for gym (setup.py) ... \u001b[?25lerror\n",
            "\u001b[31m  ERROR: Failed building wheel for gym\u001b[0m\u001b[31m\n",
            "\u001b[0m\u001b[?25h  Running setup.py clean for gym\n",
            "  Building wheel for elegantrl (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for elegantrl: filename=elegantrl-0.3.6-py3-none-any.whl size=195067 sha256=65055639a7cc93a7eb32581edce2b18617dece40b0b444983088a5fb0da536aa\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-x5y5swzs/wheels/c0/51/a5/b05f165548221bc570f7223babd33e2992fa873cdcebe2d229\n",
            "  Building wheel for gputil (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for gputil: filename=GPUtil-1.4.0-py3-none-any.whl size=7408 sha256=9fbdbcbbb7c98f5970e41c01cf33571e63052958e62fc3e79d7999fb092751e5\n",
            "  Stored in directory: /root/.cache/pip/wheels/a9/8a/bd/81082387151853ab8b6b3ef33426e98f5cbfebc3c397a9d4d0\n",
            "  Building wheel for pyfolio (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pyfolio: filename=pyfolio-0.9.2+75.g4b901f6-py3-none-any.whl size=75775 sha256=6aecc4c845fd0b91b388a08a581ab1305f726a897e19ec66c7930260314a33fd\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-x5y5swzs/wheels/c4/1d/91/8ab5d1c88a11b06a63dcd6a69ea81547e2247123232949bb26\n",
            "  Building wheel for empyrical (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for empyrical: filename=empyrical-0.5.5-py3-none-any.whl size=39779 sha256=b325869afd8d7744074a2cf47365efdc02df338c5986047c05ff4210498afd2e\n",
            "  Stored in directory: /root/.cache/pip/wheels/0e/2e/f2/d6d2d9a1eb8fbbd9949bb5d4c00f753e3b74e5bd7ed10b1d36\n",
            "  Building wheel for gpustat (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for gpustat: filename=gpustat-1.1-py3-none-any.whl size=26298 sha256=f43c60b1bc3c4eff849b620c003f096354dce7bc70c782a0d52d4ff541dd3b07\n",
            "  Stored in directory: /root/.cache/pip/wheels/ee/d0/2c/1e02440645c2318ba03aea99993a44a9108dc8f74de0bd370b\n",
            "  Building wheel for thriftpy2 (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for thriftpy2: filename=thriftpy2-0.4.16-cp310-cp310-linux_x86_64.whl size=522576 sha256=b0737fac74edb66e823bd0f6bf7f854507e9c480c539544b86f1012e298e69e1\n",
            "  Stored in directory: /root/.cache/pip/wheels/95/f5/1d/fe404692e1c8aaea45220c322d1d0f32c9fd40eb0e2bdd571e\n",
            "  Building wheel for box2d-py (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for box2d-py: filename=box2d_py-2.3.5-cp310-cp310-linux_x86_64.whl size=494649 sha256=af832b09d298d9fdcf49031f3a9b5776216a61d27183b7b9472bb58ee4ce6c03\n",
            "  Stored in directory: /root/.cache/pip/wheels/db/8f/6a/eaaadf056fba10a98d986f6dce954e6201ba3126926fc5ad9e\n",
            "  Building wheel for lit (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for lit: filename=lit-16.0.2-py3-none-any.whl size=88190 sha256=8555a38666b7a2846fb8e3e43dc4102210943eb3527d6bbcfce9ad3d0bdc75d6\n",
            "  Stored in directory: /root/.cache/pip/wheels/4d/31/6f/140862d5c69ddd665b3ed9485f4c93aa9c84cb34b359cba3ce\n",
            "Successfully built finrl exchange_calendars elegantrl gputil pyfolio empyrical gpustat thriftpy2 box2d-py lit\n",
            "Failed to build gym\n",
            "Installing collected packages: webencodings, wcwidth, pytz, pyglet, py-spy, pure-eval, ptyprocess, ply, pickleshare, opencensus-context, nvidia-ml-py, multitasking, msgpack, mpmath, lit, korean_lunar_calendar, gputil, executing, distlib, colorful, cmake, box2d-py, backcall, appdirs, zipp, websockets, websocket-client, tzdata, typing-extensions, traitlets, threadpoolctl, tabulate, sympy, soupsieve, smart-open, six, PyYAML, pyrsistent, pyparsing, pymysql, pyluach, pygments, pyasn1, psycopg2-binary, psutil, protobuf, prompt-toolkit, prometheus-client, platformdirs, pillow, pexpect, parso, packaging, nvidia-nvtx-cu11, nvidia-nccl-cu11, nvidia-cusparse-cu11, nvidia-curand-cu11, nvidia-cufft-cu11, nvidia-cuda-runtime-cu11, nvidia-cuda-nvrtc-cu11, nvidia-cuda-cupti-cu11, nvidia-cublas-cu11, numpy, networkx, multidict, MarkupSafe, lz4, lxml, kiwisolver, joblib, grpcio, greenlet, frozenlist, frozendict, fonttools, filelock, decorator, cycler, cloudpickle, click, charset-normalizer, cachetools, attrs, async-timeout, yarl, virtualenv, thriftpy2, tensorboardX, sqlalchemy, scipy, rsa, python-dateutil, pydantic, pycares, pyasn1-modules, nvidia-cusolver-cu11, nvidia-cudnn-cu11, matplotlib-inline, jsonschema, jinja2, jedi, importlib-metadata, html5lib, gym, googleapis-common-protos, deprecation, contourpy, blessed, beautifulsoup4, asttokens, aiosignal, stack-data, scikit-learn, ray, pandas, matplotlib, gpustat, google-auth, aiohttp, aiodns, yfinance, wrds, stockstats, seaborn, pandas-datareader, jqdatasdk, ipython, google-api-core, exchange_calendars, ccxt, alpaca_trade_api, aiohttp-cors, opencensus, empyrical, pyfolio, triton, torch, stable-baselines3, elegantrl, finrl\n",
            "  Attempting uninstall: charset-normalizer\n",
            "    Found existing installation: charset-normalizer 3.1.0\n",
            "    Uninstalling charset-normalizer-3.1.0:\n",
            "      Successfully uninstalled charset-normalizer-3.1.0\n",
            "  Running setup.py install for gym ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[33m  DEPRECATION: gym was installed using the legacy 'setup.py install' method, because a wheel could not be built for it. pip 23.1 will enforce this behaviour change. A possible replacement is to fix the wheel build issue reported above. Discussion can be found at https://github.com/pypa/pip/issues/8368\u001b[0m\u001b[33m\n",
            "\u001b[0mSuccessfully installed MarkupSafe-2.1.2 PyYAML-6.0 aiodns-3.0.0 aiohttp-3.8.1 aiohttp-cors-0.7.0 aiosignal-1.3.1 alpaca_trade_api-3.0.0 appdirs-1.4.4 asttokens-2.2.1 async-timeout-4.0.2 attrs-23.1.0 backcall-0.2.0 beautifulsoup4-4.12.2 blessed-1.20.0 box2d-py-2.3.5 cachetools-5.3.0 ccxt-3.0.86 charset-normalizer-2.1.1 click-8.1.3 cloudpickle-2.2.1 cmake-3.26.3 colorful-0.5.5 contourpy-1.0.7 cycler-0.11.0 decorator-5.1.1 deprecation-2.1.0 distlib-0.3.6 elegantrl-0.3.6 empyrical-0.5.5 exchange_calendars-3.6.3 executing-1.2.0 filelock-3.12.0 finrl-0.3.5 fonttools-4.39.3 frozendict-2.3.8 frozenlist-1.3.3 google-api-core-2.11.0 google-auth-2.17.3 googleapis-common-protos-1.59.0 gpustat-1.1 gputil-1.4.0 greenlet-2.0.2 grpcio-1.51.3 gym-0.21.0 html5lib-1.1 importlib-metadata-4.13.0 ipython-8.13.1 jedi-0.18.2 jinja2-3.1.2 joblib-1.2.0 jqdatasdk-1.8.11 jsonschema-4.17.3 kiwisolver-1.4.4 korean_lunar_calendar-0.3.1 lit-16.0.2 lxml-4.9.2 lz4-4.3.2 matplotlib-3.7.1 matplotlib-inline-0.1.6 mpmath-1.3.0 msgpack-1.0.3 multidict-6.0.4 multitasking-0.0.11 networkx-3.1 numpy-1.24.3 nvidia-cublas-cu11-11.10.3.66 nvidia-cuda-cupti-cu11-11.7.101 nvidia-cuda-nvrtc-cu11-11.7.99 nvidia-cuda-runtime-cu11-11.7.99 nvidia-cudnn-cu11-8.5.0.96 nvidia-cufft-cu11-10.9.0.58 nvidia-curand-cu11-10.2.10.91 nvidia-cusolver-cu11-11.4.0.1 nvidia-cusparse-cu11-11.7.4.91 nvidia-ml-py-11.525.112 nvidia-nccl-cu11-2.14.3 nvidia-nvtx-cu11-11.7.91 opencensus-0.11.2 opencensus-context-0.1.3 packaging-23.1 pandas-2.0.1 pandas-datareader-0.10.0 parso-0.8.3 pexpect-4.8.0 pickleshare-0.7.5 pillow-9.5.0 platformdirs-3.5.0 ply-3.11 prometheus-client-0.16.0 prompt-toolkit-3.0.38 protobuf-3.20.3 psutil-5.9.5 psycopg2-binary-2.9.6 ptyprocess-0.7.0 pure-eval-0.2.2 py-spy-0.3.14 pyasn1-0.5.0 pyasn1-modules-0.3.0 pycares-4.3.0 pydantic-1.10.7 pyfolio-0.9.2+75.g4b901f6 pyglet-2.0.6 pygments-2.15.1 pyluach-2.2.0 pymysql-1.0.3 pyparsing-3.0.9 pyrsistent-0.19.3 python-dateutil-2.8.2 pytz-2023.3 ray-2.4.0 rsa-4.9 scikit-learn-1.2.2 scipy-1.10.1 seaborn-0.12.2 six-1.16.0 smart-open-6.3.0 soupsieve-2.4.1 sqlalchemy-1.4.48 stable-baselines3-1.8.0 stack-data-0.6.2 stockstats-0.5.2 sympy-1.11.1 tabulate-0.9.0 tensorboardX-2.6 threadpoolctl-3.1.0 thriftpy2-0.4.16 torch-2.0.0 traitlets-5.9.0 triton-2.0.0 typing-extensions-4.5.0 tzdata-2023.3 virtualenv-20.21.0 wcwidth-0.2.6 webencodings-0.5.1 websocket-client-1.5.1 websockets-10.4 wrds-3.1.6 yarl-1.9.2 yfinance-0.2.18 zipp-3.15.0\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
            "\u001b[0m"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 1.2 Check if the additional packages needed are present, if not install them"
      ],
      "metadata": {
        "id": "DMaIlOcVTJfp"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# !pip install trading_calendars\n",
        "# !pip install alpaca_trade_api\n",
        "# !pip install ccxt\n",
        "# !pip install jqdatasdk\n",
        "# !pip install wrds\n",
        "\n",
        "# !pip install lz4\n",
        "# !pip install ray[tune]\n",
        "# !pip install tensorboardX\n",
        "# !pip install gputil\n",
        "\n",
        "#%%capture\n",
        "if True:\n",
        "    # installing packages\n",
        "    !pip install pyfolio-reloaded  #original pyfolio no longer maintained\n",
        "    !pip install optuna\n",
        "    !pip install -U \"ray[rllib]\"\n",
        "    !pip install plotly\n",
        "    !pip install ipywidgets\n",
        "    !pip install -U kaleido   # enables saving plots to file\n",
        "    \n",
        "    "
      ],
      "metadata": {
        "id": "UfmXkH1rTFM7",
        "outputId": "0afe9559-e211-490c-e367-42f03ef71f7c",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting pyfolio-reloaded\n",
            "  Downloading pyfolio_reloaded-0.9.5-py3-none-any.whl (6.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.3/6.3 MB\u001b[0m \u001b[31m2.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: ipython>=3.2.3 in /usr/local/lib/python3.10/site-packages (from pyfolio-reloaded) (8.13.1)\n",
            "Requirement already satisfied: scikit-learn>=0.16.1 in /usr/local/lib/python3.10/site-packages (from pyfolio-reloaded) (1.2.2)\n",
            "Requirement already satisfied: pytz>=2014.10 in /usr/local/lib/python3.10/site-packages (from pyfolio-reloaded) (2023.3)\n",
            "Requirement already satisfied: matplotlib>=1.4.0 in /usr/local/lib/python3.10/site-packages (from pyfolio-reloaded) (3.7.1)\n",
            "Requirement already satisfied: scipy>=0.14.0 in /usr/local/lib/python3.10/site-packages (from pyfolio-reloaded) (1.10.1)\n",
            "Requirement already satisfied: numpy>=1.11.1 in /usr/local/lib/python3.10/site-packages (from pyfolio-reloaded) (1.24.3)\n",
            "Requirement already satisfied: seaborn>=0.7.1 in /usr/local/lib/python3.10/site-packages (from pyfolio-reloaded) (0.12.2)\n",
            "Collecting empyrical-reloaded>=0.5.8\n",
            "  Downloading empyrical_reloaded-0.5.9-py3-none-any.whl (32 kB)\n",
            "Requirement already satisfied: pandas>=0.18.1 in /usr/local/lib/python3.10/site-packages (from pyfolio-reloaded) (2.0.1)\n",
            "Requirement already satisfied: yfinance>=0.1.63 in /usr/local/lib/python3.10/site-packages (from empyrical-reloaded>=0.5.8->pyfolio-reloaded) (0.2.18)\n",
            "Collecting bottleneck>=1.3.0\n",
            "  Downloading Bottleneck-1.3.7-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (354 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m354.0/354.0 kB\u001b[0m \u001b[31m43.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pandas-datareader>=0.4 in /usr/local/lib/python3.10/site-packages (from empyrical-reloaded>=0.5.8->pyfolio-reloaded) (0.10.0)\n",
            "Requirement already satisfied: prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30 in /usr/local/lib/python3.10/site-packages (from ipython>=3.2.3->pyfolio-reloaded) (3.0.38)\n",
            "Requirement already satisfied: backcall in /usr/local/lib/python3.10/site-packages (from ipython>=3.2.3->pyfolio-reloaded) (0.2.0)\n",
            "Requirement already satisfied: traitlets>=5 in /usr/local/lib/python3.10/site-packages (from ipython>=3.2.3->pyfolio-reloaded) (5.9.0)\n",
            "Requirement already satisfied: jedi>=0.16 in /usr/local/lib/python3.10/site-packages (from ipython>=3.2.3->pyfolio-reloaded) (0.18.2)\n",
            "Requirement already satisfied: pygments>=2.4.0 in /usr/local/lib/python3.10/site-packages (from ipython>=3.2.3->pyfolio-reloaded) (2.15.1)\n",
            "Requirement already satisfied: pickleshare in /usr/local/lib/python3.10/site-packages (from ipython>=3.2.3->pyfolio-reloaded) (0.7.5)\n",
            "Requirement already satisfied: decorator in /usr/local/lib/python3.10/site-packages (from ipython>=3.2.3->pyfolio-reloaded) (5.1.1)\n",
            "Requirement already satisfied: pexpect>4.3 in /usr/local/lib/python3.10/site-packages (from ipython>=3.2.3->pyfolio-reloaded) (4.8.0)\n",
            "Requirement already satisfied: matplotlib-inline in /usr/local/lib/python3.10/site-packages (from ipython>=3.2.3->pyfolio-reloaded) (0.1.6)\n",
            "Requirement already satisfied: stack-data in /usr/local/lib/python3.10/site-packages (from ipython>=3.2.3->pyfolio-reloaded) (0.6.2)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/site-packages (from matplotlib>=1.4.0->pyfolio-reloaded) (23.1)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/site-packages (from matplotlib>=1.4.0->pyfolio-reloaded) (0.11.0)\n",
            "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/site-packages (from matplotlib>=1.4.0->pyfolio-reloaded) (4.39.3)\n",
            "Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.10/site-packages (from matplotlib>=1.4.0->pyfolio-reloaded) (9.5.0)\n",
            "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.10/site-packages (from matplotlib>=1.4.0->pyfolio-reloaded) (2.8.2)\n",
            "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/site-packages (from matplotlib>=1.4.0->pyfolio-reloaded) (3.0.9)\n",
            "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/site-packages (from matplotlib>=1.4.0->pyfolio-reloaded) (1.0.7)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/site-packages (from matplotlib>=1.4.0->pyfolio-reloaded) (1.4.4)\n",
            "Requirement already satisfied: tzdata>=2022.1 in /usr/local/lib/python3.10/site-packages (from pandas>=0.18.1->pyfolio-reloaded) (2023.3)\n",
            "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/site-packages (from scikit-learn>=0.16.1->pyfolio-reloaded) (1.2.0)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/site-packages (from scikit-learn>=0.16.1->pyfolio-reloaded) (3.1.0)\n",
            "Requirement already satisfied: parso<0.9.0,>=0.8.0 in /usr/local/lib/python3.10/site-packages (from jedi>=0.16->ipython>=3.2.3->pyfolio-reloaded) (0.8.3)\n",
            "Requirement already satisfied: lxml in /usr/local/lib/python3.10/site-packages (from pandas-datareader>=0.4->empyrical-reloaded>=0.5.8->pyfolio-reloaded) (4.9.2)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.10/site-packages (from pandas-datareader>=0.4->empyrical-reloaded>=0.5.8->pyfolio-reloaded) (2.28.2)\n",
            "Requirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.10/site-packages (from pexpect>4.3->ipython>=3.2.3->pyfolio-reloaded) (0.7.0)\n",
            "Requirement already satisfied: wcwidth in /usr/local/lib/python3.10/site-packages (from prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30->ipython>=3.2.3->pyfolio-reloaded) (0.2.6)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/site-packages (from python-dateutil>=2.7->matplotlib>=1.4.0->pyfolio-reloaded) (1.16.0)\n",
            "Requirement already satisfied: beautifulsoup4>=4.11.1 in /usr/local/lib/python3.10/site-packages (from yfinance>=0.1.63->empyrical-reloaded>=0.5.8->pyfolio-reloaded) (4.12.2)\n",
            "Requirement already satisfied: cryptography>=3.3.2 in /usr/local/lib/python3.10/site-packages (from yfinance>=0.1.63->empyrical-reloaded>=0.5.8->pyfolio-reloaded) (40.0.1)\n",
            "Requirement already satisfied: multitasking>=0.0.7 in /usr/local/lib/python3.10/site-packages (from yfinance>=0.1.63->empyrical-reloaded>=0.5.8->pyfolio-reloaded) (0.0.11)\n",
            "Requirement already satisfied: frozendict>=2.3.4 in /usr/local/lib/python3.10/site-packages (from yfinance>=0.1.63->empyrical-reloaded>=0.5.8->pyfolio-reloaded) (2.3.8)\n",
            "Requirement already satisfied: appdirs>=1.4.4 in /usr/local/lib/python3.10/site-packages (from yfinance>=0.1.63->empyrical-reloaded>=0.5.8->pyfolio-reloaded) (1.4.4)\n",
            "Requirement already satisfied: html5lib>=1.1 in /usr/local/lib/python3.10/site-packages (from yfinance>=0.1.63->empyrical-reloaded>=0.5.8->pyfolio-reloaded) (1.1)\n",
            "Requirement already satisfied: asttokens>=2.1.0 in /usr/local/lib/python3.10/site-packages (from stack-data->ipython>=3.2.3->pyfolio-reloaded) (2.2.1)\n",
            "Requirement already satisfied: pure-eval in /usr/local/lib/python3.10/site-packages (from stack-data->ipython>=3.2.3->pyfolio-reloaded) (0.2.2)\n",
            "Requirement already satisfied: executing>=1.2.0 in /usr/local/lib/python3.10/site-packages (from stack-data->ipython>=3.2.3->pyfolio-reloaded) (1.2.0)\n",
            "Requirement already satisfied: soupsieve>1.2 in /usr/local/lib/python3.10/site-packages (from beautifulsoup4>=4.11.1->yfinance>=0.1.63->empyrical-reloaded>=0.5.8->pyfolio-reloaded) (2.4.1)\n",
            "Requirement already satisfied: cffi>=1.12 in /usr/local/lib/python3.10/site-packages (from cryptography>=3.3.2->yfinance>=0.1.63->empyrical-reloaded>=0.5.8->pyfolio-reloaded) (1.15.1)\n",
            "Requirement already satisfied: webencodings in /usr/local/lib/python3.10/site-packages (from html5lib>=1.1->yfinance>=0.1.63->empyrical-reloaded>=0.5.8->pyfolio-reloaded) (0.5.1)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/site-packages (from requests>=2.19.0->pandas-datareader>=0.4->empyrical-reloaded>=0.5.8->pyfolio-reloaded) (3.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/site-packages (from requests>=2.19.0->pandas-datareader>=0.4->empyrical-reloaded>=0.5.8->pyfolio-reloaded) (2022.12.7)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/site-packages (from requests>=2.19.0->pandas-datareader>=0.4->empyrical-reloaded>=0.5.8->pyfolio-reloaded) (2.1.1)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/site-packages (from requests>=2.19.0->pandas-datareader>=0.4->empyrical-reloaded>=0.5.8->pyfolio-reloaded) (1.26.15)\n",
            "Requirement already satisfied: pycparser in /usr/local/lib/python3.10/site-packages (from cffi>=1.12->cryptography>=3.3.2->yfinance>=0.1.63->empyrical-reloaded>=0.5.8->pyfolio-reloaded) (2.21)\n",
            "Installing collected packages: bottleneck, empyrical-reloaded, pyfolio-reloaded\n",
            "Successfully installed bottleneck-1.3.7 empyrical-reloaded-0.5.9 pyfolio-reloaded-0.9.5\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
            "\u001b[0mLooking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting optuna\n",
            "  Downloading optuna-3.1.1-py3-none-any.whl (365 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m365.7/365.7 kB\u001b[0m \u001b[31m19.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: sqlalchemy>=1.3.0 in /usr/local/lib/python3.10/site-packages (from optuna) (1.4.48)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/site-packages (from optuna) (4.65.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/site-packages (from optuna) (1.24.3)\n",
            "Requirement already satisfied: PyYAML in /usr/local/lib/python3.10/site-packages (from optuna) (6.0)\n",
            "Collecting colorlog\n",
            "  Downloading colorlog-6.7.0-py2.py3-none-any.whl (11 kB)\n",
            "Collecting alembic>=1.5.0\n",
            "  Downloading alembic-1.10.4-py3-none-any.whl (212 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m212.9/212.9 kB\u001b[0m \u001b[31m28.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting cmaes>=0.9.1\n",
            "  Downloading cmaes-0.9.1-py3-none-any.whl (21 kB)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/site-packages (from optuna) (23.1)\n",
            "Requirement already satisfied: typing-extensions>=4 in /usr/local/lib/python3.10/site-packages (from alembic>=1.5.0->optuna) (4.5.0)\n",
            "Collecting Mako\n",
            "  Downloading Mako-1.2.4-py3-none-any.whl (78 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m78.7/78.7 kB\u001b[0m \u001b[31m7.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.10/site-packages (from sqlalchemy>=1.3.0->optuna) (2.0.2)\n",
            "Requirement already satisfied: MarkupSafe>=0.9.2 in /usr/local/lib/python3.10/site-packages (from Mako->alembic>=1.5.0->optuna) (2.1.2)\n",
            "Installing collected packages: Mako, colorlog, cmaes, alembic, optuna\n",
            "Successfully installed Mako-1.2.4 alembic-1.10.4 cmaes-0.9.1 colorlog-6.7.0 optuna-3.1.1\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
            "\u001b[0mLooking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: ray[rllib] in /usr/local/lib/python3.10/site-packages (2.4.0)\n",
            "Requirement already satisfied: virtualenv<20.21.1,>=20.0.24 in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (20.21.0)\n",
            "Requirement already satisfied: protobuf!=3.19.5,>=3.15.3 in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (3.20.3)\n",
            "Requirement already satisfied: jsonschema in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (4.17.3)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (3.12.0)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (6.0)\n",
            "Requirement already satisfied: attrs in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (23.1.0)\n",
            "Requirement already satisfied: aiosignal in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (1.3.1)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (2.28.2)\n",
            "Requirement already satisfied: grpcio<=1.51.3,>=1.42.0 in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (1.51.3)\n",
            "Requirement already satisfied: numpy>=1.19.3 in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (1.24.3)\n",
            "Requirement already satisfied: msgpack<2.0.0,>=1.0.0 in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (1.0.3)\n",
            "Requirement already satisfied: frozenlist in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (1.3.3)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (23.1)\n",
            "Requirement already satisfied: click>=7.0 in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (8.1.3)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (2.0.1)\n",
            "Collecting scikit-image\n",
            "  Downloading scikit_image-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (13.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m13.2/13.2 MB\u001b[0m \u001b[31m94.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting dm-tree\n",
            "  Downloading dm_tree-0.1.8-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (152 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m152.8/152.8 kB\u001b[0m \u001b[31m22.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting rich\n",
            "  Downloading rich-13.3.5-py3-none-any.whl (238 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m238.7/238.7 kB\u001b[0m \u001b[31m29.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: lz4 in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (4.3.2)\n",
            "Collecting gymnasium==0.26.3\n",
            "  Downloading Gymnasium-0.26.3-py3-none-any.whl (836 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m836.9/836.9 kB\u001b[0m \u001b[31m61.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting typer\n",
            "  Downloading typer-0.9.0-py3-none-any.whl (45 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m45.9/45.9 kB\u001b[0m \u001b[31m6.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: scipy in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (1.10.1)\n",
            "Requirement already satisfied: tensorboardX>=1.9 in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (2.6)\n",
            "Requirement already satisfied: tabulate in /usr/local/lib/python3.10/site-packages (from ray[rllib]) (0.9.0)\n",
            "Collecting gymnasium-notices>=0.0.1\n",
            "  Downloading gymnasium_notices-0.0.1-py3-none-any.whl (2.8 kB)\n",
            "Requirement already satisfied: cloudpickle>=1.2.0 in /usr/local/lib/python3.10/site-packages (from gymnasium==0.26.3->ray[rllib]) (2.2.1)\n",
            "Requirement already satisfied: distlib<1,>=0.3.6 in /usr/local/lib/python3.10/site-packages (from virtualenv<20.21.1,>=20.0.24->ray[rllib]) (0.3.6)\n",
            "Requirement already satisfied: platformdirs<4,>=2.4 in /usr/local/lib/python3.10/site-packages (from virtualenv<20.21.1,>=20.0.24->ray[rllib]) (3.5.0)\n",
            "Requirement already satisfied: pyrsistent!=0.17.0,!=0.17.1,!=0.17.2,>=0.14.0 in /usr/local/lib/python3.10/site-packages (from jsonschema->ray[rllib]) (0.19.3)\n",
            "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/site-packages (from pandas->ray[rllib]) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/site-packages (from pandas->ray[rllib]) (2023.3)\n",
            "Requirement already satisfied: tzdata>=2022.1 in /usr/local/lib/python3.10/site-packages (from pandas->ray[rllib]) (2023.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/site-packages (from requests->ray[rllib]) (2022.12.7)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/site-packages (from requests->ray[rllib]) (2.1.1)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/site-packages (from requests->ray[rllib]) (3.4)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/site-packages (from requests->ray[rllib]) (1.26.15)\n",
            "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/site-packages (from rich->ray[rllib]) (2.15.1)\n",
            "Collecting markdown-it-py<3.0.0,>=2.2.0\n",
            "  Downloading markdown_it_py-2.2.0-py3-none-any.whl (84 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m84.5/84.5 kB\u001b[0m \u001b[31m13.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pillow>=9.0.1 in /usr/local/lib/python3.10/site-packages (from scikit-image->ray[rllib]) (9.5.0)\n",
            "Requirement already satisfied: networkx>=2.8 in /usr/local/lib/python3.10/site-packages (from scikit-image->ray[rllib]) (3.1)\n",
            "Collecting imageio>=2.4.1\n",
            "  Downloading imageio-2.28.1-py3-none-any.whl (3.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.4/3.4 MB\u001b[0m \u001b[31m106.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting PyWavelets>=1.1.1\n",
            "  Downloading PyWavelets-1.4.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (6.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.8/6.8 MB\u001b[0m \u001b[31m86.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting lazy_loader>=0.1\n",
            "  Downloading lazy_loader-0.2-py3-none-any.whl (8.6 kB)\n",
            "Collecting tifffile>=2019.7.26\n",
            "  Downloading tifffile-2023.4.12-py3-none-any.whl (219 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m219.4/219.4 kB\u001b[0m \u001b[31m29.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/site-packages (from typer->ray[rllib]) (4.5.0)\n",
            "Collecting mdurl~=0.1\n",
            "  Downloading mdurl-0.1.2-py3-none-any.whl (10.0 kB)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/site-packages (from python-dateutil>=2.8.2->pandas->ray[rllib]) (1.16.0)\n",
            "Installing collected packages: gymnasium-notices, dm-tree, typer, tifffile, PyWavelets, mdurl, lazy_loader, imageio, gymnasium, scikit-image, markdown-it-py, rich\n",
            "Successfully installed PyWavelets-1.4.1 dm-tree-0.1.8 gymnasium-0.26.3 gymnasium-notices-0.0.1 imageio-2.28.1 lazy_loader-0.2 markdown-it-py-2.2.0 mdurl-0.1.2 rich-13.3.5 scikit-image-0.20.0 tifffile-2023.4.12 typer-0.9.0\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
            "\u001b[0mLooking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting plotly\n",
            "  Downloading plotly-5.14.1-py2.py3-none-any.whl (15.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m15.3/15.3 MB\u001b[0m \u001b[31m90.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting tenacity>=6.2.0\n",
            "  Downloading tenacity-8.2.2-py3-none-any.whl (24 kB)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/site-packages (from plotly) (23.1)\n",
            "Installing collected packages: tenacity, plotly\n",
            "Successfully installed plotly-5.14.1 tenacity-8.2.2\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
            "\u001b[0mLooking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting ipywidgets\n",
            "  Downloading ipywidgets-8.0.6-py3-none-any.whl (138 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m138.3/138.3 kB\u001b[0m \u001b[31m7.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting widgetsnbextension~=4.0.7\n",
            "  Downloading widgetsnbextension-4.0.7-py3-none-any.whl (2.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.1/2.1 MB\u001b[0m \u001b[31m65.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: traitlets>=4.3.1 in /usr/local/lib/python3.10/site-packages (from ipywidgets) (5.9.0)\n",
            "Collecting jupyterlab-widgets~=3.0.7\n",
            "  Downloading jupyterlab_widgets-3.0.7-py3-none-any.whl (198 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m198.2/198.2 kB\u001b[0m \u001b[31m13.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: ipython>=6.1.0 in /usr/local/lib/python3.10/site-packages (from ipywidgets) (8.13.1)\n",
            "Collecting ipykernel>=4.5.1\n",
            "  Downloading ipykernel-6.22.0-py3-none-any.whl (149 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m150.0/150.0 kB\u001b[0m \u001b[31m19.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting jupyter-client>=6.1.12\n",
            "  Downloading jupyter_client-8.2.0-py3-none-any.whl (103 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m103.2/103.2 kB\u001b[0m \u001b[31m534.5 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: psutil in /usr/local/lib/python3.10/site-packages (from ipykernel>=4.5.1->ipywidgets) (5.9.5)\n",
            "Collecting jupyter-core!=5.0.*,>=4.12\n",
            "  Downloading jupyter_core-5.3.0-py3-none-any.whl (93 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m93.2/93.2 kB\u001b[0m \u001b[31m13.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.10/site-packages (from ipykernel>=4.5.1->ipywidgets) (23.1)\n",
            "Collecting debugpy>=1.6.5\n",
            "  Downloading debugpy-1.6.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.0/3.0 MB\u001b[0m \u001b[31m58.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nest-asyncio\n",
            "  Downloading nest_asyncio-1.5.6-py3-none-any.whl (5.2 kB)\n",
            "Collecting comm>=0.1.1\n",
            "  Downloading comm-0.1.3-py3-none-any.whl (6.6 kB)\n",
            "Collecting pyzmq>=20\n",
            "  Downloading pyzmq-25.0.2-cp310-cp310-manylinux_2_28_x86_64.whl (1.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.1 MB\u001b[0m \u001b[31m57.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting tornado>=6.1\n",
            "  Downloading tornado-6.3.1-cp38-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (426 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m426.8/426.8 kB\u001b[0m \u001b[31m41.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: matplotlib-inline>=0.1 in /usr/local/lib/python3.10/site-packages (from ipykernel>=4.5.1->ipywidgets) (0.1.6)\n",
            "Requirement already satisfied: decorator in /usr/local/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets) (5.1.1)\n",
            "Requirement already satisfied: pexpect>4.3 in /usr/local/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets) (4.8.0)\n",
            "Requirement already satisfied: jedi>=0.16 in /usr/local/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets) (0.18.2)\n",
            "Requirement already satisfied: pickleshare in /usr/local/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets) (0.7.5)\n",
            "Requirement already satisfied: backcall in /usr/local/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets) (0.2.0)\n",
            "Requirement already satisfied: prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30 in /usr/local/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets) (3.0.38)\n",
            "Requirement already satisfied: stack-data in /usr/local/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets) (0.6.2)\n",
            "Requirement already satisfied: pygments>=2.4.0 in /usr/local/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets) (2.15.1)\n",
            "Requirement already satisfied: parso<0.9.0,>=0.8.0 in /usr/local/lib/python3.10/site-packages (from jedi>=0.16->ipython>=6.1.0->ipywidgets) (0.8.3)\n",
            "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/site-packages (from jupyter-client>=6.1.12->ipykernel>=4.5.1->ipywidgets) (2.8.2)\n",
            "Requirement already satisfied: platformdirs>=2.5 in /usr/local/lib/python3.10/site-packages (from jupyter-core!=5.0.*,>=4.12->ipykernel>=4.5.1->ipywidgets) (3.5.0)\n",
            "Requirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.10/site-packages (from pexpect>4.3->ipython>=6.1.0->ipywidgets) (0.7.0)\n",
            "Requirement already satisfied: wcwidth in /usr/local/lib/python3.10/site-packages (from prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30->ipython>=6.1.0->ipywidgets) (0.2.6)\n",
            "Requirement already satisfied: pure-eval in /usr/local/lib/python3.10/site-packages (from stack-data->ipython>=6.1.0->ipywidgets) (0.2.2)\n",
            "Requirement already satisfied: asttokens>=2.1.0 in /usr/local/lib/python3.10/site-packages (from stack-data->ipython>=6.1.0->ipywidgets) (2.2.1)\n",
            "Requirement already satisfied: executing>=1.2.0 in /usr/local/lib/python3.10/site-packages (from stack-data->ipython>=6.1.0->ipywidgets) (1.2.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.10/site-packages (from asttokens>=2.1.0->stack-data->ipython>=6.1.0->ipywidgets) (1.16.0)\n",
            "Installing collected packages: widgetsnbextension, tornado, pyzmq, nest-asyncio, jupyterlab-widgets, jupyter-core, debugpy, comm, jupyter-client, ipykernel, ipywidgets\n",
            "Successfully installed comm-0.1.3 debugpy-1.6.7 ipykernel-6.22.0 ipywidgets-8.0.6 jupyter-client-8.2.0 jupyter-core-5.3.0 jupyterlab-widgets-3.0.7 nest-asyncio-1.5.6 pyzmq-25.0.2 tornado-6.3.1 widgetsnbextension-4.0.7\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
            "\u001b[0mLooking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting kaleido\n",
            "  Downloading kaleido-0.2.1-py2.py3-none-manylinux1_x86_64.whl (79.9 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m79.9/79.9 MB\u001b[0m \u001b[31m15.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: kaleido\n",
            "Successfully installed kaleido-0.2.1\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
            "\u001b[0m"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install pandas==1.5.3\n",
        "!pip install gymnasium"
      ],
      "metadata": {
        "id": "9Zkmtvr-TFn_",
        "outputId": "432ff578-a0a2-4fda-a514-8ae351e33991",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: pandas==1.5.3 in /usr/local/lib/python3.9/site-packages (1.5.3)\n",
            "Requirement already satisfied: numpy>=1.20.3 in /usr/local/lib/python3.9/site-packages (from pandas==1.5.3) (1.24.3)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.9/site-packages (from pandas==1.5.3) (2023.3)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.9/site-packages (from pandas==1.5.3) (2.8.2)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.9/site-packages (from python-dateutil>=2.8.1->pandas==1.5.3) (1.16.0)\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
            "\u001b[0mLooking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: gymnasium in /usr/local/lib/python3.9/site-packages (0.26.3)\n",
            "Requirement already satisfied: importlib-metadata>=4.8.0 in /usr/local/lib/python3.9/site-packages (from gymnasium) (4.13.0)\n",
            "Requirement already satisfied: numpy>=1.18.0 in /usr/local/lib/python3.9/site-packages (from gymnasium) (1.24.3)\n",
            "Requirement already satisfied: cloudpickle>=1.2.0 in /usr/local/lib/python3.9/site-packages (from gymnasium) (2.2.1)\n",
            "Requirement already satisfied: gymnasium-notices>=0.0.1 in /usr/local/lib/python3.9/site-packages (from gymnasium) (0.0.1)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.9/site-packages (from importlib-metadata>=4.8.0->gymnasium) (3.15.0)\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
            "\u001b[0m"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import sys\n",
        "sys.path.append(\"../FinRL-Library\")"
      ],
      "metadata": {
        "id": "cMn1VNKCR0ld"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 1.3 Import packages"
      ],
      "metadata": {
        "id": "BbOMJnGdTRYX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#Importing the libraries\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import matplotlib\n",
        "import matplotlib.pyplot as plt\n",
        "# matplotlib.use('Agg')\n",
        "import datetime\n",
        "import optuna\n",
        "from pathlib import Path\n",
        "from google.colab import files\n",
        "%matplotlib inline\n",
        "from finrl import config\n",
        "from finrl import config_tickers\n",
        "from optuna.integration import PyTorchLightningPruningCallback\n",
        "from finrl.meta.preprocessor.yahoodownloader import YahooDownloader\n",
        "from finrl.meta.preprocessor.preprocessors import FeatureEngineer, data_split\n",
        "from finrl.meta.env_stock_trading.env_stocktrading import StockTradingEnv\n",
        "from finrl.meta.env_stock_trading.env_stocktrading_np import StockTradingEnv as StockTradingEnv_numpy\n",
        "from finrl.agents.stablebaselines3.models import DRLAgent\n",
        "from finrl.agents.rllib.models import DRLAgent as DRLAgent_rllib\n",
        "from finrl.meta.data_processor import DataProcessor\n",
        "import joblib\n",
        "from finrl.plot import backtest_stats, backtest_plot, get_daily_return, get_baseline\n",
        "import ray\n",
        "from pprint import pprint\n",
        "import kaleido\n",
        "\n",
        "\n",
        "\n",
        "import itertools\n",
        "\n",
        "import torch\n",
        "if torch.cuda.is_available():\n",
        "  device = torch.device(\"cuda\")\n",
        "else:\n",
        "  device = torch.device(\"cpu\")\n",
        "print(f'Torch device: {device}')"
      ],
      "metadata": {
        "id": "tG5qIkVHTNv8",
        "outputId": "4bd85d52-a54f-4963-e57e-e29f9cab78f2",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Torch device: cuda\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/site-packages/pyfolio/pos.py:25: UserWarning: Module \"zipline.assets\" not found; multipliers will not be applied to position notionals.\n",
            "  warnings.warn(\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "config_tickers.DOW_30_TICKER"
      ],
      "metadata": {
        "id": "DB11ygQLVLUD",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d31470b0-481f-463f-95a7-ac4568c1017c"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['AXP',\n",
              " 'AMGN',\n",
              " 'AAPL',\n",
              " 'BA',\n",
              " 'CAT',\n",
              " 'CSCO',\n",
              " 'CVX',\n",
              " 'GS',\n",
              " 'HD',\n",
              " 'HON',\n",
              " 'IBM',\n",
              " 'INTC',\n",
              " 'JNJ',\n",
              " 'KO',\n",
              " 'JPM',\n",
              " 'MCD',\n",
              " 'MMM',\n",
              " 'MRK',\n",
              " 'MSFT',\n",
              " 'NKE',\n",
              " 'PG',\n",
              " 'TRV',\n",
              " 'UNH',\n",
              " 'CRM',\n",
              " 'VZ',\n",
              " 'V',\n",
              " 'WBA',\n",
              " 'WMT',\n",
              " 'DIS',\n",
              " 'DOW']"
            ]
          },
          "metadata": {},
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "fMaPm8MsUo0R"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "if not os.path.exists(\"./\" + config.DATA_SAVE_DIR):\n",
        "    os.makedirs(\"./\" + config.DATA_SAVE_DIR)\n",
        "if not os.path.exists(\"./\" + config.TRAINED_MODEL_DIR):\n",
        "    os.makedirs(\"./\" + config.TRAINED_MODEL_DIR)\n",
        "if not os.path.exists(\"./\" + config.TENSORBOARD_LOG_DIR):\n",
        "    os.makedirs(\"./\" + config.TENSORBOARD_LOG_DIR)\n",
        "if not os.path.exists(\"./\" + config.RESULTS_DIR):\n",
        "    os.makedirs(\"./\" + config.RESULTS_DIR)"
      ],
      "metadata": {
        "id": "FAZpwTH3VVi-"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Collecting data and preprocessing"
      ],
      "metadata": {
        "id": "cqJ8ngFWVcNG"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#config_tickers.DOW_30_TICKER = [\"PYPL\"]"
      ],
      "metadata": {
        "id": "4M4UwcP4Ap_o"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#Custom ticker list dataframe download\n",
        "#TODO save df to avoid download\n",
        "path_pf = '/content/ticker_data.csv'\n",
        "if Path(path_pf).is_file():\n",
        "  print('Reading ticker data')\n",
        "  df = pd.read_csv(path_pf)\n",
        "  \n",
        "else:\n",
        "  print('Downloading ticker data')\n",
        "  ticker_list = config_tickers.DOW_30_TICKER\n",
        "  df = YahooDownloader(start_date = '2009-01-01',\n",
        "                     end_date = '2023-04-30',\n",
        "                     ticker_list = ticker_list).fetch_data()\n",
        "  df.to_csv('ticker_data.csv')"
      ],
      "metadata": {
        "id": "7BJNlsAuVZ-o",
        "outputId": "4c8cff4f-66f5-4fac-c824-41768a74121f",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Reading ticker data\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def create_processed_full(processed):\n",
        "  list_ticker = processed[\"tic\"].unique().tolist()\n",
        "  list_date = list(pd.date_range(processed['date'].min(),processed['date'].max()).astype(str))\n",
        "  combination = list(itertools.product(list_date,list_ticker))\n",
        "\n",
        "  processed_full = pd.DataFrame(combination,columns=[\"date\",\"tic\"]).merge(processed,on=[\"date\",\"tic\"],how=\"left\")\n",
        "  processed_full = processed_full[processed_full['date'].isin(processed['date'])]\n",
        "  processed_full = processed_full.sort_values(['date','tic'])\n",
        "\n",
        "  processed_full = processed_full.fillna(0)\n",
        "  processed_full.sort_values(['date','tic'],ignore_index=True).head(5)\n",
        "\n",
        "  processed_full.to_csv('processed_full.csv')\n",
        "  return processed_full"
      ],
      "metadata": {
        "id": "RD6wwtGSVZVU"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#You can add technical indicators and turbulence factor to dataframe\n",
        "#Just set the use_technical_indicator=True, use_vix=True and use_turbulence=True\n",
        "def create_techind():\n",
        "  fe = FeatureEngineer(\n",
        "                    use_technical_indicator=True,\n",
        "                    tech_indicator_list = config.INDICATORS,\n",
        "                    use_vix=True,\n",
        "                    use_turbulence=True,\n",
        "                    user_defined_feature = False)\n",
        "\n",
        "  processed = fe.preprocess_data(df)\n",
        "  return processed"
      ],
      "metadata": {
        "id": "UNqEALmbVoJo"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#Load price and technical indicator data from file if available\n",
        "path_pf = '/content/processed_full.csv'\n",
        "if Path(path_pf).is_file():\n",
        "  print('Reading processed_full data')\n",
        "  processed_full = pd.read_csv(path_pf)\n",
        "\n",
        "else:\n",
        "  print('Creating processed_full file')\n",
        "  processed=create_techind()\n",
        "  processed_full=create_processed_full(processed)"
      ],
      "metadata": {
        "id": "Qr1Qd-lMVpzg",
        "outputId": "5206ed3d-f848-4a65-9783-9be6bbf7380e",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Reading processed_full data\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "date_col = \"date\"\n",
        "tic_col = \"tic\"\n",
        "\n",
        "init_train_trade_data = processed_full.sort_values([date_col, tic_col])\n",
        "\n",
        "init_train_trade_data = processed_full.fillna(0)\n",
        "\n",
        "init_train_data = data_split(\n",
        "    init_train_trade_data, '2020-01-01', '2020-05-01')\n",
        "init_trade_data = data_split(\n",
        "    init_train_trade_data, '2021-05-01','2021-10-01')\n",
        "\n",
        "print(f'Number of training samples: {len(init_train_data)}')\n",
        "print(f'Number of testing samples: {len(init_train_trade_data)}')"
      ],
      "metadata": {
        "id": "KFs4AjWIVr2s",
        "outputId": "dae08f54-ddd1-41d3-b83d-741bf93b70ab",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Number of training samples: 2407\n",
            "Number of testing samples: 104516\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "init_trade_data.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 250
        },
        "id": "GI2qgPGFTerU",
        "outputId": "71be646c-caf3-4dce-924a-f04df2359f72"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   Unnamed: 0        date   tic        open        high         low   \n",
              "0      130616  2021-05-03  AAPL  132.039993  134.070007  131.830002  \\\n",
              "0      130617  2021-05-03  AMGN  240.669998  247.020004  240.550003   \n",
              "0      130618  2021-05-03   AXP  154.589996  156.050003  154.009995   \n",
              "0      130619  2021-05-03    BA  234.110001  237.100006  233.809998   \n",
              "0      130620  2021-05-03   CAT  230.000000  230.929993  227.210007   \n",
              "\n",
              "        close      volume  day      macd     boll_ub     boll_lb     rsi_30   \n",
              "0  130.963547  75135100.0  0.0  1.805508  135.539558  126.477489  53.856509  \\\n",
              "0  230.412491   3587700.0  0.0 -0.410712  248.223094  221.790977  50.842033   \n",
              "0  150.957825   2726200.0  0.0  2.754003  151.027338  139.179359  62.209616   \n",
              "0  235.190002   9887800.0  0.0 -2.593339  260.290298  227.831702  49.445883   \n",
              "0  218.290497   3182900.0  0.0  1.167004  223.005763  216.688283  56.081916   \n",
              "\n",
              "       cci_30      dx_30  close_30_sma  close_60_sma        vix  turbulence  \n",
              "0   50.478409  20.583396    127.527261    126.185906  18.309999   20.080286  \n",
              "0  -81.617889   5.785497    234.734412    226.585522  18.309999   20.080286  \n",
              "0  175.795921  22.874057    142.745663    138.440773  18.309999   20.080286  \n",
              "0 -102.331978  17.827725    245.819000    237.101333  18.309999   20.080286  \n",
              "0  -17.678502   2.780977    218.974400    211.601993  18.309999   20.080286  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-af489a9d-4aa2-4cf1-8436-94f18cef3777\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>date</th>\n",
              "      <th>tic</th>\n",
              "      <th>open</th>\n",
              "      <th>high</th>\n",
              "      <th>low</th>\n",
              "      <th>close</th>\n",
              "      <th>volume</th>\n",
              "      <th>day</th>\n",
              "      <th>macd</th>\n",
              "      <th>boll_ub</th>\n",
              "      <th>boll_lb</th>\n",
              "      <th>rsi_30</th>\n",
              "      <th>cci_30</th>\n",
              "      <th>dx_30</th>\n",
              "      <th>close_30_sma</th>\n",
              "      <th>close_60_sma</th>\n",
              "      <th>vix</th>\n",
              "      <th>turbulence</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>130616</td>\n",
              "      <td>2021-05-03</td>\n",
              "      <td>AAPL</td>\n",
              "      <td>132.039993</td>\n",
              "      <td>134.070007</td>\n",
              "      <td>131.830002</td>\n",
              "      <td>130.963547</td>\n",
              "      <td>75135100.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.805508</td>\n",
              "      <td>135.539558</td>\n",
              "      <td>126.477489</td>\n",
              "      <td>53.856509</td>\n",
              "      <td>50.478409</td>\n",
              "      <td>20.583396</td>\n",
              "      <td>127.527261</td>\n",
              "      <td>126.185906</td>\n",
              "      <td>18.309999</td>\n",
              "      <td>20.080286</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>130617</td>\n",
              "      <td>2021-05-03</td>\n",
              "      <td>AMGN</td>\n",
              "      <td>240.669998</td>\n",
              "      <td>247.020004</td>\n",
              "      <td>240.550003</td>\n",
              "      <td>230.412491</td>\n",
              "      <td>3587700.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>-0.410712</td>\n",
              "      <td>248.223094</td>\n",
              "      <td>221.790977</td>\n",
              "      <td>50.842033</td>\n",
              "      <td>-81.617889</td>\n",
              "      <td>5.785497</td>\n",
              "      <td>234.734412</td>\n",
              "      <td>226.585522</td>\n",
              "      <td>18.309999</td>\n",
              "      <td>20.080286</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>130618</td>\n",
              "      <td>2021-05-03</td>\n",
              "      <td>AXP</td>\n",
              "      <td>154.589996</td>\n",
              "      <td>156.050003</td>\n",
              "      <td>154.009995</td>\n",
              "      <td>150.957825</td>\n",
              "      <td>2726200.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>2.754003</td>\n",
              "      <td>151.027338</td>\n",
              "      <td>139.179359</td>\n",
              "      <td>62.209616</td>\n",
              "      <td>175.795921</td>\n",
              "      <td>22.874057</td>\n",
              "      <td>142.745663</td>\n",
              "      <td>138.440773</td>\n",
              "      <td>18.309999</td>\n",
              "      <td>20.080286</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>130619</td>\n",
              "      <td>2021-05-03</td>\n",
              "      <td>BA</td>\n",
              "      <td>234.110001</td>\n",
              "      <td>237.100006</td>\n",
              "      <td>233.809998</td>\n",
              "      <td>235.190002</td>\n",
              "      <td>9887800.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>-2.593339</td>\n",
              "      <td>260.290298</td>\n",
              "      <td>227.831702</td>\n",
              "      <td>49.445883</td>\n",
              "      <td>-102.331978</td>\n",
              "      <td>17.827725</td>\n",
              "      <td>245.819000</td>\n",
              "      <td>237.101333</td>\n",
              "      <td>18.309999</td>\n",
              "      <td>20.080286</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>130620</td>\n",
              "      <td>2021-05-03</td>\n",
              "      <td>CAT</td>\n",
              "      <td>230.000000</td>\n",
              "      <td>230.929993</td>\n",
              "      <td>227.210007</td>\n",
              "      <td>218.290497</td>\n",
              "      <td>3182900.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.167004</td>\n",
              "      <td>223.005763</td>\n",
              "      <td>216.688283</td>\n",
              "      <td>56.081916</td>\n",
              "      <td>-17.678502</td>\n",
              "      <td>2.780977</td>\n",
              "      <td>218.974400</td>\n",
              "      <td>211.601993</td>\n",
              "      <td>18.309999</td>\n",
              "      <td>20.080286</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-af489a9d-4aa2-4cf1-8436-94f18cef3777')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-af489a9d-4aa2-4cf1-8436-94f18cef3777 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-af489a9d-4aa2-4cf1-8436-94f18cef3777');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Creating Environment"
      ],
      "metadata": {
        "id": "L3VlkfBMW8e7"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from __future__ import annotations\n",
        "\n",
        "from typing import List\n",
        "\n",
        "import gym\n",
        "import matplotlib\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "np.set_printoptions(linewidth=np.inf)\n",
        "import pandas as pd\n",
        "from gym import spaces\n",
        "from gym.utils import seeding\n",
        "from stable_baselines3.common.vec_env import DummyVecEnv\n",
        "\n",
        "matplotlib.use(\"Agg\")\n",
        "\n",
        "# from stable_baselines3.common.logger import Logger, KVWriter, CSVOutputFormat\n",
        "\n",
        "\n",
        "class GokuEnv(gym.Env):\n",
        "    \"\"\"A stock trading environment for OpenAI gym\"\"\"\n",
        "\n",
        "    metadata = {\"render.modes\": [\"human\"]}\n",
        "\n",
        "    def __init__(\n",
        "        self,\n",
        "        df: pd.DataFrame,\n",
        "        stock_dim: int,\n",
        "        hmax: int,\n",
        "        initial_amount: int,\n",
        "        num_stock_shares: list[int],\n",
        "        buy_cost_pct: list[float],\n",
        "        sell_cost_pct: list[float],\n",
        "        reward_scaling: float,\n",
        "        state_space: int,\n",
        "        action_space: int,\n",
        "        tech_indicator_list: list[str],\n",
        "        turbulence_threshold=None,\n",
        "        risk_indicator_col=\"turbulence\",\n",
        "        make_plots: bool = False,\n",
        "        print_verbosity=10,\n",
        "        day=0,\n",
        "        initial=True,\n",
        "        previous_state=[],\n",
        "        model_name=\"\",\n",
        "        mode=\"\",\n",
        "        iteration=\"\",\n",
        "    ):\n",
        "        self.day = day\n",
        "        self.df = df\n",
        "        self.stock_dim = stock_dim\n",
        "        self.hmax = hmax\n",
        "        self.num_stock_shares = num_stock_shares\n",
        "        self.initial_amount = initial_amount  # get the initial cash\n",
        "        self.buy_cost_pct = buy_cost_pct\n",
        "        self.sell_cost_pct = sell_cost_pct\n",
        "        self.reward_scaling = reward_scaling\n",
        "        self.state_space = state_space\n",
        "        self.action_space = action_space\n",
        "        self.tech_indicator_list = tech_indicator_list\n",
        "        self.action_space = spaces.Box(low=-1, high=1, shape=(self.action_space,))\n",
        "        self.observation_space = spaces.Box(\n",
        "            low=-np.inf, high=np.inf, shape=(self.state_space,)\n",
        "        )\n",
        "        self.data = self.df.loc[self.day, :]\n",
        "        self.terminal = False\n",
        "        self.make_plots = make_plots\n",
        "        self.print_verbosity = print_verbosity\n",
        "        self.turbulence_threshold = turbulence_threshold\n",
        "        self.risk_indicator_col = risk_indicator_col\n",
        "        self.initial = initial\n",
        "        self.previous_state = previous_state\n",
        "        self.model_name = model_name\n",
        "        self.mode = mode\n",
        "        self.iteration = iteration\n",
        "        # initalize state\n",
        "        self.state = self._initiate_state()\n",
        "\n",
        "        # initialize reward\n",
        "        self.reward = 0\n",
        "        self.turbulence = 0\n",
        "        self.cost = 0\n",
        "        self.trades = 0\n",
        "        self.episode = 0\n",
        "        # memorize all the total balance change\n",
        "        self.asset_memory = [\n",
        "            self.initial_amount\n",
        "            + np.sum(\n",
        "                np.array(self.num_stock_shares)\n",
        "                * np.array(self.state[1 : 1 + self.stock_dim])\n",
        "            )\n",
        "        ]  # the initial total asset is calculated by cash + sum (num_share_stock_i * price_stock_i)\n",
        "        self.rewards_memory = []\n",
        "        self.actions_memory = []\n",
        "        self.state_memory = (\n",
        "            []\n",
        "        )  # we need sometimes to preserve the state in the middle of trading process\n",
        "        self.date_memory = [self._get_date()]\n",
        "        #         self.logger = Logger('results',[CSVOutputFormat])\n",
        "        # self.reset()\n",
        "        self._seed()\n",
        "        self.total_price = np.array([0.0] * 29)\n",
        "        self.avg_price = np.array([0.0] * 29)\n",
        "        self.total_stockss = np.array([0.0] * 29)\n",
        "\n",
        "    def _sell_stock(self, index, action):\n",
        "        def _do_sell_normal():\n",
        "            if (\n",
        "                self.state[index + 2 * self.stock_dim + 1] != True\n",
        "            ):  # check if the stock is able to sell, for simlicity we just add it in techical index\n",
        "                # if self.state[index + 1] > 0: # if we use price<0 to denote a stock is unable to trade in that day, the total asset calculation may be wrong for the price is unreasonable\n",
        "                # Sell only if the price is > 0 (no missing data in this particular date)\n",
        "                # perform sell action based on the sign of the action\n",
        "                if self.state[index + self.stock_dim + 1] > 0:\n",
        "                    # Sell only if current asset is > 0\n",
        "                    sell_num_shares = min(\n",
        "                        abs(action), self.state[index + self.stock_dim + 1]\n",
        "                    )\n",
        "                    sell_amount = (\n",
        "                        self.state[index + 1]\n",
        "                        * sell_num_shares\n",
        "                        * (1 - self.sell_cost_pct[index])\n",
        "                    )\n",
        "                    # update balance\n",
        "                    self.state[0] += sell_amount\n",
        "\n",
        "                    self.state[index + self.stock_dim + 1] -= sell_num_shares\n",
        "                    self.cost += (\n",
        "                        self.state[index + 1]\n",
        "                        * sell_num_shares\n",
        "                        * self.sell_cost_pct[index]\n",
        "                    )\n",
        "                    self.trades += 1\n",
        "                    #if sell_num_shares >0:\n",
        "                    #  print(\"stocks Sold sell_num_shares\", sell_num_shares)\n",
        "                else:\n",
        "                    sell_num_shares = 0\n",
        "            else:\n",
        "                sell_num_shares = 0\n",
        "            return sell_num_shares\n",
        "\n",
        "        # perform sell action based on the sign of the action\n",
        "        if self.turbulence_threshold is not None:\n",
        "            if self.turbulence >= self.turbulence_threshold:\n",
        "                if self.state[index + 1] > 0:\n",
        "                    # Sell only if the price is > 0 (no missing data in this particular date)\n",
        "                    # if turbulence goes over threshold, just clear out all positions\n",
        "                    if self.state[index + self.stock_dim + 1] > 0:\n",
        "                        # Sell only if current asset is > 0\n",
        "                        sell_num_shares = self.state[index + self.stock_dim + 1]\n",
        "                        sell_amount = (\n",
        "                            self.state[index + 1]\n",
        "                            * sell_num_shares\n",
        "                            * (1 - self.sell_cost_pct[index])\n",
        "                        )\n",
        "                        # update balance\n",
        "                        self.state[0] += sell_amount\n",
        "                        self.state[index + self.stock_dim + 1] = 0\n",
        "                        self.cost += (\n",
        "                            self.state[index + 1]\n",
        "                            * sell_num_shares\n",
        "                            * self.sell_cost_pct[index]\n",
        "                        )\n",
        "                        self.trades += 1\n",
        "                    else:\n",
        "                        sell_num_shares = 0\n",
        "                else:\n",
        "                    sell_num_shares = 0\n",
        "            else:\n",
        "                sell_num_shares = _do_sell_normal()\n",
        "        else:\n",
        "            sell_num_shares = _do_sell_normal()\n",
        "        #print(\"stocks Sold sell_num_shares\", sell_num_shares)\n",
        "        return sell_num_shares\n",
        "\n",
        "    def _buy_stock(self, index, action):\n",
        "        def _do_buy():\n",
        "            if (self.state[index + 2 * self.stock_dim + 1] != True):  # check if the stock is able to buy\n",
        "                # if self.state[index + 1] >0:\n",
        "                # Buy only if the price is > 0 (no missing data in this particular date)\n",
        "                available_amount = self.state[0] // (self.state[index + 1] * (1 + self.buy_cost_pct[index]))\n",
        "                # when buying stocks, we should consider the cost of trading when calculating available_amount, or we may be have cash<0\n",
        "                # print('available_amount:{}'.format(available_amount))\n",
        "                # update balance\n",
        "                buy_num_shares = min(available_amount, action)\n",
        "                buy_amount = (self.state[index + 1] * buy_num_shares * (1 + self.buy_cost_pct[index]))\n",
        "                self.state[0] -= buy_amount\n",
        "                self.state[index + self.stock_dim + 1] += buy_num_shares\n",
        "                self.cost += (self.state[index + 1] * buy_num_shares * self.buy_cost_pct[index])\n",
        "                self.trades += 1\n",
        "            else:\n",
        "                buy_num_shares = 0\n",
        "\n",
        "            return buy_num_shares\n",
        "\n",
        "        # perform buy action based on the sign of the action\n",
        "        if self.turbulence_threshold is None:\n",
        "            buy_num_shares = _do_buy()\n",
        "        else:\n",
        "            if self.turbulence < self.turbulence_threshold:\n",
        "                buy_num_shares = _do_buy()\n",
        "            else:\n",
        "                buy_num_shares = 0\n",
        "                pass\n",
        "\n",
        "        return buy_num_shares\n",
        "\n",
        "    def _make_plot(self):\n",
        "        plt.plot(self.asset_memory, \"r\")\n",
        "        plt.savefig(f\"results/account_value_trade_{self.episode}.png\")\n",
        "        plt.close()\n",
        "\n",
        "    def step(self, actions):\n",
        "        self.terminal = self.day >= len(self.df.index.unique()) - 1\n",
        "        if self.terminal:\n",
        "            # print(f\"Episode: {self.episode}\")\n",
        "            if self.make_plots:\n",
        "                self._make_plot()\n",
        "            end_total_asset = self.state[0] + sum(np.array(self.state[1 : (self.stock_dim + 1)]) * np.array(self.state[(self.stock_dim + 1) : (self.stock_dim * 2 + 1)]))\n",
        "            df_total_value = pd.DataFrame(self.asset_memory)\n",
        "            tot_reward = (self.state[0] + \n",
        "                          sum(np.array(self.state[1 : (self.stock_dim + 1)]) \n",
        "                          * np.array(self.state[(self.stock_dim + 1) : (self.stock_dim * 2 + 1)])) \n",
        "                          - self.asset_memory[0])  # initial_amount is only cash part of our initial asset\n",
        "            df_total_value.columns = [\"account_value\"]\n",
        "            df_total_value[\"date\"] = self.date_memory\n",
        "            df_total_value[\"daily_return\"] = df_total_value[\"account_value\"].pct_change(1)\n",
        "            if df_total_value[\"daily_return\"].std() != 0:\n",
        "                sharpe = ((252**0.5) \n",
        "                * df_total_value[\"daily_return\"].mean() \n",
        "                / df_total_value[\"daily_return\"].std())\n",
        "            df_rewards = pd.DataFrame(self.rewards_memory)\n",
        "            df_rewards.columns = [\"account_rewards\"]\n",
        "            df_rewards[\"date\"] = self.date_memory[:-1]\n",
        "            \n",
        "            if self.episode % self.print_verbosity == 0:\n",
        "                print(f\"day: {self.day}, episode: {self.episode}\")\n",
        "                print(f\"begin_total_asset: {self.asset_memory[0]:0.2f}\")\n",
        "                print(f\"end_total_asset: {end_total_asset:0.2f}\")\n",
        "                print(f\"total_reward: {tot_reward:0.2f}\")\n",
        "                print(f\"total_cost: {self.cost:0.2f}\")\n",
        "                print(f\"total_trades: {self.trades}\")\n",
        "                if df_total_value[\"daily_return\"].std() != 0:\n",
        "                    print(f\"Sharpe: {sharpe:0.3f}\")\n",
        "                print(\"=================================\")\n",
        "\n",
        "            if (self.model_name != \"\") and (self.mode != \"\"):\n",
        "                df_actions = self.save_action_memory()\n",
        "                df_actions.to_csv(\"results/actions_{}_{}_{}.csv\".format(self.mode, self.model_name, self.iteration))\n",
        "                df_total_value.to_csv(\"results/account_value_{}_{}_{}.csv\".format(self.mode, self.model_name, self.iteration),index=False,)\n",
        "                df_rewards.to_csv(\"results/account_rewards_{}_{}_{}.csv\".format(self.mode, self.model_name, self.iteration),index=False,)\n",
        "                plt.plot(self.asset_memory, \"r\")\n",
        "                plt.savefig(\"results/account_value_{}_{}_{}.png\".format(self.mode, self.model_name, self.iteration))\n",
        "                plt.close()\n",
        "            return self.state, self.reward, self.terminal, {}\n",
        "\n",
        "        else:\n",
        "            actions = actions * self.hmax  # actions initially is scaled between 0 to 1\n",
        "            actions = actions.astype(int)  # convert into integer because we can't by fraction of shares\n",
        "\n",
        "\n",
        "            if self.turbulence_threshold is not None:\n",
        "                if self.turbulence >= self.turbulence_threshold:\n",
        "                    actions = np.array([-self.hmax] * self.stock_dim)\n",
        "            \n",
        "            current_price = np.array(self.state[1 : (self.stock_dim + 1)])\n",
        "            actions = np.where(((current_price > ( self.avg_price * 0.3  + self.avg_price))& (self.avg_price >0.0)), self.total_stockss*-1,actions  )\n",
        "            actions = np.where(((current_price < (self.avg_price - self.avg_price * .1))& (self.avg_price >0)), self.total_stockss*-1,actions  )\n",
        "            \n",
        "            \n",
        "            # Mandatory Selling Profits and Losses\n",
        "            # for i in range(0,len(actions)):\n",
        "            #   # self.total_price\n",
        "            #   # self.total_stockss\n",
        "            #   # self.avg_price\n",
        "            #   # print(\"Current Price\")\n",
        "            #   # print(np.array(self.state[1 : (self.stock_dim + 1)])[i])\n",
        "            #   current_price  = np.array(self.state[1 : (self.stock_dim + 1)])[i]\n",
        "            #   avg_price = self.avg_price[i]\n",
        "            #   #print(avg_price)\n",
        "            #   if (current_price > ( avg_price * 0.3  + avg_price))  and (avg_price > 0.0):\n",
        "            #     actions[i] = self.total_stockss[i] * -1\n",
        "            #     #print(\"Updated actions\")\n",
        "            #     #self.total_price[i] = 0.0\n",
        "            #     print(round(current_price) , \"|\", \" avg_price\", avg_price, \"Profit\", current_price -  avg_price)\n",
        "            #   elif current_price < (avg_price - avg_price * 1)  and avg_price > 0.0:\n",
        "            #     actions[i] = self.total_stockss[i] * -1\n",
        "            #     print(round(current_price) , \"|\", \" avg_price\", avg_price,  \"loss\", current_price -  avg_price )\n",
        "            #     #self.total_price[i] = 0.0\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "            begin_total_asset = self.state[0] + sum(np.array(self.state[1 : (self.stock_dim + 1)])* np.array(self.state[(self.stock_dim + 1) : (self.stock_dim * 2 + 1)]))\n",
        "            # print(\"begin_total_asset:{}\".format(begin_total_asset))\n",
        "            #print(\"*\" * 20 )\n",
        "            #print(np.array(self.state[1 : (self.stock_dim + 1)]))\n",
        "            \n",
        "            #print(np.array(self.state[(self.stock_dim + 1) : (self.stock_dim * 2 + 1)]))\n",
        "            #print(\"*\" * 20 )\n",
        "            \n",
        "            #print(\"AVG PRices\")  \n",
        "            #print(self.avg_price)\n",
        "            #print(np.where(np.array(self.state[1 : (self.stock_dim + 1)]) > (self.avg_price + self.avg_price*.15))) \n",
        "            #print(self.avg_price)\n",
        "            #print(self.total_stockss)\n",
        "            #print(\"*\"*20)\n",
        "\n",
        "            # if sum(self.total_stockss) !=0:\n",
        "            #   for i in np.where(np.array(self.state[1 : (self.stock_dim + 1)]) > (self.avg_price + self.avg_price*.15)):\n",
        "            #     for a in i:\n",
        "            #       actions[a] = actions[a]*-1\n",
        "\n",
        "            # for a in actions:\n",
        "            #   print(a)\n",
        "\n",
        "            argsort_actions = np.argsort(actions)\n",
        "            sell_index = argsort_actions[: np.where(actions < 0)[0].shape[0]]\n",
        "            buy_index = argsort_actions[::-1][: np.where(actions > 0)[0].shape[0]]\n",
        "\n",
        "            for index in sell_index:\n",
        "              actions[index] = self._sell_stock(index, actions[index]) * (-1)\n",
        "\n",
        "            for index in buy_index:\n",
        "                actions[index] = self._buy_stock(index, actions[index])\n",
        "\n",
        "\n",
        "            self.actions_memory.append(actions)\n",
        "\n",
        "            # state: s -> s+1\n",
        "            self.day += 1\n",
        "            self.data = self.df.loc[self.day, :]\n",
        "            if self.turbulence_threshold is not None:\n",
        "                if len(self.df.tic.unique()) == 1:\n",
        "                    self.turbulence = self.data[self.risk_indicator_col]\n",
        "                elif len(self.df.tic.unique()) > 1:\n",
        "                    self.turbulence = self.data[self.risk_indicator_col].values[0]\n",
        "\n",
        "\n",
        "\n",
        "            \n",
        "            \n",
        "            recent_price = self.state[1 : (self.stock_dim + 1)]\n",
        "            recent_buy_sell = actions\n",
        "            recent_buy_sell_price = np.where(recent_buy_sell != 0.0, recent_buy_sell * recent_price, 0.0)\n",
        "            #print(recent_price)\n",
        "            #print(recent_buy_sell)\n",
        "            #print(recent_buy_sell_price)\n",
        "            \n",
        "            \n",
        "            self.total_stockss = actions + self.total_stockss\n",
        "            self.total_price = recent_buy_sell_price + self.total_price\n",
        "            self.avg_price = np.divide(self.total_price,\n",
        "                                       self.total_stockss,\n",
        "                                       out=np.zeros_like(self.total_price),\n",
        "                                       where=(recent_buy_sell!=0.0))\n",
        "\n",
        "            self.avg_price = np.where(self.avg_price<0.0, 0.0, self.avg_price)\n",
        "            # dx = np.where(((self.avg_price < 0.00)))\n",
        "            # print(dx)\n",
        "            #if len(dx) > 0:\n",
        "            #print(self.total_stockss)\n",
        "            #print(self.total_price )\n",
        "            #print(self.avg_price)\n",
        "\n",
        "\n",
        "            \n",
        "\n",
        "\n",
        "\n",
        "\n",
        "            # self.total_price = np.add(recent_buy_sell_price,\n",
        "            #                           self.total_price,\n",
        "            #                           out=np.zeros_like(self.total_price),\n",
        "            #                                where=(recent_buy_sell!=0.0  )\n",
        "            \n",
        "            # #self.total_price = np.where(self.total_price<0.0, 0.0, self.total_price)\n",
        "            # self.total_stockss = actions + self.total_stockss\n",
        "            \n",
        "            # self.avg_price = np.divide(self.total_price,\n",
        "            #                            self.total_stockss,\n",
        "            #                            out=np.zeros_like(self.total_price),\n",
        "            #                            where=recent_buy_sell!=0.0)\n",
        "          \n",
        "            # print(\"Total Stocks\")\n",
        "            # print(self.total_stockss)\n",
        "            # print(\"Price\")\n",
        "            # print(recent_price)\n",
        "\n",
        "            # print(\"Avg Price\")\n",
        "            # print(self.avg_price)\n",
        "            # print(\"Total Price\")\n",
        "            # print(self.total_price)\n",
        "\n",
        "\n",
        "            self.state = self._update_state()\n",
        "\n",
        "            end_total_asset = self.state[0] + sum(\n",
        "                np.array(self.state[1 : (self.stock_dim + 1)])\n",
        "                * np.array(self.state[(self.stock_dim + 1) : (self.stock_dim * 2 + 1)])\n",
        "            )\n",
        "            self.asset_memory.append(end_total_asset)\n",
        "            self.date_memory.append(self._get_date())\n",
        "            self.reward = end_total_asset - begin_total_asset\n",
        "            self.rewards_memory.append(self.reward)\n",
        "            self.reward = self.reward * self.reward_scaling\n",
        "            self.state_memory.append(\n",
        "                self.state\n",
        "            )  # add current state in state_recorder for each step\n",
        "\n",
        "        return self.state, self.reward, self.terminal, {}\n",
        "\n",
        "    def reset(self):\n",
        "        # initiate state\n",
        "        self.state = self._initiate_state()\n",
        "\n",
        "        if self.initial:\n",
        "            self.asset_memory = [\n",
        "                self.initial_amount\n",
        "                + np.sum(\n",
        "                    np.array(self.num_stock_shares)\n",
        "                    * np.array(self.state[1 : 1 + self.stock_dim])\n",
        "                )\n",
        "            ]\n",
        "        else:\n",
        "            previous_total_asset = self.previous_state[0] + sum(\n",
        "                np.array(self.state[1 : (self.stock_dim + 1)])\n",
        "                * np.array(\n",
        "                    self.previous_state[(self.stock_dim + 1) : (self.stock_dim * 2 + 1)]\n",
        "                )\n",
        "            )\n",
        "            self.asset_memory = [previous_total_asset]\n",
        "\n",
        "        self.day = 0\n",
        "        self.data = self.df.loc[self.day, :]\n",
        "        self.turbulence = 0\n",
        "        self.cost = 0\n",
        "        self.trades = 0\n",
        "        self.terminal = False\n",
        "        # self.iteration=self.iteration\n",
        "        self.rewards_memory = []\n",
        "        self.actions_memory = []\n",
        "        self.date_memory = [self._get_date()]\n",
        "        if self.total_price[5] > 0.0:\n",
        "          dd = pd.DataFrame(data = self.avg_price)\n",
        "          dd.to_csv(\"avg\")\n",
        "          dd = pd.DataFrame(data = self.total_price)\n",
        "          dd.to_csv(\"tp\")\n",
        "        \n",
        "        self.episode += 1\n",
        "        self.total_price = np.array([0.0] * 29)\n",
        "        self.avg_price = np.array([0.0] * 29)\n",
        "        self.total_stockss = np.array([0.0] * 29)\n",
        "        #print(\"Resteting Account\")\n",
        "        return self.state\n",
        "\n",
        "    def render(self, mode=\"human\", close=False):\n",
        "        return self.state\n",
        "\n",
        "    def _initiate_state(self):\n",
        "        if self.initial:\n",
        "            # For Initial State\n",
        "            if len(self.df.tic.unique()) > 1:\n",
        "                # for multiple stock\n",
        "                state = (\n",
        "                    [self.initial_amount]\n",
        "                    + self.data.close.values.tolist()\n",
        "                    + self.num_stock_shares\n",
        "                    + sum(\n",
        "                        (\n",
        "                            self.data[tech].values.tolist()\n",
        "                            for tech in self.tech_indicator_list\n",
        "                        ),\n",
        "                        [],\n",
        "                    )\n",
        "                )  # append initial stocks_share to initial state, instead of all zero\n",
        "            else:\n",
        "                # for single stock\n",
        "                state = (\n",
        "                    [self.initial_amount]\n",
        "                    + [self.data.close]\n",
        "                    + [0] * self.stock_dim\n",
        "                    + sum(([self.data[tech]] for tech in self.tech_indicator_list), [])\n",
        "                )\n",
        "        else:\n",
        "            # Using Previous State\n",
        "            if len(self.df.tic.unique()) > 1:\n",
        "                # for multiple stock\n",
        "                state = (\n",
        "                    [self.previous_state[0]]\n",
        "                    + self.data.close.values.tolist()\n",
        "                    + self.previous_state[\n",
        "                        (self.stock_dim + 1) : (self.stock_dim * 2 + 1)\n",
        "                    ]\n",
        "                    + sum(\n",
        "                        (\n",
        "                            self.data[tech].values.tolist()\n",
        "                            for tech in self.tech_indicator_list\n",
        "                        ),\n",
        "                        [],\n",
        "                    )\n",
        "                )\n",
        "            else:\n",
        "                # for single stock\n",
        "                state = (\n",
        "                    [self.previous_state[0]]\n",
        "                    + [self.data.close]\n",
        "                    + self.previous_state[\n",
        "                        (self.stock_dim + 1) : (self.stock_dim * 2 + 1)\n",
        "                    ]\n",
        "                    + sum(([self.data[tech]] for tech in self.tech_indicator_list), [])\n",
        "                )\n",
        "        return state\n",
        "\n",
        "    def _update_state(self):\n",
        "        if len(self.df.tic.unique()) > 1:\n",
        "            # for multiple stock\n",
        "            state = (\n",
        "                [self.state[0]]\n",
        "                + self.data.close.values.tolist()\n",
        "                + list(self.state[(self.stock_dim + 1) : (self.stock_dim * 2 + 1)])\n",
        "                + sum((self.data[tech].values.tolist() for tech in self.tech_indicator_list),[],))\n",
        "\n",
        "        else:\n",
        "            # for single stock\n",
        "            state = ([self.state[0]] + [self.data.close] \n",
        "                     + list(self.state[(self.stock_dim + 1) : (self.stock_dim * 2 + 1)])\n",
        "                     + sum(([self.data[tech]] for tech in self.tech_indicator_list), []))\n",
        "\n",
        "        return state\n",
        "\n",
        "    def _get_date(self):\n",
        "        if len(self.df.tic.unique()) > 1:\n",
        "            date = self.data.date.unique()[0]\n",
        "        else:\n",
        "            date = self.data.date\n",
        "        return date\n",
        "\n",
        "\n",
        "    def save_asset_memory(self):\n",
        "        date_list = self.date_memory\n",
        "        asset_list = self.asset_memory\n",
        "        # print(len(date_list))\n",
        "        # print(len(asset_list))\n",
        "        df_account_value = pd.DataFrame({\"date\": date_list, \"account_value\": asset_list})\n",
        "        return df_account_value\n",
        "\n",
        "    def save_action_memory(self):\n",
        "        if len(self.df.tic.unique()) > 1:\n",
        "            # date and close price length must match actions length\n",
        "            date_list = self.date_memory[:-1]\n",
        "            df_date = pd.DataFrame(date_list)\n",
        "            df_date.columns = [\"date\"]\n",
        "\n",
        "            action_list = self.actions_memory\n",
        "            df_actions = pd.DataFrame(action_list)\n",
        "            df_actions.columns = self.data.tic.values\n",
        "            df_actions.index = df_date.date\n",
        "            # df_actions = pd.DataFrame({'date':date_list,'actions':action_list})\n",
        "        else:\n",
        "            date_list = self.date_memory[:-1]\n",
        "            action_list = self.actions_memory\n",
        "            df_actions = pd.DataFrame({\"date\": date_list, \"actions\": action_list})\n",
        "        return df_actions\n",
        "\n",
        "    def _seed(self, seed=None):\n",
        "        self.np_random, seed = seeding.np_random(seed)\n",
        "        return [seed]\n",
        "\n",
        "    def get_sb_env(self):\n",
        "        e = DummyVecEnv([lambda: self])\n",
        "        obs = e.reset()\n",
        "        return e, obs\n"
      ],
      "metadata": {
        "id": "4p02b2cx_dj5"
      },
      "execution_count": 64,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "stock_dimension = len(init_train_data.tic.unique())\n",
        "state_space = 1 + 2 * stock_dimension + len(config.INDICATORS) * stock_dimension\n",
        "print(f\"Stock Dimension: {stock_dimension}, State Space: {state_space}\")\n",
        "buy_cost_list = sell_cost_list = [0.001] * stock_dimension\n",
        "num_stock_shares = [0] * stock_dimension"
      ],
      "metadata": {
        "id": "rg7DTZbVV10n",
        "outputId": "f3cd6a02-d22e-4bd6-f389-68e3366e29ea",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 65,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Stock Dimension: 29, State Space: 291\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Defining the environment kwargs\n",
        "\n",
        "initial_amount = 500000\n",
        "env_kwargs = {\n",
        "    \"hmax\": 500,\n",
        "    \"initial_amount\": initial_amount,\n",
        "    \"num_stock_shares\": num_stock_shares,\n",
        "    \"buy_cost_pct\": buy_cost_list,\n",
        "    \"sell_cost_pct\": sell_cost_list,\n",
        "    \"state_space\": state_space,\n",
        "    \"stock_dim\": stock_dimension,\n",
        "    \"tech_indicator_list\": config.INDICATORS,\n",
        "    \"action_space\": stock_dimension,\n",
        "    \"reward_scaling\": 1e-4,\n",
        "}"
      ],
      "metadata": {
        "id": "zODPmc5hV42J"
      },
      "execution_count": 66,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#Instantiate the training gym compatible environment\n",
        "e_train_gym = GokuEnv(df = init_train_data, **env_kwargs)\n",
        "env_train, _ = e_train_gym.get_sb_env()\n",
        "agent = DRLAgent(env = env_train)"
      ],
      "metadata": {
        "id": "7n2QYGP5V58p"
      },
      "execution_count": 67,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#Instantiate the trading environment\n",
        "e_trade_gym = GokuEnv(df = init_trade_data, turbulence_threshold = None, **env_kwargs)"
      ],
      "metadata": {
        "id": "xoBM74XEX1oI"
      },
      "execution_count": 68,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Trade performance code\n",
        "The following code calculates trade performance metrics, which are then used as an objective for optimizing hyperparameter values.\n",
        "\n",
        "There are several available metrics. In this tutorial, the default choice is the ratio of average value of winning to losing trades."
      ],
      "metadata": {
        "id": "Dq6zJ14ZYsTK"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "#Main method\n",
        "# Calculates Trade Performance for Objective\n",
        "# Called from objective method\n",
        "# Returns selected trade perf metric(s)\n",
        "# Requires actions and associated prices\n",
        "\n",
        "def calc_trade_perf_metric(df_actions, \n",
        "                           df_prices_trade,\n",
        "                           tp_metric,\n",
        "                           dbg=False):\n",
        "  \n",
        "    df_actions_p, df_prices_p, tics = prep_data(df_actions.copy(),\n",
        "                                                df_prices_trade.copy())\n",
        "    # actions predicted by trained model on trade data\n",
        "    df_actions_p.to_csv('df_actions.csv') \n",
        "\n",
        "    \n",
        "    # Confirms that actions, prices and tics are consistent\n",
        "    df_actions_s, df_prices_s, tics_prtfl = \\\n",
        "        sync_tickers(df_actions_p.copy(),df_prices_p.copy(),tics)\n",
        "    \n",
        "    # copy to ensure that tics from portfolio remains unchanged\n",
        "    tics = tics_prtfl.copy()\n",
        "    \n",
        "    # Analysis is performed on each portfolio ticker\n",
        "    perf_data= collect_performance_data(df_actions_s, df_prices_s, tics)\n",
        "    # profit/loss for each ticker\n",
        "    pnl_all = calc_pnl_all(perf_data, tics)\n",
        "    # values for trade performance metrics\n",
        "    perf_results = calc_trade_perf(pnl_all)\n",
        "    df = pd.DataFrame.from_dict(perf_results, orient='index')\n",
        "    \n",
        "    # calculate and return trade metric value as objective\n",
        "    m = calc_trade_metric(df,tp_metric)\n",
        "    print(f'Ratio Avg Win/Avg Loss: {m}')\n",
        "    k = str(len(tpm_hist)+1)\n",
        "    # save metric value\n",
        "    tpm_hist[k] = m\n",
        "    return m\n",
        "\n",
        "\n",
        "# Supporting methods\n",
        "def calc_trade_metric(df,metric='avgwl'):\n",
        "    '''# trades', '# wins', '# losses', 'wins total value', 'wins avg value',\n",
        "       'losses total value', 'losses avg value'''\n",
        "    # For this tutorial, the only metric available is the ratio of \n",
        "    #  average values of winning to losing trades. Others are in development.\n",
        "    \n",
        "    # some test cases produce no losing trades.\n",
        "    # The code below assigns a value as a multiple of the highest value during\n",
        "    # previous hp optimization runs. If the first run experiences no losses,\n",
        "    # a fixed value is assigned for the ratio\n",
        "    tpm_mult = 1.0\n",
        "    avgwl_no_losses = 25\n",
        "    if metric == 'avgwl':\n",
        "        if sum(df['# losses']) == 0:\n",
        "          try:\n",
        "            return max(tpm_hist.values())*tpm_mult\n",
        "          except ValueError:\n",
        "            return avgwl_no_losses\n",
        "        avg_w = sum(df['wins total value'])/sum(df['# wins'])\n",
        "        avg_l = sum(df['losses total value'])/sum(df['# losses'])\n",
        "        m = abs(avg_w/avg_l)\n",
        "\n",
        "    return m\n",
        "\n",
        "\n",
        "def prep_data(df_actions,\n",
        "              df_prices_trade):\n",
        "    \n",
        "    df=df_prices_trade[['date','close','tic']]\n",
        "    df['Date'] = pd.to_datetime(df['date'])\n",
        "    df = df.set_index('Date')\n",
        "    # set indices on both df to datetime\n",
        "    idx = pd.to_datetime(df_actions.index, infer_datetime_format=True)\n",
        "    df_actions.index=idx\n",
        "    tics = np.unique(df.tic)\n",
        "    n_tics = len(tics)\n",
        "    print(f'Number of tickers: {n_tics}')\n",
        "    print(f'Tickers: {tics}')\n",
        "    dategr = df.groupby('tic')\n",
        "    p_d={t:dategr.get_group(t).loc[:,'close'] for t in tics}\n",
        "    df_prices = pd.DataFrame.from_dict(p_d)\n",
        "    df_prices.index = df_prices.index.normalize()\n",
        "    return df_actions, df_prices, tics\n",
        "\n",
        "\n",
        "# prepares for integrating action and price files\n",
        "def link_prices_actions(df_a,\n",
        "                        df_p):\n",
        "    cols_a = [t + '_a' for t in df_a.columns]\n",
        "    df_a.columns = cols_a\n",
        "    cols_p = [t + '_p' for t in df_p.columns]\n",
        "    df_p.columns = cols_p\n",
        "    return df_a, df_p\n",
        "\n",
        "\n",
        "def sync_tickers(df_actions,df_tickers_p,tickers):\n",
        "    # Some DOW30 components may not be included in portfolio\n",
        "    # passed tickers includes all DOW30 components\n",
        "    # actions and ticker files may have different length indices\n",
        "    if len(df_actions) != len(df_tickers_p):\n",
        "      msng_dates = set(df_actions.index)^set(df_tickers_p.index)\n",
        "      try:\n",
        "        #assumption is prices has one additional timestamp (row)\n",
        "        df_tickers_p.drop(msng_dates,inplace=True)\n",
        "      except:\n",
        "        df_actions.drop(msng_dates,inplace=True)\n",
        "    df_actions, df_tickers_p = link_prices_actions(df_actions,df_tickers_p)\n",
        "    # identify any DOW components not in portfolio\n",
        "    t_not_in_a = [t for t in tickers if t + '_a' not in list(df_actions.columns)]\n",
        "  \n",
        "    # remove t_not_in_a from df_tickers_p\n",
        "    drop_cols = [t + '_p' for t in t_not_in_a]\n",
        "    df_tickers_p.drop(columns=drop_cols,inplace=True)\n",
        "    \n",
        "    # Tickers in portfolio\n",
        "    tickers_prtfl = [c.split('_')[0] for c in df_actions.columns]\n",
        "    return df_actions,df_tickers_p, tickers_prtfl\n",
        "\n",
        "def collect_performance_data(dfa,dfp,tics, dbg=False):\n",
        "    \n",
        "    perf_data = {}\n",
        "    # In current version, files columns include secondary identifier\n",
        "    for t in tics:\n",
        "        # actions: purchase/sale of DOW equities\n",
        "        acts = dfa['_'.join([t,'a'])].values\n",
        "        # ticker prices\n",
        "        prices = dfp['_'.join([t,'p'])].values\n",
        "        # market value of purchases/sales\n",
        "        tvals_init = np.multiply(acts,prices)\n",
        "        d={'actions':acts, 'prices':prices,'init_values':tvals_init}\n",
        "        perf_data[t]=d\n",
        "\n",
        "    return perf_data\n",
        "\n",
        "\n",
        "def calc_pnl_all(perf_dict, tics_all):\n",
        "    # calculate profit/loss for each ticker\n",
        "    print(f'Calculating profit/loss for each ticker')\n",
        "    pnl_all = {}\n",
        "    for tic in tics_all:\n",
        "        pnl_t = []\n",
        "        tic_data = perf_dict[tic]\n",
        "        init_values = tic_data['init_values']\n",
        "        acts = tic_data['actions']\n",
        "        prices = tic_data['prices']\n",
        "        cs = np.cumsum(acts)\n",
        "        args_s = [i + 1 for i in range(len(cs) - 1) if cs[i + 1] < cs[i]]\n",
        "        # tic actions with no sales\n",
        "        if not args_s:\n",
        "            pnl = complete_calc_buyonly(acts, prices, init_values)\n",
        "            pnl_all[tic] = pnl\n",
        "            continue\n",
        "        # copy acts: acts_rev will be revised based on closing/reducing init positions\n",
        "        pnl_all = execute_position_sales(tic,acts,prices,args_s,pnl_all)\n",
        "\n",
        "    return pnl_all\n",
        "\n",
        "\n",
        "def complete_calc_buyonly(actions, prices, init_values):\n",
        "    # calculate final pnl for each ticker assuming no sales\n",
        "    fnl_price = prices[-1]\n",
        "    final_values = np.multiply(fnl_price, actions)\n",
        "    pnl = np.subtract(final_values, init_values)\n",
        "    return pnl\n",
        "\n",
        "\n",
        "def execute_position_sales(tic,acts,prices,args_s,pnl_all):\n",
        "  # calculate final pnl for each ticker with sales\n",
        "    pnl_t = []\n",
        "    acts_rev = acts.copy()\n",
        "    # location of sales transactions\n",
        "    for s in args_s:  # s is scaler\n",
        "        # price_s = [prices[s]]\n",
        "        act_s = [acts_rev[s]]\n",
        "        args_b = [i for i in range(s) if acts_rev[i] > 0]\n",
        "        prcs_init_trades = prices[args_b]\n",
        "        acts_init_trades = acts_rev[args_b]\n",
        "  \n",
        "        # update actions for sales\n",
        "        # reduce/eliminate init values through trades\n",
        "        # always start with earliest purchase that has not been closed through sale\n",
        "        # selectors for purchase and sales trades\n",
        "        # find earliest remaining purchase\n",
        "        arg_sel = min(args_b)\n",
        "        # sel_s = len(acts_trades) - 1\n",
        "\n",
        "        # closing part/all of earliest init trade not yet closed\n",
        "        # sales actions are negative\n",
        "        # in this test case, abs_val of init and sales share counts are same\n",
        "        # zero-out sales actions\n",
        "        # market value of sale\n",
        "        # max number of shares to be closed: may be less than # originally purchased\n",
        "        acts_shares = min(abs(act_s.pop()), acts_rev[arg_sel])\n",
        "\n",
        "        # mv of shares when purchased\n",
        "        mv_p = abs(acts_shares * prices[arg_sel])\n",
        "        # mv of sold shares\n",
        "        mv_s = abs(acts_shares * prices[s])\n",
        "\n",
        "        # calc pnl\n",
        "        pnl = mv_s - mv_p\n",
        "        # reduce init share count\n",
        "        # close all/part of init purchase\n",
        "        acts_rev[arg_sel] -= acts_shares\n",
        "        acts_rev[s] += acts_shares\n",
        "        # calculate pnl for trade\n",
        "        # value of associated purchase\n",
        "        \n",
        "        # find earliest non-zero positive act in acts_revs\n",
        "        pnl_t.append(pnl)\n",
        "    \n",
        "    pnl_op = calc_pnl_for_open_positions(acts_rev, prices)\n",
        "    #pnl_op is list\n",
        "    # add pnl_op results (if any) to pnl_t (both lists)\n",
        "    pnl_t.extend(pnl_op)\n",
        "    #print(f'Total pnl for {tic}: {np.sum(pnl_t)}')\n",
        "    pnl_all[tic] = np.array(pnl_t)\n",
        "    return pnl_all\n",
        "\n",
        "\n",
        "def calc_pnl_for_open_positions(acts,prices):\n",
        "    # identify any positive share values after accounting for sales\n",
        "    pnl = []\n",
        "    fp = prices[-1] # last price\n",
        "    open_pos_arg = np.argwhere(acts>0)\n",
        "    if len(open_pos_arg)==0:return pnl # no open positions\n",
        "\n",
        "    mkt_vals_open = np.multiply(acts[open_pos_arg], prices[open_pos_arg])\n",
        "    # mkt val at end of testing period\n",
        "    # treat as trades for purposes of calculating pnl at end of testing period\n",
        "    mkt_vals_final = np.multiply(fp, acts[open_pos_arg])\n",
        "    pnl_a = np.subtract(mkt_vals_final, mkt_vals_open)\n",
        "    #convert to list\n",
        "    pnl = [i[0] for i in pnl_a.tolist()]\n",
        "    #print(f'Market value of open positions at end of testing {pnl}')\n",
        "    return pnl\n",
        "\n",
        "\n",
        "def calc_trade_perf(pnl_d):\n",
        "    # calculate trade performance metrics\n",
        "    perf_results = {}\n",
        "    for t,pnl in pnl_d.items():\n",
        "        wins = pnl[pnl>0]  # total val\n",
        "        losses = pnl[pnl<0]\n",
        "        n_wins = len(wins)\n",
        "        n_losses = len(losses)\n",
        "        n_trades = n_wins + n_losses\n",
        "        wins_val = np.sum(wins)\n",
        "        losses_val = np.sum(losses)\n",
        "        wins_avg = 0 if n_wins==0 else np.mean(wins)\n",
        "        #print(f'{t} n_wins: {n_wins} n_losses: {n_losses}')\n",
        "        losses_avg = 0 if n_losses==0 else np.mean(losses)\n",
        "        d = {'# trades':n_trades,'# wins':n_wins,'# losses':n_losses,\n",
        "             'wins total value':wins_val, 'wins avg value':wins_avg,\n",
        "             'losses total value':losses_val, 'losses avg value':losses_avg,}\n",
        "        perf_results[t] = d\n",
        "    return perf_results"
      ],
      "metadata": {
        "id": "QydaFexDX5BQ"
      },
      "execution_count": 69,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Tuning hyperparameters using Optuna"
      ],
      "metadata": {
        "id": "c_AT1dsuZApj"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def sample_ddpg_params(trial:optuna.Trial):\n",
        "  # Size of the replay buffer\n",
        "  buffer_size = trial.suggest_categorical(\"buffer_size\", [int(1e4), int(1e5), int(1e6)])\n",
        "  learning_rate = trial.suggest_loguniform(\"learning_rate\", 1e-5, 1)\n",
        "  batch_size = trial.suggest_categorical(\"batch_size\", [32, 64, 128, 256, 512])\n",
        "  \n",
        "  return {\"buffer_size\": buffer_size,\n",
        "          \"learning_rate\":learning_rate,\n",
        "          \"batch_size\":batch_size}"
      ],
      "metadata": {
        "id": "fKGkKHV7Y9AA"
      },
      "execution_count": 70,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Set Variables\n",
        "## Fixed\n",
        "tpm_hist = {}  # record tp metric values for trials\n",
        "tp_metric = 'avgwl'  # specified trade_param_metric: ratio avg value win/loss\n",
        "## Settable by User\n",
        "n_trials = 100  # number of HP optimization runs\n",
        "total_timesteps = 2000 # per HP optimization run\n",
        "## Logging callback params\n",
        "lc_threshold=1e-5\n",
        "lc_patience=15\n",
        "lc_trial_number=5"
      ],
      "metadata": {
        "id": "N6blMWpz-pR0"
      },
      "execution_count": 71,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "OPTIONAL CODE FOR SAMPLING HYPERPARAMETERS\n",
        "\n",
        "Replace current call in function objective with\n",
        "\n",
        "hyperparameters = sample_ddpg_params_all(trial)"
      ],
      "metadata": {
        "id": "fHET-odKZShg"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def sample_ddpg_params_all(trial:optuna.Trial,\n",
        "                           # fixed values from previous study\n",
        "                           learning_rate=0.0103,\n",
        "                           batch_size=128,\n",
        "                           buffer_size=int(1e6)):\n",
        "\n",
        "    gamma = trial.suggest_categorical(\"gamma\", [0.94, 0.96, 0.98])\n",
        "    # Polyak coeff\n",
        "    tau = trial.suggest_categorical(\"tau\", [0.08, 0.1, 0.12])\n",
        "\n",
        "    train_freq = trial.suggest_categorical(\"train_freq\", [512,768,1024])\n",
        "    gradient_steps = train_freq\n",
        "    \n",
        "    noise_type = trial.suggest_categorical(\"noise_type\", [\"ornstein-uhlenbeck\", \"normal\", None])\n",
        "    noise_std = trial.suggest_categorical(\"noise_std\", [.1,.2,.3] )\n",
        "\n",
        "    # NOTE: Add \"verybig\" to net_arch when tuning HER (see TD3)\n",
        "    net_arch = trial.suggest_categorical(\"net_arch\", [\"small\", \"big\"])\n",
        "    # activation_fn = trial.suggest_categorical('activation_fn', [nn.Tanh, nn.ReLU, nn.ELU, nn.LeakyReLU])\n",
        "\n",
        "    net_arch = {\n",
        "        \"small\": [64, 64],\n",
        "        \"medium\": [256, 256],\n",
        "        \"big\": [512, 512],\n",
        "    }[net_arch]\n",
        "  \n",
        "    hyperparams = {\n",
        "        \"batch_size\": batch_size,\n",
        "        \"buffer_size\": buffer_size,\n",
        "        \"gamma\": gamma,\n",
        "        \"gradient_steps\": gradient_steps,\n",
        "        \"learning_rate\": learning_rate,\n",
        "        \"tau\": tau,\n",
        "        \"train_freq\": train_freq,\n",
        "        #\"noise_std\": noise_std,\n",
        "        #\"noise_type\": noise_type,\n",
        "        \n",
        "        \"policy_kwargs\": dict(net_arch=net_arch)\n",
        "    }\n",
        "    return hyperparams"
      ],
      "metadata": {
        "id": "htLdZHKTZKPJ"
      },
      "execution_count": 72,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def sample_ddpg_params_all(trial:optuna.Trial,\n",
        "                           # fixed values from previous study\n",
        "                           learning_rate=0.0103,\n",
        "                           batch_size=128,\n",
        "                           buffer_size=int(1e6)):\n",
        "\n",
        "    gamma = trial.suggest_categorical(\"gamma\", [0.90, 0.92, 0.94, 0.96, 0.98])\n",
        "    # Polyak coeff\n",
        "    tau = trial.suggest_categorical(\"tau\", [0.02,0.04,0.06,0.08, 0.1, 0.12])\n",
        "\n",
        "    train_freq = trial.suggest_categorical(\"train_freq\", [512,768,1024])\n",
        "    gradient_steps = train_freq\n",
        "    \n",
        "    noise_type = trial.suggest_categorical(\"noise_type\", [\"ornstein-uhlenbeck\", \"normal\", None])\n",
        "    noise_std = trial.suggest_categorical(\"noise_std\", [.1,.2,.3,.4,.5] )\n",
        "\n",
        "    # NOTE: Add \"verybig\" to net_arch when tuning HER (see TD3)\n",
        "    net_arch = trial.suggest_categorical(\"net_arch\", [\"small\", \"big\"])\n",
        "    # activation_fn = trial.suggest_categorical('activation_fn', [nn.Tanh, nn.ReLU, nn.ELU, nn.LeakyReLU])\n",
        "\n",
        "    net_arch = {\n",
        "        \"small\": [32, 32],\n",
        "        \"medium\": [64, 64],\n",
        "        \"big\": [256, 256],\n",
        "    }[net_arch]\n",
        "  \n",
        "    hyperparams = {\n",
        "        \"batch_size\": batch_size,\n",
        "        \"buffer_size\": buffer_size,\n",
        "        \"gamma\": gamma,\n",
        "        \"gradient_steps\": gradient_steps,\n",
        "        \"learning_rate\": learning_rate,\n",
        "        \"tau\": tau,\n",
        "        \"train_freq\": train_freq,\n",
        "        #\"noise_std\": noise_std,\n",
        "        #\"noise_type\": noise_type,\n",
        "        \n",
        "        \"policy_kwargs\": dict(net_arch=net_arch)\n",
        "    }\n",
        "    return hyperparams"
      ],
      "metadata": {
        "id": "dJYdh-E4ZQKP"
      },
      "execution_count": 60,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Callbacks\n",
        "\n",
        "\n",
        "1. The callback will terminate if the improvement margin is below certain point\n",
        "2. It will terminate after certain number of trial_number are reached, not before that\n",
        "3. It will hold its patience to reach the threshold"
      ],
      "metadata": {
        "id": "mWEoJMPLZY4o"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class LoggingCallback:\n",
        "    def __init__(self,threshold,trial_number,patience):\n",
        "      '''\n",
        "      threshold:int tolerance for increase in objective\n",
        "      trial_number: int Prune after minimum number of trials\n",
        "      patience: int patience for the threshold\n",
        "      '''\n",
        "      self.threshold = threshold\n",
        "      self.trial_number  = trial_number\n",
        "      self.patience = patience\n",
        "      print(f'Callback threshold {self.threshold}, \\\n",
        "            trial_number {self.trial_number}, \\\n",
        "            patience {self.patience}')\n",
        "      self.cb_list = [] #Trials list for which threshold is reached\n",
        "    def __call__(self,study:optuna.study, frozen_trial:optuna.Trial):\n",
        "      #Setting the best value in the current trial\n",
        "      study.set_user_attr(\"previous_best_value\", study.best_value)\n",
        "      \n",
        "      #Checking if the minimum number of trials have pass\n",
        "      if frozen_trial.number >self.trial_number:\n",
        "          previous_best_value = study.user_attrs.get(\"previous_best_value\",None)\n",
        "          #Checking if the previous and current objective values have the same sign\n",
        "          if previous_best_value * study.best_value >=0:\n",
        "              #Checking for the threshold condition\n",
        "              if abs(previous_best_value-study.best_value) < self.threshold: \n",
        "                  self.cb_list.append(frozen_trial.number)\n",
        "                  #If threshold is achieved for the patience amount of time\n",
        "                  if len(self.cb_list)>self.patience:\n",
        "                      print('The study stops now...')\n",
        "                      print('With number',frozen_trial.number ,'and value ',frozen_trial.value)\n",
        "                      print('The previous and current best values are {} and {} respectively'\n",
        "                              .format(previous_best_value, study.best_value))\n",
        "                      study.stop()"
      ],
      "metadata": {
        "id": "He7GTZZUZWP8"
      },
      "execution_count": 61,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#Calculate the Sharpe ratio\n",
        "#This is our objective for tuning\n",
        "def calculate_sharpe(df):\n",
        "  df['daily_return'] = df['account_value'].pct_change(1)\n",
        "  if df['daily_return'].std() !=0:\n",
        "    sharpe = (252**0.5)*df['daily_return'].mean()/ \\\n",
        "          df['daily_return'].std()\n",
        "    return sharpe\n",
        "  else:\n",
        "    return 0"
      ],
      "metadata": {
        "id": "s9orU-WplQ70"
      },
      "execution_count": 62,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from IPython.display import clear_output\n",
        "import sys   \n",
        "\n",
        "os.makedirs(\"models\",exist_ok=True)\n",
        "\n",
        "def objective(trial:optuna.Trial):\n",
        "  #Trial will suggest a set of hyperparamters from the specified range\n",
        "\n",
        "  # Optional to optimize larger set of parameters\n",
        "  # hyperparameters = sample_ddpg_params_all(trial)\n",
        "  \n",
        "  # Optimize buffer size, batch size, learning rate\n",
        "  hyperparameters = sample_ddpg_params_all(trial)\n",
        "  print(f'Hyperparameters from objective: {hyperparameters.keys()}')\n",
        "  policy_kwargs = None  # default\n",
        "  if 'policy_kwargs' in hyperparameters.keys():\n",
        "    policy_kwargs = hyperparameters['policy_kwargs']\n",
        "    del hyperparameters['policy_kwargs']\n",
        "    print(f'Policy keyword arguments {policy_kwargs}')\n",
        "  model_ddpg = agent.get_model(\"ddpg\",\n",
        "                               policy_kwargs = policy_kwargs,\n",
        "                               model_kwargs = hyperparameters )\n",
        "  \n",
        "  #You can increase it for better comparison\n",
        "  trained_ddpg = agent.train_model(model=model_ddpg,\n",
        "                                   tb_log_name=\"ddpg\",\n",
        "                                   total_timesteps=total_timesteps)\n",
        "  trained_ddpg.save('models/ddpg_{}.pth'.format(trial.number))\n",
        "  clear_output(wait=True)\n",
        "  \n",
        "  #For the given hyperparamters, determine the account value in the trading period\n",
        "  df_account_value, df_actions = DRLAgent.DRL_prediction(\n",
        "    model=trained_ddpg, \n",
        "    environment = e_trade_gym)\n",
        " \n",
        "  # Calculate trade performance metric\n",
        "  # Currently ratio of average win and loss market values\n",
        "  #tpm = calc_trade_perf_metric(df_actions,init_trade_data,tp_metric)\n",
        "  tpm = calculate_sharpe(df_account_value)\n",
        "  return tpm\n",
        "\n",
        "#Create a study object and specify the direction as 'maximize'\n",
        "#As you want to maximize sharpe\n",
        "#Pruner stops not promising iterations\n",
        "#Use a pruner, else you will get error related to divergence of model\n",
        "#You can also use Multivariate samplere\n",
        "#sampler = optuna.samplers.TPESampler(multivarite=True,seed=42)\n",
        "sampler = optuna.samplers.TPESampler()\n",
        "\n",
        "study = optuna.create_study(study_name=\"ddpg_study\",direction='maximize',\n",
        "                            sampler = sampler, pruner=optuna.pruners.HyperbandPruner())\n",
        "\n",
        "logging_callback = LoggingCallback(threshold=lc_threshold,\n",
        "                                   patience=lc_patience,\n",
        "                                   trial_number=lc_trial_number)\n",
        "#You can increase the n_trials for a better search space scanning\n",
        "study.optimize(objective, n_trials=n_trials,catch=(ValueError,),callbacks=[logging_callback])"
      ],
      "metadata": {
        "id": "b6EiwiiIZpRU",
        "outputId": "aadc5c25-bce2-4675-d6a5-805ae2587174",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "execution_count": 63,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\u001b[32m[I 2023-05-02 13:54:28,425]\u001b[0m Trial 0 finished with value: 0.18324508785463875 and parameters: {'gamma': 0.9, 'tau': 0.06, 'train_freq': 512, 'noise_type': None, 'noise_std': 0.5, 'net_arch': 'big'}. Best is trial 0 with value: 0.18324508785463875.\u001b[0m\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "hit end!\n",
            "Hyperparameters from objective: dict_keys(['batch_size', 'buffer_size', 'gamma', 'gradient_steps', 'learning_rate', 'tau', 'train_freq', 'policy_kwargs'])\n",
            "Policy keyword arguments {'net_arch': [256, 256]}\n",
            "{'batch_size': 128, 'buffer_size': 1000000, 'gamma': 0.9, 'gradient_steps': 768, 'learning_rate': 0.0103, 'tau': 0.08, 'train_freq': 768}\n",
            "Using cuda device\n",
            "137 |  avg_price 93.8512191772461 Profit 42.661827087402344\n",
            "146 |  avg_price 98.12000274658205 Profit 48.379997253417955\n",
            "125 |  avg_price 79.98869323730469 Profit 45.145721435546875\n",
            "49 |  avg_price 35.95546340942383 Profit 13.456661224365234\n",
            "158 |  avg_price 118.48297119140622 Profit 39.39479064941409\n",
            "78 |  avg_price 59.78041458129883 Profit 18.572490692138672\n",
            "127 |  avg_price 87.46646118164062 Profit 39.543724060058594\n",
            "276 |  avg_price 210.7080535888672 Profit 65.65815734863281\n",
            "132 |  avg_price 95.19500637054445 Profit 37.18731784820555\n",
            "157 |  avg_price 116.95700300500745 Profit 40.434201828976924\n",
            "192 |  avg_price 74.62757237752278 Profit 116.9248415629069\n",
            "47 |  avg_price 36.00785195560198 Profit 11.03319037227888\n",
            "47 |  avg_price 32.85527835777419 Profit 14.499015465467998\n",
            "51 |  avg_price 28.335840497698104 Profit 22.79660960606166\n",
            "124 |  avg_price 7.82255762273615 Profit 115.81033117120917\n",
            "78 |  avg_price 58.486481529414455 Profit 19.579939979374608\n",
            "116 |  avg_price 86.3361209538429 Profit 29.96071589918445\n",
            "45 |  avg_price 33.29050986663154 Profit 11.99280769928643\n",
            "109 |  avg_price 63.554178873697914 Profit 45.01836140950521\n",
            "75 |  avg_price 51.42445182800293 Profit 24.0571231842041\n",
            "80 |  avg_price 46.77432388649847 Profit 32.75448470725153\n",
            "210 |  avg_price 119.46704620675942 Profit 90.45865874929527\n",
            "128 |  avg_price 96.84704466040188 Profit 30.75630311791845\n",
            "127 |  avg_price 40.897464058615945 Profit 85.63070366599342\n",
            "183 |  avg_price 70.92313563570063 Profit 112.0693265225025\n",
            "220 |  avg_price 167.80454545862582 Profit 51.70630354039761\n",
            "144 |  avg_price 99.03055117828674 Profit 44.691784637142945\n",
            "61 |  avg_price 42.50426026641351 Profit 18.745281969914615\n",
            "100 |  avg_price 62.1561348393279 Profit 38.107674364773665\n",
            "140 |  avg_price 101.79420455406452 Profit 37.74578873206829\n",
            "124 |  avg_price 92.19772106598094 Profit 32.14071887542529\n",
            "169 |  avg_price 85.46289996711575 Profit 83.88635845573582\n",
            "54 |  avg_price 33.93427823384603 Profit 20.067598597208658\n",
            "200 |  avg_price 92.83569973618236 Profit 106.66602450698173\n",
            "46 |  avg_price 20.89427763186624 Profit 24.63567155636618\n",
            "140 |  avg_price 48.87129019357103 Profit 91.22465707205397\n",
            "174 |  avg_price 62.041816412233835 Profit 111.61948424694586\n",
            "202 |  avg_price 124.60969161987406 Profit 77.4023323059072\n",
            "196 |  avg_price 67.3639156765408 Profit 128.60564303927953\n",
            "106 |  avg_price 71.72262642251788 Profit 34.07266929281417\n",
            "137 |  avg_price 90.12925789572974 Profit 47.3331291892312\n",
            "43 |  avg_price 30.286506574380663 Profit 12.452125627767774\n",
            "194 |  avg_price 109.82434866361528 Profit 83.93465402193159\n",
            "107 |  avg_price 73.23268677206599 Profit 33.32278655557073\n",
            "152 |  avg_price 96.94934794108073 Profit 55.269432576497394\n",
            "89 |  avg_price 65.70654371448026 Profit 23.08282396130099\n",
            "126 |  avg_price 72.48444416708573 Profit 53.99736735635177\n",
            "154 |  avg_price 92.89030969443442 Profit 61.556330320214016\n",
            "126 |  avg_price 87.06921609242757 Profit 39.2139259974162\n",
            "85 |  avg_price 62.94962722263979 Profit 21.860271153825053\n",
            "113 |  avg_price 64.86235591343471 Profit 47.74130467006138\n",
            "41 |  avg_price 26.981618715025096 Profit 13.895494627260067\n",
            "46 |  avg_price 31.212024300456342 Profit 15.143028647541705\n",
            "78 |  avg_price 18.037311214031416 Profit 59.91143451350764\n",
            "60 |  avg_price 34.474198896332666 Profit 25.709230821684912\n",
            "82 |  avg_price 34.63282964983557 Profit 47.79813195905115\n",
            "123 |  avg_price 89.6472337480025 Profit 33.32607100053265\n",
            "41 |  avg_price 19.499713871214137 Profit 21.514118221071026\n",
            "144 |  avg_price 104.88328640156813 Profit 39.096709325970934\n",
            "40 |  avg_price 26.55236284025423 Profit 13.605626661698896\n",
            "108 |  avg_price 24.44410998480667 Profit 83.66907971245895\n",
            "151 |"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-51-6274c5b7f4e2>:354: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-51-6274c5b7f4e2>:286: RuntimeWarning: invalid value encountered in double_scalars\n",
            "  elif current_price < (avg_price - avg_price * 1)  and avg_price > 0.0:\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "  avg_price 73.16513684331154 Profit 77.8100218480947\n",
            "119 |  avg_price 77.3418387011223 Profit 42.0963479443855\n",
            "113 |  avg_price 72.22510650734499 Profit 40.685286253885494\n",
            "83 |  avg_price 59.56927114819724 Profit 23.2704352727012\n",
            "82 |  avg_price 6.788358688354492 Profit 75.06916999816895\n",
            "38 |  avg_price 24.105569679506363 Profit 13.471784751645973\n",
            "132 |  avg_price 83.4647117251442 Profit 48.98514789399643\n",
            "199 |  avg_price 107.62932987803036 Profit 91.69258479482123\n",
            "100 |  avg_price 23.48621190737363 Profit 76.16021143247013\n",
            "38 |  avg_price 27.40570558298129 Profit 10.454413282985506\n",
            "95 |  avg_price 72.40596905876609 Profit 22.926313961253427\n",
            "109 |  avg_price 66.55582974947416 Profit 42.13870455228364\n",
            "130 |  avg_price 99.60807921988832 Profit 30.609617923666363\n",
            "44 |  avg_price 33.02993776851569 Profit 10.77283475345697\n",
            "72 |  avg_price 50.53310388731725 Profit 21.722847040417122\n",
            "38 |  avg_price 23.039059563366052 Profit 14.573393897327307\n",
            "42 |  avg_price 27.845777723524307 Profit 13.787809160020615\n",
            "86 |  avg_price 64.62493382895865 Profit 21.71647394692026\n",
            "50 |  avg_price 38.27889288995499 Profit 11.56411705877548\n",
            "216 |  avg_price 163.30960520108542 Profit 52.51813832918802\n",
            "39 |  avg_price 12.906580309495858 Profit 26.35778584040648\n",
            "169 |  avg_price 37.62835342819626 Profit 131.61825912063188\n",
            "331 |  avg_price 141.02000427246094 Profit 189.7718963623047\n",
            "127 |  avg_price 94.4365234375 Profit 32.57366180419922\n",
            "127 |  avg_price 33.33032861220098 Profit 93.29979834092401\n",
            "168 |  avg_price 35.83595478103822 Profit 131.83723247482115\n",
            "169 |  avg_price 16.976561920552314 Profit 151.9887701106977\n",
            "57 |  avg_price 27.873710491039 Profit 29.352192066334048\n",
            "74 |  avg_price 28.57624053955078 Profit 45.35455322265625\n",
            "315 |  avg_price 103.58017575609811 Profit 211.10753786694875\n",
            "138 |  avg_price 102.29700960431781 Profit 36.001589638846255\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-51-6274c5b7f4e2>:354: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-51-6274c5b7f4e2>:286: RuntimeWarning: invalid value encountered in double_scalars\n",
            "  elif current_price < (avg_price - avg_price * 1)  and avg_price > 0.0:\n",
            "<ipython-input-51-6274c5b7f4e2>:354: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-51-6274c5b7f4e2>:286: RuntimeWarning: invalid value encountered in double_scalars\n",
            "  elif current_price < (avg_price - avg_price * 1)  and avg_price > 0.0:\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "331 |  avg_price 141.02000427246094 Profit 189.7718963623047\n",
            "104 |  avg_price 80.07931518554688 Profit 24.034225463867188\n",
            "125 |  avg_price 87.51908874511719 Profit 37.615325927734375\n",
            "176 |  avg_price 124.30469780294304 Profit 51.69530219705696\n",
            "49 |  avg_price 34.636893798828126 Profit 14.468044708251952\n",
            "331 |  avg_price 141.02000427246094 Profit 189.7718963623047\n",
            "104 |  avg_price 80.07931518554688 Profit 24.034225463867188\n",
            "125 |  avg_price 87.51908874511719 Profit 37.615325927734375\n",
            "176 |  avg_price 124.30469780294304 Profit 51.69530219705696\n",
            "49 |  avg_price 34.636893798828126 Profit 14.468044708251952\n",
            "day: 82, episode: 30\n",
            "begin_total_asset: 500000.00\n",
            "end_total_asset: 657400.16\n",
            "total_reward: 157400.16\n",
            "total_cost: 1943.03\n",
            "total_trades: 1187\n",
            "Sharpe: 1.587\n",
            "=================================\n",
            "------------------------------------\n",
            "| time/              |             |\n",
            "|    episodes        | 4           |\n",
            "|    fps             | 454         |\n",
            "|    time_elapsed    | 0           |\n",
            "|    total_timesteps | 332         |\n",
            "| train/             |             |\n",
            "|    reward          | -0.81137323 |\n",
            "------------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-51-6274c5b7f4e2>:354: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-51-6274c5b7f4e2>:286: RuntimeWarning: invalid value encountered in double_scalars\n",
            "  elif current_price < (avg_price - avg_price * 1)  and avg_price > 0.0:\n",
            "<ipython-input-51-6274c5b7f4e2>:354: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-51-6274c5b7f4e2>:286: RuntimeWarning: invalid value encountered in double_scalars\n",
            "  elif current_price < (avg_price - avg_price * 1)  and avg_price > 0.0:\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "331 |  avg_price 141.02000427246094 Profit 189.7718963623047\n",
            "104 |  avg_price 80.07931518554688 Profit 24.034225463867188\n",
            "125 |  avg_price 87.51908874511719 Profit 37.615325927734375\n",
            "176 |  avg_price 124.30469780294304 Profit 51.69530219705696\n",
            "49 |  avg_price 34.636893798828126 Profit 14.468044708251952\n",
            "331 |  avg_price 141.02000427246094 Profit 189.7718963623047\n",
            "104 |  avg_price 80.07931518554688 Profit 24.034225463867188\n",
            "125 |  avg_price 87.51908874511719 Profit 37.615325927734375\n",
            "176 |  avg_price 124.30469780294304 Profit 51.69530219705696\n",
            "49 |  avg_price 34.636893798828126 Profit 14.468044708251952\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-51-6274c5b7f4e2>:354: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-51-6274c5b7f4e2>:286: RuntimeWarning: invalid value encountered in double_scalars\n",
            "  elif current_price < (avg_price - avg_price * 1)  and avg_price > 0.0:\n",
            "<ipython-input-51-6274c5b7f4e2>:354: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-51-6274c5b7f4e2>:286: RuntimeWarning: invalid value encountered in double_scalars\n",
            "  elif current_price < (avg_price - avg_price * 1)  and avg_price > 0.0:\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "331 |  avg_price 141.02000427246094 Profit 189.7718963623047\n",
            "104 |  avg_price 80.07931518554688 Profit 24.034225463867188\n",
            "125 |  avg_price 87.51908874511719 Profit 37.615325927734375\n",
            "176 |  avg_price 124.30469780294304 Profit 51.69530219705696\n",
            "49 |  avg_price 34.636893798828126 Profit 14.468044708251952\n",
            "331 |  avg_price 141.02000427246094 Profit 189.7718963623047\n",
            "104 |  avg_price 80.07931518554688 Profit 24.034225463867188\n",
            "125 |  avg_price 87.51908874511719 Profit 37.615325927734375\n",
            "176 |  avg_price 124.30469780294304 Profit 51.69530219705696\n",
            "49 |  avg_price 34.636893798828126 Profit 14.468044708251952\n",
            "------------------------------------\n",
            "| time/              |             |\n",
            "|    episodes        | 8           |\n",
            "|    fps             | 480         |\n",
            "|    time_elapsed    | 1           |\n",
            "|    total_timesteps | 664         |\n",
            "| train/             |             |\n",
            "|    reward          | -0.81137323 |\n",
            "------------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-51-6274c5b7f4e2>:354: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-51-6274c5b7f4e2>:286: RuntimeWarning: invalid value encountered in double_scalars\n",
            "  elif current_price < (avg_price - avg_price * 1)  and avg_price > 0.0:\n",
            "<ipython-input-51-6274c5b7f4e2>:354: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-51-6274c5b7f4e2>:286: RuntimeWarning: invalid value encountered in double_scalars\n",
            "  elif current_price < (avg_price - avg_price * 1)  and avg_price > 0.0:\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "331 |  avg_price 141.02000427246094 Profit 189.7718963623047\n",
            "104 |  avg_price 80.07931518554688 Profit 24.034225463867188\n",
            "125 |  avg_price 87.51908874511719 Profit 37.615325927734375\n",
            "176 |  avg_price 124.30469780294304 Profit 51.69530219705696\n",
            "49 |  avg_price 34.636893798828126 Profit 14.468044708251952\n",
            "331 |  avg_price 141.02000427246094 Profit 189.7718963623047\n",
            "104 |  avg_price 80.07931518554688 Profit 24.034225463867188\n",
            "125 |  avg_price 87.51908874511719 Profit 37.615325927734375\n",
            "176 |  avg_price 124.30469780294304 Profit 51.69530219705696\n",
            "49 |  avg_price 34.636893798828126 Profit 14.468044708251952\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-51-6274c5b7f4e2>:354: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-51-6274c5b7f4e2>:286: RuntimeWarning: invalid value encountered in double_scalars\n",
            "  elif current_price < (avg_price - avg_price * 1)  and avg_price > 0.0:\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "51 |  avg_price 37.99867630004883 Profit 13.41058349609375\n",
            "51 |  avg_price 37.99867630004883 Profit 13.41058349609375\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-51-6274c5b7f4e2>:354: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-51-6274c5b7f4e2>:354: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "------------------------------------\n",
            "| time/              |             |\n",
            "|    episodes        | 12          |\n",
            "|    fps             | 156         |\n",
            "|    time_elapsed    | 6           |\n",
            "|    total_timesteps | 996         |\n",
            "| train/             |             |\n",
            "|    actor_loss      | -2.47       |\n",
            "|    critic_loss     | 1.53e+05    |\n",
            "|    learning_rate   | 0.0103      |\n",
            "|    n_updates       | 768         |\n",
            "|    reward          | -0.49049777 |\n",
            "------------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-51-6274c5b7f4e2>:354: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "\u001b[33m[W 2023-05-02 13:54:34,832]\u001b[0m Trial 1 failed with parameters: {'gamma': 0.9, 'tau': 0.08, 'train_freq': 768, 'noise_type': 'normal', 'noise_std': 0.3, 'net_arch': 'big'} because of the following error: KeyboardInterrupt().\u001b[0m\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/site-packages/optuna/study/_optimize.py\", line 200, in _run_trial\n",
            "    value_or_values = func(trial)\n",
            "  File \"<ipython-input-63-26bf33718e81>\", line 25, in objective\n",
            "    trained_ddpg = agent.train_model(model=model_ddpg,\n",
            "  File \"/usr/local/lib/python3.10/site-packages/finrl/agents/stablebaselines3/models.py\", line 103, in train_model\n",
            "    model = model.learn(\n",
            "  File \"/usr/local/lib/python3.10/site-packages/stable_baselines3/ddpg/ddpg.py\", line 123, in learn\n",
            "    return super().learn(\n",
            "  File \"/usr/local/lib/python3.10/site-packages/stable_baselines3/td3/td3.py\", line 216, in learn\n",
            "    return super().learn(\n",
            "  File \"/usr/local/lib/python3.10/site-packages/stable_baselines3/common/off_policy_algorithm.py\", line 311, in learn\n",
            "    rollout = self.collect_rollouts(\n",
            "  File \"/usr/local/lib/python3.10/site-packages/stable_baselines3/common/off_policy_algorithm.py\", line 540, in collect_rollouts\n",
            "    actions, buffer_actions = self._sample_action(learning_starts, action_noise, env.num_envs)\n",
            "  File \"/usr/local/lib/python3.10/site-packages/stable_baselines3/common/off_policy_algorithm.py\", line 372, in _sample_action\n",
            "    unscaled_action, _ = self.predict(self._last_obs, deterministic=False)\n",
            "  File \"/usr/local/lib/python3.10/site-packages/stable_baselines3/common/base_class.py\", line 539, in predict\n",
            "    return self.policy.predict(observation, state, episode_start, deterministic)\n",
            "  File \"/usr/local/lib/python3.10/site-packages/stable_baselines3/common/policies.py\", line 348, in predict\n",
            "    actions = actions.cpu().numpy().reshape((-1, *self.action_space.shape))\n",
            "KeyboardInterrupt\n",
            "\u001b[33m[W 2023-05-02 13:54:34,834]\u001b[0m Trial 1 failed with value None.\u001b[0m\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "51 |  avg_price 37.99867630004883 Profit 13.41058349609375\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-63-26bf33718e81>\u001b[0m in \u001b[0;36m<cell line: 57>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     55\u001b[0m                                    trial_number=lc_trial_number)\n\u001b[1;32m     56\u001b[0m \u001b[0;31m#You can increase the n_trials for a better search space scanning\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 57\u001b[0;31m \u001b[0mstudy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobjective\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_trials\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mn_trials\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mcatch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mValueError\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mlogging_callback\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/optuna/study/study.py\u001b[0m in \u001b[0;36moptimize\u001b[0;34m(self, func, n_trials, timeout, n_jobs, catch, callbacks, gc_after_trial, show_progress_bar)\u001b[0m\n\u001b[1;32m    423\u001b[0m         \"\"\"\n\u001b[1;32m    424\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 425\u001b[0;31m         _optimize(\n\u001b[0m\u001b[1;32m    426\u001b[0m             \u001b[0mstudy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    427\u001b[0m             \u001b[0mfunc\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/optuna/study/_optimize.py\u001b[0m in \u001b[0;36m_optimize\u001b[0;34m(study, func, n_trials, timeout, n_jobs, catch, callbacks, gc_after_trial, show_progress_bar)\u001b[0m\n\u001b[1;32m     64\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     65\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mn_jobs\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 66\u001b[0;31m             _optimize_sequential(\n\u001b[0m\u001b[1;32m     67\u001b[0m                 \u001b[0mstudy\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     68\u001b[0m                 \u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/optuna/study/_optimize.py\u001b[0m in \u001b[0;36m_optimize_sequential\u001b[0;34m(study, func, n_trials, timeout, catch, callbacks, gc_after_trial, reseed_sampler_rng, time_start, progress_bar)\u001b[0m\n\u001b[1;32m    161\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    162\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 163\u001b[0;31m             \u001b[0mfrozen_trial\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_run_trial\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstudy\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcatch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    164\u001b[0m         \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    165\u001b[0m             \u001b[0;31m# The following line mitigates memory problems that can be occurred in some\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/optuna/study/_optimize.py\u001b[0m in \u001b[0;36m_run_trial\u001b[0;34m(study, func, catch)\u001b[0m\n\u001b[1;32m    249\u001b[0m         \u001b[0;32mand\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc_err\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcatch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    250\u001b[0m     ):\n\u001b[0;32m--> 251\u001b[0;31m         \u001b[0;32mraise\u001b[0m \u001b[0mfunc_err\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    252\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mfrozen_trial\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    253\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/optuna/study/_optimize.py\u001b[0m in \u001b[0;36m_run_trial\u001b[0;34m(study, func, catch)\u001b[0m\n\u001b[1;32m    198\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mget_heartbeat_thread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrial\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_trial_id\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstudy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_storage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    199\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 200\u001b[0;31m             \u001b[0mvalue_or_values\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrial\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    201\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mexceptions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTrialPruned\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    202\u001b[0m             \u001b[0;31m# TODO(mamu): Handle multi-objective cases.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-63-26bf33718e81>\u001b[0m in \u001b[0;36mobjective\u001b[0;34m(trial)\u001b[0m\n\u001b[1;32m     23\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m   \u001b[0;31m#You can increase it for better comparison\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 25\u001b[0;31m   trained_ddpg = agent.train_model(model=model_ddpg,\n\u001b[0m\u001b[1;32m     26\u001b[0m                                    \u001b[0mtb_log_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"ddpg\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     27\u001b[0m                                    total_timesteps=total_timesteps)\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/finrl/agents/stablebaselines3/models.py\u001b[0m in \u001b[0;36mtrain_model\u001b[0;34m(self, model, tb_log_name, total_timesteps)\u001b[0m\n\u001b[1;32m    101\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    102\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mtrain_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtb_log_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m5000\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 103\u001b[0;31m         model = model.learn(\n\u001b[0m\u001b[1;32m    104\u001b[0m             \u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    105\u001b[0m             \u001b[0mtb_log_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtb_log_name\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/stable_baselines3/ddpg/ddpg.py\u001b[0m in \u001b[0;36mlearn\u001b[0;34m(self, total_timesteps, callback, log_interval, tb_log_name, reset_num_timesteps, progress_bar)\u001b[0m\n\u001b[1;32m    121\u001b[0m         \u001b[0mprogress_bar\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mbool\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    122\u001b[0m     ) -> SelfDDPG:\n\u001b[0;32m--> 123\u001b[0;31m         return super().learn(\n\u001b[0m\u001b[1;32m    124\u001b[0m             \u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    125\u001b[0m             \u001b[0mcallback\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallback\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/stable_baselines3/td3/td3.py\u001b[0m in \u001b[0;36mlearn\u001b[0;34m(self, total_timesteps, callback, log_interval, tb_log_name, reset_num_timesteps, progress_bar)\u001b[0m\n\u001b[1;32m    214\u001b[0m         \u001b[0mprogress_bar\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mbool\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    215\u001b[0m     ) -> SelfTD3:\n\u001b[0;32m--> 216\u001b[0;31m         return super().learn(\n\u001b[0m\u001b[1;32m    217\u001b[0m             \u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    218\u001b[0m             \u001b[0mcallback\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallback\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/stable_baselines3/common/off_policy_algorithm.py\u001b[0m in \u001b[0;36mlearn\u001b[0;34m(self, total_timesteps, callback, log_interval, tb_log_name, reset_num_timesteps, progress_bar)\u001b[0m\n\u001b[1;32m    309\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    310\u001b[0m         \u001b[0;32mwhile\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnum_timesteps\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 311\u001b[0;31m             rollout = self.collect_rollouts(\n\u001b[0m\u001b[1;32m    312\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menv\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    313\u001b[0m                 \u001b[0mtrain_freq\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_freq\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/stable_baselines3/common/off_policy_algorithm.py\u001b[0m in \u001b[0;36mcollect_rollouts\u001b[0;34m(self, env, callback, train_freq, replay_buffer, action_noise, learning_starts, log_interval)\u001b[0m\n\u001b[1;32m    538\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    539\u001b[0m             \u001b[0;31m# Select action randomly or according to policy\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 540\u001b[0;31m             \u001b[0mactions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbuffer_actions\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sample_action\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlearning_starts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maction_noise\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnum_envs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    541\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    542\u001b[0m             \u001b[0;31m# Rescale and perform action\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/stable_baselines3/common/off_policy_algorithm.py\u001b[0m in \u001b[0;36m_sample_action\u001b[0;34m(self, learning_starts, action_noise, n_envs)\u001b[0m\n\u001b[1;32m    370\u001b[0m             \u001b[0;31m# we assume that the policy uses tanh to scale the action\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    371\u001b[0m             \u001b[0;31m# We use non-deterministic action in the case of SAC, for TD3, it does not matter\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 372\u001b[0;31m             \u001b[0munscaled_action\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_last_obs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdeterministic\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    373\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    374\u001b[0m         \u001b[0;31m# Rescale the action from [low, high] to [-1, 1]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/stable_baselines3/common/base_class.py\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(self, observation, state, episode_start, deterministic)\u001b[0m\n\u001b[1;32m    537\u001b[0m             \u001b[0;34m(\u001b[0m\u001b[0mused\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrecurrent\u001b[0m \u001b[0mpolicies\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    538\u001b[0m         \"\"\"\n\u001b[0;32m--> 539\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpolicy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobservation\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepisode_start\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdeterministic\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    540\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    541\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mset_random_seed\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mseed\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mOptional\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/stable_baselines3/common/policies.py\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(self, observation, state, episode_start, deterministic)\u001b[0m\n\u001b[1;32m    346\u001b[0m             \u001b[0mactions\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_predict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobservation\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdeterministic\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdeterministic\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    347\u001b[0m         \u001b[0;31m# Convert to numpy, and reshape to the original action shape\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 348\u001b[0;31m         \u001b[0mactions\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mactions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcpu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maction_space\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    349\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    350\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maction_space\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mspaces\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mBox\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "joblib.dump(study, \"final_ddpg_study__.pkl\")"
      ],
      "metadata": {
        "id": "1yIXC6W2ZqPP",
        "outputId": "9343c42b-652a-4755-b8a4-08e09227d402",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 331,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['final_ddpg_study__.pkl']"
            ]
          },
          "metadata": {},
          "execution_count": 331
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Get the best hyperparamters\n",
        "print('Hyperparameters after tuning',study.best_params)\n",
        "print('Hyperparameters before tuning',config.DDPG_PARAMS)"
      ],
      "metadata": {
        "id": "gXzfenQTdqq_",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "307ce707-340b-427b-d8ed-a68ca2d99e1f"
      },
      "execution_count": 332,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Hyperparameters after tuning {'gamma': 0.96, 'tau': 0.06, 'train_freq': 512, 'noise_type': None, 'noise_std': 0.4, 'net_arch': 'small'}\n",
            "Hyperparameters before tuning {'batch_size': 128, 'buffer_size': 50000, 'learning_rate': 0.001}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "study.best_trial"
      ],
      "metadata": {
        "id": "VQcuj-yidu0O",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "dc84fbf7-444c-4561-99f3-fada7b9fff3b"
      },
      "execution_count": 333,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "FrozenTrial(number=4, state=TrialState.COMPLETE, values=[0.0323970214377839], datetime_start=datetime.datetime(2023, 5, 2, 13, 25, 43, 607970), datetime_complete=datetime.datetime(2023, 5, 2, 13, 25, 58, 95930), params={'gamma': 0.96, 'tau': 0.06, 'train_freq': 512, 'noise_type': None, 'noise_std': 0.4, 'net_arch': 'small'}, user_attrs={}, system_attrs={}, intermediate_values={}, distributions={'gamma': CategoricalDistribution(choices=(0.9, 0.92, 0.94, 0.96, 0.98)), 'tau': CategoricalDistribution(choices=(0.02, 0.04, 0.06, 0.08, 0.1, 0.12)), 'train_freq': CategoricalDistribution(choices=(512, 768, 1024)), 'noise_type': CategoricalDistribution(choices=('ornstein-uhlenbeck', 'normal', None)), 'noise_std': CategoricalDistribution(choices=(0.1, 0.2, 0.3, 0.4, 0.5)), 'net_arch': CategoricalDistribution(choices=('small', 'big'))}, trial_id=4, value=None)"
            ]
          },
          "metadata": {},
          "execution_count": 333
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from stable_baselines3 import DDPG\n",
        "tuned_model_ddpg = DDPG.load('models/ddpg_{}.pth'.format(study.best_trial.number),env=env_train)"
      ],
      "metadata": {
        "id": "ZjYe9lOIdw3i"
      },
      "execution_count": 361,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#Trading period account value with tuned model\n",
        "df_account_value_tuned, df_actions_tuned = DRLAgent.DRL_prediction(\n",
        "    model=tuned_model_ddpg, \n",
        "    environment = e_trade_gym)"
      ],
      "metadata": {
        "id": "3bnbx87vdzHZ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "226735d3-f555-4204-9f4d-5f96006bd6f4"
      },
      "execution_count": 362,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "hit end!\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_account_value_tuned[105:106][\"account_value\"]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "G1Y1jjMLf0qF",
        "outputId": "e7d9c458-6981-48a4-d7ae-c6308f42f8c6"
      },
      "execution_count": 363,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "105    508978.064416\n",
              "Name: account_value, dtype: float64"
            ]
          },
          "metadata": {},
          "execution_count": 363
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Backtesting with our pruned model\n",
        "print(\"==============Get Backtest Results===========\")\n",
        "now = datetime.datetime.now().strftime('%Y%m%d-%Hh%M')\n",
        "\n",
        "perf_stats_all_tuned = backtest_stats(account_value=df_account_value_tuned)\n",
        "perf_stats_all_tuned = pd.DataFrame(perf_stats_all_tuned)\n",
        "perf_stats_all_tuned.to_csv(\"./\"+config.RESULTS_DIR+\"/perf_stats_all_tuned_\"+now+'.csv')"
      ],
      "metadata": {
        "id": "FJrv144od3G_",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a754ff06-e41a-4402-e4ce-929256f946b8"
      },
      "execution_count": 345,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==============Get Backtest Results===========\n",
            "Annual return         -0.392959\n",
            "Cumulative returns    -0.189386\n",
            "Annual volatility      0.126233\n",
            "Sharpe ratio          -3.925389\n",
            "Calmar ratio          -2.026607\n",
            "Stability              0.968984\n",
            "Max drawdown          -0.193900\n",
            "Omega ratio            0.513490\n",
            "Sortino ratio         -4.724054\n",
            "Skew                        NaN\n",
            "Kurtosis                    NaN\n",
            "Tail ratio             0.655493\n",
            "Daily value at risk   -0.017870\n",
            "dtype: float64\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Now train with not tuned hyperaparameters\n",
        "#Default config.ddpg_PARAMS\n",
        "non_tuned_model_ddpg = agent.get_model(\"ddpg\",model_kwargs = config.DDPG_PARAMS )\n",
        "trained_ddpg = agent.train_model(model=non_tuned_model_ddpg, \n",
        "                             tb_log_name='ddpg',\n",
        "                             total_timesteps=50000)"
      ],
      "metadata": {
        "id": "asWsWzemd5dk",
        "outputId": "5bfa4cdf-d304-46d8-adae-99c5dc6f4ab5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'batch_size': 128, 'buffer_size': 50000, 'learning_rate': 0.001}\n",
            "Using cuda device\n",
            "------------------------------------\n",
            "| time/              |             |\n",
            "|    episodes        | 4           |\n",
            "|    fps             | 129         |\n",
            "|    time_elapsed    | 10          |\n",
            "|    total_timesteps | 1340        |\n",
            "| train/             |             |\n",
            "|    actor_loss      | -71         |\n",
            "|    critic_loss     | 2.97e+03    |\n",
            "|    learning_rate   | 0.001       |\n",
            "|    n_updates       | 1005        |\n",
            "|    reward          | -0.60931635 |\n",
            "------------------------------------\n",
            "day: 334, episode: 160\n",
            "begin_total_asset: 1000000.00\n",
            "end_total_asset: 1143478.33\n",
            "total_reward: 143478.33\n",
            "total_cost: 999.00\n",
            "total_trades: 4008\n",
            "Sharpe: 0.458\n",
            "=================================\n",
            "------------------------------------\n",
            "| time/              |             |\n",
            "|    episodes        | 8           |\n",
            "|    fps             | 119         |\n",
            "|    time_elapsed    | 22          |\n",
            "|    total_timesteps | 2680        |\n",
            "| train/             |             |\n",
            "|    actor_loss      | -15.4       |\n",
            "|    critic_loss     | 2.79e+03    |\n",
            "|    learning_rate   | 0.001       |\n",
            "|    n_updates       | 2345        |\n",
            "|    reward          | -0.60931635 |\n",
            "------------------------------------\n",
            "------------------------------------\n",
            "| time/              |             |\n",
            "|    episodes        | 12          |\n",
            "|    fps             | 115         |\n",
            "|    time_elapsed    | 34          |\n",
            "|    total_timesteps | 4020        |\n",
            "| train/             |             |\n",
            "|    actor_loss      | -7.04       |\n",
            "|    critic_loss     | 44.5        |\n",
            "|    learning_rate   | 0.001       |\n",
            "|    n_updates       | 3685        |\n",
            "|    reward          | -0.60931635 |\n",
            "------------------------------------\n",
            "day: 334, episode: 170\n",
            "begin_total_asset: 1000000.00\n",
            "end_total_asset: 1143478.33\n",
            "total_reward: 143478.33\n",
            "total_cost: 999.00\n",
            "total_trades: 4008\n",
            "Sharpe: 0.458\n",
            "=================================\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-81-3b1b0264cc27>\u001b[0m in \u001b[0;36m<cell line: 4>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;31m#Default config.ddpg_PARAMS\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mnon_tuned_model_ddpg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0magent\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"ddpg\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mmodel_kwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDDPG_PARAMS\u001b[0m \u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m trained_ddpg = agent.train_model(model=non_tuned_model_ddpg, \n\u001b[0m\u001b[1;32m      5\u001b[0m                              \u001b[0mtb_log_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'ddpg'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m                              total_timesteps=50000)\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/finrl/agents/stablebaselines3/models.py\u001b[0m in \u001b[0;36mtrain_model\u001b[0;34m(self, model, tb_log_name, total_timesteps)\u001b[0m\n\u001b[1;32m    101\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    102\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mtrain_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtb_log_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m5000\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 103\u001b[0;31m         model = model.learn(\n\u001b[0m\u001b[1;32m    104\u001b[0m             \u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    105\u001b[0m             \u001b[0mtb_log_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtb_log_name\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/stable_baselines3/ddpg/ddpg.py\u001b[0m in \u001b[0;36mlearn\u001b[0;34m(self, total_timesteps, callback, log_interval, tb_log_name, reset_num_timesteps, progress_bar)\u001b[0m\n\u001b[1;32m    121\u001b[0m         \u001b[0mprogress_bar\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mbool\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    122\u001b[0m     ) -> SelfDDPG:\n\u001b[0;32m--> 123\u001b[0;31m         return super().learn(\n\u001b[0m\u001b[1;32m    124\u001b[0m             \u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    125\u001b[0m             \u001b[0mcallback\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallback\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/stable_baselines3/td3/td3.py\u001b[0m in \u001b[0;36mlearn\u001b[0;34m(self, total_timesteps, callback, log_interval, tb_log_name, reset_num_timesteps, progress_bar)\u001b[0m\n\u001b[1;32m    214\u001b[0m         \u001b[0mprogress_bar\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mbool\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    215\u001b[0m     ) -> SelfTD3:\n\u001b[0;32m--> 216\u001b[0;31m         return super().learn(\n\u001b[0m\u001b[1;32m    217\u001b[0m             \u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    218\u001b[0m             \u001b[0mcallback\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallback\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/stable_baselines3/common/off_policy_algorithm.py\u001b[0m in \u001b[0;36mlearn\u001b[0;34m(self, total_timesteps, callback, log_interval, tb_log_name, reset_num_timesteps, progress_bar)\u001b[0m\n\u001b[1;32m    328\u001b[0m                 \u001b[0;31m# Special case when the user passes `gradient_steps=0`\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    329\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mgradient_steps\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 330\u001b[0;31m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient_steps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mgradient_steps\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    331\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    332\u001b[0m         \u001b[0mcallback\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_training_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/stable_baselines3/td3/td3.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self, gradient_steps, batch_size)\u001b[0m\n\u001b[1;32m    192\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mactor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    193\u001b[0m                 \u001b[0mactor_loss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 194\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mactor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    195\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    196\u001b[0m                 \u001b[0mpolyak_update\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcritic\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcritic_target\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtau\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/torch/optim/optimizer.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    278\u001b[0m                                                f\"but got {result}.\")\n\u001b[1;32m    279\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 280\u001b[0;31m                 \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    281\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_optimizer_step_code\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    282\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/torch/optim/optimizer.py\u001b[0m in \u001b[0;36m_use_grad\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m     31\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     32\u001b[0m             \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_grad_enabled\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdefaults\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'differentiable'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 33\u001b[0;31m             \u001b[0mret\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     34\u001b[0m         \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     35\u001b[0m             \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_grad_enabled\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprev_grad\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/torch/optim/adam.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, closure)\u001b[0m\n\u001b[1;32m    139\u001b[0m                 state_steps)\n\u001b[1;32m    140\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 141\u001b[0;31m             adam(\n\u001b[0m\u001b[1;32m    142\u001b[0m                 \u001b[0mparams_with_grad\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    143\u001b[0m                 \u001b[0mgrads\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/torch/optim/adam.py\u001b[0m in \u001b[0;36madam\u001b[0;34m(params, grads, exp_avgs, exp_avg_sqs, max_exp_avg_sqs, state_steps, foreach, capturable, differentiable, fused, grad_scale, found_inf, amsgrad, beta1, beta2, lr, weight_decay, eps, maximize)\u001b[0m\n\u001b[1;32m    279\u001b[0m         \u001b[0mfunc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_single_tensor_adam\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    280\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 281\u001b[0;31m     func(params,\n\u001b[0m\u001b[1;32m    282\u001b[0m          \u001b[0mgrads\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    283\u001b[0m          \u001b[0mexp_avgs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/torch/optim/adam.py\u001b[0m in \u001b[0;36m_multi_tensor_adam\u001b[0;34m(params, grads, exp_avgs, exp_avg_sqs, max_exp_avg_sqs, state_steps, grad_scale, found_inf, amsgrad, beta1, beta2, lr, weight_decay, eps, maximize, capturable, differentiable)\u001b[0m\n\u001b[1;32m    444\u001b[0m         \u001b[0;31m# Decay the first and second moment running average coefficient\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    445\u001b[0m         \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_foreach_mul_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice_exp_avgs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbeta1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 446\u001b[0;31m         \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_foreach_add_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice_exp_avgs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice_grads\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0malpha\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mbeta1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    447\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    448\u001b[0m         \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_foreach_mul_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice_exp_avg_sqs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbeta2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_account_value, df_actions = DRLAgent.DRL_prediction(\n",
        "    model=trained_ddpg, \n",
        "    environment = e_trade_gym)"
      ],
      "metadata": {
        "id": "mv1zgwOgd9FY",
        "outputId": "3fd9a94a-98d4-41c0-8870-74b8271b6494",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-35-38368194789e>\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m df_account_value, df_actions = DRLAgent.DRL_prediction(\n\u001b[0;32m----> 2\u001b[0;31m     \u001b[0mmodel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrained_ddpg\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m     environment = e_trade_gym)\n",
            "\u001b[0;31mNameError\u001b[0m: name 'trained_ddpg' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Backtesting for not tuned hyperparamters\n",
        "print(\"==============Get Backtest Results===========\")\n",
        "now = datetime.datetime.now().strftime('%Y%m%d-%Hh%M')\n",
        "\n",
        "perf_stats_all = backtest_stats(account_value=df_account_value)\n",
        "perf_stats_all = pd.DataFrame(perf_stats_all)\n",
        "# perf_stats_all.to_csv(\"./\"+config.RESULTS_DIR+\"/perf_stats_all_\"+now+'.csv')"
      ],
      "metadata": {
        "id": "DuZL52qDeEz6",
        "outputId": "f4e9cd19-27cd-4dba-956e-353df0dabffa",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==============Get Backtest Results===========\n",
            "Annual return          0.247826\n",
            "Cumulative returns     0.368416\n",
            "Annual volatility      0.167226\n",
            "Sharpe ratio           1.412093\n",
            "Calmar ratio           2.531845\n",
            "Stability              0.897149\n",
            "Max drawdown          -0.097883\n",
            "Omega ratio            1.280406\n",
            "Sortino ratio          2.010344\n",
            "Skew                        NaN\n",
            "Kurtosis                    NaN\n",
            "Tail ratio             1.033938\n",
            "Daily value at risk   -0.020131\n",
            "dtype: float64\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#You can see with trial, our sharpe ratio is increasing\n",
        "#Certainly you can afford more number of trials for further optimization\n",
        "from optuna.visualization import plot_optimization_history\n",
        "plot_optimization_history(study)"
      ],
      "metadata": {
        "id": "5aSsEmpZeI5C",
        "outputId": "4e4acb29-dc3a-4b5b-fabe-736db19aea9d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 542
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<html>\n",
              "<head><meta charset=\"utf-8\" /></head>\n",
              "<body>\n",
              "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
              "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.20.0.min.js\"></script>                <div id=\"6afac010-8c60-495c-b9f7-1749179d9521\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"6afac010-8c60-495c-b9f7-1749179d9521\")) {                    Plotly.newPlot(                        \"6afac010-8c60-495c-b9f7-1749179d9521\",                        [{\"mode\":\"markers\",\"name\":\"Objective Value\",\"x\":[0,1,2,3,4],\"y\":[25.0,5.458908059384231,9.181461931558035,26.496666465769017,7.580685200661261],\"type\":\"scatter\"},{\"name\":\"Best Value\",\"x\":[0,1,2,3,4],\"y\":[25.0,25.0,25.0,26.496666465769017,26.496666465769017],\"type\":\"scatter\"}],                        {\"title\":{\"text\":\"Optimization History Plot\"},\"xaxis\":{\"title\":{\"text\":\"Trial\"}},\"yaxis\":{\"title\":{\"text\":\"Objective Value\"}},\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}}},                        {\"responsive\": true}                    ).then(function(){\n",
              "                            \n",
              "var gd = document.getElementById('6afac010-8c60-495c-b9f7-1749179d9521');\n",
              "var x = new MutationObserver(function (mutations, observer) {{\n",
              "        var display = window.getComputedStyle(gd).display;\n",
              "        if (!display || display === 'none') {{\n",
              "            console.log([gd, 'removed!']);\n",
              "            Plotly.purge(gd);\n",
              "            observer.disconnect();\n",
              "        }}\n",
              "}});\n",
              "\n",
              "// Listen for the removal of the full notebook cells\n",
              "var notebookContainer = gd.closest('#notebook-container');\n",
              "if (notebookContainer) {{\n",
              "    x.observe(notebookContainer, {childList: true});\n",
              "}}\n",
              "\n",
              "// Listen for the clearing of the current output cell\n",
              "var outputEl = gd.closest('.output');\n",
              "if (outputEl) {{\n",
              "    x.observe(outputEl, {childList: true});\n",
              "}}\n",
              "\n",
              "                        })                };                            </script>        </div>\n",
              "</body>\n",
              "</html>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from optuna.visualization import plot_contour\n",
        "from optuna.visualization import plot_edf\n",
        "from optuna.visualization import plot_intermediate_values\n",
        "from optuna.visualization import plot_optimization_history\n",
        "from optuna.visualization import plot_parallel_coordinate\n",
        "from optuna.visualization import plot_param_importances\n",
        "from optuna.visualization import plot_slice"
      ],
      "metadata": {
        "id": "K9zcey_BeJMe"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#Hyperparamters importance\n",
        "#Ent_coef is the most important\n",
        "plot_param_importances(study)"
      ],
      "metadata": {
        "id": "J6WmB5DneOgh",
        "outputId": "9510c118-7601-45ac-adba-0226c3c76277",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 542
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<html>\n",
              "<head><meta charset=\"utf-8\" /></head>\n",
              "<body>\n",
              "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
              "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.20.0.min.js\"></script>                <div id=\"79c56b7b-1dac-42d7-a7e1-58bd16ed9c0e\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"79c56b7b-1dac-42d7-a7e1-58bd16ed9c0e\")) {                    Plotly.newPlot(                        \"79c56b7b-1dac-42d7-a7e1-58bd16ed9c0e\",                        [{\"cliponaxis\":false,\"hovertemplate\":[\"gamma (CategoricalDistribution): 0.051402280785507216<extra></extra>\",\"noise_std (CategoricalDistribution): 0.10120568535574027<extra></extra>\",\"noise_type (CategoricalDistribution): 0.16624172227685763<extra></extra>\",\"net_arch (CategoricalDistribution): 0.20489031403705613<extra></extra>\",\"train_freq (CategoricalDistribution): 0.22282841915371404<extra></extra>\",\"tau (CategoricalDistribution): 0.25343157839112473<extra></extra>\"],\"marker\":{\"color\":\"rgb(66,146,198)\"},\"orientation\":\"h\",\"text\":[\"0.05\",\"0.10\",\"0.17\",\"0.20\",\"0.22\",\"0.25\"],\"textposition\":\"outside\",\"x\":[0.051402280785507216,0.10120568535574027,0.16624172227685763,0.20489031403705613,0.22282841915371404,0.25343157839112473],\"y\":[\"gamma\",\"noise_std\",\"noise_type\",\"net_arch\",\"train_freq\",\"tau\"],\"type\":\"bar\"}],                        {\"showlegend\":false,\"title\":{\"text\":\"Hyperparameter Importances\"},\"xaxis\":{\"title\":{\"text\":\"Importance for Objective Value\"}},\"yaxis\":{\"title\":{\"text\":\"Hyperparameter\"}},\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}}},                        {\"responsive\": true}                    ).then(function(){\n",
              "                            \n",
              "var gd = document.getElementById('79c56b7b-1dac-42d7-a7e1-58bd16ed9c0e');\n",
              "var x = new MutationObserver(function (mutations, observer) {{\n",
              "        var display = window.getComputedStyle(gd).display;\n",
              "        if (!display || display === 'none') {{\n",
              "            console.log([gd, 'removed!']);\n",
              "            Plotly.purge(gd);\n",
              "            observer.disconnect();\n",
              "        }}\n",
              "}});\n",
              "\n",
              "// Listen for the removal of the full notebook cells\n",
              "var notebookContainer = gd.closest('#notebook-container');\n",
              "if (notebookContainer) {{\n",
              "    x.observe(notebookContainer, {childList: true});\n",
              "}}\n",
              "\n",
              "// Listen for the clearing of the current output cell\n",
              "var outputEl = gd.closest('.output');\n",
              "if (outputEl) {{\n",
              "    x.observe(outputEl, {childList: true});\n",
              "}}\n",
              "\n",
              "                        })                };                            </script>        </div>\n",
              "</body>\n",
              "</html>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def main():\n",
        "    import warnings\n",
        "    warnings.filterwarnings(\"ignore\")\n",
        "    import pandas as pd\n",
        "    import numpy as np\n",
        "    import matplotlib\n",
        "    import matplotlib.pyplot as plt\n",
        "    # matplotlib.use('Agg')\n",
        "    import datetime\n",
        "\n",
        "\n",
        "    from finrl.config_tickers import DOW_30_TICKER\n",
        "    from finrl.meta.preprocessor.yahoodownloader import YahooDownloader\n",
        "    from finrl.meta.preprocessor.preprocessors import FeatureEngineer, data_split\n",
        "    from finrl.meta.env_stock_trading.env_stocktrading import StockTradingEnv\n",
        "    from finrl.agents.stablebaselines3.models import DRLAgent, DRLEnsembleAgent\n",
        "    from finrl.plot import backtest_stats, backtest_plot, get_daily_return, get_baseline\n",
        "\n",
        "    from pprint import pprint\n",
        "\n",
        "    import sys\n",
        "    sys.path.append(\"../FinRL-Library\")\n",
        "\n",
        "    import itertools\n",
        "\n",
        "    import os\n",
        "    from finrl.main import check_and_make_directories\n",
        "    from finrl.config import (\n",
        "        DATA_SAVE_DIR,\n",
        "        TRAINED_MODEL_DIR,\n",
        "        TENSORBOARD_LOG_DIR,\n",
        "        RESULTS_DIR,\n",
        "        INDICATORS,\n",
        "        TRAIN_START_DATE,\n",
        "        TRAIN_END_DATE,\n",
        "        TEST_START_DATE,\n",
        "        TEST_END_DATE,\n",
        "        TRADE_START_DATE,\n",
        "        TRADE_END_DATE,\n",
        "    )\n",
        "\n",
        "    check_and_make_directories([DATA_SAVE_DIR, TRAINED_MODEL_DIR, TENSORBOARD_LOG_DIR, RESULTS_DIR])\n",
        "    print(DOW_30_TICKER)\n",
        "    TRAIN_START_DATE = '2009-04-01'\n",
        "    TRAIN_END_DATE = '2021-01-01'\n",
        "    TEST_START_DATE = '2021-01-01'\n",
        "    TEST_END_DATE = '2022-06-01'\n",
        "\n",
        "    df = YahooDownloader(start_date=TRAIN_START_DATE,\n",
        "                         end_date=TEST_END_DATE,\n",
        "                         ticker_list=DOW_30_TICKER).fetch_data()\n",
        "\n",
        "    df.sort_values(['date', 'tic']).head()\n",
        "\n",
        "    fe = FeatureEngineer(use_technical_indicator=True,\n",
        "                         tech_indicator_list=INDICATORS,\n",
        "                         use_turbulence=True,\n",
        "                         user_defined_feature=False)\n",
        "\n",
        "    processed = fe.preprocess_data(df)\n",
        "    processed = processed.copy()\n",
        "    processed = processed.fillna(0)\n",
        "    processed = processed.replace(np.inf, 0)\n",
        "\n",
        "    stock_dimension = len(processed.tic.unique())\n",
        "    state_space = 1 + 2 * stock_dimension + len(INDICATORS) * stock_dimension\n",
        "    print(f\"Stock Dimension: {stock_dimension}, State Space: {state_space}\")\n",
        "\n",
        "    env_kwargs = {\n",
        "        \"hmax\": 100,\n",
        "        \"initial_amount\": 1000000,\n",
        "        \"buy_cost_pct\": 0.001,\n",
        "        \"sell_cost_pct\": 0.001,\n",
        "        \"state_space\": state_space,\n",
        "        \"stock_dim\": stock_dimension,\n",
        "        \"tech_indicator_list\": INDICATORS,\n",
        "        \"action_space\": stock_dimension,\n",
        "        \"reward_scaling\": 1e-4,\n",
        "        \"print_verbosity\": 5\n",
        "\n",
        "    }\n",
        "\n",
        "    rebalance_window = 63  # rebalance_window is the number of days to retrain the model\n",
        "    validation_window = 63  # validation_window is the number of days to do validation and trading (e.g. if validation_window=63, then both validation and trading period will be 63 days)\n",
        "\n",
        "    ensemble_agent = DRLEnsembleAgent(df=processed,\n",
        "                                      train_period=(TRAIN_START_DATE, TRAIN_END_DATE),\n",
        "                                      val_test_period=(TEST_START_DATE, TEST_END_DATE),\n",
        "                                      rebalance_window=rebalance_window,\n",
        "                                      validation_window=validation_window,\n",
        "                                      **env_kwargs)\n",
        "\n",
        "    A2C_model_kwargs = {\n",
        "        'n_steps': 5,\n",
        "        'ent_coef': 0.005,\n",
        "        'learning_rate': 0.0007\n",
        "    }\n",
        "\n",
        "    PPO_model_kwargs = {\n",
        "        \"ent_coef\": 0.01,\n",
        "        \"n_steps\": 2048,\n",
        "        \"learning_rate\": 0.00025,\n",
        "        \"batch_size\": 128\n",
        "    }\n",
        "\n",
        "    DDPG_model_kwargs = {\n",
        "        # \"action_noise\":\"ornstein_uhlenbeck\",\n",
        "        \"buffer_size\": 10_000,\n",
        "        \"learning_rate\": 0.0005,\n",
        "        \"batch_size\": 64\n",
        "    }\n",
        "\n",
        "    timesteps_dict = {'a2c': 10_000,\n",
        "                      'ppo': 10_000,\n",
        "                      'ddpg': 10_000\n",
        "                      }\n",
        "    df_summary = ensemble_agent.run_ensemble_strategy(A2C_model_kwargs,\n",
        "                                                      PPO_model_kwargs,\n",
        "                                                      DDPG_model_kwargs,\n",
        "                                                      timesteps_dict)\n",
        "\n",
        "    unique_trade_date = processed[(processed.date > TEST_START_DATE) & (processed.date <= TEST_END_DATE)].date.unique()\n",
        "\n",
        "    df_trade_date = pd.DataFrame({'datadate': unique_trade_date})\n",
        "\n",
        "    df_account_value = pd.DataFrame()\n",
        "    for i in range(rebalance_window + validation_window, len(unique_trade_date) + 1, rebalance_window):\n",
        "        temp = pd.read_csv('results/account_value_trade_{}_{}.csv'.format('ensemble', i))\n",
        "        df_account_value = df_account_value.append(temp, ignore_index=True)\n",
        "    sharpe = (252 ** 0.5) * df_account_value.account_value.pct_change(\n",
        "        1).mean() / df_account_value.account_value.pct_change(1).std()\n",
        "    print('Sharpe Ratio: ', sharpe)\n",
        "    df_account_value = df_account_value.join(df_trade_date[validation_window:].reset_index(drop=True))\n",
        "\n",
        "    df_account_value.account_value.plot()\n",
        "\n",
        "    print(\"==============Get Backtest Results===========\")\n",
        "    now = datetime.datetime.now().strftime('%Y%m%d-%Hh%M')\n",
        "\n",
        "    perf_stats_all = backtest_stats(account_value=df_account_value)\n",
        "    perf_stats_all = pd.DataFrame(perf_stats_all)\n",
        "\n",
        "    # baseline stats\n",
        "    print(\"==============Get Baseline Stats===========\")\n",
        "    baseline_df = get_baseline(\n",
        "        ticker=\"^DJI\",\n",
        "        start=df_account_value.loc[0, 'date'],\n",
        "        end=df_account_value.loc[len(df_account_value) - 1, 'date'])\n",
        "\n",
        "    stats = backtest_stats(baseline_df, value_col_name='close')\n",
        "\n",
        "    print(\"==============Compare to DJIA===========\")\n",
        "\n",
        "    # S&P 500: ^GSPC\n",
        "    # Dow Jones Index: ^DJI\n",
        "    # NASDAQ 100: ^NDX\n",
        "    backtest_plot(df_account_value,\n",
        "                  baseline_ticker='^DJI',\n",
        "                  baseline_start=df_account_value.loc[0, 'date'],\n",
        "                  baseline_end=df_account_value.loc[len(df_account_value) - 1, 'date'])\n",
        "\n",
        "\n",
        "if __name__ == '__main__':\n",
        "    main()"
      ],
      "metadata": {
        "id": "s_xcVpBSeOyE",
        "outputId": "52b3d82a-1f8d-4af4-a405-6b5b9778781d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['AXP', 'AMGN', 'AAPL', 'BA', 'CAT', 'CSCO', 'CVX', 'GS', 'HD', 'HON', 'IBM', 'INTC', 'JNJ', 'KO', 'JPM', 'MCD', 'MMM', 'MRK', 'MSFT', 'NKE', 'PG', 'TRV', 'UNH', 'CRM', 'VZ', 'V', 'WBA', 'WMT', 'DIS', 'DOW']\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "Shape of DataFrame:  (96942, 8)\n",
            "Successfully added technical indicators\n",
            "Successfully added turbulence index\n",
            "Stock Dimension: 29, State Space: 291\n",
            "============Start Ensemble Strategy============\n",
            "============================================\n",
            "turbulence_threshold:  203.40117769841157\n",
            "======Model training from:  2009-04-01 to  2021-01-04\n",
            "======A2C Training========\n",
            "{'n_steps': 5, 'ent_coef': 0.005, 'learning_rate': 0.0007}\n",
            "Using cuda device\n",
            "Logging to tensorboard_log/a2c/a2c_126_1\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 100        |\n",
            "|    iterations         | 100        |\n",
            "|    time_elapsed       | 4          |\n",
            "|    total_timesteps    | 500        |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.1      |\n",
            "|    explained_variance | 0.00295    |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 99         |\n",
            "|    policy_loss        | -71.2      |\n",
            "|    reward             | 0.78650117 |\n",
            "|    std                | 1          |\n",
            "|    value_loss         | 12.2       |\n",
            "--------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 101        |\n",
            "|    iterations         | 200        |\n",
            "|    time_elapsed       | 9          |\n",
            "|    total_timesteps    | 1000       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.1      |\n",
            "|    explained_variance | 0.00111    |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 199        |\n",
            "|    policy_loss        | 6.59       |\n",
            "|    reward             | 0.66569686 |\n",
            "|    std                | 0.998      |\n",
            "|    value_loss         | 1.03       |\n",
            "--------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 102       |\n",
            "|    iterations         | 300       |\n",
            "|    time_elapsed       | 14        |\n",
            "|    total_timesteps    | 1500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.1     |\n",
            "|    explained_variance | 0         |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 299       |\n",
            "|    policy_loss        | -123      |\n",
            "|    reward             | 2.9206326 |\n",
            "|    std                | 0.998     |\n",
            "|    value_loss         | 15.8      |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 102        |\n",
            "|    iterations         | 400        |\n",
            "|    time_elapsed       | 19         |\n",
            "|    total_timesteps    | 2000       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.1      |\n",
            "|    explained_variance | -1.19e-07  |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 399        |\n",
            "|    policy_loss        | -6.35      |\n",
            "|    reward             | -1.8242201 |\n",
            "|    std                | 0.999      |\n",
            "|    value_loss         | 3.27       |\n",
            "--------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                 |               |\n",
            "|    fps                | 102           |\n",
            "|    iterations         | 500           |\n",
            "|    time_elapsed       | 24            |\n",
            "|    total_timesteps    | 2500          |\n",
            "| train/                |               |\n",
            "|    entropy_loss       | -41.2         |\n",
            "|    explained_variance | -1.19e-07     |\n",
            "|    learning_rate      | 0.0007        |\n",
            "|    n_updates          | 499           |\n",
            "|    policy_loss        | 16            |\n",
            "|    reward             | -0.0061751665 |\n",
            "|    std                | 1             |\n",
            "|    value_loss         | 0.657         |\n",
            "-----------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 102       |\n",
            "|    iterations         | 600       |\n",
            "|    time_elapsed       | 29        |\n",
            "|    total_timesteps    | 3000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.2     |\n",
            "|    explained_variance | -1.19e-07 |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 599       |\n",
            "|    policy_loss        | -33.6     |\n",
            "|    reward             | 0.5337091 |\n",
            "|    std                | 1         |\n",
            "|    value_loss         | 0.837     |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 102        |\n",
            "|    iterations         | 700        |\n",
            "|    time_elapsed       | 34         |\n",
            "|    total_timesteps    | 3500       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.2      |\n",
            "|    explained_variance | -1.97      |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 699        |\n",
            "|    policy_loss        | -76.4      |\n",
            "|    reward             | -1.3246601 |\n",
            "|    std                | 1          |\n",
            "|    value_loss         | 3.85       |\n",
            "--------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 101        |\n",
            "|    iterations         | 800        |\n",
            "|    time_elapsed       | 39         |\n",
            "|    total_timesteps    | 4000       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.2      |\n",
            "|    explained_variance | 0.0632     |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 799        |\n",
            "|    policy_loss        | 44.6       |\n",
            "|    reward             | -1.6959082 |\n",
            "|    std                | 1          |\n",
            "|    value_loss         | 1.53       |\n",
            "--------------------------------------\n",
            "------------------------------------\n",
            "| time/                 |          |\n",
            "|    fps                | 100      |\n",
            "|    iterations         | 900      |\n",
            "|    time_elapsed       | 44       |\n",
            "|    total_timesteps    | 4500     |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -41.2    |\n",
            "|    explained_variance | 0.0211   |\n",
            "|    learning_rate      | 0.0007   |\n",
            "|    n_updates          | 899      |\n",
            "|    policy_loss        | -33.2    |\n",
            "|    reward             | 3.994608 |\n",
            "|    std                | 1        |\n",
            "|    value_loss         | 3.53     |\n",
            "------------------------------------\n",
            "---------------------------------------\n",
            "| time/                 |             |\n",
            "|    fps                | 101         |\n",
            "|    iterations         | 1000        |\n",
            "|    time_elapsed       | 49          |\n",
            "|    total_timesteps    | 5000        |\n",
            "| train/                |             |\n",
            "|    entropy_loss       | -41.2       |\n",
            "|    explained_variance | 1.79e-07    |\n",
            "|    learning_rate      | 0.0007      |\n",
            "|    n_updates          | 999         |\n",
            "|    policy_loss        | 36.3        |\n",
            "|    reward             | -0.20298508 |\n",
            "|    std                | 1           |\n",
            "|    value_loss         | 1.68        |\n",
            "---------------------------------------\n",
            "---------------------------------------\n",
            "| time/                 |             |\n",
            "|    fps                | 101         |\n",
            "|    iterations         | 1100        |\n",
            "|    time_elapsed       | 54          |\n",
            "|    total_timesteps    | 5500        |\n",
            "| train/                |             |\n",
            "|    entropy_loss       | -41.1       |\n",
            "|    explained_variance | 0.0114      |\n",
            "|    learning_rate      | 0.0007      |\n",
            "|    n_updates          | 1099        |\n",
            "|    policy_loss        | 393         |\n",
            "|    reward             | -0.92543656 |\n",
            "|    std                | 1           |\n",
            "|    value_loss         | 120         |\n",
            "---------------------------------------\n",
            "----------------------------------------\n",
            "| time/                 |              |\n",
            "|    fps                | 101          |\n",
            "|    iterations         | 1200         |\n",
            "|    time_elapsed       | 59           |\n",
            "|    total_timesteps    | 6000         |\n",
            "| train/                |              |\n",
            "|    entropy_loss       | -41.2        |\n",
            "|    explained_variance | 0.0236       |\n",
            "|    learning_rate      | 0.0007       |\n",
            "|    n_updates          | 1199         |\n",
            "|    policy_loss        | 151          |\n",
            "|    reward             | -0.099366836 |\n",
            "|    std                | 1            |\n",
            "|    value_loss         | 19.4         |\n",
            "----------------------------------------\n",
            "---------------------------------------\n",
            "| time/                 |             |\n",
            "|    fps                | 101         |\n",
            "|    iterations         | 1300        |\n",
            "|    time_elapsed       | 64          |\n",
            "|    total_timesteps    | 6500        |\n",
            "| train/                |             |\n",
            "|    entropy_loss       | -41.2       |\n",
            "|    explained_variance | -0.000106   |\n",
            "|    learning_rate      | 0.0007      |\n",
            "|    n_updates          | 1299        |\n",
            "|    policy_loss        | -47.6       |\n",
            "|    reward             | -0.39146426 |\n",
            "|    std                | 1           |\n",
            "|    value_loss         | 3.15        |\n",
            "---------------------------------------\n",
            "---------------------------------------\n",
            "| time/                 |             |\n",
            "|    fps                | 101         |\n",
            "|    iterations         | 1400        |\n",
            "|    time_elapsed       | 69          |\n",
            "|    total_timesteps    | 7000        |\n",
            "| train/                |             |\n",
            "|    entropy_loss       | -41.2       |\n",
            "|    explained_variance | 0           |\n",
            "|    learning_rate      | 0.0007      |\n",
            "|    n_updates          | 1399        |\n",
            "|    policy_loss        | 31.8        |\n",
            "|    reward             | -0.25985184 |\n",
            "|    std                | 1           |\n",
            "|    value_loss         | 1.84        |\n",
            "---------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 101       |\n",
            "|    iterations         | 1500      |\n",
            "|    time_elapsed       | 74        |\n",
            "|    total_timesteps    | 7500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.2     |\n",
            "|    explained_variance | 0         |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1499      |\n",
            "|    policy_loss        | -30.8     |\n",
            "|    reward             | 3.1766434 |\n",
            "|    std                | 1         |\n",
            "|    value_loss         | 4.93      |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 101        |\n",
            "|    iterations         | 1600       |\n",
            "|    time_elapsed       | 78         |\n",
            "|    total_timesteps    | 8000       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.2      |\n",
            "|    explained_variance | 0.00673    |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 1599       |\n",
            "|    policy_loss        | 133        |\n",
            "|    reward             | -2.0720153 |\n",
            "|    std                | 1          |\n",
            "|    value_loss         | 10.9       |\n",
            "--------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 101       |\n",
            "|    iterations         | 1700      |\n",
            "|    time_elapsed       | 83        |\n",
            "|    total_timesteps    | 8500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.2     |\n",
            "|    explained_variance | 5.96e-08  |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1699      |\n",
            "|    policy_loss        | -84.2     |\n",
            "|    reward             | 3.8356218 |\n",
            "|    std                | 1         |\n",
            "|    value_loss         | 7         |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 101        |\n",
            "|    iterations         | 1800       |\n",
            "|    time_elapsed       | 88         |\n",
            "|    total_timesteps    | 9000       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.2      |\n",
            "|    explained_variance | 0.063      |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 1799       |\n",
            "|    policy_loss        | 42         |\n",
            "|    reward             | 0.97460705 |\n",
            "|    std                | 1          |\n",
            "|    value_loss         | 2.23       |\n",
            "--------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 101       |\n",
            "|    iterations         | 1900      |\n",
            "|    time_elapsed       | 93        |\n",
            "|    total_timesteps    | 9500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.2     |\n",
            "|    explained_variance | 0.0122    |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1899      |\n",
            "|    policy_loss        | -84.7     |\n",
            "|    reward             | 2.2624621 |\n",
            "|    std                | 1         |\n",
            "|    value_loss         | 8.34      |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 101        |\n",
            "|    iterations         | 2000       |\n",
            "|    time_elapsed       | 98         |\n",
            "|    total_timesteps    | 10000      |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.2      |\n",
            "|    explained_variance | 0          |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 1999       |\n",
            "|    policy_loss        | 45.9       |\n",
            "|    reward             | -1.0189797 |\n",
            "|    std                | 1          |\n",
            "|    value_loss         | 1.77       |\n",
            "--------------------------------------\n",
            "======A2C Validation from:  2021-01-04 to  2021-04-06\n",
            "A2C Sharpe Ratio:  0.26168708794170054\n",
            "======PPO Training========\n",
            "{'ent_coef': 0.01, 'n_steps': 2048, 'learning_rate': 0.00025, 'batch_size': 128}\n",
            "Using cuda device\n",
            "Logging to tensorboard_log/ppo/ppo_126_1\n",
            "----------------------------------\n",
            "| time/              |           |\n",
            "|    fps             | 111       |\n",
            "|    iterations      | 1         |\n",
            "|    time_elapsed    | 18        |\n",
            "|    total_timesteps | 2048      |\n",
            "| train/             |           |\n",
            "|    reward          | 1.8877938 |\n",
            "----------------------------------\n",
            "day: 2959, episode: 5\n",
            "begin_total_asset: 1000000.00\n",
            "end_total_asset: 3881161.61\n",
            "total_reward: 2881161.61\n",
            "total_cost: 376867.37\n",
            "total_trades: 82873\n",
            "Sharpe: 0.776\n",
            "=================================\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 109         |\n",
            "|    iterations           | 2           |\n",
            "|    time_elapsed         | 37          |\n",
            "|    total_timesteps      | 4096        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.016203282 |\n",
            "|    clip_fraction        | 0.192       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.2       |\n",
            "|    explained_variance   | -0.0104     |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 4           |\n",
            "|    n_updates            | 10          |\n",
            "|    policy_gradient_loss | -0.0191     |\n",
            "|    reward               | 1.0358981   |\n",
            "|    std                  | 1           |\n",
            "|    value_loss           | 11.3        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 109         |\n",
            "|    iterations           | 3           |\n",
            "|    time_elapsed         | 56          |\n",
            "|    total_timesteps      | 6144        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.012528071 |\n",
            "|    clip_fraction        | 0.126       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.2       |\n",
            "|    explained_variance   | -0.00217    |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 57          |\n",
            "|    n_updates            | 20          |\n",
            "|    policy_gradient_loss | -0.0176     |\n",
            "|    reward               | 0.07821791  |\n",
            "|    std                  | 1           |\n",
            "|    value_loss           | 55.7        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 107         |\n",
            "|    iterations           | 4           |\n",
            "|    time_elapsed         | 76          |\n",
            "|    total_timesteps      | 8192        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.012139827 |\n",
            "|    clip_fraction        | 0.146       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.2       |\n",
            "|    explained_variance   | -0.00454    |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 13.7        |\n",
            "|    n_updates            | 30          |\n",
            "|    policy_gradient_loss | -0.018      |\n",
            "|    reward               | -1.5925834  |\n",
            "|    std                  | 1           |\n",
            "|    value_loss           | 43.4        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 107         |\n",
            "|    iterations           | 5           |\n",
            "|    time_elapsed         | 94          |\n",
            "|    total_timesteps      | 10240       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.019309906 |\n",
            "|    clip_fraction        | 0.17        |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.3       |\n",
            "|    explained_variance   | -0.0161     |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 5.07        |\n",
            "|    n_updates            | 40          |\n",
            "|    policy_gradient_loss | -0.0225     |\n",
            "|    reward               | 0.0615912   |\n",
            "|    std                  | 1.01        |\n",
            "|    value_loss           | 11.3        |\n",
            "-----------------------------------------\n",
            "======PPO Validation from:  2021-01-04 to  2021-04-06\n",
            "PPO Sharpe Ratio:  0.3016310407987196\n",
            "======DDPG Training========\n",
            "{'buffer_size': 10000, 'learning_rate': 0.0005, 'batch_size': 64}\n",
            "Using cuda device\n",
            "Logging to tensorboard_log/ddpg/ddpg_126_1\n",
            "day: 2959, episode: 10\n",
            "begin_total_asset: 1000000.00\n",
            "end_total_asset: 4424085.63\n",
            "total_reward: 3424085.63\n",
            "total_cost: 1037.34\n",
            "total_trades: 50241\n",
            "Sharpe: 0.797\n",
            "=================================\n",
            "----------------------------------\n",
            "| time/              |           |\n",
            "|    episodes        | 4         |\n",
            "|    fps             | 78        |\n",
            "|    time_elapsed    | 151       |\n",
            "|    total_timesteps | 11840     |\n",
            "| train/             |           |\n",
            "|    actor_loss      | -60.3     |\n",
            "|    critic_loss     | 441       |\n",
            "|    learning_rate   | 0.0005    |\n",
            "|    n_updates       | 8880      |\n",
            "|    reward          | 3.0541244 |\n",
            "----------------------------------\n",
            "======DDPG Validation from:  2021-01-04 to  2021-04-06\n",
            "======Best Model Retraining from:  2009-04-01 to  2021-04-06\n",
            "======Trading from:  2021-04-06 to  2021-07-06\n",
            "============================================\n",
            "turbulence_threshold:  203.40117769841157\n",
            "======Model training from:  2009-04-01 to  2021-04-06\n",
            "======A2C Training========\n",
            "{'n_steps': 5, 'ent_coef': 0.005, 'learning_rate': 0.0007}\n",
            "Using cuda device\n",
            "Logging to tensorboard_log/a2c/a2c_189_1\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 99         |\n",
            "|    iterations         | 100        |\n",
            "|    time_elapsed       | 5          |\n",
            "|    total_timesteps    | 500        |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41        |\n",
            "|    explained_variance | -0.446     |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 99         |\n",
            "|    policy_loss        | -85.9      |\n",
            "|    reward             | -0.6058027 |\n",
            "|    std                | 0.996      |\n",
            "|    value_loss         | 9.16       |\n",
            "--------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 100       |\n",
            "|    iterations         | 200       |\n",
            "|    time_elapsed       | 9         |\n",
            "|    total_timesteps    | 1000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41       |\n",
            "|    explained_variance | 0         |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 199       |\n",
            "|    policy_loss        | -19.4     |\n",
            "|    reward             | 1.8647146 |\n",
            "|    std                | 0.995     |\n",
            "|    value_loss         | 0.253     |\n",
            "-------------------------------------\n",
            "------------------------------------\n",
            "| time/                 |          |\n",
            "|    fps                | 100      |\n",
            "|    iterations         | 300      |\n",
            "|    time_elapsed       | 14       |\n",
            "|    total_timesteps    | 1500     |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -41      |\n",
            "|    explained_variance | 0.0122   |\n",
            "|    learning_rate      | 0.0007   |\n",
            "|    n_updates          | 299      |\n",
            "|    policy_loss        | -179     |\n",
            "|    reward             | 3.291392 |\n",
            "|    std                | 0.996    |\n",
            "|    value_loss         | 22.1     |\n",
            "------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 100       |\n",
            "|    iterations         | 400       |\n",
            "|    time_elapsed       | 19        |\n",
            "|    total_timesteps    | 2000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41       |\n",
            "|    explained_variance | 0.0507    |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 399       |\n",
            "|    policy_loss        | -11.1     |\n",
            "|    reward             | 0.3743614 |\n",
            "|    std                | 0.996     |\n",
            "|    value_loss         | 1.09      |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 100        |\n",
            "|    iterations         | 500        |\n",
            "|    time_elapsed       | 24         |\n",
            "|    total_timesteps    | 2500       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41        |\n",
            "|    explained_variance | 0          |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 499        |\n",
            "|    policy_loss        | 169        |\n",
            "|    reward             | -0.3291204 |\n",
            "|    std                | 0.995      |\n",
            "|    value_loss         | 16         |\n",
            "--------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 100        |\n",
            "|    iterations         | 600        |\n",
            "|    time_elapsed       | 29         |\n",
            "|    total_timesteps    | 3000       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41        |\n",
            "|    explained_variance | 0.054      |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 599        |\n",
            "|    policy_loss        | 905        |\n",
            "|    reward             | -1.0711567 |\n",
            "|    std                | 0.994      |\n",
            "|    value_loss         | 521        |\n",
            "--------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 100        |\n",
            "|    iterations         | 700        |\n",
            "|    time_elapsed       | 34         |\n",
            "|    total_timesteps    | 3500       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41        |\n",
            "|    explained_variance | -0.0433    |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 699        |\n",
            "|    policy_loss        | -32.9      |\n",
            "|    reward             | -2.9666545 |\n",
            "|    std                | 0.995      |\n",
            "|    value_loss         | 0.946      |\n",
            "--------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 100        |\n",
            "|    iterations         | 800        |\n",
            "|    time_elapsed       | 39         |\n",
            "|    total_timesteps    | 4000       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41        |\n",
            "|    explained_variance | -1.19e-07  |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 799        |\n",
            "|    policy_loss        | -3.25      |\n",
            "|    reward             | -0.5585839 |\n",
            "|    std                | 0.994      |\n",
            "|    value_loss         | 1.05       |\n",
            "--------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 100       |\n",
            "|    iterations         | 900       |\n",
            "|    time_elapsed       | 44        |\n",
            "|    total_timesteps    | 4500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -40.9     |\n",
            "|    explained_variance | -0.66     |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 899       |\n",
            "|    policy_loss        | 306       |\n",
            "|    reward             | 2.3080251 |\n",
            "|    std                | 0.992     |\n",
            "|    value_loss         | 70.3      |\n",
            "-------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 100       |\n",
            "|    iterations         | 1000      |\n",
            "|    time_elapsed       | 49        |\n",
            "|    total_timesteps    | 5000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41       |\n",
            "|    explained_variance | 0         |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 999       |\n",
            "|    policy_loss        | -108      |\n",
            "|    reward             | 1.1761196 |\n",
            "|    std                | 0.994     |\n",
            "|    value_loss         | 10.8      |\n",
            "-------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 100       |\n",
            "|    iterations         | 1100      |\n",
            "|    time_elapsed       | 54        |\n",
            "|    total_timesteps    | 5500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41       |\n",
            "|    explained_variance | 0         |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1099      |\n",
            "|    policy_loss        | -117      |\n",
            "|    reward             | 3.2501192 |\n",
            "|    std                | 0.995     |\n",
            "|    value_loss         | 10.7      |\n",
            "-------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 100       |\n",
            "|    iterations         | 1200      |\n",
            "|    time_elapsed       | 59        |\n",
            "|    total_timesteps    | 6000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41       |\n",
            "|    explained_variance | -1.19e-07 |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1199      |\n",
            "|    policy_loss        | 362       |\n",
            "|    reward             | 6.932239  |\n",
            "|    std                | 0.994     |\n",
            "|    value_loss         | 84.6      |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 100        |\n",
            "|    iterations         | 1300       |\n",
            "|    time_elapsed       | 64         |\n",
            "|    total_timesteps    | 6500       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41        |\n",
            "|    explained_variance | 0          |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 1299       |\n",
            "|    policy_loss        | 44.8       |\n",
            "|    reward             | -1.5383602 |\n",
            "|    std                | 0.994      |\n",
            "|    value_loss         | 2.08       |\n",
            "--------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 100       |\n",
            "|    iterations         | 1400      |\n",
            "|    time_elapsed       | 69        |\n",
            "|    total_timesteps    | 7000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41       |\n",
            "|    explained_variance | 0         |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1399      |\n",
            "|    policy_loss        | -143      |\n",
            "|    reward             | 0.7461217 |\n",
            "|    std                | 0.994     |\n",
            "|    value_loss         | 17        |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 100        |\n",
            "|    iterations         | 1500       |\n",
            "|    time_elapsed       | 74         |\n",
            "|    total_timesteps    | 7500       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41        |\n",
            "|    explained_variance | 0          |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 1499       |\n",
            "|    policy_loss        | -145       |\n",
            "|    reward             | -1.6722031 |\n",
            "|    std                | 0.995      |\n",
            "|    value_loss         | 14.8       |\n",
            "--------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 100       |\n",
            "|    iterations         | 1600      |\n",
            "|    time_elapsed       | 79        |\n",
            "|    total_timesteps    | 8000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41       |\n",
            "|    explained_variance | 1.19e-07  |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1599      |\n",
            "|    policy_loss        | 4.35      |\n",
            "|    reward             | 3.1992574 |\n",
            "|    std                | 0.995     |\n",
            "|    value_loss         | 0.916     |\n",
            "-------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 100       |\n",
            "|    iterations         | 1700      |\n",
            "|    time_elapsed       | 84        |\n",
            "|    total_timesteps    | 8500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -40.9     |\n",
            "|    explained_variance | -0.00566  |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1699      |\n",
            "|    policy_loss        | -1.16e+03 |\n",
            "|    reward             | 11.774156 |\n",
            "|    std                | 0.993     |\n",
            "|    value_loss         | 936       |\n",
            "-------------------------------------\n",
            "------------------------------------\n",
            "| time/                 |          |\n",
            "|    fps                | 100      |\n",
            "|    iterations         | 1800     |\n",
            "|    time_elapsed       | 89       |\n",
            "|    total_timesteps    | 9000     |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -40.9    |\n",
            "|    explained_variance | -0.00464 |\n",
            "|    learning_rate      | 0.0007   |\n",
            "|    n_updates          | 1799     |\n",
            "|    policy_loss        | -185     |\n",
            "|    reward             | 4.000574 |\n",
            "|    std                | 0.993    |\n",
            "|    value_loss         | 41.6     |\n",
            "------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 100        |\n",
            "|    iterations         | 1900       |\n",
            "|    time_elapsed       | 94         |\n",
            "|    total_timesteps    | 9500       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -40.9      |\n",
            "|    explained_variance | 0          |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 1899       |\n",
            "|    policy_loss        | 12.7       |\n",
            "|    reward             | -1.3208354 |\n",
            "|    std                | 0.991      |\n",
            "|    value_loss         | 0.306      |\n",
            "--------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 100       |\n",
            "|    iterations         | 2000      |\n",
            "|    time_elapsed       | 99        |\n",
            "|    total_timesteps    | 10000     |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -40.9     |\n",
            "|    explained_variance | 0         |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1999      |\n",
            "|    policy_loss        | -10.6     |\n",
            "|    reward             | 0.7043803 |\n",
            "|    std                | 0.992     |\n",
            "|    value_loss         | 0.431     |\n",
            "-------------------------------------\n",
            "======A2C Validation from:  2021-04-06 to  2021-07-06\n",
            "A2C Sharpe Ratio:  0.23121777505990648\n",
            "======PPO Training========\n",
            "{'ent_coef': 0.01, 'n_steps': 2048, 'learning_rate': 0.00025, 'batch_size': 128}\n",
            "Using cuda device\n",
            "Logging to tensorboard_log/ppo/ppo_189_1\n",
            "----------------------------------\n",
            "| time/              |           |\n",
            "|    fps             | 104       |\n",
            "|    iterations      | 1         |\n",
            "|    time_elapsed    | 19        |\n",
            "|    total_timesteps | 2048      |\n",
            "| train/             |           |\n",
            "|    reward          | 1.1126131 |\n",
            "----------------------------------\n",
            "day: 3022, episode: 5\n",
            "begin_total_asset: 1000000.00\n",
            "end_total_asset: 3636477.12\n",
            "total_reward: 2636477.12\n",
            "total_cost: 386575.15\n",
            "total_trades: 84625\n",
            "Sharpe: 0.731\n",
            "=================================\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 105         |\n",
            "|    iterations           | 2           |\n",
            "|    time_elapsed         | 38          |\n",
            "|    total_timesteps      | 4096        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.015871149 |\n",
            "|    clip_fraction        | 0.234       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.2       |\n",
            "|    explained_variance   | -0.0129     |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 3.2         |\n",
            "|    n_updates            | 10          |\n",
            "|    policy_gradient_loss | -0.0282     |\n",
            "|    reward               | 0.88015693  |\n",
            "|    std                  | 1           |\n",
            "|    value_loss           | 10.1        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 105         |\n",
            "|    iterations           | 3           |\n",
            "|    time_elapsed         | 58          |\n",
            "|    total_timesteps      | 6144        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.019049045 |\n",
            "|    clip_fraction        | 0.166       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.2       |\n",
            "|    explained_variance   | 0.00262     |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 32.1        |\n",
            "|    n_updates            | 20          |\n",
            "|    policy_gradient_loss | -0.023      |\n",
            "|    reward               | 0.49640504  |\n",
            "|    std                  | 1           |\n",
            "|    value_loss           | 53.7        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 105         |\n",
            "|    iterations           | 4           |\n",
            "|    time_elapsed         | 77          |\n",
            "|    total_timesteps      | 8192        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.015561214 |\n",
            "|    clip_fraction        | 0.161       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.2       |\n",
            "|    explained_variance   | -0.0133     |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 31.9        |\n",
            "|    n_updates            | 30          |\n",
            "|    policy_gradient_loss | -0.0201     |\n",
            "|    reward               | -0.7624933  |\n",
            "|    std                  | 1           |\n",
            "|    value_loss           | 45.6        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 105         |\n",
            "|    iterations           | 5           |\n",
            "|    time_elapsed         | 96          |\n",
            "|    total_timesteps      | 10240       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.014756277 |\n",
            "|    clip_fraction        | 0.141       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.2       |\n",
            "|    explained_variance   | -0.00517    |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 6.56        |\n",
            "|    n_updates            | 40          |\n",
            "|    policy_gradient_loss | -0.0204     |\n",
            "|    reward               | 1.2296445   |\n",
            "|    std                  | 1           |\n",
            "|    value_loss           | 14.8        |\n",
            "-----------------------------------------\n",
            "======PPO Validation from:  2021-04-06 to  2021-07-06\n",
            "PPO Sharpe Ratio:  0.010011405217654536\n",
            "======DDPG Training========\n",
            "{'buffer_size': 10000, 'learning_rate': 0.0005, 'batch_size': 64}\n",
            "Using cuda device\n",
            "Logging to tensorboard_log/ddpg/ddpg_189_1\n",
            "day: 3022, episode: 10\n",
            "begin_total_asset: 1000000.00\n",
            "end_total_asset: 5009064.34\n",
            "total_reward: 4009064.34\n",
            "total_cost: 1282.47\n",
            "total_trades: 51375\n",
            "Sharpe: 0.880\n",
            "=================================\n",
            "---------------------------------\n",
            "| time/              |          |\n",
            "|    episodes        | 4        |\n",
            "|    fps             | 76       |\n",
            "|    time_elapsed    | 157      |\n",
            "|    total_timesteps | 12092    |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -65.5    |\n",
            "|    critic_loss     | 3.14e+03 |\n",
            "|    learning_rate   | 0.0005   |\n",
            "|    n_updates       | 9069     |\n",
            "|    reward          | 8.301459 |\n",
            "---------------------------------\n",
            "======DDPG Validation from:  2021-04-06 to  2021-07-06\n",
            "======Best Model Retraining from:  2009-04-01 to  2021-07-06\n",
            "======Trading from:  2021-07-06 to  2021-10-04\n",
            "============================================\n",
            "turbulence_threshold:  203.40117769841157\n",
            "======Model training from:  2009-04-01 to  2021-07-06\n",
            "======A2C Training========\n",
            "{'n_steps': 5, 'ent_coef': 0.005, 'learning_rate': 0.0007}\n",
            "Using cuda device\n",
            "Logging to tensorboard_log/a2c/a2c_252_1\n",
            "---------------------------------------\n",
            "| time/                 |             |\n",
            "|    fps                | 98          |\n",
            "|    iterations         | 100         |\n",
            "|    time_elapsed       | 5           |\n",
            "|    total_timesteps    | 500         |\n",
            "| train/                |             |\n",
            "|    entropy_loss       | -41.1       |\n",
            "|    explained_variance | -0.0351     |\n",
            "|    learning_rate      | 0.0007      |\n",
            "|    n_updates          | 99          |\n",
            "|    policy_loss        | -28.6       |\n",
            "|    reward             | -0.15436654 |\n",
            "|    std                | 0.998       |\n",
            "|    value_loss         | 5.15        |\n",
            "---------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 97        |\n",
            "|    iterations         | 200       |\n",
            "|    time_elapsed       | 10        |\n",
            "|    total_timesteps    | 1000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41       |\n",
            "|    explained_variance | 0         |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 199       |\n",
            "|    policy_loss        | 50.5      |\n",
            "|    reward             | 1.3676119 |\n",
            "|    std                | 0.996     |\n",
            "|    value_loss         | 2.12      |\n",
            "-------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 98        |\n",
            "|    iterations         | 300       |\n",
            "|    time_elapsed       | 15        |\n",
            "|    total_timesteps    | 1500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41       |\n",
            "|    explained_variance | 0         |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 299       |\n",
            "|    policy_loss        | -179      |\n",
            "|    reward             | 5.1952586 |\n",
            "|    std                | 0.995     |\n",
            "|    value_loss         | 22.2      |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 400        |\n",
            "|    time_elapsed       | 20         |\n",
            "|    total_timesteps    | 2000       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41        |\n",
            "|    explained_variance | 0          |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 399        |\n",
            "|    policy_loss        | 79.1       |\n",
            "|    reward             | -0.3978011 |\n",
            "|    std                | 0.995      |\n",
            "|    value_loss         | 7.82       |\n",
            "--------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 500        |\n",
            "|    time_elapsed       | 25         |\n",
            "|    total_timesteps    | 2500       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -40.9      |\n",
            "|    explained_variance | 0.19       |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 499        |\n",
            "|    policy_loss        | -8.81      |\n",
            "|    reward             | 0.19138733 |\n",
            "|    std                | 0.993      |\n",
            "|    value_loss         | 6.38       |\n",
            "--------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 600        |\n",
            "|    time_elapsed       | 30         |\n",
            "|    total_timesteps    | 3000       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41        |\n",
            "|    explained_variance | -0.00925   |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 599        |\n",
            "|    policy_loss        | -498       |\n",
            "|    reward             | -11.956305 |\n",
            "|    std                | 0.996      |\n",
            "|    value_loss         | 222        |\n",
            "--------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 700        |\n",
            "|    time_elapsed       | 35         |\n",
            "|    total_timesteps    | 3500       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41        |\n",
            "|    explained_variance | -0.169     |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 699        |\n",
            "|    policy_loss        | -68.7      |\n",
            "|    reward             | 0.39965674 |\n",
            "|    std                | 0.996      |\n",
            "|    value_loss         | 3.86       |\n",
            "--------------------------------------\n",
            "---------------------------------------\n",
            "| time/                 |             |\n",
            "|    fps                | 98          |\n",
            "|    iterations         | 800         |\n",
            "|    time_elapsed       | 40          |\n",
            "|    total_timesteps    | 4000        |\n",
            "| train/                |             |\n",
            "|    entropy_loss       | -41         |\n",
            "|    explained_variance | -0.0865     |\n",
            "|    learning_rate      | 0.0007      |\n",
            "|    n_updates          | 799         |\n",
            "|    policy_loss        | -164        |\n",
            "|    reward             | -0.33571205 |\n",
            "|    std                | 0.996       |\n",
            "|    value_loss         | 16.1        |\n",
            "---------------------------------------\n",
            "---------------------------------------\n",
            "| time/                 |             |\n",
            "|    fps                | 98          |\n",
            "|    iterations         | 900         |\n",
            "|    time_elapsed       | 45          |\n",
            "|    total_timesteps    | 4500        |\n",
            "| train/                |             |\n",
            "|    entropy_loss       | -41         |\n",
            "|    explained_variance | 0.159       |\n",
            "|    learning_rate      | 0.0007      |\n",
            "|    n_updates          | 899         |\n",
            "|    policy_loss        | 46.5        |\n",
            "|    reward             | -0.19355251 |\n",
            "|    std                | 0.996       |\n",
            "|    value_loss         | 1.99        |\n",
            "---------------------------------------\n",
            "---------------------------------------\n",
            "| time/                 |             |\n",
            "|    fps                | 98          |\n",
            "|    iterations         | 1000        |\n",
            "|    time_elapsed       | 50          |\n",
            "|    total_timesteps    | 5000        |\n",
            "| train/                |             |\n",
            "|    entropy_loss       | -41.1       |\n",
            "|    explained_variance | 0.0618      |\n",
            "|    learning_rate      | 0.0007      |\n",
            "|    n_updates          | 999         |\n",
            "|    policy_loss        | -15.1       |\n",
            "|    reward             | -0.48996487 |\n",
            "|    std                | 0.997       |\n",
            "|    value_loss         | 0.677       |\n",
            "---------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 98        |\n",
            "|    iterations         | 1100      |\n",
            "|    time_elapsed       | 55        |\n",
            "|    total_timesteps    | 5500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41       |\n",
            "|    explained_variance | -0.192    |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1099      |\n",
            "|    policy_loss        | -233      |\n",
            "|    reward             | 1.1524396 |\n",
            "|    std                | 0.997     |\n",
            "|    value_loss         | 35.2      |\n",
            "-------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 98        |\n",
            "|    iterations         | 1200      |\n",
            "|    time_elapsed       | 60        |\n",
            "|    total_timesteps    | 6000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.1     |\n",
            "|    explained_variance | -0.214    |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1199      |\n",
            "|    policy_loss        | 14.9      |\n",
            "|    reward             | -2.838138 |\n",
            "|    std                | 0.998     |\n",
            "|    value_loss         | 3.95      |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 1300       |\n",
            "|    time_elapsed       | 65         |\n",
            "|    total_timesteps    | 6500       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.2      |\n",
            "|    explained_variance | 0.167      |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 1299       |\n",
            "|    policy_loss        | 43.7       |\n",
            "|    reward             | -1.0078373 |\n",
            "|    std                | 1          |\n",
            "|    value_loss         | 1.86       |\n",
            "--------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 98        |\n",
            "|    iterations         | 1400      |\n",
            "|    time_elapsed       | 70        |\n",
            "|    total_timesteps    | 7000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.2     |\n",
            "|    explained_variance | -0.0727   |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1399      |\n",
            "|    policy_loss        | -17.9     |\n",
            "|    reward             | 2.1590974 |\n",
            "|    std                | 1         |\n",
            "|    value_loss         | 1.71      |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 1500       |\n",
            "|    time_elapsed       | 75         |\n",
            "|    total_timesteps    | 7500       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.2      |\n",
            "|    explained_variance | 0          |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 1499       |\n",
            "|    policy_loss        | 42.5       |\n",
            "|    reward             | 0.47163484 |\n",
            "|    std                | 1          |\n",
            "|    value_loss         | 1.17       |\n",
            "--------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 98        |\n",
            "|    iterations         | 1600      |\n",
            "|    time_elapsed       | 80        |\n",
            "|    total_timesteps    | 8000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.2     |\n",
            "|    explained_variance | 0.000262  |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1599      |\n",
            "|    policy_loss        | -73.2     |\n",
            "|    reward             | 0.8027305 |\n",
            "|    std                | 1         |\n",
            "|    value_loss         | 19.5      |\n",
            "-------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 98        |\n",
            "|    iterations         | 1700      |\n",
            "|    time_elapsed       | 85        |\n",
            "|    total_timesteps    | 8500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.2     |\n",
            "|    explained_variance | 0         |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1699      |\n",
            "|    policy_loss        | -184      |\n",
            "|    reward             | 0.3686392 |\n",
            "|    std                | 1         |\n",
            "|    value_loss         | 29.3      |\n",
            "-------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 98        |\n",
            "|    iterations         | 1800      |\n",
            "|    time_elapsed       | 90        |\n",
            "|    total_timesteps    | 9000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.2     |\n",
            "|    explained_variance | -1.19e-07 |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1799      |\n",
            "|    policy_loss        | 116       |\n",
            "|    reward             | 1.6162095 |\n",
            "|    std                | 1         |\n",
            "|    value_loss         | 21.6      |\n",
            "-------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 98        |\n",
            "|    iterations         | 1900      |\n",
            "|    time_elapsed       | 96        |\n",
            "|    total_timesteps    | 9500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.2     |\n",
            "|    explained_variance | -0.00951  |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1899      |\n",
            "|    policy_loss        | -1.74     |\n",
            "|    reward             | 1.1433389 |\n",
            "|    std                | 1         |\n",
            "|    value_loss         | 0.195     |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 2000       |\n",
            "|    time_elapsed       | 101        |\n",
            "|    total_timesteps    | 10000      |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.2      |\n",
            "|    explained_variance | -0.0113    |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 1999       |\n",
            "|    policy_loss        | -38.9      |\n",
            "|    reward             | 0.97623837 |\n",
            "|    std                | 1          |\n",
            "|    value_loss         | 0.806      |\n",
            "--------------------------------------\n",
            "======A2C Validation from:  2021-07-06 to  2021-10-04\n",
            "A2C Sharpe Ratio:  -0.0528373614471615\n",
            "======PPO Training========\n",
            "{'ent_coef': 0.01, 'n_steps': 2048, 'learning_rate': 0.00025, 'batch_size': 128}\n",
            "Using cuda device\n",
            "Logging to tensorboard_log/ppo/ppo_252_1\n",
            "----------------------------------\n",
            "| time/              |           |\n",
            "|    fps             | 101       |\n",
            "|    iterations      | 1         |\n",
            "|    time_elapsed    | 20        |\n",
            "|    total_timesteps | 2048      |\n",
            "| train/             |           |\n",
            "|    reward          | 1.2801732 |\n",
            "----------------------------------\n",
            "day: 3085, episode: 5\n",
            "begin_total_asset: 1000000.00\n",
            "end_total_asset: 4550937.53\n",
            "total_reward: 3550937.53\n",
            "total_cost: 408098.28\n",
            "total_trades: 86397\n",
            "Sharpe: 0.873\n",
            "=================================\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 102         |\n",
            "|    iterations           | 2           |\n",
            "|    time_elapsed         | 39          |\n",
            "|    total_timesteps      | 4096        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.018978704 |\n",
            "|    clip_fraction        | 0.233       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.2       |\n",
            "|    explained_variance   | -0.00994    |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 5.48        |\n",
            "|    n_updates            | 10          |\n",
            "|    policy_gradient_loss | -0.0284     |\n",
            "|    reward               | 0.49718028  |\n",
            "|    std                  | 1           |\n",
            "|    value_loss           | 10.8        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 103         |\n",
            "|    iterations           | 3           |\n",
            "|    time_elapsed         | 59          |\n",
            "|    total_timesteps      | 6144        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.012346724 |\n",
            "|    clip_fraction        | 0.155       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.2       |\n",
            "|    explained_variance   | -0.00278    |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 19.6        |\n",
            "|    n_updates            | 20          |\n",
            "|    policy_gradient_loss | -0.0154     |\n",
            "|    reward               | -0.8982904  |\n",
            "|    std                  | 1           |\n",
            "|    value_loss           | 50.6        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 103         |\n",
            "|    iterations           | 4           |\n",
            "|    time_elapsed         | 78          |\n",
            "|    total_timesteps      | 8192        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.014507987 |\n",
            "|    clip_fraction        | 0.123       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.2       |\n",
            "|    explained_variance   | -0.012      |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 43.8        |\n",
            "|    n_updates            | 30          |\n",
            "|    policy_gradient_loss | -0.016      |\n",
            "|    reward               | -0.0212465  |\n",
            "|    std                  | 1           |\n",
            "|    value_loss           | 74.7        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 104         |\n",
            "|    iterations           | 5           |\n",
            "|    time_elapsed         | 98          |\n",
            "|    total_timesteps      | 10240       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.016703494 |\n",
            "|    clip_fraction        | 0.186       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.3       |\n",
            "|    explained_variance   | -0.0496     |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 4.09        |\n",
            "|    n_updates            | 40          |\n",
            "|    policy_gradient_loss | -0.0285     |\n",
            "|    reward               | 1.0109037   |\n",
            "|    std                  | 1.01        |\n",
            "|    value_loss           | 14.2        |\n",
            "-----------------------------------------\n",
            "======PPO Validation from:  2021-07-06 to  2021-10-04\n",
            "PPO Sharpe Ratio:  -0.05262682094979255\n",
            "======DDPG Training========\n",
            "{'buffer_size': 10000, 'learning_rate': 0.0005, 'batch_size': 64}\n",
            "Using cuda device\n",
            "Logging to tensorboard_log/ddpg/ddpg_252_1\n",
            "day: 3085, episode: 10\n",
            "begin_total_asset: 1000000.00\n",
            "end_total_asset: 4501430.61\n",
            "total_reward: 3501430.61\n",
            "total_cost: 1082.97\n",
            "total_trades: 61741\n",
            "Sharpe: 0.737\n",
            "=================================\n",
            "----------------------------------\n",
            "| time/              |           |\n",
            "|    episodes        | 4         |\n",
            "|    fps             | 76        |\n",
            "|    time_elapsed    | 161       |\n",
            "|    total_timesteps | 12344     |\n",
            "| train/             |           |\n",
            "|    actor_loss      | -3.66     |\n",
            "|    critic_loss     | 176       |\n",
            "|    learning_rate   | 0.0005    |\n",
            "|    n_updates       | 9258      |\n",
            "|    reward          | 2.1645079 |\n",
            "----------------------------------\n",
            "======DDPG Validation from:  2021-07-06 to  2021-10-04\n",
            "======Best Model Retraining from:  2009-04-01 to  2021-10-04\n",
            "======Trading from:  2021-10-04 to  2022-01-03\n",
            "============================================\n",
            "turbulence_threshold:  203.40117769841157\n",
            "======Model training from:  2009-04-01 to  2021-10-04\n",
            "======A2C Training========\n",
            "{'n_steps': 5, 'ent_coef': 0.005, 'learning_rate': 0.0007}\n",
            "Using cuda device\n",
            "Logging to tensorboard_log/a2c/a2c_315_1\n",
            "----------------------------------------\n",
            "| time/                 |              |\n",
            "|    fps                | 98           |\n",
            "|    iterations         | 100          |\n",
            "|    time_elapsed       | 5            |\n",
            "|    total_timesteps    | 500          |\n",
            "| train/                |              |\n",
            "|    entropy_loss       | -41.3        |\n",
            "|    explained_variance | 0            |\n",
            "|    learning_rate      | 0.0007       |\n",
            "|    n_updates          | 99           |\n",
            "|    policy_loss        | -51.2        |\n",
            "|    reward             | -0.096114956 |\n",
            "|    std                | 1.01         |\n",
            "|    value_loss         | 4.56         |\n",
            "----------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 98        |\n",
            "|    iterations         | 200       |\n",
            "|    time_elapsed       | 10        |\n",
            "|    total_timesteps    | 1000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.3     |\n",
            "|    explained_variance | 5.96e-08  |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 199       |\n",
            "|    policy_loss        | -36.9     |\n",
            "|    reward             | 2.9646184 |\n",
            "|    std                | 1         |\n",
            "|    value_loss         | 0.756     |\n",
            "-------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 97        |\n",
            "|    iterations         | 300       |\n",
            "|    time_elapsed       | 15        |\n",
            "|    total_timesteps    | 1500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.3     |\n",
            "|    explained_variance | 0         |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 299       |\n",
            "|    policy_loss        | -236      |\n",
            "|    reward             | 1.1201329 |\n",
            "|    std                | 1.01      |\n",
            "|    value_loss         | 36.4      |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 400        |\n",
            "|    time_elapsed       | 20         |\n",
            "|    total_timesteps    | 2000       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.4      |\n",
            "|    explained_variance | 0          |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 399        |\n",
            "|    policy_loss        | -53.9      |\n",
            "|    reward             | 0.03644575 |\n",
            "|    std                | 1.01       |\n",
            "|    value_loss         | 2.41       |\n",
            "--------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 98        |\n",
            "|    iterations         | 500       |\n",
            "|    time_elapsed       | 25        |\n",
            "|    total_timesteps    | 2500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.3     |\n",
            "|    explained_variance | 0         |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 499       |\n",
            "|    policy_loss        | 34.1      |\n",
            "|    reward             | 0.3948357 |\n",
            "|    std                | 1         |\n",
            "|    value_loss         | 1.08      |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 600        |\n",
            "|    time_elapsed       | 30         |\n",
            "|    total_timesteps    | 3000       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.3      |\n",
            "|    explained_variance | -1.19e-07  |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 599        |\n",
            "|    policy_loss        | -618       |\n",
            "|    reward             | -12.502208 |\n",
            "|    std                | 1          |\n",
            "|    value_loss         | 338        |\n",
            "--------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 700        |\n",
            "|    time_elapsed       | 35         |\n",
            "|    total_timesteps    | 3500       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.2      |\n",
            "|    explained_variance | -0.0164    |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 699        |\n",
            "|    policy_loss        | -105       |\n",
            "|    reward             | 0.30092302 |\n",
            "|    std                | 1          |\n",
            "|    value_loss         | 10.8       |\n",
            "--------------------------------------\n",
            "---------------------------------------\n",
            "| time/                 |             |\n",
            "|    fps                | 98          |\n",
            "|    iterations         | 800         |\n",
            "|    time_elapsed       | 40          |\n",
            "|    total_timesteps    | 4000        |\n",
            "| train/                |             |\n",
            "|    entropy_loss       | -41.3       |\n",
            "|    explained_variance | 1.19e-07    |\n",
            "|    learning_rate      | 0.0007      |\n",
            "|    n_updates          | 799         |\n",
            "|    policy_loss        | 74.6        |\n",
            "|    reward             | 0.009111182 |\n",
            "|    std                | 1           |\n",
            "|    value_loss         | 4.73        |\n",
            "---------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 98        |\n",
            "|    iterations         | 900       |\n",
            "|    time_elapsed       | 45        |\n",
            "|    total_timesteps    | 4500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.3     |\n",
            "|    explained_variance | -0.0124   |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 899       |\n",
            "|    policy_loss        | -131      |\n",
            "|    reward             | 2.0478728 |\n",
            "|    std                | 1         |\n",
            "|    value_loss         | 16.5      |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 1000       |\n",
            "|    time_elapsed       | 50         |\n",
            "|    total_timesteps    | 5000       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.3      |\n",
            "|    explained_variance | 0          |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 999        |\n",
            "|    policy_loss        | -34.9      |\n",
            "|    reward             | 0.04952446 |\n",
            "|    std                | 1.01       |\n",
            "|    value_loss         | 2.24       |\n",
            "--------------------------------------\n",
            "---------------------------------------\n",
            "| time/                 |             |\n",
            "|    fps                | 98          |\n",
            "|    iterations         | 1100        |\n",
            "|    time_elapsed       | 55          |\n",
            "|    total_timesteps    | 5500        |\n",
            "| train/                |             |\n",
            "|    entropy_loss       | -41.3       |\n",
            "|    explained_variance | -0.0188     |\n",
            "|    learning_rate      | 0.0007      |\n",
            "|    n_updates          | 1099        |\n",
            "|    policy_loss        | 344         |\n",
            "|    reward             | -0.11844914 |\n",
            "|    std                | 1.01        |\n",
            "|    value_loss         | 64.7        |\n",
            "---------------------------------------\n",
            "---------------------------------------\n",
            "| time/                 |             |\n",
            "|    fps                | 98          |\n",
            "|    iterations         | 1200        |\n",
            "|    time_elapsed       | 60          |\n",
            "|    total_timesteps    | 6000        |\n",
            "| train/                |             |\n",
            "|    entropy_loss       | -41.4       |\n",
            "|    explained_variance | 0           |\n",
            "|    learning_rate      | 0.0007      |\n",
            "|    n_updates          | 1199        |\n",
            "|    policy_loss        | 58.9        |\n",
            "|    reward             | -0.28185675 |\n",
            "|    std                | 1.01        |\n",
            "|    value_loss         | 6.99        |\n",
            "---------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 1300       |\n",
            "|    time_elapsed       | 66         |\n",
            "|    total_timesteps    | 6500       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.4      |\n",
            "|    explained_variance | 0.549      |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 1299       |\n",
            "|    policy_loss        | -11.7      |\n",
            "|    reward             | -0.7957832 |\n",
            "|    std                | 1.01       |\n",
            "|    value_loss         | 0.135      |\n",
            "--------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 1400       |\n",
            "|    time_elapsed       | 71         |\n",
            "|    total_timesteps    | 7000       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.4      |\n",
            "|    explained_variance | 1.19e-07   |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 1399       |\n",
            "|    policy_loss        | -29.7      |\n",
            "|    reward             | 0.16117185 |\n",
            "|    std                | 1.01       |\n",
            "|    value_loss         | 0.859      |\n",
            "--------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 1500       |\n",
            "|    time_elapsed       | 76         |\n",
            "|    total_timesteps    | 7500       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.4      |\n",
            "|    explained_variance | 1.79e-07   |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 1499       |\n",
            "|    policy_loss        | 15.7       |\n",
            "|    reward             | 0.17504984 |\n",
            "|    std                | 1.01       |\n",
            "|    value_loss         | 0.528      |\n",
            "--------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 98        |\n",
            "|    iterations         | 1600      |\n",
            "|    time_elapsed       | 81        |\n",
            "|    total_timesteps    | 8000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.4     |\n",
            "|    explained_variance | 0         |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1599      |\n",
            "|    policy_loss        | 48.7      |\n",
            "|    reward             | 3.1540215 |\n",
            "|    std                | 1.01      |\n",
            "|    value_loss         | 6.42      |\n",
            "-------------------------------------\n",
            "---------------------------------------\n",
            "| time/                 |             |\n",
            "|    fps                | 98          |\n",
            "|    iterations         | 1700        |\n",
            "|    time_elapsed       | 86          |\n",
            "|    total_timesteps    | 8500        |\n",
            "| train/                |             |\n",
            "|    entropy_loss       | -41.4       |\n",
            "|    explained_variance | 0           |\n",
            "|    learning_rate      | 0.0007      |\n",
            "|    n_updates          | 1699        |\n",
            "|    policy_loss        | 22.9        |\n",
            "|    reward             | -0.09754947 |\n",
            "|    std                | 1.01        |\n",
            "|    value_loss         | 0.677       |\n",
            "---------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 98        |\n",
            "|    iterations         | 1800      |\n",
            "|    time_elapsed       | 91        |\n",
            "|    total_timesteps    | 9000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.4     |\n",
            "|    explained_variance | -0.171    |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1799      |\n",
            "|    policy_loss        | 99        |\n",
            "|    reward             | -1.175583 |\n",
            "|    std                | 1.01      |\n",
            "|    value_loss         | 9.98      |\n",
            "-------------------------------------\n",
            "------------------------------------\n",
            "| time/                 |          |\n",
            "|    fps                | 98       |\n",
            "|    iterations         | 1900     |\n",
            "|    time_elapsed       | 96       |\n",
            "|    total_timesteps    | 9500     |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -41.4    |\n",
            "|    explained_variance | 0        |\n",
            "|    learning_rate      | 0.0007   |\n",
            "|    n_updates          | 1899     |\n",
            "|    policy_loss        | -24.6    |\n",
            "|    reward             | 0.362812 |\n",
            "|    std                | 1.01     |\n",
            "|    value_loss         | 0.367    |\n",
            "------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 2000       |\n",
            "|    time_elapsed       | 101        |\n",
            "|    total_timesteps    | 10000      |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.4      |\n",
            "|    explained_variance | 0          |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 1999       |\n",
            "|    policy_loss        | -146       |\n",
            "|    reward             | -1.7740997 |\n",
            "|    std                | 1.01       |\n",
            "|    value_loss         | 13.5       |\n",
            "--------------------------------------\n",
            "======A2C Validation from:  2021-10-04 to  2022-01-03\n",
            "A2C Sharpe Ratio:  0.3744625013790845\n",
            "======PPO Training========\n",
            "{'ent_coef': 0.01, 'n_steps': 2048, 'learning_rate': 0.00025, 'batch_size': 128}\n",
            "Using cuda device\n",
            "Logging to tensorboard_log/ppo/ppo_315_1\n",
            "----------------------------------\n",
            "| time/              |           |\n",
            "|    fps             | 106       |\n",
            "|    iterations      | 1         |\n",
            "|    time_elapsed    | 19        |\n",
            "|    total_timesteps | 2048      |\n",
            "| train/             |           |\n",
            "|    reward          | 0.7561091 |\n",
            "----------------------------------\n",
            "day: 3148, episode: 5\n",
            "begin_total_asset: 1000000.00\n",
            "end_total_asset: 4259003.47\n",
            "total_reward: 3259003.47\n",
            "total_cost: 426068.70\n",
            "total_trades: 87901\n",
            "Sharpe: 0.790\n",
            "=================================\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 101         |\n",
            "|    iterations           | 2           |\n",
            "|    time_elapsed         | 40          |\n",
            "|    total_timesteps      | 4096        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.016951367 |\n",
            "|    clip_fraction        | 0.229       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.2       |\n",
            "|    explained_variance   | -0.000697   |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 5.25        |\n",
            "|    n_updates            | 10          |\n",
            "|    policy_gradient_loss | -0.0307     |\n",
            "|    reward               | 0.9373541   |\n",
            "|    std                  | 1           |\n",
            "|    value_loss           | 9.91        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 102         |\n",
            "|    iterations           | 3           |\n",
            "|    time_elapsed         | 60          |\n",
            "|    total_timesteps      | 6144        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.017283382 |\n",
            "|    clip_fraction        | 0.176       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.3       |\n",
            "|    explained_variance   | 0.00633     |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 33.4        |\n",
            "|    n_updates            | 20          |\n",
            "|    policy_gradient_loss | -0.0207     |\n",
            "|    reward               | 7.712371    |\n",
            "|    std                  | 1           |\n",
            "|    value_loss           | 56.7        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 102         |\n",
            "|    iterations           | 4           |\n",
            "|    time_elapsed         | 79          |\n",
            "|    total_timesteps      | 8192        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.010827523 |\n",
            "|    clip_fraction        | 0.0962      |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.3       |\n",
            "|    explained_variance   | 0.00682     |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 73.4        |\n",
            "|    n_updates            | 30          |\n",
            "|    policy_gradient_loss | -0.018      |\n",
            "|    reward               | -0.83698237 |\n",
            "|    std                  | 1.01        |\n",
            "|    value_loss           | 134         |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 102         |\n",
            "|    iterations           | 5           |\n",
            "|    time_elapsed         | 99          |\n",
            "|    total_timesteps      | 10240       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.013603406 |\n",
            "|    clip_fraction        | 0.172       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.4       |\n",
            "|    explained_variance   | 0.0243      |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 4.64        |\n",
            "|    n_updates            | 40          |\n",
            "|    policy_gradient_loss | -0.0243     |\n",
            "|    reward               | 0.5904208   |\n",
            "|    std                  | 1.01        |\n",
            "|    value_loss           | 12.7        |\n",
            "-----------------------------------------\n",
            "======PPO Validation from:  2021-10-04 to  2022-01-03\n",
            "PPO Sharpe Ratio:  0.0593551480877188\n",
            "======DDPG Training========\n",
            "{'buffer_size': 10000, 'learning_rate': 0.0005, 'batch_size': 64}\n",
            "Using cuda device\n",
            "Logging to tensorboard_log/ddpg/ddpg_315_1\n",
            "day: 3148, episode: 10\n",
            "begin_total_asset: 1000000.00\n",
            "end_total_asset: 4617783.42\n",
            "total_reward: 3617783.42\n",
            "total_cost: 1110.92\n",
            "total_trades: 49019\n",
            "Sharpe: 0.745\n",
            "=================================\n",
            "----------------------------------\n",
            "| time/              |           |\n",
            "|    episodes        | 4         |\n",
            "|    fps             | 75        |\n",
            "|    time_elapsed    | 167       |\n",
            "|    total_timesteps | 12596     |\n",
            "| train/             |           |\n",
            "|    actor_loss      | 36.1      |\n",
            "|    critic_loss     | 656       |\n",
            "|    learning_rate   | 0.0005    |\n",
            "|    n_updates       | 9447      |\n",
            "|    reward          | 5.4972343 |\n",
            "----------------------------------\n",
            "======DDPG Validation from:  2021-10-04 to  2022-01-03\n",
            "======Best Model Retraining from:  2009-04-01 to  2022-01-03\n",
            "======Trading from:  2022-01-03 to  2022-04-04\n",
            "Ensemble Strategy took:  25.210217595100403  minutes\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "AttributeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-30-4153cda81a48>\u001b[0m in \u001b[0;36m<cell line: 163>\u001b[0;34m()\u001b[0m\n\u001b[1;32m    162\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    163\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0m__name__\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'__main__'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 164\u001b[0;31m     \u001b[0mmain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-30-4153cda81a48>\u001b[0m in \u001b[0;36mmain\u001b[0;34m()\u001b[0m\n\u001b[1;32m    127\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrebalance_window\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mvalidation_window\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0munique_trade_date\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrebalance_window\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    128\u001b[0m         \u001b[0mtemp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'results/account_value_trade_{}_{}.csv'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'ensemble'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 129\u001b[0;31m         \u001b[0mdf_account_value\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdf_account_value\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtemp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mignore_index\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    130\u001b[0m     sharpe = (252 ** 0.5) * df_account_value.account_value.pct_change(\n\u001b[1;32m    131\u001b[0m         1).mean() / df_account_value.account_value.pct_change(1).std()\n",
            "\u001b[0;32m/usr/local/lib/python3.9/site-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36m__getattr__\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m   5987\u001b[0m         ):\n\u001b[1;32m   5988\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 5989\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mobject\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__getattribute__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   5990\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   5991\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__setattr__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mAttributeError\u001b[0m: 'DataFrame' object has no attribute 'append'"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "current_price  = 110\n",
        "avg_price = 130\n",
        "print(( current_price * 0.3 ) + current_price)\n",
        "if current_price > ( avg_price * 0.3  + avg_price)  and avg_price > 0:\n",
        "  #actions[i] = self.total_stockss[i] * -1\n",
        "  #print(\"Updated actions\")\n",
        "  print(\"Current Price\", current_price , \" \", \" avg_price\", \" \", avg_price, \"Profit\", current_price -  avg_price)\n",
        "elif current_price < (avg_price - avg_price * 0.15)  and avg_price > 0:\n",
        "  #actions[i] = self.total_stockss[i] * -1\n",
        "  print(\"Current Price\", current_price , \" \", \" avg_price\", \" \", avg_price, \"loss\", current_price -  avg_price )"
      ],
      "metadata": {
        "id": "R6wjk6Qwg-ar",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "65204f2e-8f07-4ca7-8505-b1dfcad5f769"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "143.0\n",
            "Current Price 110    avg_price   130 loss -20\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "avg_price < ( current_price * 0.3  + current_price)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lZm_y4awCWO9",
        "outputId": "89222edb-bc50-4871-c67d-ee4300ed49db"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 153
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "current_price - current_price * 0.15"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wr3yRoNwCvV7",
        "outputId": "60d7ebbb-13fb-4b17-ece3-b008217d2987"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "93.5"
            ]
          },
          "metadata": {},
          "execution_count": 163
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "JtmhGYiRC07B",
        "outputId": "401210c0-edca-413f-c39b-716e9fedeee3",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 401,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "False"
            ]
          },
          "metadata": {},
          "execution_count": 401
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "S15p3oh8smm1"
      },
      "execution_count": 401,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "IJO0W8Uxsr2i"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "colab": {
      "name": "Making the most of your colab subscription",
      "provenance": [],
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "accelerator": "GPU",
    "gpuClass": "standard"
  },
  "nbformat": 4,
  "nbformat_minor": 0
}