V12_Goku.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/abdulazeezb/1_SparkFundsInvestmentAnalysis/blob/master/V12_Goku.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Checking GPU Version"
      ],
      "metadata": {
        "id": "NAjscLW7Sk9V"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "gpu_info = !nvidia-smi\n",
        "gpu_info = '\\n'.join(gpu_info)\n",
        "if gpu_info.find('failed') >= 0:\n",
        "  print('Not connected to a GPU')\n",
        "else:\n",
        "  print(gpu_info)"
      ],
      "metadata": {
        "id": "t8nxH9ZvSaY6",
        "outputId": "5bc536f7-4490-4eca-8a4e-7cf8365270f3",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Wed May  3 09:41:11 2023       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 525.85.12    Driver Version: 525.85.12    CUDA Version: 12.0     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla T4            Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   38C    P8     9W /  70W |      0MiB / 15360MiB |      0%      Default |\n",
            "|                               |                      |                  N/A |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from psutil import virtual_memory\n",
        "ram_gb = virtual_memory().total / 1e9\n",
        "print('Your runtime has {:.1f} gigabytes of available RAM\\n'.format(ram_gb))\n",
        "\n",
        "if ram_gb < 20:\n",
        "  print('Not using a high-RAM runtime')\n",
        "else:\n",
        "  print('You are using a high-RAM runtime!')"
      ],
      "metadata": {
        "id": "KEPzY7qCS2Lc",
        "outputId": "eedf8e8b-52cb-455b-c0bb-1d3532561943",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Your runtime has 27.3 gigabytes of available RAM\n",
            "\n",
            "You are using a high-RAM runtime!\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QMMqmdiYMkvi"
      },
      "source": [
        "## Part 1: Getting Started - Install Python Packages"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "23TOba33L4qf",
        "outputId": "b76eb491-5e83-4c9d-ee76-ada1f191398b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: wrds in /usr/local/lib/python3.10/site-packages (3.1.6)\n",
            "Requirement already satisfied: sqlalchemy<2 in /usr/local/lib/python3.10/site-packages (from wrds) (1.4.48)\n",
            "Requirement already satisfied: psycopg2-binary in /usr/local/lib/python3.10/site-packages (from wrds) (2.9.6)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.10/site-packages (from wrds) (1.10.1)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/site-packages (from wrds) (2.0.1)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/site-packages (from wrds) (1.24.3)\n",
            "Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.10/site-packages (from sqlalchemy<2->wrds) (2.0.2)\n",
            "Requirement already satisfied: tzdata>=2022.1 in /usr/local/lib/python3.10/site-packages (from pandas->wrds) (2023.3)\n",
            "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/site-packages (from pandas->wrds) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/site-packages (from pandas->wrds) (2023.3)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/site-packages (from python-dateutil>=2.8.2->pandas->wrds) (1.16.0)\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
            "\u001b[0mLooking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting swig\n",
            "  Using cached swig-4.1.1-py2.py3-none-manylinux_2_5_x86_64.manylinux1_x86_64.whl (1.8 MB)\n",
            "Installing collected packages: swig\n",
            "Successfully installed swig-4.1.1\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
            "\u001b[0m\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
            "\u001b[0m✨🍰✨ Everything looks OK!\n"
          ]
        }
      ],
      "source": [
        "## install finrl library\n",
        "!pip install wrds\n",
        "!pip install swig\n",
        "!pip install -q condacolab\n",
        "import condacolab\n",
        "condacolab.install()\n",
        "!apt-get update -y -qq && apt-get install -y -qq cmake libopenmpi-dev python3-dev zlib1g-dev libgl1-mesa-glx swig"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install git+https://github.com/AI4Finance-Foundation/FinRL.git\n"
      ],
      "metadata": {
        "id": "haJsmd15TCdH",
        "outputId": "e6fd8da6-0ce6-40d4-e409-918d46efa160",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting git+https://github.com/AI4Finance-Foundation/FinRL.git\n",
            "  Cloning https://github.com/AI4Finance-Foundation/FinRL.git to /tmp/pip-req-build-_g5uf4lm\n",
            "  Running command git clone --filter=blob:none --quiet https://github.com/AI4Finance-Foundation/FinRL.git /tmp/pip-req-build-_g5uf4lm\n",
            "  Resolved https://github.com/AI4Finance-Foundation/FinRL.git to commit a3598dae504bcd834d12b17110b5aa91c1c5305d\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting elegantrl@ git+https://github.com/AI4Finance-Foundation/ElegantRL.git#egg=elegantrl\n",
            "  Cloning https://github.com/AI4Finance-Foundation/ElegantRL.git to /tmp/pip-install-6g6w51wi/elegantrl_67faa23541f14c1d8ed6088e20a24aee\n",
            "  Running command git clone --filter=blob:none --quiet https://github.com/AI4Finance-Foundation/ElegantRL.git /tmp/pip-install-6g6w51wi/elegantrl_67faa23541f14c1d8ed6088e20a24aee\n",
            "  Resolved https://github.com/AI4Finance-Foundation/ElegantRL.git to commit b974a806e6235f59055c954418e54640fa549331\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting pyfolio@ git+https://github.com/quantopian/pyfolio.git#egg=pyfolio-0.9.2\n",
            "  Cloning https://github.com/quantopian/pyfolio.git to /tmp/pip-install-6g6w51wi/pyfolio_76d2e0c60d6e407fa5a8d42e9249a894\n",
            "  Running command git clone --filter=blob:none --quiet https://github.com/quantopian/pyfolio.git /tmp/pip-install-6g6w51wi/pyfolio_76d2e0c60d6e407fa5a8d42e9249a894\n",
            "  Resolved https://github.com/quantopian/pyfolio.git to commit 4b901f6d73aa02ceb6d04b7d83502e5c6f2e81aa\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting gputil\n",
            "  Using cached GPUtil-1.4.0.tar.gz (5.5 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: pandas>=1.1.5 in /usr/local/lib/python3.10/dist-packages (from finrl==0.3.5) (1.5.3)\n",
            "Collecting stable-baselines3<2.0.0,>=1.6.2\n",
            "  Using cached stable_baselines3-1.8.0-py3-none-any.whl (174 kB)\n",
            "Requirement already satisfied: yfinance in /usr/local/lib/python3.10/dist-packages (from finrl==0.3.5) (0.2.18)\n",
            "Collecting lz4\n",
            "  Using cached lz4-4.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "Collecting exchange_calendars==3.6.3\n",
            "  Using cached exchange_calendars-3.6.3.tar.gz (152 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.10/dist-packages (from finrl==0.3.5) (3.7.1)\n",
            "Requirement already satisfied: scikit-learn>=0.21.0 in /usr/local/lib/python3.10/dist-packages (from finrl==0.3.5) (1.2.2)\n",
            "Requirement already satisfied: numpy>=1.17.3 in /usr/local/lib/python3.10/dist-packages (from finrl==0.3.5) (1.22.4)\n",
            "Collecting ray[default,tune]>=2.0.0\n",
            "  Using cached ray-2.4.0-cp310-cp310-manylinux2014_x86_64.whl (58.6 MB)\n",
            "Collecting tensorboardX\n",
            "  Using cached tensorboardX-2.6-py2.py3-none-any.whl (114 kB)\n",
            "Collecting alpaca_trade_api>=2.1.0\n",
            "  Using cached alpaca_trade_api-3.0.0-py3-none-any.whl (33 kB)\n",
            "Requirement already satisfied: gym>=0.17 in /usr/local/lib/python3.10/dist-packages (from finrl==0.3.5) (0.25.2)\n",
            "Collecting ccxt>=1.66.32\n",
            "  Using cached ccxt-3.0.90-py2.py3-none-any.whl (3.8 MB)\n",
            "Collecting stockstats>=0.4.0\n",
            "  Using cached stockstats-0.5.2-py2.py3-none-any.whl (20 kB)\n",
            "Collecting wrds>=3.1.6\n",
            "  Using cached wrds-3.1.6-py3-none-any.whl (12 kB)\n",
            "Collecting importlib-metadata==4.13.0\n",
            "  Using cached importlib_metadata-4.13.0-py3-none-any.whl (23 kB)\n",
            "Collecting jqdatasdk\n",
            "  Using cached jqdatasdk-1.8.11-py3-none-any.whl (158 kB)\n",
            "Collecting pyluach\n",
            "  Using cached pyluach-2.2.0-py3-none-any.whl (25 kB)\n",
            "Requirement already satisfied: python-dateutil in /usr/local/lib/python3.10/dist-packages (from exchange_calendars==3.6.3->finrl==0.3.5) (2.8.2)\n",
            "Requirement already satisfied: pytz in /usr/local/lib/python3.10/dist-packages (from exchange_calendars==3.6.3->finrl==0.3.5) (2022.7.1)\n",
            "Requirement already satisfied: toolz in /usr/local/lib/python3.10/dist-packages (from exchange_calendars==3.6.3->finrl==0.3.5) (0.12.0)\n",
            "Requirement already satisfied: korean_lunar_calendar in /usr/local/lib/python3.10/dist-packages (from exchange_calendars==3.6.3->finrl==0.3.5) (0.3.1)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.10/dist-packages (from importlib-metadata==4.13.0->finrl==0.3.5) (3.15.0)\n",
            "Requirement already satisfied: urllib3<2,>1.24 in /usr/local/lib/python3.10/dist-packages (from alpaca_trade_api>=2.1.0->finrl==0.3.5) (1.26.15)\n",
            "Requirement already satisfied: websocket-client<2,>=0.56.0 in /usr/local/lib/python3.10/dist-packages (from alpaca_trade_api>=2.1.0->finrl==0.3.5) (1.5.1)\n",
            "Collecting websockets<11,>=9.0\n",
            "  Using cached websockets-10.4-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (106 kB)\n",
            "Collecting aiohttp==3.8.1\n",
            "  Using cached aiohttp-3.8.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (1.2 MB)\n",
            "Requirement already satisfied: requests<3,>2 in /usr/local/lib/python3.10/dist-packages (from alpaca_trade_api>=2.1.0->finrl==0.3.5) (2.27.1)\n",
            "Collecting deprecation==2.1.0\n",
            "  Using cached deprecation-2.1.0-py2.py3-none-any.whl (11 kB)\n",
            "Requirement already satisfied: PyYAML==6.0 in /usr/local/lib/python3.10/dist-packages (from alpaca_trade_api>=2.1.0->finrl==0.3.5) (6.0)\n",
            "Collecting msgpack==1.0.3\n",
            "  Using cached msgpack-1.0.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (323 kB)\n",
            "Collecting yarl<2.0,>=1.0\n",
            "  Using cached yarl-1.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (268 kB)\n",
            "Collecting multidict<7.0,>=4.5\n",
            "  Using cached multidict-6.0.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (114 kB)\n",
            "Collecting frozenlist>=1.1.1\n",
            "  Using cached frozenlist-1.3.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (149 kB)\n",
            "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp==3.8.1->alpaca_trade_api>=2.1.0->finrl==0.3.5) (2.0.12)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp==3.8.1->alpaca_trade_api>=2.1.0->finrl==0.3.5) (23.1.0)\n",
            "Collecting async-timeout<5.0,>=4.0.0a3\n",
            "  Using cached async_timeout-4.0.2-py3-none-any.whl (5.8 kB)\n",
            "Collecting aiosignal>=1.1.2\n",
            "  Using cached aiosignal-1.3.1-py3-none-any.whl (7.6 kB)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from deprecation==2.1.0->alpaca_trade_api>=2.1.0->finrl==0.3.5) (23.1)\n",
            "Requirement already satisfied: cryptography>=2.6.1 in /usr/local/lib/python3.10/dist-packages (from ccxt>=1.66.32->finrl==0.3.5) (40.0.2)\n",
            "Collecting aiodns>=1.1.1\n",
            "  Using cached aiodns-3.0.0-py3-none-any.whl (5.0 kB)\n",
            "Requirement already satisfied: setuptools>=60.9.0 in /usr/local/lib/python3.10/dist-packages (from ccxt>=1.66.32->finrl==0.3.5) (67.7.2)\n",
            "Requirement already satisfied: certifi>=2018.1.18 in /usr/local/lib/python3.10/dist-packages (from ccxt>=1.66.32->finrl==0.3.5) (2022.12.7)\n",
            "Requirement already satisfied: cloudpickle>=1.2.0 in /usr/local/lib/python3.10/dist-packages (from gym>=0.17->finrl==0.3.5) (2.2.1)\n",
            "Requirement already satisfied: gym-notices>=0.0.4 in /usr/local/lib/python3.10/dist-packages (from gym>=0.17->finrl==0.3.5) (0.0.8)\n",
            "Requirement already satisfied: protobuf!=3.19.5,>=3.15.3 in /usr/local/lib/python3.10/dist-packages (from ray[default,tune]>=2.0.0->finrl==0.3.5) (3.20.3)\n",
            "Collecting virtualenv<20.21.1,>=20.0.24\n",
            "  Using cached virtualenv-20.21.0-py3-none-any.whl (8.7 MB)\n",
            "Collecting grpcio<=1.51.3,>=1.42.0\n",
            "  Using cached grpcio-1.51.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (4.8 MB)\n",
            "Requirement already satisfied: jsonschema in /usr/local/lib/python3.10/dist-packages (from ray[default,tune]>=2.0.0->finrl==0.3.5) (4.3.3)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from ray[default,tune]>=2.0.0->finrl==0.3.5) (3.12.0)\n",
            "Requirement already satisfied: click>=7.0 in /usr/local/lib/python3.10/dist-packages (from ray[default,tune]>=2.0.0->finrl==0.3.5) (8.1.3)\n",
            "Collecting opencensus\n",
            "  Using cached opencensus-0.11.2-py2.py3-none-any.whl (128 kB)\n",
            "Collecting gpustat>=1.0.0\n",
            "  Using cached gpustat-1.1.tar.gz (97 kB)\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: prometheus-client>=0.7.1 in /usr/local/lib/python3.10/dist-packages (from ray[default,tune]>=2.0.0->finrl==0.3.5) (0.16.0)\n",
            "Requirement already satisfied: smart-open in /usr/local/lib/python3.10/dist-packages (from ray[default,tune]>=2.0.0->finrl==0.3.5) (6.3.0)\n",
            "Requirement already satisfied: pydantic in /usr/local/lib/python3.10/dist-packages (from ray[default,tune]>=2.0.0->finrl==0.3.5) (1.10.7)\n",
            "Collecting colorful\n",
            "  Using cached colorful-0.5.5-py2.py3-none-any.whl (201 kB)\n",
            "Collecting aiohttp-cors\n",
            "  Using cached aiohttp_cors-0.7.0-py3-none-any.whl (27 kB)\n",
            "Collecting py-spy>=0.2.0\n",
            "  Using cached py_spy-0.3.14-py2.py3-none-manylinux_2_5_x86_64.manylinux1_x86_64.whl (3.0 MB)\n",
            "Requirement already satisfied: tabulate in /usr/local/lib/python3.10/dist-packages (from ray[default,tune]>=2.0.0->finrl==0.3.5) (0.8.10)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.21.0->finrl==0.3.5) (3.1.0)\n",
            "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.21.0->finrl==0.3.5) (1.2.0)\n",
            "Requirement already satisfied: scipy>=1.3.2 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.21.0->finrl==0.3.5) (1.10.1)\n",
            "Collecting gym>=0.17\n",
            "  Using cached gym-0.21.0.tar.gz (1.5 MB)\n",
            "  \u001b[1;31merror\u001b[0m: \u001b[1msubprocess-exited-with-error\u001b[0m\n",
            "  \n",
            "  \u001b[31m×\u001b[0m \u001b[32mpython setup.py egg_info\u001b[0m did not run successfully.\n",
            "  \u001b[31m│\u001b[0m exit code: \u001b[1;36m1\u001b[0m\n",
            "  \u001b[31m╰─>\u001b[0m See above for output.\n",
            "  \n",
            "  \u001b[1;35mnote\u001b[0m: This error originates from a subprocess, and is likely not a problem with pip.\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25herror\n",
            "\u001b[1;31merror\u001b[0m: \u001b[1mmetadata-generation-failed\u001b[0m\n",
            "\n",
            "\u001b[31m×\u001b[0m Encountered error while generating package metadata.\n",
            "\u001b[31m╰─>\u001b[0m See above for output.\n",
            "\n",
            "\u001b[1;35mnote\u001b[0m: This is an issue with the package mentioned above, not pip.\n",
            "\u001b[1;36mhint\u001b[0m: See above for details.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 1.2 Check if the additional packages needed are present, if not install them"
      ],
      "metadata": {
        "id": "DMaIlOcVTJfp"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# !pip install trading_calendars\n",
        "# !pip install alpaca_trade_api\n",
        "# !pip install ccxt\n",
        "# !pip install jqdatasdk\n",
        "# !pip install wrds\n",
        "\n",
        "# !pip install lz4\n",
        "# !pip install ray[tune]\n",
        "# !pip install tensorboardX\n",
        "# !pip install gputil\n",
        "\n",
        "#%%capture\n",
        "if True:\n",
        "    # installing packages\n",
        "    !pip install pyfolio-reloaded  #original pyfolio no longer maintained\n",
        "    !pip install optuna\n",
        "    !pip install -U \"ray[rllib]\"\n",
        "    !pip install plotly\n",
        "    !pip install ipywidgets\n",
        "    !pip install -U kaleido   # enables saving plots to file\n",
        "    \n",
        "    "
      ],
      "metadata": {
        "id": "UfmXkH1rTFM7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install pandas==1.5.3\n",
        "!pip install gymnasium"
      ],
      "metadata": {
        "id": "9Zkmtvr-TFn_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import sys\n",
        "sys.path.append(\"../FinRL-Library\")"
      ],
      "metadata": {
        "id": "cMn1VNKCR0ld"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 1.3 Import packages"
      ],
      "metadata": {
        "id": "BbOMJnGdTRYX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#Importing the libraries\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import matplotlib\n",
        "import matplotlib.pyplot as plt\n",
        "# matplotlib.use('Agg')\n",
        "import datetime\n",
        "import optuna\n",
        "from pathlib import Path\n",
        "from google.colab import files\n",
        "%matplotlib inline\n",
        "from finrl import config\n",
        "from finrl import config_tickers\n",
        "from optuna.integration import PyTorchLightningPruningCallback\n",
        "from finrl.meta.preprocessor.yahoodownloader import YahooDownloader\n",
        "from finrl.meta.preprocessor.preprocessors import FeatureEngineer, data_split\n",
        "from finrl.meta.env_stock_trading.env_stocktrading import StockTradingEnv\n",
        "from finrl.meta.env_stock_trading.env_stocktrading_np import StockTradingEnv as StockTradingEnv_numpy\n",
        "from finrl.agents.stablebaselines3.models import DRLAgent\n",
        "from finrl.agents.rllib.models import DRLAgent as DRLAgent_rllib\n",
        "from finrl.meta.data_processor import DataProcessor\n",
        "import joblib\n",
        "from finrl.plot import backtest_stats, backtest_plot, get_daily_return, get_baseline\n",
        "import ray\n",
        "from pprint import pprint\n",
        "import kaleido\n",
        "\n",
        "\n",
        "\n",
        "import itertools\n",
        "\n",
        "import torch\n",
        "if torch.cuda.is_available():\n",
        "  device = torch.device(\"cuda\")\n",
        "else:\n",
        "  device = torch.device(\"cpu\")\n",
        "print(f'Torch device: {device}')"
      ],
      "metadata": {
        "id": "tG5qIkVHTNv8",
        "outputId": "3fc5ee2e-3777-459c-d009-c5d7c1d95a08",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 383
        }
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "error",
          "ename": "ModuleNotFoundError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-16-5ba25c8ae617>\u001b[0m in \u001b[0;36m<cell line: 8>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;31m# matplotlib.use('Agg')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mdatetime\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0moptuna\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      9\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mpathlib\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mPath\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mgoogle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolab\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mfiles\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'optuna'",
            "",
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0;32m\nNOTE: If your import is failing due to a missing package, you can\nmanually install dependencies using either !pip or !apt.\n\nTo view examples of installing some common dependencies, click the\n\"Open Examples\" button below.\n\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n"
          ],
          "errorDetails": {
            "actions": [
              {
                "action": "open_url",
                "actionText": "Open Examples",
                "url": "/notebooks/snippets/importing_libraries.ipynb"
              }
            ]
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "config_tickers.DOW_30_TICKER"
      ],
      "metadata": {
        "id": "DB11ygQLVLUD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "fMaPm8MsUo0R"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "if not os.path.exists(\"./\" + config.DATA_SAVE_DIR):\n",
        "    os.makedirs(\"./\" + config.DATA_SAVE_DIR)\n",
        "if not os.path.exists(\"./\" + config.TRAINED_MODEL_DIR):\n",
        "    os.makedirs(\"./\" + config.TRAINED_MODEL_DIR)\n",
        "if not os.path.exists(\"./\" + config.TENSORBOARD_LOG_DIR):\n",
        "    os.makedirs(\"./\" + config.TENSORBOARD_LOG_DIR)\n",
        "if not os.path.exists(\"./\" + config.RESULTS_DIR):\n",
        "    os.makedirs(\"./\" + config.RESULTS_DIR)"
      ],
      "metadata": {
        "id": "FAZpwTH3VVi-"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Collecting data and preprocessing"
      ],
      "metadata": {
        "id": "cqJ8ngFWVcNG"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#config_tickers.DOW_30_TICKER = [\"PYPL\"]"
      ],
      "metadata": {
        "id": "4M4UwcP4Ap_o"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#Custom ticker list dataframe download\n",
        "#TODO save df to avoid download\n",
        "path_pf = '/content/ticker_data.csv'\n",
        "if Path(path_pf).is_file():\n",
        "  print('Reading ticker data')\n",
        "  df = pd.read_csv(path_pf)\n",
        "  \n",
        "else:\n",
        "  print('Downloading ticker data')\n",
        "  ticker_list = config_tickers.DOW_30_TICKER\n",
        "  df = YahooDownloader(start_date = '2009-01-01',\n",
        "                     end_date = '2023-04-30',\n",
        "                     ticker_list = ticker_list).fetch_data()\n",
        "  df.to_csv('ticker_data.csv')"
      ],
      "metadata": {
        "id": "7BJNlsAuVZ-o",
        "outputId": "c5d9ff13-0c22-489b-ff98-b70a9b4551b3",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading ticker data\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "Shape of DataFrame:  (105581, 8)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def create_processed_full(processed):\n",
        "  list_ticker = processed[\"tic\"].unique().tolist()\n",
        "  list_date = list(pd.date_range(processed['date'].min(),processed['date'].max()).astype(str))\n",
        "  combination = list(itertools.product(list_date,list_ticker))\n",
        "\n",
        "  processed_full = pd.DataFrame(combination,columns=[\"date\",\"tic\"]).merge(processed,on=[\"date\",\"tic\"],how=\"left\")\n",
        "  processed_full = processed_full[processed_full['date'].isin(processed['date'])]\n",
        "  processed_full = processed_full.sort_values(['date','tic'])\n",
        "\n",
        "  processed_full = processed_full.fillna(0)\n",
        "  processed_full.sort_values(['date','tic'],ignore_index=True).head(5)\n",
        "\n",
        "  processed_full.to_csv('processed_full.csv')\n",
        "  return processed_full"
      ],
      "metadata": {
        "id": "RD6wwtGSVZVU"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#You can add technical indicators and turbulence factor to dataframe\n",
        "#Just set the use_technical_indicator=True, use_vix=True and use_turbulence=True\n",
        "def create_techind():\n",
        "  fe = FeatureEngineer(\n",
        "                    use_technical_indicator=True,\n",
        "                    tech_indicator_list = config.INDICATORS,\n",
        "                    use_vix=True,\n",
        "                    use_turbulence=True,\n",
        "                    user_defined_feature = False)\n",
        "\n",
        "  processed = fe.preprocess_data(df)\n",
        "  return processed"
      ],
      "metadata": {
        "id": "UNqEALmbVoJo"
      },
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#Load price and technical indicator data from file if available\n",
        "path_pf = '/content/processed_full.csv'\n",
        "if Path(path_pf).is_file():\n",
        "  print('Reading processed_full data')\n",
        "  processed_full = pd.read_csv(path_pf)\n",
        "\n",
        "else:\n",
        "  print('Creating processed_full file')\n",
        "  processed=create_techind()\n",
        "  processed_full=create_processed_full(processed)"
      ],
      "metadata": {
        "id": "Qr1Qd-lMVpzg",
        "outputId": "6564155b-d99c-419d-dfbf-9cce3abb3eb7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 240
        }
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-15-594d83fdbf74>\u001b[0m in \u001b[0;36m<cell line: 3>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m#Load price and technical indicator data from file if available\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mpath_pf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'/content/processed_full.csv'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0;32mif\u001b[0m \u001b[0mPath\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath_pf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_file\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m   \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Reading processed_full data'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m   \u001b[0mprocessed_full\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath_pf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'Path' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "date_col = \"date\"\n",
        "tic_col = \"tic\"\n",
        "\n",
        "init_train_trade_data = processed_full.sort_values([date_col, tic_col])\n",
        "\n",
        "init_train_trade_data = processed_full.fillna(0)\n",
        "\n",
        "init_train_data = data_split(\n",
        "    init_train_trade_data, '2020-01-01', '2020-05-01')\n",
        "init_trade_data = data_split(\n",
        "    init_train_trade_data, '2021-05-01','2021-10-01')\n",
        "\n",
        "print(f'Number of training samples: {len(init_train_data)}')\n",
        "print(f'Number of testing samples: {len(init_train_trade_data)}')"
      ],
      "metadata": {
        "id": "KFs4AjWIVr2s"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "init_trade_data.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 357
        },
        "id": "GI2qgPGFTerU",
        "outputId": "d8cf7fa9-fc9d-4ee9-99b0-4786ea3971d9"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "         date   tic        open        high         low       close  \\\n",
              "0  2021-05-03  AAPL  132.039993  134.070007  131.830002  130.963547   \n",
              "0  2021-05-03  AMGN  240.669998  247.020004  240.550003  230.412506   \n",
              "0  2021-05-03   AXP  154.589996  156.050003  154.009995  150.957855   \n",
              "0  2021-05-03    BA  234.110001  237.100006  233.809998  235.190002   \n",
              "0  2021-05-03   CAT  230.000000  230.929993  227.210007  218.290497   \n",
              "\n",
              "       volume  day      macd     boll_ub     boll_lb     rsi_30      cci_30  \\\n",
              "0  75135100.0  0.0  1.805509  135.539557  126.477487  53.856505   50.478419   \n",
              "0   3587700.0  0.0 -0.410710  248.223100  221.790974  50.842043  -81.617824   \n",
              "0   2726200.0  0.0  2.754007  151.027344  139.179348  62.209674  175.796227   \n",
              "0   9887800.0  0.0 -2.593339  260.290298  227.831702  49.445883 -102.331978   \n",
              "0   3182900.0  0.0  1.167005  223.005759  216.688283  56.081904  -17.678519   \n",
              "\n",
              "       dx_30  close_30_sma  close_60_sma        vix  turbulence  \n",
              "0  20.583396    127.527261    126.185906  18.309999   20.080323  \n",
              "0   5.785497    234.734412    226.585524  18.309999   20.080323  \n",
              "0  22.874057    142.745664    138.440773  18.309999   20.080323  \n",
              "0  17.827725    245.819000    237.101333  18.309999   20.080323  \n",
              "0   2.780977    218.974401    211.601992  18.309999   20.080323  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-8366af60-9a79-45d8-915b-6e7ddc017d63\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>date</th>\n",
              "      <th>tic</th>\n",
              "      <th>open</th>\n",
              "      <th>high</th>\n",
              "      <th>low</th>\n",
              "      <th>close</th>\n",
              "      <th>volume</th>\n",
              "      <th>day</th>\n",
              "      <th>macd</th>\n",
              "      <th>boll_ub</th>\n",
              "      <th>boll_lb</th>\n",
              "      <th>rsi_30</th>\n",
              "      <th>cci_30</th>\n",
              "      <th>dx_30</th>\n",
              "      <th>close_30_sma</th>\n",
              "      <th>close_60_sma</th>\n",
              "      <th>vix</th>\n",
              "      <th>turbulence</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2021-05-03</td>\n",
              "      <td>AAPL</td>\n",
              "      <td>132.039993</td>\n",
              "      <td>134.070007</td>\n",
              "      <td>131.830002</td>\n",
              "      <td>130.963547</td>\n",
              "      <td>75135100.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.805509</td>\n",
              "      <td>135.539557</td>\n",
              "      <td>126.477487</td>\n",
              "      <td>53.856505</td>\n",
              "      <td>50.478419</td>\n",
              "      <td>20.583396</td>\n",
              "      <td>127.527261</td>\n",
              "      <td>126.185906</td>\n",
              "      <td>18.309999</td>\n",
              "      <td>20.080323</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2021-05-03</td>\n",
              "      <td>AMGN</td>\n",
              "      <td>240.669998</td>\n",
              "      <td>247.020004</td>\n",
              "      <td>240.550003</td>\n",
              "      <td>230.412506</td>\n",
              "      <td>3587700.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>-0.410710</td>\n",
              "      <td>248.223100</td>\n",
              "      <td>221.790974</td>\n",
              "      <td>50.842043</td>\n",
              "      <td>-81.617824</td>\n",
              "      <td>5.785497</td>\n",
              "      <td>234.734412</td>\n",
              "      <td>226.585524</td>\n",
              "      <td>18.309999</td>\n",
              "      <td>20.080323</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2021-05-03</td>\n",
              "      <td>AXP</td>\n",
              "      <td>154.589996</td>\n",
              "      <td>156.050003</td>\n",
              "      <td>154.009995</td>\n",
              "      <td>150.957855</td>\n",
              "      <td>2726200.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>2.754007</td>\n",
              "      <td>151.027344</td>\n",
              "      <td>139.179348</td>\n",
              "      <td>62.209674</td>\n",
              "      <td>175.796227</td>\n",
              "      <td>22.874057</td>\n",
              "      <td>142.745664</td>\n",
              "      <td>138.440773</td>\n",
              "      <td>18.309999</td>\n",
              "      <td>20.080323</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2021-05-03</td>\n",
              "      <td>BA</td>\n",
              "      <td>234.110001</td>\n",
              "      <td>237.100006</td>\n",
              "      <td>233.809998</td>\n",
              "      <td>235.190002</td>\n",
              "      <td>9887800.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>-2.593339</td>\n",
              "      <td>260.290298</td>\n",
              "      <td>227.831702</td>\n",
              "      <td>49.445883</td>\n",
              "      <td>-102.331978</td>\n",
              "      <td>17.827725</td>\n",
              "      <td>245.819000</td>\n",
              "      <td>237.101333</td>\n",
              "      <td>18.309999</td>\n",
              "      <td>20.080323</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2021-05-03</td>\n",
              "      <td>CAT</td>\n",
              "      <td>230.000000</td>\n",
              "      <td>230.929993</td>\n",
              "      <td>227.210007</td>\n",
              "      <td>218.290497</td>\n",
              "      <td>3182900.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.167005</td>\n",
              "      <td>223.005759</td>\n",
              "      <td>216.688283</td>\n",
              "      <td>56.081904</td>\n",
              "      <td>-17.678519</td>\n",
              "      <td>2.780977</td>\n",
              "      <td>218.974401</td>\n",
              "      <td>211.601992</td>\n",
              "      <td>18.309999</td>\n",
              "      <td>20.080323</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-8366af60-9a79-45d8-915b-6e7ddc017d63')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-8366af60-9a79-45d8-915b-6e7ddc017d63 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-8366af60-9a79-45d8-915b-6e7ddc017d63');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Creating Environment"
      ],
      "metadata": {
        "id": "L3VlkfBMW8e7"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class Portfolio:\n",
        "    def __init__(self):\n",
        "        self.num_shares = 0\n",
        "        self.total_cost = 0.0\n",
        "        self.avg_cost = 0.0\n",
        "\n",
        "    def buy(self, num_shares, price):\n",
        "        self.total_cost += num_shares * price\n",
        "        self.num_shares += num_shares\n",
        "        self.avg_cost = self.total_cost / self.num_shares\n",
        "\n",
        "    def sell(self, num_shares):\n",
        "        self.total_cost -= self.avg_cost * num_shares\n",
        "        self.num_shares -= num_shares\n",
        "\n",
        "    def get_num_shares(self):\n",
        "        return self.num_shares\n",
        "\n",
        "    def get_total_cost(self):\n",
        "        return self.total_cost\n",
        "\n",
        "    def get_avg_cost(self):\n",
        "        return self.avg_cost\n",
        "\n",
        "# Example usage\n",
        "portfolio = Portfolio()\n",
        "\n",
        "# Buy 50 shares at $126.81\n",
        "portfolio.buy(50, 126.81)\n",
        "\n",
        "# Sell 25 shares\n",
        "portfolio.sell(25)\n",
        "\n",
        "# Buy 30 shares at $125.03\n",
        "portfolio.buy(30, 125.03)\n",
        "\n",
        "# Sell 10 shares\n",
        "portfolio.sell(10)\n",
        "\n",
        "\n",
        "print(\"Number of shares: \", portfolio.get_num_shares())\n",
        "print(\"Total cost: \", round(portfolio.get_total_cost(), 2))\n",
        "print(\"Average cost: \", round(portfolio.get_avg_cost(), 2))\n"
      ],
      "metadata": {
        "id": "_henJDi45WQV",
        "outputId": "c8609bf9-8032-4b97-b0dc-c471743ca458",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Number of shares:  45\n",
            "Total cost:  5662.76\n",
            "Average cost:  125.84\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from __future__ import annotations\n",
        "import copy\n",
        "from typing import List\n",
        "\n",
        "import gym\n",
        "import matplotlib\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "np.set_printoptions(linewidth=np.inf)\n",
        "import pandas as pd\n",
        "from gym import spaces\n",
        "from gym.utils import seeding\n",
        "from stable_baselines3.common.vec_env import DummyVecEnv\n",
        "\n",
        "matplotlib.use(\"Agg\")\n",
        "\n",
        "# from stable_baselines3.common.logger import Logger, KVWriter, CSVOutputFormat\n",
        "\n",
        "\n",
        "class GokuEnv(gym.Env):\n",
        "    \"\"\"A stock trading environment for OpenAI gym\"\"\"\n",
        "\n",
        "    metadata = {\"render.modes\": [\"human\"]}\n",
        "\n",
        "    def __init__(\n",
        "        self,\n",
        "        df: pd.DataFrame,\n",
        "        stock_dim: int,\n",
        "        hmax: int,\n",
        "        initial_amount: int,\n",
        "        num_stock_shares: list[int],\n",
        "        buy_cost_pct: list[float],\n",
        "        sell_cost_pct: list[float],\n",
        "        reward_scaling: float,\n",
        "        state_space: int,\n",
        "        action_space: int,\n",
        "        tech_indicator_list: list[str],\n",
        "        turbulence_threshold=None,\n",
        "        risk_indicator_col=\"turbulence\",\n",
        "        make_plots: bool = False,\n",
        "        print_verbosity=10,\n",
        "        day=0,\n",
        "        initial=True,\n",
        "        previous_state=[],\n",
        "        model_name=\"\",\n",
        "        mode=\"\",\n",
        "        iteration=\"\",\n",
        "    ):\n",
        "        self.day = day\n",
        "        self.df = df\n",
        "        self.stock_dim = stock_dim\n",
        "        self.hmax = hmax\n",
        "        self.num_stock_shares = num_stock_shares\n",
        "        self.initial_amount = initial_amount  # get the initial cash\n",
        "        self.buy_cost_pct = buy_cost_pct\n",
        "        self.sell_cost_pct = sell_cost_pct\n",
        "        self.reward_scaling = reward_scaling\n",
        "        self.state_space = state_space\n",
        "        self.action_space = action_space\n",
        "        self.tech_indicator_list = tech_indicator_list\n",
        "        self.action_space = spaces.Box(low=-1, high=1, shape=(self.action_space,))\n",
        "        self.observation_space = spaces.Box(\n",
        "            low=-np.inf, high=np.inf, shape=(self.state_space,)\n",
        "        )\n",
        "        self.data = self.df.loc[self.day, :]\n",
        "        self.terminal = False\n",
        "        self.make_plots = make_plots\n",
        "        self.print_verbosity = print_verbosity\n",
        "        self.turbulence_threshold = turbulence_threshold\n",
        "        self.risk_indicator_col = risk_indicator_col\n",
        "        self.initial = initial\n",
        "        self.previous_state = previous_state\n",
        "        self.model_name = model_name\n",
        "        self.mode = mode\n",
        "        self.iteration = iteration\n",
        "        # initalize state\n",
        "        self.state = self._initiate_state()\n",
        "\n",
        "        # initialize reward\n",
        "        self.reward = 0\n",
        "        self.turbulence = 0\n",
        "        self.cost = 0\n",
        "        self.trades = 0\n",
        "        self.episode = 0\n",
        "        # memorize all the total balance change\n",
        "        self.asset_memory = [\n",
        "            self.initial_amount\n",
        "            + np.sum(\n",
        "                np.array(self.num_stock_shares)\n",
        "                * np.array(self.state[1 : 1 + self.stock_dim])\n",
        "            )\n",
        "        ]  # the initial total asset is calculated by cash + sum (num_share_stock_i * price_stock_i)\n",
        "        self.rewards_memory = []\n",
        "        self.actions_memory = []\n",
        "        self.state_memory = (\n",
        "            []\n",
        "        )  # we need sometimes to preserve the state in the middle of trading process\n",
        "        self.date_memory = [self._get_date()]\n",
        "        #         self.logger = Logger('results',[CSVOutputFormat])\n",
        "        # self.reset()\n",
        "        self._seed()\n",
        "        self.total_price = np.array([0.0] * 29)\n",
        "        self.avg_price = np.array([0.0] * 29)\n",
        "        self.total_stockss = np.array([0.0] * 29)\n",
        "        self.frame  = pd.DataFrame()\n",
        "        self.portfolios = [Portfolio() for _ in range(29)]\n",
        "\n",
        "    def _sell_stock(self, index, action):\n",
        "        def _do_sell_normal():\n",
        "            if (\n",
        "                self.state[index + 2 * self.stock_dim + 1] != True\n",
        "            ):  # check if the stock is able to sell, for simlicity we just add it in techical index\n",
        "                # if self.state[index + 1] > 0: # if we use price<0 to denote a stock is unable to trade in that day, the total asset calculation may be wrong for the price is unreasonable\n",
        "                # Sell only if the price is > 0 (no missing data in this particular date)\n",
        "                # perform sell action based on the sign of the action\n",
        "                if self.state[index + self.stock_dim + 1] > 0:\n",
        "                    # Sell only if current asset is > 0\n",
        "                    sell_num_shares = min(\n",
        "                        abs(action), self.state[index + self.stock_dim + 1]\n",
        "                    )\n",
        "                    sell_amount = (\n",
        "                        self.state[index + 1]\n",
        "                        * sell_num_shares\n",
        "                        * (1 - self.sell_cost_pct[index])\n",
        "                    )\n",
        "                    # update balance\n",
        "                    self.state[0] += sell_amount\n",
        "\n",
        "                    self.state[index + self.stock_dim + 1] -= sell_num_shares\n",
        "                    self.cost += (\n",
        "                        self.state[index + 1]\n",
        "                        * sell_num_shares\n",
        "                        * self.sell_cost_pct[index]\n",
        "                    )\n",
        "                    self.trades += 1\n",
        "                    #if sell_num_shares >0:\n",
        "                    #  print(\"stocks Sold sell_num_shares\", sell_num_shares)\n",
        "                else:\n",
        "                    sell_num_shares = 0\n",
        "            else:\n",
        "                sell_num_shares = 0\n",
        "            return sell_num_shares\n",
        "\n",
        "        # perform sell action based on the sign of the action\n",
        "        if self.turbulence_threshold is not None:\n",
        "            if self.turbulence >= self.turbulence_threshold:\n",
        "                if self.state[index + 1] > 0:\n",
        "                    # Sell only if the price is > 0 (no missing data in this particular date)\n",
        "                    # if turbulence goes over threshold, just clear out all positions\n",
        "                    if self.state[index + self.stock_dim + 1] > 0:\n",
        "                        # Sell only if current asset is > 0\n",
        "                        sell_num_shares = self.state[index + self.stock_dim + 1]\n",
        "                        sell_amount = (\n",
        "                            self.state[index + 1]\n",
        "                            * sell_num_shares\n",
        "                            * (1 - self.sell_cost_pct[index])\n",
        "                        )\n",
        "                        # update balance\n",
        "                        self.state[0] += sell_amount\n",
        "                        self.state[index + self.stock_dim + 1] = 0\n",
        "                        self.cost += (\n",
        "                            self.state[index + 1]\n",
        "                            * sell_num_shares\n",
        "                            * self.sell_cost_pct[index]\n",
        "                        )\n",
        "                        self.trades += 1\n",
        "                    else:\n",
        "                        sell_num_shares = 0\n",
        "                else:\n",
        "                    sell_num_shares = 0\n",
        "            else:\n",
        "                sell_num_shares = _do_sell_normal()\n",
        "        else:\n",
        "            sell_num_shares = _do_sell_normal()\n",
        "        #print(\"stocks Sold sell_num_shares\", sell_num_shares)\n",
        "        return sell_num_shares\n",
        "\n",
        "    def _buy_stock(self, index, action):\n",
        "        def _do_buy():\n",
        "            if (self.state[index + 2 * self.stock_dim + 1] != True):  # check if the stock is able to buy\n",
        "                # if self.state[index + 1] >0:\n",
        "                # Buy only if the price is > 0 (no missing data in this particular date)\n",
        "                available_amount = self.state[0] // (self.state[index + 1] * (1 + self.buy_cost_pct[index]))\n",
        "                # when buying stocks, we should consider the cost of trading when calculating available_amount, or we may be have cash<0\n",
        "                # print('available_amount:{}'.format(available_amount))\n",
        "                # update balance\n",
        "                buy_num_shares = min(available_amount, action)\n",
        "                buy_amount = (self.state[index + 1] * buy_num_shares * (1 + self.buy_cost_pct[index]))\n",
        "                self.state[0] -= buy_amount\n",
        "                self.state[index + self.stock_dim + 1] += buy_num_shares\n",
        "                self.cost += (self.state[index + 1] * buy_num_shares * self.buy_cost_pct[index])\n",
        "                self.trades += 1\n",
        "            else:\n",
        "                buy_num_shares = 0\n",
        "\n",
        "            return buy_num_shares\n",
        "\n",
        "        # perform buy action based on the sign of the action\n",
        "        if self.turbulence_threshold is None:\n",
        "            buy_num_shares = _do_buy()\n",
        "        else:\n",
        "            if self.turbulence < self.turbulence_threshold:\n",
        "                buy_num_shares = _do_buy()\n",
        "            else:\n",
        "                buy_num_shares = 0\n",
        "                pass\n",
        "\n",
        "        return buy_num_shares\n",
        "\n",
        "    def _make_plot(self):\n",
        "        plt.plot(self.asset_memory, \"r\")\n",
        "        plt.savefig(f\"results/account_value_trade_{self.episode}.png\")\n",
        "        plt.close()\n",
        "\n",
        "    def step(self, actions):\n",
        "        self.terminal = self.day >= len(self.df.index.unique()) - 1\n",
        "        if self.terminal:\n",
        "            # print(f\"Episode: {self.episode}\")\n",
        "            if self.make_plots:\n",
        "                self._make_plot()\n",
        "            end_total_asset = self.state[0] + sum(np.array(self.state[1 : (self.stock_dim + 1)]) * np.array(self.state[(self.stock_dim + 1) : (self.stock_dim * 2 + 1)]))\n",
        "            df_total_value = pd.DataFrame(self.asset_memory)\n",
        "            tot_reward = (self.state[0] + \n",
        "                          sum(np.array(self.state[1 : (self.stock_dim + 1)]) \n",
        "                          * np.array(self.state[(self.stock_dim + 1) : (self.stock_dim * 2 + 1)])) \n",
        "                          - self.asset_memory[0])  # initial_amount is only cash part of our initial asset\n",
        "            df_total_value.columns = [\"account_value\"]\n",
        "            df_total_value[\"date\"] = self.date_memory\n",
        "            df_total_value[\"daily_return\"] = df_total_value[\"account_value\"].pct_change(1)\n",
        "            if df_total_value[\"daily_return\"].std() != 0:\n",
        "                sharpe = ((252**0.5) \n",
        "                * df_total_value[\"daily_return\"].mean() \n",
        "                / df_total_value[\"daily_return\"].std())\n",
        "            df_rewards = pd.DataFrame(self.rewards_memory)\n",
        "            df_rewards.columns = [\"account_rewards\"]\n",
        "            df_rewards[\"date\"] = self.date_memory[:-1]\n",
        "            \n",
        "            if self.episode % self.print_verbosity == 0:\n",
        "                print(f\"day: {self.day}, episode: {self.episode}\")\n",
        "                print(f\"begin_total_asset: {self.asset_memory[0]:0.2f}\")\n",
        "                print(f\"end_total_asset: {end_total_asset:0.2f}\")\n",
        "                print(f\"total_reward: {tot_reward:0.2f}\")\n",
        "                print(f\"total_cost: {self.cost:0.2f}\")\n",
        "                print(f\"total_trades: {self.trades}\")\n",
        "                if df_total_value[\"daily_return\"].std() != 0:\n",
        "                    print(f\"Sharpe: {sharpe:0.3f}\")\n",
        "                print(\"=================================\")\n",
        "\n",
        "            if (self.model_name != \"\") and (self.mode != \"\"):\n",
        "                df_actions = self.save_action_memory()\n",
        "                df_actions.to_csv(\"results/actions_{}_{}_{}.csv\".format(self.mode, self.model_name, self.iteration))\n",
        "                df_total_value.to_csv(\"results/account_value_{}_{}_{}.csv\".format(self.mode, self.model_name, self.iteration),index=False,)\n",
        "                df_rewards.to_csv(\"results/account_rewards_{}_{}_{}.csv\".format(self.mode, self.model_name, self.iteration),index=False,)\n",
        "                plt.plot(self.asset_memory, \"r\")\n",
        "                plt.savefig(\"results/account_value_{}_{}_{}.png\".format(self.mode, self.model_name, self.iteration))\n",
        "                plt.close()\n",
        "            return self.state, self.reward, self.terminal, {}\n",
        "\n",
        "        else:\n",
        "            actions = actions * self.hmax  # actions initially is scaled between 0 to 1\n",
        "            actions = actions.astype(int)  # convert into integer because we can't by fraction of shares\n",
        "\n",
        "\n",
        "            if self.turbulence_threshold is not None:\n",
        "                if self.turbulence >= self.turbulence_threshold:\n",
        "                    actions = np.array([-self.hmax] * self.stock_dim)\n",
        "            \n",
        "            current_price = np.array(self.state[1 : (self.stock_dim + 1)])\n",
        "            #actions = np.where(((current_price > ( self.avg_price * 0.6  + self.avg_price))& (self.avg_price >0.0)), self.total_stockss*-1,actions  )\n",
        "            #actions = np.where(((current_price < (self.avg_price - self.avg_price * .2))& (self.avg_price >0)), self.total_stockss*-1,actions  )\n",
        "            \n",
        "            \n",
        "            # # Mandatory Selling Profits and Losses\n",
        "            for i in range(0,len(actions)):\n",
        "              # self.total_price\n",
        "              # self.total_stockss\n",
        "              # self.avg_price\n",
        "              # print(\"Current Price\")\n",
        "              # print(np.array(self.state[1 : (self.stock_dim + 1)])[i])\n",
        "              current_price  = np.array(self.state[1 : (self.stock_dim + 1)])[i]\n",
        "              avg_price = self.avg_price[i]\n",
        "              #print(avg_price)\n",
        "              if (current_price > ( avg_price * 0.6  + avg_price))  and (avg_price > 0.0):\n",
        "                actions[i] = self.total_stockss[i] * -1\n",
        "                #print(\"Updated actions\")\n",
        "                #self.total_price[i] = 0.0\n",
        "                print(round(current_price) , \"|\", \" avg_price\", avg_price, \"Profit\", current_price -  avg_price)\n",
        "              elif current_price < (avg_price - avg_price * 0.2)  and avg_price > 0.0:\n",
        "                actions[i] = self.total_stockss[i] * -1\n",
        "                print(round(current_price) , \"|\", \" avg_price\", avg_price,  \"loss\", current_price -  avg_price )\n",
        "                #self.total_price[i] = 0.0\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "            begin_total_asset = self.state[0] + sum(np.array(self.state[1 : (self.stock_dim + 1)])* np.array(self.state[(self.stock_dim + 1) : (self.stock_dim * 2 + 1)]))\n",
        "            # print(\"begin_total_asset:{}\".format(begin_total_asset))\n",
        "\n",
        "\n",
        "            argsort_actions = np.argsort(actions)\n",
        "            sell_index = argsort_actions[: np.where(actions < 0)[0].shape[0]]\n",
        "            buy_index = argsort_actions[::-1][: np.where(actions > 0)[0].shape[0]]\n",
        "\n",
        "            for index in sell_index:\n",
        "              actions[index] = self._sell_stock(index, actions[index]) * (-1)\n",
        "\n",
        "            for index in buy_index:\n",
        "                actions[index] = self._buy_stock(index, actions[index])\n",
        "\n",
        "\n",
        "            self.actions_memory.append(actions)\n",
        "\n",
        "            # state: s -> s+1\n",
        "            self.day += 1\n",
        "            self.data = self.df.loc[self.day, :]\n",
        "            if self.turbulence_threshold is not None:\n",
        "                if len(self.df.tic.unique()) == 1:\n",
        "                    self.turbulence = self.data[self.risk_indicator_col]\n",
        "                elif len(self.df.tic.unique()) > 1:\n",
        "                    self.turbulence = self.data[self.risk_indicator_col].values[0]\n",
        "\n",
        "\n",
        "\n",
        "            \n",
        "            #self.frame.loc[self.frame.shape[0]+1] = recent_price\n",
        "            recent_price = copy.deepcopy(self.state[1 : (self.stock_dim + 1)])\n",
        "            recent_buy_sell = copy.deepcopy(actions)\n",
        "\n",
        "            recent_buy_sell_price = np.where(recent_buy_sell != 0.0, recent_buy_sell * recent_price, 0.0)\n",
        "            #print(recent_price)\n",
        "            #print(recent_buy_sell)\n",
        "            #print(recent_buy_sell_price)\n",
        "            #self.frame.append(pd.DataFrame( recent_price))\n",
        "            #self.frame.loc[self.frame.shape[0]+1] = recent_buy_sell\n",
        "            #self.frame.loc[self.frame.shape[0]+1] = recent_buy_sell_price\n",
        "\n",
        "            \n",
        "            self.total_stockss = copy.deepcopy(recent_buy_sell) + copy.deepcopy(self.total_stockss)\n",
        "            self.total_price = copy.deepcopy(recent_buy_sell_price)+ copy.deepcopy(self.total_price)\n",
        "            def _def_find_avg(x):\n",
        "                index, val = x\n",
        "                recent_price, recent_action = val\n",
        "                if recent_action > 0:\n",
        "                  self.portfolios[index].buy(recent_action, recent_price)\n",
        "                elif recent_action < 0:\n",
        "                  self.portfolios[index].sell(abs(recent_action))\n",
        "                return round(self.portfolios[index].get_avg_cost(), 2)\n",
        "              \n",
        "              \n",
        "            indexed = enumerate(zip(recent_buy_sell, recent_price))\n",
        "            \n",
        "            self.avg_price = np.fromiter(map(_def_find_avg, indexed, ), dtype=float)\n",
        "            \n",
        "            print(self.avg_price )\n",
        "            # self.avg_price = np.divide(self.total_price,\n",
        "            #                            self.total_stockss,\n",
        "            #                            out=np.zeros_like(self.total_price),\n",
        "            #                            #where=((recent_buy_sell!=0.0)) &(self.total_stockss >0.0) )\n",
        "            #                            where=recent_buy_sell!=0.0)\n",
        "\n",
        "            self.avg_price = np.where(~np.isfinite(self.avg_price), 0.0, self.avg_price)\n",
        "            self.avg_price = np.where(self.avg_price<0.0, 0.0, self.avg_price)\n",
        "\n",
        "            self.avg_price = np.where(self.total_stockss <=0.0,0.0,self.avg_price)\n",
        "\n",
        "\n",
        "            new_df = pd.DataFrame([recent_price])\n",
        "            self.frame = pd.concat([self.frame, pd.DataFrame(new_df)],ignore_index=True,axis = 0)\n",
        "            \n",
        "            new_df = pd.DataFrame([recent_buy_sell_price])\n",
        "            self.frame = pd.concat([self.frame, pd.DataFrame(new_df)],ignore_index=True,axis = 0)\n",
        "            \n",
        "            new_df = pd.DataFrame([self.total_stockss])\n",
        "            self.frame = pd.concat([self.frame, pd.DataFrame(new_df)],ignore_index=True,axis = 0)\n",
        "\n",
        "            new_df = pd.DataFrame([actions])\n",
        "            self.frame = pd.concat([self.frame, pd.DataFrame(new_df)],ignore_index=True,axis = 0)\n",
        "\n",
        "            new_df = pd.DataFrame([self.total_price])\n",
        "            self.frame = pd.concat([self.frame, pd.DataFrame(new_df)],ignore_index=True,axis = 0)\n",
        "\n",
        "            new_df = pd.DataFrame([self.avg_price])\n",
        "            self.frame = pd.concat([self.frame, pd.DataFrame(new_df)],ignore_index=True,axis = 0)\n",
        "\n",
        "            self.frame.to_csv(\"test.csv\")\n",
        "            \n",
        "            \n",
        "            \n",
        "            # dx = np.where(((self.avg_price < 0.00)))\n",
        "            # print(dx)\n",
        "            #if len(dx) > 0:\n",
        "            #print(self.total_stockss)\n",
        "            #print(self.total_price )\n",
        "            #print(self.avg_price)\n",
        "\n",
        "\n",
        "            \n",
        "\n",
        "\n",
        "\n",
        "\n",
        "            # self.total_price = np.add(recent_buy_sell_price,\n",
        "            #                           self.total_price,\n",
        "            #                           out=np.zeros_like(self.total_price),\n",
        "            #                                where=(recent_buy_sell!=0.0  )\n",
        "            \n",
        "            # #self.total_price = np.where(self.total_price<0.0, 0.0, self.total_price)\n",
        "            # self.total_stockss = actions + self.total_stockss\n",
        "            \n",
        "            # self.avg_price = np.divide(self.total_price,\n",
        "            #                            self.total_stockss,\n",
        "            #                            out=np.zeros_like(self.total_price),\n",
        "            #                            where=recent_buy_sell!=0.0)\n",
        "          \n",
        "            # print(\"Total Stocks\")\n",
        "            # print(self.total_stockss)\n",
        "            # print(\"Price\")\n",
        "            # print(recent_price)\n",
        "\n",
        "            # print(\"Avg Price\")\n",
        "            # print(self.avg_price)\n",
        "            # print(\"Total Price\")\n",
        "            # print(self.total_price)\n",
        "\n",
        "\n",
        "            self.state = self._update_state()\n",
        "\n",
        "            end_total_asset = self.state[0] + sum(\n",
        "                np.array(self.state[1 : (self.stock_dim + 1)])\n",
        "                * np.array(self.state[(self.stock_dim + 1) : (self.stock_dim * 2 + 1)])\n",
        "            )\n",
        "            self.asset_memory.append(end_total_asset)\n",
        "            self.date_memory.append(self._get_date())\n",
        "            self.reward = end_total_asset - begin_total_asset\n",
        "            self.rewards_memory.append(self.reward)\n",
        "            self.reward = self.reward * self.reward_scaling\n",
        "            self.state_memory.append(\n",
        "                self.state\n",
        "            )  # add current state in state_recorder for each step\n",
        "\n",
        "        return self.state, self.reward, self.terminal, {}\n",
        "\n",
        "    def reset(self):\n",
        "        # initiate state\n",
        "        self.state = self._initiate_state()\n",
        "\n",
        "        if self.initial:\n",
        "            self.asset_memory = [\n",
        "                self.initial_amount\n",
        "                + np.sum(\n",
        "                    np.array(self.num_stock_shares)\n",
        "                    * np.array(self.state[1 : 1 + self.stock_dim])\n",
        "                )\n",
        "            ]\n",
        "        else:\n",
        "            previous_total_asset = self.previous_state[0] + sum(\n",
        "                np.array(self.state[1 : (self.stock_dim + 1)])\n",
        "                * np.array(\n",
        "                    self.previous_state[(self.stock_dim + 1) : (self.stock_dim * 2 + 1)]\n",
        "                )\n",
        "            )\n",
        "            self.asset_memory = [previous_total_asset]\n",
        "\n",
        "        self.day = 0\n",
        "        self.data = self.df.loc[self.day, :]\n",
        "        self.turbulence = 0\n",
        "        self.cost = 0\n",
        "        self.trades = 0\n",
        "        self.terminal = False\n",
        "        # self.iteration=self.iteration\n",
        "        self.rewards_memory = []\n",
        "        self.actions_memory = []\n",
        "        self.date_memory = [self._get_date()]\n",
        "        if self.total_price[5] > 0.0:\n",
        "          dd = pd.DataFrame(data = self.avg_price)\n",
        "          dd.to_csv(\"avg\")\n",
        "          dd = pd.DataFrame(data = self.total_price)\n",
        "          dd.to_csv(\"tp\")\n",
        "        \n",
        "        self.episode += 1\n",
        "        self.total_price = np.array([0.0] * 29)\n",
        "        self.avg_price = np.array([0.0] * 29)\n",
        "        self.total_stockss = np.array([0.0] * 29)\n",
        "        #print(\"Resteting Account\")\n",
        "        return self.state\n",
        "\n",
        "    def render(self, mode=\"human\", close=False):\n",
        "        return self.state\n",
        "\n",
        "    def _initiate_state(self):\n",
        "        if self.initial:\n",
        "            # For Initial State\n",
        "            if len(self.df.tic.unique()) > 1:\n",
        "                # for multiple stock\n",
        "                state = (\n",
        "                    [self.initial_amount]\n",
        "                    + self.data.close.values.tolist()\n",
        "                    + self.num_stock_shares\n",
        "                    + sum(\n",
        "                        (\n",
        "                            self.data[tech].values.tolist()\n",
        "                            for tech in self.tech_indicator_list\n",
        "                        ),\n",
        "                        [],\n",
        "                    )\n",
        "                )  # append initial stocks_share to initial state, instead of all zero\n",
        "            else:\n",
        "                # for single stock\n",
        "                state = (\n",
        "                    [self.initial_amount]\n",
        "                    + [self.data.close]\n",
        "                    + [0] * self.stock_dim\n",
        "                    + sum(([self.data[tech]] for tech in self.tech_indicator_list), [])\n",
        "                )\n",
        "        else:\n",
        "            # Using Previous State\n",
        "            if len(self.df.tic.unique()) > 1:\n",
        "                # for multiple stock\n",
        "                state = (\n",
        "                    [self.previous_state[0]]\n",
        "                    + self.data.close.values.tolist()\n",
        "                    + self.previous_state[\n",
        "                        (self.stock_dim + 1) : (self.stock_dim * 2 + 1)\n",
        "                    ]\n",
        "                    + sum(\n",
        "                        (\n",
        "                            self.data[tech].values.tolist()\n",
        "                            for tech in self.tech_indicator_list\n",
        "                        ),\n",
        "                        [],\n",
        "                    )\n",
        "                )\n",
        "            else:\n",
        "                # for single stock\n",
        "                state = (\n",
        "                    [self.previous_state[0]]\n",
        "                    + [self.data.close]\n",
        "                    + self.previous_state[\n",
        "                        (self.stock_dim + 1) : (self.stock_dim * 2 + 1)\n",
        "                    ]\n",
        "                    + sum(([self.data[tech]] for tech in self.tech_indicator_list), [])\n",
        "                )\n",
        "        return state\n",
        "\n",
        "    def _update_state(self):\n",
        "        if len(self.df.tic.unique()) > 1:\n",
        "            # for multiple stock\n",
        "            state = (\n",
        "                [self.state[0]]\n",
        "                + self.data.close.values.tolist()\n",
        "                + list(self.state[(self.stock_dim + 1) : (self.stock_dim * 2 + 1)])\n",
        "                + sum((self.data[tech].values.tolist() for tech in self.tech_indicator_list),[],))\n",
        "\n",
        "        else:\n",
        "            # for single stock\n",
        "            state = ([self.state[0]] + [self.data.close] \n",
        "                     + list(self.state[(self.stock_dim + 1) : (self.stock_dim * 2 + 1)])\n",
        "                     + sum(([self.data[tech]] for tech in self.tech_indicator_list), []))\n",
        "\n",
        "        return state\n",
        "\n",
        "    def _get_date(self):\n",
        "        if len(self.df.tic.unique()) > 1:\n",
        "            date = self.data.date.unique()[0]\n",
        "        else:\n",
        "            date = self.data.date\n",
        "        return date\n",
        "\n",
        "\n",
        "    def save_asset_memory(self):\n",
        "        date_list = self.date_memory\n",
        "        asset_list = self.asset_memory\n",
        "        # print(len(date_list))\n",
        "        # print(len(asset_list))\n",
        "        df_account_value = pd.DataFrame({\"date\": date_list, \"account_value\": asset_list})\n",
        "        return df_account_value\n",
        "\n",
        "    def save_action_memory(self):\n",
        "        if len(self.df.tic.unique()) > 1:\n",
        "            # date and close price length must match actions length\n",
        "            date_list = self.date_memory[:-1]\n",
        "            df_date = pd.DataFrame(date_list)\n",
        "            df_date.columns = [\"date\"]\n",
        "\n",
        "            action_list = self.actions_memory\n",
        "            df_actions = pd.DataFrame(action_list)\n",
        "            df_actions.columns = self.data.tic.values\n",
        "            df_actions.index = df_date.date\n",
        "            # df_actions = pd.DataFrame({'date':date_list,'actions':action_list})\n",
        "        else:\n",
        "            date_list = self.date_memory[:-1]\n",
        "            action_list = self.actions_memory\n",
        "            df_actions = pd.DataFrame({\"date\": date_list, \"actions\": action_list})\n",
        "        return df_actions\n",
        "\n",
        "    def _seed(self, seed=None):\n",
        "        self.np_random, seed = seeding.np_random(seed)\n",
        "        return [seed]\n",
        "\n",
        "    def get_sb_env(self):\n",
        "        e = DummyVecEnv([lambda: self])\n",
        "        obs = e.reset()\n",
        "        return e, obs\n"
      ],
      "metadata": {
        "id": "4p02b2cx_dj5"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "k5e9VO3mBLhm"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "stock_dimension = len(init_train_data.tic.unique())\n",
        "state_space = 1 + 2 * stock_dimension + len(config.INDICATORS) * stock_dimension\n",
        "print(f\"Stock Dimension: {stock_dimension}, State Space: {state_space}\")\n",
        "buy_cost_list = sell_cost_list = [0.001] * stock_dimension\n",
        "num_stock_shares = [0] * stock_dimension"
      ],
      "metadata": {
        "id": "rg7DTZbVV10n",
        "outputId": "ed9feb6b-d6b1-41df-9dce-5d7c41f48a1e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 240
        }
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-12-44ef56146891>\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mstock_dimension\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minit_train_data\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtic\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munique\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mstate_space\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m1\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m2\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mstock_dimension\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mINDICATORS\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mstock_dimension\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"Stock Dimension: {stock_dimension}, State Space: {state_space}\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mbuy_cost_list\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msell_cost_list\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;36m0.001\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mstock_dimension\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mnum_stock_shares\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mstock_dimension\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'init_train_data' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Defining the environment kwargs\n",
        "\n",
        "initial_amount = 500000\n",
        "env_kwargs = {\n",
        "    \"hmax\": 100,\n",
        "    \"initial_amount\": initial_amount,\n",
        "    \"num_stock_shares\": num_stock_shares,\n",
        "    \"buy_cost_pct\": buy_cost_list,\n",
        "    \"sell_cost_pct\": sell_cost_list,\n",
        "    \"state_space\": state_space,\n",
        "    \"stock_dim\": stock_dimension,\n",
        "    \"tech_indicator_list\": config.INDICATORS,\n",
        "    \"action_space\": stock_dimension,\n",
        "    \"reward_scaling\": 1e-4\n",
        "}"
      ],
      "metadata": {
        "id": "zODPmc5hV42J"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#Instantiate the training gym compatible environment\n",
        "e_train_gym = GokuEnv(df = init_train_data, **env_kwargs)\n",
        "env_train, _ = e_train_gym.get_sb_env()\n",
        "agent = DRLAgent(env = env_train)"
      ],
      "metadata": {
        "id": "7n2QYGP5V58p"
      },
      "execution_count": 202,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#Instantiate the trading environment\n",
        "e_trade_gym = GokuEnv(df = init_trade_data, turbulence_threshold = None, **env_kwargs)"
      ],
      "metadata": {
        "id": "xoBM74XEX1oI"
      },
      "execution_count": 203,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Trade performance code\n",
        "The following code calculates trade performance metrics, which are then used as an objective for optimizing hyperparameter values.\n",
        "\n",
        "There are several available metrics. In this tutorial, the default choice is the ratio of average value of winning to losing trades."
      ],
      "metadata": {
        "id": "Dq6zJ14ZYsTK"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "#Main method\n",
        "# Calculates Trade Performance for Objective\n",
        "# Called from objective method\n",
        "# Returns selected trade perf metric(s)\n",
        "# Requires actions and associated prices\n",
        "\n",
        "def calc_trade_perf_metric(df_actions, \n",
        "                           df_prices_trade,\n",
        "                           tp_metric,\n",
        "                           dbg=False):\n",
        "  \n",
        "    df_actions_p, df_prices_p, tics = prep_data(df_actions.copy(),\n",
        "                                                df_prices_trade.copy())\n",
        "    # actions predicted by trained model on trade data\n",
        "    df_actions_p.to_csv('df_actions.csv') \n",
        "\n",
        "    \n",
        "    # Confirms that actions, prices and tics are consistent\n",
        "    df_actions_s, df_prices_s, tics_prtfl = \\\n",
        "        sync_tickers(df_actions_p.copy(),df_prices_p.copy(),tics)\n",
        "    \n",
        "    # copy to ensure that tics from portfolio remains unchanged\n",
        "    tics = tics_prtfl.copy()\n",
        "    \n",
        "    # Analysis is performed on each portfolio ticker\n",
        "    perf_data= collect_performance_data(df_actions_s, df_prices_s, tics)\n",
        "    # profit/loss for each ticker\n",
        "    pnl_all = calc_pnl_all(perf_data, tics)\n",
        "    # values for trade performance metrics\n",
        "    perf_results = calc_trade_perf(pnl_all)\n",
        "    df = pd.DataFrame.from_dict(perf_results, orient='index')\n",
        "    \n",
        "    # calculate and return trade metric value as objective\n",
        "    m = calc_trade_metric(df,tp_metric)\n",
        "    print(f'Ratio Avg Win/Avg Loss: {m}')\n",
        "    k = str(len(tpm_hist)+1)\n",
        "    # save metric value\n",
        "    tpm_hist[k] = m\n",
        "    return m\n",
        "\n",
        "\n",
        "# Supporting methods\n",
        "def calc_trade_metric(df,metric='avgwl'):\n",
        "    '''# trades', '# wins', '# losses', 'wins total value', 'wins avg value',\n",
        "       'losses total value', 'losses avg value'''\n",
        "    # For this tutorial, the only metric available is the ratio of \n",
        "    #  average values of winning to losing trades. Others are in development.\n",
        "    \n",
        "    # some test cases produce no losing trades.\n",
        "    # The code below assigns a value as a multiple of the highest value during\n",
        "    # previous hp optimization runs. If the first run experiences no losses,\n",
        "    # a fixed value is assigned for the ratio\n",
        "    tpm_mult = 1.0\n",
        "    avgwl_no_losses = 25\n",
        "    if metric == 'avgwl':\n",
        "        if sum(df['# losses']) == 0:\n",
        "          try:\n",
        "            return max(tpm_hist.values())*tpm_mult\n",
        "          except ValueError:\n",
        "            return avgwl_no_losses\n",
        "        avg_w = sum(df['wins total value'])/sum(df['# wins'])\n",
        "        avg_l = sum(df['losses total value'])/sum(df['# losses'])\n",
        "        m = abs(avg_w/avg_l)\n",
        "\n",
        "    return m\n",
        "\n",
        "\n",
        "def prep_data(df_actions,\n",
        "              df_prices_trade):\n",
        "    \n",
        "    df=df_prices_trade[['date','close','tic']]\n",
        "    df['Date'] = pd.to_datetime(df['date'])\n",
        "    df = df.set_index('Date')\n",
        "    # set indices on both df to datetime\n",
        "    idx = pd.to_datetime(df_actions.index, infer_datetime_format=True)\n",
        "    df_actions.index=idx\n",
        "    tics = np.unique(df.tic)\n",
        "    n_tics = len(tics)\n",
        "    print(f'Number of tickers: {n_tics}')\n",
        "    print(f'Tickers: {tics}')\n",
        "    dategr = df.groupby('tic')\n",
        "    p_d={t:dategr.get_group(t).loc[:,'close'] for t in tics}\n",
        "    df_prices = pd.DataFrame.from_dict(p_d)\n",
        "    df_prices.index = df_prices.index.normalize()\n",
        "    return df_actions, df_prices, tics\n",
        "\n",
        "\n",
        "# prepares for integrating action and price files\n",
        "def link_prices_actions(df_a,\n",
        "                        df_p):\n",
        "    cols_a = [t + '_a' for t in df_a.columns]\n",
        "    df_a.columns = cols_a\n",
        "    cols_p = [t + '_p' for t in df_p.columns]\n",
        "    df_p.columns = cols_p\n",
        "    return df_a, df_p\n",
        "\n",
        "\n",
        "def sync_tickers(df_actions,df_tickers_p,tickers):\n",
        "    # Some DOW30 components may not be included in portfolio\n",
        "    # passed tickers includes all DOW30 components\n",
        "    # actions and ticker files may have different length indices\n",
        "    if len(df_actions) != len(df_tickers_p):\n",
        "      msng_dates = set(df_actions.index)^set(df_tickers_p.index)\n",
        "      try:\n",
        "        #assumption is prices has one additional timestamp (row)\n",
        "        df_tickers_p.drop(msng_dates,inplace=True)\n",
        "      except:\n",
        "        df_actions.drop(msng_dates,inplace=True)\n",
        "    df_actions, df_tickers_p = link_prices_actions(df_actions,df_tickers_p)\n",
        "    # identify any DOW components not in portfolio\n",
        "    t_not_in_a = [t for t in tickers if t + '_a' not in list(df_actions.columns)]\n",
        "  \n",
        "    # remove t_not_in_a from df_tickers_p\n",
        "    drop_cols = [t + '_p' for t in t_not_in_a]\n",
        "    df_tickers_p.drop(columns=drop_cols,inplace=True)\n",
        "    \n",
        "    # Tickers in portfolio\n",
        "    tickers_prtfl = [c.split('_')[0] for c in df_actions.columns]\n",
        "    return df_actions,df_tickers_p, tickers_prtfl\n",
        "\n",
        "def collect_performance_data(dfa,dfp,tics, dbg=False):\n",
        "    \n",
        "    perf_data = {}\n",
        "    # In current version, files columns include secondary identifier\n",
        "    for t in tics:\n",
        "        # actions: purchase/sale of DOW equities\n",
        "        acts = dfa['_'.join([t,'a'])].values\n",
        "        # ticker prices\n",
        "        prices = dfp['_'.join([t,'p'])].values\n",
        "        # market value of purchases/sales\n",
        "        tvals_init = np.multiply(acts,prices)\n",
        "        d={'actions':acts, 'prices':prices,'init_values':tvals_init}\n",
        "        perf_data[t]=d\n",
        "\n",
        "    return perf_data\n",
        "\n",
        "\n",
        "def calc_pnl_all(perf_dict, tics_all):\n",
        "    # calculate profit/loss for each ticker\n",
        "    print(f'Calculating profit/loss for each ticker')\n",
        "    pnl_all = {}\n",
        "    for tic in tics_all:\n",
        "        pnl_t = []\n",
        "        tic_data = perf_dict[tic]\n",
        "        init_values = tic_data['init_values']\n",
        "        acts = tic_data['actions']\n",
        "        prices = tic_data['prices']\n",
        "        cs = np.cumsum(acts)\n",
        "        args_s = [i + 1 for i in range(len(cs) - 1) if cs[i + 1] < cs[i]]\n",
        "        # tic actions with no sales\n",
        "        if not args_s:\n",
        "            pnl = complete_calc_buyonly(acts, prices, init_values)\n",
        "            pnl_all[tic] = pnl\n",
        "            continue\n",
        "        # copy acts: acts_rev will be revised based on closing/reducing init positions\n",
        "        pnl_all = execute_position_sales(tic,acts,prices,args_s,pnl_all)\n",
        "\n",
        "    return pnl_all\n",
        "\n",
        "\n",
        "def complete_calc_buyonly(actions, prices, init_values):\n",
        "    # calculate final pnl for each ticker assuming no sales\n",
        "    fnl_price = prices[-1]\n",
        "    final_values = np.multiply(fnl_price, actions)\n",
        "    pnl = np.subtract(final_values, init_values)\n",
        "    return pnl\n",
        "\n",
        "\n",
        "def execute_position_sales(tic,acts,prices,args_s,pnl_all):\n",
        "  # calculate final pnl for each ticker with sales\n",
        "    pnl_t = []\n",
        "    acts_rev = acts.copy()\n",
        "    # location of sales transactions\n",
        "    for s in args_s:  # s is scaler\n",
        "        # price_s = [prices[s]]\n",
        "        act_s = [acts_rev[s]]\n",
        "        args_b = [i for i in range(s) if acts_rev[i] > 0]\n",
        "        prcs_init_trades = prices[args_b]\n",
        "        acts_init_trades = acts_rev[args_b]\n",
        "  \n",
        "        # update actions for sales\n",
        "        # reduce/eliminate init values through trades\n",
        "        # always start with earliest purchase that has not been closed through sale\n",
        "        # selectors for purchase and sales trades\n",
        "        # find earliest remaining purchase\n",
        "        arg_sel = min(args_b)\n",
        "        # sel_s = len(acts_trades) - 1\n",
        "\n",
        "        # closing part/all of earliest init trade not yet closed\n",
        "        # sales actions are negative\n",
        "        # in this test case, abs_val of init and sales share counts are same\n",
        "        # zero-out sales actions\n",
        "        # market value of sale\n",
        "        # max number of shares to be closed: may be less than # originally purchased\n",
        "        acts_shares = min(abs(act_s.pop()), acts_rev[arg_sel])\n",
        "\n",
        "        # mv of shares when purchased\n",
        "        mv_p = abs(acts_shares * prices[arg_sel])\n",
        "        # mv of sold shares\n",
        "        mv_s = abs(acts_shares * prices[s])\n",
        "\n",
        "        # calc pnl\n",
        "        pnl = mv_s - mv_p\n",
        "        # reduce init share count\n",
        "        # close all/part of init purchase\n",
        "        acts_rev[arg_sel] -= acts_shares\n",
        "        acts_rev[s] += acts_shares\n",
        "        # calculate pnl for trade\n",
        "        # value of associated purchase\n",
        "        \n",
        "        # find earliest non-zero positive act in acts_revs\n",
        "        pnl_t.append(pnl)\n",
        "    \n",
        "    pnl_op = calc_pnl_for_open_positions(acts_rev, prices)\n",
        "    #pnl_op is list\n",
        "    # add pnl_op results (if any) to pnl_t (both lists)\n",
        "    pnl_t.extend(pnl_op)\n",
        "    #print(f'Total pnl for {tic}: {np.sum(pnl_t)}')\n",
        "    pnl_all[tic] = np.array(pnl_t)\n",
        "    return pnl_all\n",
        "\n",
        "\n",
        "def calc_pnl_for_open_positions(acts,prices):\n",
        "    # identify any positive share values after accounting for sales\n",
        "    pnl = []\n",
        "    fp = prices[-1] # last price\n",
        "    open_pos_arg = np.argwhere(acts>0)\n",
        "    if len(open_pos_arg)==0:return pnl # no open positions\n",
        "\n",
        "    mkt_vals_open = np.multiply(acts[open_pos_arg], prices[open_pos_arg])\n",
        "    # mkt val at end of testing period\n",
        "    # treat as trades for purposes of calculating pnl at end of testing period\n",
        "    mkt_vals_final = np.multiply(fp, acts[open_pos_arg])\n",
        "    pnl_a = np.subtract(mkt_vals_final, mkt_vals_open)\n",
        "    #convert to list\n",
        "    pnl = [i[0] for i in pnl_a.tolist()]\n",
        "    #print(f'Market value of open positions at end of testing {pnl}')\n",
        "    return pnl\n",
        "\n",
        "\n",
        "def calc_trade_perf(pnl_d):\n",
        "    # calculate trade performance metrics\n",
        "    perf_results = {}\n",
        "    for t,pnl in pnl_d.items():\n",
        "        wins = pnl[pnl>0]  # total val\n",
        "        losses = pnl[pnl<0]\n",
        "        n_wins = len(wins)\n",
        "        n_losses = len(losses)\n",
        "        n_trades = n_wins + n_losses\n",
        "        wins_val = np.sum(wins)\n",
        "        losses_val = np.sum(losses)\n",
        "        wins_avg = 0 if n_wins==0 else np.mean(wins)\n",
        "        #print(f'{t} n_wins: {n_wins} n_losses: {n_losses}')\n",
        "        losses_avg = 0 if n_losses==0 else np.mean(losses)\n",
        "        d = {'# trades':n_trades,'# wins':n_wins,'# losses':n_losses,\n",
        "             'wins total value':wins_val, 'wins avg value':wins_avg,\n",
        "             'losses total value':losses_val, 'losses avg value':losses_avg,}\n",
        "        perf_results[t] = d\n",
        "    return perf_results"
      ],
      "metadata": {
        "id": "QydaFexDX5BQ"
      },
      "execution_count": 204,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Tuning hyperparameters using Optuna"
      ],
      "metadata": {
        "id": "c_AT1dsuZApj"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def sample_ddpg_params(trial:optuna.Trial):\n",
        "  # Size of the replay buffer\n",
        "  buffer_size = trial.suggest_categorical(\"buffer_size\", [int(1e4), int(1e5), int(1e6)])\n",
        "  learning_rate = trial.suggest_loguniform(\"learning_rate\", 1e-5, 1)\n",
        "  batch_size = trial.suggest_categorical(\"batch_size\", [32, 64, 128, 256, 512])\n",
        "  \n",
        "  return {\"buffer_size\": buffer_size,\n",
        "          \"learning_rate\":learning_rate,\n",
        "          \"batch_size\":batch_size}"
      ],
      "metadata": {
        "id": "fKGkKHV7Y9AA"
      },
      "execution_count": 205,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Set Variables\n",
        "## Fixed\n",
        "tpm_hist = {}  # record tp metric values for trials\n",
        "tp_metric = 'avgwl'  # specified trade_param_metric: ratio avg value win/loss\n",
        "## Settable by User\n",
        "n_trials = 100  # number of HP optimization runs\n",
        "total_timesteps = 2000 # per HP optimization run\n",
        "## Logging callback params\n",
        "lc_threshold=1e-5\n",
        "lc_patience=15\n",
        "lc_trial_number=5"
      ],
      "metadata": {
        "id": "N6blMWpz-pR0"
      },
      "execution_count": 206,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "OPTIONAL CODE FOR SAMPLING HYPERPARAMETERS\n",
        "\n",
        "Replace current call in function objective with\n",
        "\n",
        "hyperparameters = sample_ddpg_params_all(trial)"
      ],
      "metadata": {
        "id": "fHET-odKZShg"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def sample_ddpg_params_all(trial:optuna.Trial,\n",
        "                           # fixed values from previous study\n",
        "                           learning_rate=0.0103,\n",
        "                           batch_size=128,\n",
        "                           buffer_size=int(1e6)):\n",
        "\n",
        "    gamma = trial.suggest_categorical(\"gamma\", [0.94, 0.96, 0.98])\n",
        "    # Polyak coeff\n",
        "    tau = trial.suggest_categorical(\"tau\", [0.08, 0.1, 0.12])\n",
        "\n",
        "    train_freq = trial.suggest_categorical(\"train_freq\", [512,768,1024])\n",
        "    gradient_steps = train_freq\n",
        "    \n",
        "    noise_type = trial.suggest_categorical(\"noise_type\", [\"ornstein-uhlenbeck\", \"normal\", None])\n",
        "    noise_std = trial.suggest_categorical(\"noise_std\", [.1,.2,.3] )\n",
        "\n",
        "    # NOTE: Add \"verybig\" to net_arch when tuning HER (see TD3)\n",
        "    net_arch = trial.suggest_categorical(\"net_arch\", [\"small\", \"big\"])\n",
        "    # activation_fn = trial.suggest_categorical('activation_fn', [nn.Tanh, nn.ReLU, nn.ELU, nn.LeakyReLU])\n",
        "\n",
        "    net_arch = {\n",
        "        \"small\": [64, 64],\n",
        "        \"medium\": [256, 256],\n",
        "        \"big\": [512, 512],\n",
        "    }[net_arch]\n",
        "  \n",
        "    hyperparams = {\n",
        "        \"batch_size\": batch_size,\n",
        "        \"buffer_size\": buffer_size,\n",
        "        \"gamma\": gamma,\n",
        "        \"gradient_steps\": gradient_steps,\n",
        "        \"learning_rate\": learning_rate,\n",
        "        \"tau\": tau,\n",
        "        \"train_freq\": train_freq,\n",
        "        #\"noise_std\": noise_std,\n",
        "        #\"noise_type\": noise_type,\n",
        "        \n",
        "        \"policy_kwargs\": dict(net_arch=net_arch)\n",
        "    }\n",
        "    return hyperparams"
      ],
      "metadata": {
        "id": "htLdZHKTZKPJ"
      },
      "execution_count": 207,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def sample_ddpg_params_all(trial:optuna.Trial,\n",
        "                           # fixed values from previous study\n",
        "                           learning_rate=0.0103,\n",
        "                           batch_size=128,\n",
        "                           buffer_size=int(1e6)):\n",
        "\n",
        "    gamma = trial.suggest_categorical(\"gamma\", [0.90, 0.92, 0.94, 0.96, 0.98])\n",
        "    # Polyak coeff\n",
        "    tau = trial.suggest_categorical(\"tau\", [0.02,0.04,0.06,0.08, 0.1, 0.12])\n",
        "\n",
        "    train_freq = trial.suggest_categorical(\"train_freq\", [512,768,1024])\n",
        "    gradient_steps = train_freq\n",
        "    \n",
        "    noise_type = trial.suggest_categorical(\"noise_type\", [\"ornstein-uhlenbeck\", \"normal\", None])\n",
        "    noise_std = trial.suggest_categorical(\"noise_std\", [.1,.2,.3,.4,.5] )\n",
        "\n",
        "    # NOTE: Add \"verybig\" to net_arch when tuning HER (see TD3)\n",
        "    net_arch = trial.suggest_categorical(\"net_arch\", [\"small\", \"big\"])\n",
        "    # activation_fn = trial.suggest_categorical('activation_fn', [nn.Tanh, nn.ReLU, nn.ELU, nn.LeakyReLU])\n",
        "\n",
        "    net_arch = {\n",
        "        \"small\": [32, 32],\n",
        "        \"medium\": [64, 64],\n",
        "        \"big\": [256, 256],\n",
        "    }[net_arch]\n",
        "  \n",
        "    hyperparams = {\n",
        "        \"batch_size\": batch_size,\n",
        "        \"buffer_size\": buffer_size,\n",
        "        \"gamma\": gamma,\n",
        "        \"gradient_steps\": gradient_steps,\n",
        "        \"learning_rate\": learning_rate,\n",
        "        \"tau\": tau,\n",
        "        \"train_freq\": train_freq,\n",
        "        #\"noise_std\": noise_std,\n",
        "        #\"noise_type\": noise_type,\n",
        "        \n",
        "        \"policy_kwargs\": dict(net_arch=net_arch)\n",
        "    }\n",
        "    return hyperparams"
      ],
      "metadata": {
        "id": "dJYdh-E4ZQKP"
      },
      "execution_count": 208,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Callbacks\n",
        "\n",
        "\n",
        "1. The callback will terminate if the improvement margin is below certain point\n",
        "2. It will terminate after certain number of trial_number are reached, not before that\n",
        "3. It will hold its patience to reach the threshold"
      ],
      "metadata": {
        "id": "mWEoJMPLZY4o"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class LoggingCallback:\n",
        "    def __init__(self,threshold,trial_number,patience):\n",
        "      '''\n",
        "      threshold:int tolerance for increase in objective\n",
        "      trial_number: int Prune after minimum number of trials\n",
        "      patience: int patience for the threshold\n",
        "      '''\n",
        "      self.threshold = threshold\n",
        "      self.trial_number  = trial_number\n",
        "      self.patience = patience\n",
        "      print(f'Callback threshold {self.threshold}, \\\n",
        "            trial_number {self.trial_number}, \\\n",
        "            patience {self.patience}')\n",
        "      self.cb_list = [] #Trials list for which threshold is reached\n",
        "    def __call__(self,study:optuna.study, frozen_trial:optuna.Trial):\n",
        "      #Setting the best value in the current trial\n",
        "      study.set_user_attr(\"previous_best_value\", study.best_value)\n",
        "      \n",
        "      #Checking if the minimum number of trials have pass\n",
        "      if frozen_trial.number >self.trial_number:\n",
        "          previous_best_value = study.user_attrs.get(\"previous_best_value\",None)\n",
        "          #Checking if the previous and current objective values have the same sign\n",
        "          if previous_best_value * study.best_value >=0:\n",
        "              #Checking for the threshold condition\n",
        "              if abs(previous_best_value-study.best_value) < self.threshold: \n",
        "                  self.cb_list.append(frozen_trial.number)\n",
        "                  #If threshold is achieved for the patience amount of time\n",
        "                  if len(self.cb_list)>self.patience:\n",
        "                      print('The study stops now...')\n",
        "                      print('With number',frozen_trial.number ,'and value ',frozen_trial.value)\n",
        "                      print('The previous and current best values are {} and {} respectively'\n",
        "                              .format(previous_best_value, study.best_value))\n",
        "                      study.stop()"
      ],
      "metadata": {
        "id": "He7GTZZUZWP8"
      },
      "execution_count": 209,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#Calculate the Sharpe ratio\n",
        "#This is our objective for tuning\n",
        "def calculate_sharpe(df):\n",
        "  df['daily_return'] = df['account_value'].pct_change(1)\n",
        "  if df['daily_return'].std() !=0:\n",
        "    sharpe = (252**0.5)*df['daily_return'].mean()/ \\\n",
        "          df['daily_return'].std()\n",
        "    return sharpe\n",
        "  else:\n",
        "    return 0"
      ],
      "metadata": {
        "id": "s9orU-WplQ70"
      },
      "execution_count": 210,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from IPython.display import clear_output\n",
        "import sys   \n",
        "\n",
        "os.makedirs(\"models\",exist_ok=True)\n",
        "\n",
        "def objective(trial:optuna.Trial):\n",
        "  #Trial will suggest a set of hyperparamters from the specified range\n",
        "\n",
        "  # Optional to optimize larger set of parameters\n",
        "  # hyperparameters = sample_ddpg_params_all(trial)\n",
        "  \n",
        "  # Optimize buffer size, batch size, learning rate\n",
        "  hyperparameters = sample_ddpg_params_all(trial)\n",
        "  print(f'Hyperparameters from objective: {hyperparameters.keys()}')\n",
        "  policy_kwargs = None  # default\n",
        "  if 'policy_kwargs' in hyperparameters.keys():\n",
        "    policy_kwargs = hyperparameters['policy_kwargs']\n",
        "    del hyperparameters['policy_kwargs']\n",
        "    print(f'Policy keyword arguments {policy_kwargs}')\n",
        "  model_ddpg = agent.get_model(\"ddpg\",\n",
        "                               policy_kwargs = policy_kwargs,\n",
        "                               model_kwargs = hyperparameters )\n",
        "  \n",
        "  #You can increase it for better comparison\n",
        "  trained_ddpg = agent.train_model(model=model_ddpg,\n",
        "                                   tb_log_name=\"ddpg\",\n",
        "                                   total_timesteps=total_timesteps)\n",
        "  trained_ddpg.save('models/ddpg_{}.pth'.format(trial.number))\n",
        "  clear_output(wait=True)\n",
        "  \n",
        "  #For the given hyperparamters, determine the account value in the trading period\n",
        "  df_account_value, df_actions = DRLAgent.DRL_prediction(\n",
        "    model=trained_ddpg, \n",
        "    environment = e_trade_gym)\n",
        " \n",
        "  # Calculate trade performance metric\n",
        "  # Currently ratio of average win and loss market values\n",
        "  #tpm = calc_trade_perf_metric(df_actions,init_trade_data,tp_metric)\n",
        "  tpm = calculate_sharpe(df_account_value)\n",
        "  return tpm\n",
        "\n",
        "#Create a study object and specify the direction as 'maximize'\n",
        "#As you want to maximize sharpe\n",
        "#Pruner stops not promising iterations\n",
        "#Use a pruner, else you will get error related to divergence of model\n",
        "#You can also use Multivariate samplere\n",
        "#sampler = optuna.samplers.TPESampler(multivarite=True,seed=42)\n",
        "sampler = optuna.samplers.TPESampler()\n",
        "\n",
        "study = optuna.create_study(study_name=\"ddpg_study\",direction='maximize',\n",
        "                            sampler = sampler, pruner=optuna.pruners.HyperbandPruner())\n",
        "\n",
        "logging_callback = LoggingCallback(threshold=lc_threshold,\n",
        "                                   patience=lc_patience,\n",
        "                                   trial_number=lc_trial_number)\n",
        "#You can increase the n_trials for a better search space scanning\n",
        "study.optimize(objective, n_trials=n_trials,catch=(ValueError,),callbacks=[logging_callback])"
      ],
      "metadata": {
        "id": "b6EiwiiIZpRU",
        "outputId": "8fc60285-8115-4498-fc8a-73ccae95ee51",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "execution_count": 211,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\u001b[32m[I 2023-05-02 20:47:36,927]\u001b[0m A new study created in memory with name: ddpg_study\u001b[0m\n",
            "<ipython-input-199-175aa58dc7db>:340: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Callback threshold 1e-05,             trial_number 5,             patience 15\n",
            "Hyperparameters from objective: dict_keys(['batch_size', 'buffer_size', 'gamma', 'gradient_steps', 'learning_rate', 'tau', 'train_freq', 'policy_kwargs'])\n",
            "Policy keyword arguments {'net_arch': [256, 256]}\n",
            "{'batch_size': 128, 'buffer_size': 1000000, 'gamma': 0.9, 'gradient_steps': 768, 'learning_rate': 0.0103, 'tau': 0.12, 'train_freq': 768}\n",
            "Using cuda device\n",
            "124 |  avg_price 168.94795036315918 loss -44.62763786315918\n",
            "47 |  avg_price 66.15374894575639 loss -19.112702803178266\n",
            "283 |  avg_price 371.0545654296875 loss -88.1309814453125\n",
            "47 |  avg_price 60.10480846058239 loss -12.750510822642937\n",
            "137 |  avg_price 25.942731221516926 Profit 110.7114741007487\n",
            "155 |  avg_price 474.82176208496094 loss -319.4388732910156\n",
            "116 |  avg_price 24.92083231608073 Profit 91.03916676839192\n",
            "144 |  avg_price 187.0030080159505 loss -42.89142049153645\n",
            "51 |  avg_price 64.11990547180176 loss -13.149023056030266\n",
            "144 |  avg_price 232.86361040387834 loss -89.14130510602678\n",
            "189 |  avg_price 84.52798868815093 Profit 104.93201802571626\n",
            "128 |  avg_price 37.53933544158936 Profit 90.20385150909424\n",
            "179 |  avg_price 56.462218064528244 Profit 122.6609356219952\n",
            "50 |  avg_price 154.70894241333002 loss -104.83502960205072\n",
            "140 |  avg_price 181.34879193987166 loss -41.80879865373885\n",
            "127 |  avg_price 242.70999438946058 loss -115.96361072246837\n",
            "123 |  avg_price 73.37981672689946 Profit 49.8642628629443\n",
            "119 |  avg_price 149.5012252579874 loss -30.03402402507723\n",
            "163 |  avg_price 7.5689849853515625 Profit 155.86268615722656\n",
            "113 |  avg_price 192.2205047607422 loss -79.0332565307617\n",
            "85 |  avg_price 118.04589515603999 loss -32.70918708719233\n",
            "157 |  avg_price 211.95501708984375 loss -55.415130615234375\n",
            "288 |  avg_price 396.0382289180049 loss -108.2782191523799\n",
            "82 |  avg_price 119.86226851519416 loss -37.702638693416816\n",
            "116 |  avg_price 153.22198486328125 loss -36.77198791503905\n",
            "283 |  avg_price 381.24178196833685 loss -98.12178685114935\n",
            "103 |  avg_price 147.90127262924656 loss -44.81055912827\n",
            "262 |  avg_price 560.507225485409 loss -298.17723891314336\n",
            "115 |  avg_price 190.99657287597657 loss -75.72657623291016\n",
            "180 |  avg_price 227.4321765170071 loss -47.31077759122584\n",
            "105 |  avg_price 61.325502137880015 Profit 43.73300768878014\n",
            "51 |  avg_price 63.83236848896947 loss -13.18161546772923\n",
            "96 |  avg_price 134.02395838687292 loss -37.64448756167759\n",
            "70 |  avg_price 171.98925823635525 loss -101.77189678615994\n",
            "261 |  avg_price 417.99348885672435 loss -156.9947705950056\n",
            "231 |  avg_price 307.6305229550316 loss -76.62052844819564\n",
            "49 |  avg_price 216.0787463717989 loss -167.05367999606648\n",
            "112 |  avg_price 142.96062563094625 loss -31.39284608993063\n",
            "33 |  avg_price 42.07747169656957 loss -8.610648118688708\n",
            "72 |  avg_price 139.39370100838798 loss -67.12213271004813\n",
            "161 |  avg_price 434.54215494791646 loss -273.997370402018\n",
            "87 |  avg_price 3048.584327697754 loss -2961.784423828125\n",
            "68 |  avg_price 23.150831604003905 Profit 44.898110961914064\n",
            "155 |  avg_price 244.56383277629985 loss -89.72383643840922\n",
            "85 |  avg_price 331.6643478393551 loss -246.2221328735348\n",
            "92 |  avg_price 690.5737330118809 loss -598.7637354532872\n",
            "41 |  avg_price 97.59722844670324 loss -56.23743383000402\n",
            "159 |  avg_price 202.39787292480477 loss -43.60292053222665\n",
            "94 |  avg_price 118.5749629758439 loss -24.59625661598062\n",
            "95 |  avg_price 1384.322732925415 loss -1288.9811267852783\n",
            "157 |  avg_price 200.09494610222018 loss -43.23273296745455\n",
            "36 |  avg_price 47.56360499549458 loss -11.557467147594188\n",
            "73 |  avg_price 106.79640484881658 loss -34.185359011414235\n",
            "165 |  avg_price 361.2691642947311 loss -195.7928611941452\n",
            "130 |  avg_price 603.8652064005533 loss -474.2552057902018\n",
            "124 |  avg_price 163.62727310812684 loss -39.327270056369045\n",
            "30 |  avg_price 112.83752337369052 loss -82.38768282803622\n",
            "95 |  avg_price 302.1079084608288 loss -207.09790632459834\n",
            "154 |  avg_price 212.04614562988283 loss -58.14447326660158\n",
            "117 |  avg_price 156.5811464566711 loss -39.71401236243281\n",
            "100 |  avg_price 184.27080605246803 loss -84.14617226340553\n",
            "216 |  avg_price 443.94776662190753 loss -228.2164586385091\n",
            "149 |  avg_price 229.54979891390414 loss -80.59536165804477\n",
            "43 |  avg_price 58.085593992663966 loss -14.72547512669717\n",
            "62 |  avg_price 100.40926876068116 loss -38.878762626647955\n",
            "148 |  avg_price 313.4521802400286 loss -165.2548383210833\n",
            "45 |  avg_price 104.68350977775374 loss -59.20045084831038\n",
            "143 |  avg_price 223.13375854492188 loss -80.6087646484375\n",
            "44 |  avg_price 207.79524421691895 loss -164.2578296661377\n",
            "70 |  avg_price 92.72943180512617 loss -22.28588932465742\n",
            "93 |  avg_price 1323.9061126708966 loss -1231.1847457885724\n",
            "34 |  avg_price 666.4518844604493 loss -632.9218284606934\n",
            "145 |  avg_price 976.0224273681645 loss -830.8253143310551\n",
            "122 |  avg_price 300.65316452518584 loss -178.33144058719756\n",
            "108 |  avg_price 145.64990856772974 loss -37.26616528159694\n",
            "46 |  avg_price 65.20372827847798 loss -19.011898597081505\n",
            "98 |  avg_price 670.9962463378906 loss -573.286247253418\n",
            "50 |  avg_price 83.66760902843257 loss -33.71378212413568\n",
            "38 |  avg_price 51.614508310953774 loss -13.489874521891274\n",
            "80 |  avg_price 173.50450249438015 loss -93.1947261882278\n",
            "149 |  avg_price 571.4994310651507 loss -422.309810093471\n",
            "43 |  avg_price 201.96772605494448 loss -159.35198051051088\n",
            "88 |  avg_price 197.9965178171792 loss -109.55376116434718\n",
            "139 |  avg_price 214.9862412539396 loss -75.63623515042397\n",
            "32 |  avg_price 192.29345684051515 loss -160.13639430999757\n",
            "86 |  avg_price 153.17628645896906 loss -67.19628310203547\n",
            "106 |  avg_price 137.32944290023045 loss -31.41930190901951\n",
            "111 |  avg_price 313.38929301294786 loss -201.89638834986192\n",
            "134 |  avg_price 340.29344177246094 loss -206.73399353027344\n",
            "44 |  avg_price 55.51582431793213 loss -11.466912269592292\n",
            "66 |  avg_price 418.0854008992513 loss -352.2171223958333\n",
            "106 |  avg_price 184.58597826957703 loss -78.96597552299498\n",
            "47 |  avg_price 235.01336956024164 loss -187.81880092620844\n",
            "126 |  avg_price 350.3901953898683 loss -224.32858802902848\n",
            "152 |  avg_price 1406.2497850501024 loss -1254.7458482825243\n",
            "34 |  avg_price 195.5594677780614 loss -161.32659195408678\n",
            "128 |  avg_price 161.92751027576963 loss -33.96186543934385\n",
            "90 |  avg_price 136.79525381059796 loss -46.63694387407452\n",
            "187 |  avg_price 295.85153307233537 loss -109.05183519635881\n",
            "38 |  avg_price 148.87779481070382 loss -110.59823090689522\n",
            "35 |  avg_price 1098.8297290802002 loss -1063.9628047943115\n",
            "118 |  avg_price 183.29255564371744 loss -64.80956156412759\n",
            "86 |  avg_price 174.2063271416558 loss -88.07894677056206\n",
            "159 |  avg_price 219.0141406691218 loss -60.284144941582724\n",
            "97 |  avg_price 134.9338567256927 loss -37.99872946739191\n",
            "60 |  avg_price 95.02695860686124 loss -34.73246016325772\n",
            "101 |  avg_price 277.3100910186766 loss -176.580087661743\n",
            "87 |  avg_price 20.790050895846544 Profit 66.27206763442689\n",
            "36 |  avg_price 221.88090066348806 loss -185.50609902774588\n",
            "37 |  avg_price 370.0151286878084 loss -333.35969121832596\n",
            "182 |  avg_price 658.7136667542541 loss -476.58985693980094\n",
            "50 |  avg_price 289.58816688939123 loss -239.14629143162756\n",
            "98 |  avg_price 183.3982686360676 loss -85.7500295003254\n",
            "146 |  avg_price 313.205916995094 loss -167.20591699509401\n",
            "96 |  avg_price 143.1985908235822 loss -46.7985892977033\n",
            "89 |  avg_price 53.72805776940768 Profit 35.12710580481107\n",
            "119 |  avg_price 1336.7510948181152 loss -1217.8039207458496\n",
            "102 |  avg_price 282.8954979391661 loss -181.2298027487364\n",
            "93 |  avg_price 2561.7267570495605 loss -2468.465908050537\n",
            "232 |  avg_price 317.5271359398252 loss -85.10597810291111\n",
            "39 |  avg_price 190.25936871483214 loss -151.63747387840635\n",
            "86 |  avg_price 216.87035802205403 loss -130.6474728902181\n",
            "152 |  avg_price 244.02596770739947 loss -91.7459689281026\n",
            "63 |  avg_price 130.20122423171992 loss -67.57396974563594\n",
            "183 |  avg_price 662.5831380383725 loss -479.62899069950527\n",
            "85 |  avg_price 130.9848198812516 loss -46.41008996181802\n",
            "107 |  avg_price 271.2266766092049 loss -163.76176419465412\n",
            "49 |  avg_price 131.53537416458124 loss -82.38316011428827\n",
            "122 |  avg_price 196.77076813668916 loss -74.90435883493136\n",
            "93 |  avg_price 448.36215373447965 loss -355.6611420767648\n",
            "239 |  avg_price 389.8213717142741 loss -150.75688807169604\n",
            "40 |  avg_price 73.88082610919912 loss -33.72283279254873\n",
            "74 |  avg_price 221.75124767848425 loss -147.401088987078\n",
            "131 |  avg_price 309.0401166643415 loss -178.3401197160993\n",
            "35 |  avg_price 563.3277737299601 loss -528.7047306696574\n",
            "60 |  avg_price 104.02508805014868 loss -44.348597266457276\n",
            "167 |  avg_price 1810.5748659021722 loss -1643.9195924646722\n",
            "38 |  avg_price 173.42469959259034 loss -135.03575687408448\n",
            "45 |  avg_price 72.74026890290091 loss -27.738945202949736\n",
            "140 |  avg_price 649.4852548414657 loss -509.4510293775985\n",
            "125 |  avg_price 437.7331992062655 loss -312.70273503390223\n",
            "123 |  avg_price 168.79417226772117 loss -45.67793844203757\n",
            "106 |  avg_price 137.13652451459106 loss -31.567341164981684\n",
            "35 |  avg_price 114.26370927764148 loss -78.87200471831531\n",
            "71 |  avg_price 163.35222376947817 loss -92.6614355004352\n",
            "134 |  avg_price 7156.968521118168 loss -7022.658523559574\n",
            "36 |  avg_price 130.98511357093926 loss -95.3681549527752\n",
            "65 |  avg_price 98.07222552921459 loss -32.69441791202709\n",
            "94 |  avg_price 284.16435841151633 loss -190.28436115809836\n",
            "87 |  avg_price 7.174079677036753 Profit 80.2910463605609\n",
            "49 |  avg_price 133.24213499589393 loss -84.08083815141151\n",
            "88 |  avg_price 920.5615603129069 loss -832.955145517985\n",
            "220 |  avg_price 286.15837496243995 loss -66.16110628568214\n",
            "47 |  avg_price 62.437221200125556 loss -15.922252328055244\n",
            "149 |  avg_price 193.87448764377172 loss -45.10448337131078\n",
            "179 |  avg_price 449.0097887443775 loss -270.5058824943775\n",
            "126 |  avg_price 718.1983562967051 loss -592.0475994607676\n",
            "37 |  avg_price 682.4932441711426 loss -645.0738945007324\n",
            "84 |  avg_price 131.9093706062041 loss -47.791191284915044\n",
            "145 |  avg_price 280.0529940623158 loss -134.98298673809705\n",
            "70 |  avg_price 130.13136940002437 loss -59.69638710021968\n",
            "155 |  avg_price 552.7346463325692 loss -397.67242099077237\n",
            "53 |  avg_price 115.55910419773406 loss -62.519755595439136\n",
            "42 |  avg_price 138.99001216888428 loss -96.59994411468506\n",
            "94 |  avg_price 158.11539067720113 loss -64.08008183931051\n",
            "238 |  avg_price 297.7963424258762 loss -60.016404681735565\n",
            "48 |  avg_price 61.64410000536815 loss -13.190303618649402\n",
            "147 |  avg_price 380.3684432085823 loss -233.49844809139478\n",
            "118 |  avg_price 175.6857357025146 loss -57.70031547546381\n",
            "106 |  avg_price 162.39595131252136 loss -56.18079475734558\n",
            "38 |  avg_price 135.31581566550514 loss -97.49268791892311\n",
            "155 |  avg_price 272.37589845818997 loss -117.82589540643215\n",
            "38 |  avg_price 315.87230417002803 loss -278.30398675669795\n",
            "172 |  avg_price 423.76725552876763 loss -251.66894315083795\n",
            "188 |  avg_price 534.129453558671 loss -346.1093577334757\n",
            "135 |  avg_price 275.955956966319 loss -140.92756035987367\n",
            "52 |  avg_price 342.7199783325193 loss -290.82497406005837\n",
            "45 |  avg_price 151.7759637451172 loss -107.11645599365235\n",
            "132 |  avg_price 202.57867384690505 loss -70.65119887131911\n",
            "104 |  avg_price 141.49973689185245 loss -37.99973689185245\n",
            "90 |  avg_price 270.85514736175537 loss -181.11517238616943\n",
            "99 |  avg_price 161.071397988193 loss -61.77354642569301\n",
            "49 |  avg_price 63.77547359466553 loss -15.06701946258545\n",
            "39 |  avg_price 79.85124221214882 loss -41.13278975853554\n",
            "141 |  avg_price 205.70461385390337 loss -64.70461385390337\n",
            "158 |  avg_price 442.155326207479 loss -284.4453194936118\n",
            "102 |  avg_price 49.204836572919525 Profit 52.7116978509086\n",
            "55 |  avg_price 110.05814212706026 loss -54.96625369932589\n",
            "45 |  avg_price 114.18383595784505 loss -69.59724995930989\n",
            "259 |  avg_price 732.7413143581814 loss -473.430431789822\n",
            "80 |  avg_price 131.02020538684934 loss -50.54222381946653\n",
            "103 |  avg_price 157.32981927054263 loss -54.04162652151919\n",
            "72 |  avg_price 143.06126182023866 loss -71.25972526018006\n",
            "185 |  avg_price 765.1148953645129 loss -579.9402737824817\n",
            "130 |  avg_price 229.7078307469686 loss -100.11931451161703\n",
            "38 |  avg_price 78.45637063419117 loss -40.326033872716565\n",
            "157 |  avg_price 408.4702888771341 loss -251.41029131854037\n",
            "165 |  avg_price 451.6454354052271 loss -286.29054648921147\n",
            "95 |  avg_price 44.911371912274994 Profit 50.27707413264688\n",
            "43 |  avg_price 109.46597772912133 loss -66.53817087487329\n",
            "112 |  avg_price 542.5821584065765 loss -430.4610417683929\n",
            "93 |  avg_price 197.87636245727538 loss -104.74615158081053\n",
            "286 |  avg_price 878.7247118268695 loss -592.6392015729632\n",
            "50 |  avg_price 71.38473622939166 loss -21.825371910544007\n",
            "39 |  avg_price 495.05557332720076 loss -456.32008062090193\n",
            "107 |  avg_price 1014.0136820475254 loss -907.3836847941075\n",
            "195 |  avg_price 528.4688482040016 loss -333.0876226180641\n",
            "131 |  avg_price 314.19205709604114 loss -183.44295736459583\n",
            "72 |  avg_price 17.19300019113641 Profit 55.06294310720344\n",
            "81 |  avg_price 130.67366224719632 loss -49.984376968876006\n",
            "54 |  avg_price 138.28670544094504 loss -84.53896374172629\n",
            "42 |  avg_price 144.88961285811203 loss -102.48131436568039\n",
            "111 |  avg_price 173.76409808072188 loss -62.47352496060468\n",
            "136 |  avg_price 201.50360247963354 loss -65.17360064857886\n",
            "151 |  avg_price 244.54406517672257 loss -93.98406761812882\n",
            "126 |  avg_price 525.7180875142416 loss -399.2809155782064\n",
            "164 |  avg_price 15335.737037658677 loss -15171.372200012192\n",
            "190 |  avg_price 472.665480728988 loss -282.182326432113\n",
            "125 |  avg_price 316.25615393597144 loss -191.01858221966285\n",
            "82 |  avg_price 2979.2293090820312 loss -2897.5777130126953\n",
            "95 |  avg_price 170.87137692216513 loss -75.89366238359091\n",
            "79 |  avg_price 136.66685937952113 loss -57.46631311487269\n",
            "138 |  avg_price 294.7504415204448 loss -157.01043602728072\n",
            "38 |  avg_price 115.98726686617223 loss -78.0359652914652\n",
            "38 |  avg_price 3465.7291107177734 loss -3427.546100616455\n",
            "107 |  avg_price 165.5724739195808 loss -58.9901527525886\n",
            "101 |  avg_price 133.40656017121813 loss -32.216557729811896\n",
            "54 |  avg_price 4618.7246398925745 loss -4564.904239654537\n",
            "82 |  avg_price 205.6871234790699 loss -123.98933142584724\n",
            "132 |  avg_price 457.0887575149536 loss -325.4024324417114\n",
            "143 |  avg_price 64.1317715364344 Profit 78.51576569501091\n",
            "86 |  avg_price 165.69163643973215 loss -79.21443307059153\n",
            "96 |  avg_price 159.53090801239014 loss -63.52681865692138\n",
            "50 |  avg_price 76.90282411575318 loss -27.214534473419192\n",
            "39 |  avg_price 84.71186349198625 loss -45.71253106400773\n",
            "78 |  avg_price 211.31074042942205 loss -133.05061622287909\n",
            "106 |  avg_price 331.57590166727704 loss -225.36590258280438\n",
            "203 |  avg_price 638.3925077793964 loss -435.35162948349796\n",
            "53 |  avg_price 102.57393853207849 loss -49.21672265073083\n",
            "141 |  avg_price 456.5925333658854 loss -315.88951517740884\n",
            "92 |  avg_price 153.17579138456887 loss -60.85518133818215\n",
            "112 |  avg_price 152.1520765395391 loss -39.94342022850395\n",
            "40 |  avg_price 107.55051471341041 loss -68.03082706082252\n",
            "70 |  avg_price 42.42508349699133 Profit 27.501987120684454\n",
            "50 |  avg_price 75.62409055800666 loss -25.669351941063297\n",
            "40 |  avg_price 208.32880496978754 loss -168.51314258575434\n",
            "331 |  avg_price 141.02000427246094 Profit 189.7718963623047\n",
            "125 |  avg_price 41.05308846866383 Profit 83.98182364071117\n",
            "109 |  avg_price 43.01923794216615 Profit 66.21203189425964\n",
            "76 |  avg_price 30.758699857271633 Profit 45.31875375600961\n",
            "228 |  avg_price 113.80142339070653 Profit 113.83224360148097\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-199-175aa58dc7db>:340: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "126 |  avg_price 59.26574221524327 Profit 66.28474148837\n",
            "47 |  avg_price 64.03195308816844 loss -16.668946496371554\n",
            "44 |  avg_price 21.281673431396484 Profit 23.024608612060547\n",
            "57 |  avg_price 107.34513092040993 loss -50.65243530273415\n",
            "221 |  avg_price 123.1882280622215 Profit 97.88219124930194\n",
            "51 |  avg_price 90.0434627532959 loss -39.09809684753418\n",
            "87 |  avg_price 110.7333560180664 loss -23.79864074707031\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\u001b[33m[W 2023-05-02 20:47:40,884]\u001b[0m Trial 0 failed with parameters: {'gamma': 0.9, 'tau': 0.12, 'train_freq': 768, 'noise_type': None, 'noise_std': 0.3, 'net_arch': 'big'} because of the following error: KeyboardInterrupt().\u001b[0m\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/site-packages/optuna/study/_optimize.py\", line 200, in _run_trial\n",
            "    value_or_values = func(trial)\n",
            "  File \"<ipython-input-211-26bf33718e81>\", line 25, in objective\n",
            "    trained_ddpg = agent.train_model(model=model_ddpg,\n",
            "  File \"/usr/local/lib/python3.10/site-packages/finrl/agents/stablebaselines3/models.py\", line 103, in train_model\n",
            "    model = model.learn(\n",
            "  File \"/usr/local/lib/python3.10/site-packages/stable_baselines3/ddpg/ddpg.py\", line 123, in learn\n",
            "    return super().learn(\n",
            "  File \"/usr/local/lib/python3.10/site-packages/stable_baselines3/td3/td3.py\", line 216, in learn\n",
            "    return super().learn(\n",
            "  File \"/usr/local/lib/python3.10/site-packages/stable_baselines3/common/off_policy_algorithm.py\", line 311, in learn\n",
            "    rollout = self.collect_rollouts(\n",
            "  File \"/usr/local/lib/python3.10/site-packages/stable_baselines3/common/off_policy_algorithm.py\", line 543, in collect_rollouts\n",
            "    new_obs, rewards, dones, infos = env.step(actions)\n",
            "  File \"/usr/local/lib/python3.10/site-packages/stable_baselines3/common/vec_env/base_vec_env.py\", line 163, in step\n",
            "    return self.step_wait()\n",
            "  File \"/usr/local/lib/python3.10/site-packages/stable_baselines3/common/vec_env/dummy_vec_env.py\", line 54, in step_wait\n",
            "    obs, self.buf_rews[env_idx], self.buf_dones[env_idx], self.buf_infos[env_idx] = self.envs[env_idx].step(\n",
            "  File \"<ipython-input-199-175aa58dc7db>\", line 370, in step\n",
            "    self.frame.to_csv(\"test.csv\")\n",
            "  File \"/usr/local/lib/python3.10/site-packages/pandas/util/_decorators.py\", line 211, in wrapper\n",
            "    return func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/site-packages/pandas/core/generic.py\", line 3720, in to_csv\n",
            "    return DataFrameRenderer(formatter).to_csv(\n",
            "  File \"/usr/local/lib/python3.10/site-packages/pandas/util/_decorators.py\", line 211, in wrapper\n",
            "    return func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/site-packages/pandas/io/formats/format.py\", line 1189, in to_csv\n",
            "    csv_formatter.save()\n",
            "  File \"/usr/local/lib/python3.10/site-packages/pandas/io/formats/csvs.py\", line 261, in save\n",
            "    self._save()\n",
            "  File \"/usr/local/lib/python3.10/site-packages/pandas/io/formats/csvs.py\", line 266, in _save\n",
            "    self._save_body()\n",
            "  File \"/usr/local/lib/python3.10/site-packages/pandas/io/formats/csvs.py\", line 304, in _save_body\n",
            "    self._save_chunk(start_i, end_i)\n",
            "  File \"/usr/local/lib/python3.10/site-packages/pandas/io/formats/csvs.py\", line 315, in _save_chunk\n",
            "    libwriters.write_csv_rows(\n",
            "  File \"pandas/_libs/writers.pyx\", line 55, in pandas._libs.writers.write_csv_rows\n",
            "KeyboardInterrupt\n",
            "\u001b[33m[W 2023-05-02 20:47:40,893]\u001b[0m Trial 0 failed with value None.\u001b[0m\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-211-26bf33718e81>\u001b[0m in \u001b[0;36m<cell line: 57>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     55\u001b[0m                                    trial_number=lc_trial_number)\n\u001b[1;32m     56\u001b[0m \u001b[0;31m#You can increase the n_trials for a better search space scanning\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 57\u001b[0;31m \u001b[0mstudy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobjective\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_trials\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mn_trials\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mcatch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mValueError\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mlogging_callback\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/optuna/study/study.py\u001b[0m in \u001b[0;36moptimize\u001b[0;34m(self, func, n_trials, timeout, n_jobs, catch, callbacks, gc_after_trial, show_progress_bar)\u001b[0m\n\u001b[1;32m    423\u001b[0m         \"\"\"\n\u001b[1;32m    424\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 425\u001b[0;31m         _optimize(\n\u001b[0m\u001b[1;32m    426\u001b[0m             \u001b[0mstudy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    427\u001b[0m             \u001b[0mfunc\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/optuna/study/_optimize.py\u001b[0m in \u001b[0;36m_optimize\u001b[0;34m(study, func, n_trials, timeout, n_jobs, catch, callbacks, gc_after_trial, show_progress_bar)\u001b[0m\n\u001b[1;32m     64\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     65\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mn_jobs\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 66\u001b[0;31m             _optimize_sequential(\n\u001b[0m\u001b[1;32m     67\u001b[0m                 \u001b[0mstudy\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     68\u001b[0m                 \u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/optuna/study/_optimize.py\u001b[0m in \u001b[0;36m_optimize_sequential\u001b[0;34m(study, func, n_trials, timeout, catch, callbacks, gc_after_trial, reseed_sampler_rng, time_start, progress_bar)\u001b[0m\n\u001b[1;32m    161\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    162\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 163\u001b[0;31m             \u001b[0mfrozen_trial\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_run_trial\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstudy\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcatch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    164\u001b[0m         \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    165\u001b[0m             \u001b[0;31m# The following line mitigates memory problems that can be occurred in some\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/optuna/study/_optimize.py\u001b[0m in \u001b[0;36m_run_trial\u001b[0;34m(study, func, catch)\u001b[0m\n\u001b[1;32m    249\u001b[0m         \u001b[0;32mand\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc_err\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcatch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    250\u001b[0m     ):\n\u001b[0;32m--> 251\u001b[0;31m         \u001b[0;32mraise\u001b[0m \u001b[0mfunc_err\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    252\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mfrozen_trial\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    253\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/optuna/study/_optimize.py\u001b[0m in \u001b[0;36m_run_trial\u001b[0;34m(study, func, catch)\u001b[0m\n\u001b[1;32m    198\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mget_heartbeat_thread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrial\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_trial_id\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstudy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_storage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    199\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 200\u001b[0;31m             \u001b[0mvalue_or_values\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrial\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    201\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mexceptions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTrialPruned\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    202\u001b[0m             \u001b[0;31m# TODO(mamu): Handle multi-objective cases.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-211-26bf33718e81>\u001b[0m in \u001b[0;36mobjective\u001b[0;34m(trial)\u001b[0m\n\u001b[1;32m     23\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m   \u001b[0;31m#You can increase it for better comparison\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 25\u001b[0;31m   trained_ddpg = agent.train_model(model=model_ddpg,\n\u001b[0m\u001b[1;32m     26\u001b[0m                                    \u001b[0mtb_log_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"ddpg\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     27\u001b[0m                                    total_timesteps=total_timesteps)\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/finrl/agents/stablebaselines3/models.py\u001b[0m in \u001b[0;36mtrain_model\u001b[0;34m(self, model, tb_log_name, total_timesteps)\u001b[0m\n\u001b[1;32m    101\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    102\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mtrain_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtb_log_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m5000\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 103\u001b[0;31m         model = model.learn(\n\u001b[0m\u001b[1;32m    104\u001b[0m             \u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    105\u001b[0m             \u001b[0mtb_log_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtb_log_name\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/stable_baselines3/ddpg/ddpg.py\u001b[0m in \u001b[0;36mlearn\u001b[0;34m(self, total_timesteps, callback, log_interval, tb_log_name, reset_num_timesteps, progress_bar)\u001b[0m\n\u001b[1;32m    121\u001b[0m         \u001b[0mprogress_bar\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mbool\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    122\u001b[0m     ) -> SelfDDPG:\n\u001b[0;32m--> 123\u001b[0;31m         return super().learn(\n\u001b[0m\u001b[1;32m    124\u001b[0m             \u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    125\u001b[0m             \u001b[0mcallback\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallback\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/stable_baselines3/td3/td3.py\u001b[0m in \u001b[0;36mlearn\u001b[0;34m(self, total_timesteps, callback, log_interval, tb_log_name, reset_num_timesteps, progress_bar)\u001b[0m\n\u001b[1;32m    214\u001b[0m         \u001b[0mprogress_bar\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mbool\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    215\u001b[0m     ) -> SelfTD3:\n\u001b[0;32m--> 216\u001b[0;31m         return super().learn(\n\u001b[0m\u001b[1;32m    217\u001b[0m             \u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    218\u001b[0m             \u001b[0mcallback\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallback\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/stable_baselines3/common/off_policy_algorithm.py\u001b[0m in \u001b[0;36mlearn\u001b[0;34m(self, total_timesteps, callback, log_interval, tb_log_name, reset_num_timesteps, progress_bar)\u001b[0m\n\u001b[1;32m    309\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    310\u001b[0m         \u001b[0;32mwhile\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnum_timesteps\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 311\u001b[0;31m             rollout = self.collect_rollouts(\n\u001b[0m\u001b[1;32m    312\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menv\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    313\u001b[0m                 \u001b[0mtrain_freq\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_freq\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/stable_baselines3/common/off_policy_algorithm.py\u001b[0m in \u001b[0;36mcollect_rollouts\u001b[0;34m(self, env, callback, train_freq, replay_buffer, action_noise, learning_starts, log_interval)\u001b[0m\n\u001b[1;32m    541\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    542\u001b[0m             \u001b[0;31m# Rescale and perform action\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 543\u001b[0;31m             \u001b[0mnew_obs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrewards\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdones\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minfos\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mactions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    544\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    545\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnum_timesteps\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnum_envs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/stable_baselines3/common/vec_env/base_vec_env.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, actions)\u001b[0m\n\u001b[1;32m    161\u001b[0m         \"\"\"\n\u001b[1;32m    162\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep_async\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mactions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 163\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep_wait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    164\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    165\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mget_images\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mSequence\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndarray\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/stable_baselines3/common/vec_env/dummy_vec_env.py\u001b[0m in \u001b[0;36mstep_wait\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     52\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mstep_wait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mVecEnvStepReturn\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     53\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0menv_idx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnum_envs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 54\u001b[0;31m             obs, self.buf_rews[env_idx], self.buf_dones[env_idx], self.buf_infos[env_idx] = self.envs[env_idx].step(\n\u001b[0m\u001b[1;32m     55\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mactions\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0menv_idx\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     56\u001b[0m             )\n",
            "\u001b[0;32m<ipython-input-199-175aa58dc7db>\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, actions)\u001b[0m\n\u001b[1;32m    368\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mframe\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconcat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mframe\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnew_df\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mignore_index\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0maxis\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    369\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 370\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mframe\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"test.csv\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    371\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    372\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/pandas/util/_decorators.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    209\u001b[0m                 \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    210\u001b[0m                     \u001b[0mkwargs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnew_arg_name\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnew_arg_value\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 211\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    212\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    213\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mF\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36mto_csv\u001b[0;34m(self, path_or_buf, sep, na_rep, float_format, columns, header, index, index_label, mode, encoding, compression, quoting, quotechar, lineterminator, chunksize, date_format, doublequote, escapechar, decimal, errors, storage_options)\u001b[0m\n\u001b[1;32m   3718\u001b[0m         )\n\u001b[1;32m   3719\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3720\u001b[0;31m         return DataFrameRenderer(formatter).to_csv(\n\u001b[0m\u001b[1;32m   3721\u001b[0m             \u001b[0mpath_or_buf\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3722\u001b[0m             \u001b[0mlineterminator\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlineterminator\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/pandas/util/_decorators.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    209\u001b[0m                 \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    210\u001b[0m                     \u001b[0mkwargs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnew_arg_name\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnew_arg_value\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 211\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    212\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    213\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mF\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/pandas/io/formats/format.py\u001b[0m in \u001b[0;36mto_csv\u001b[0;34m(self, path_or_buf, encoding, sep, columns, index_label, mode, compression, quoting, quotechar, lineterminator, chunksize, date_format, doublequote, escapechar, errors, storage_options)\u001b[0m\n\u001b[1;32m   1187\u001b[0m             \u001b[0mformatter\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfmt\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1188\u001b[0m         )\n\u001b[0;32m-> 1189\u001b[0;31m         \u001b[0mcsv_formatter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1190\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1191\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mcreated_buffer\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/pandas/io/formats/csvs.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    259\u001b[0m             )\n\u001b[1;32m    260\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 261\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_save\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    262\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    263\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_save\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/pandas/io/formats/csvs.py\u001b[0m in \u001b[0;36m_save\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    264\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_need_to_save_header\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    265\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_save_header\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 266\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_save_body\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    267\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    268\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_save_header\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/pandas/io/formats/csvs.py\u001b[0m in \u001b[0;36m_save_body\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    302\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mstart_i\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0mend_i\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    303\u001b[0m                 \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 304\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_save_chunk\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstart_i\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mend_i\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    305\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    306\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_save_chunk\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstart_i\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mend_i\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/pandas/io/formats/csvs.py\u001b[0m in \u001b[0;36m_save_chunk\u001b[0;34m(self, start_i, end_i)\u001b[0m\n\u001b[1;32m    313\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    314\u001b[0m         \u001b[0mix\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata_index\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mslicer\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_format_native_types\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_number_format\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 315\u001b[0;31m         libwriters.write_csv_rows(\n\u001b[0m\u001b[1;32m    316\u001b[0m             \u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    317\u001b[0m             \u001b[0mix\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/pandas/_libs/writers.pyx\u001b[0m in \u001b[0;36mpandas._libs.writers.write_csv_rows\u001b[0;34m()\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "joblib.dump(study, \"final_ddpg_study__.pkl\")"
      ],
      "metadata": {
        "id": "1yIXC6W2ZqPP"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#Get the best hyperparamters\n",
        "print('Hyperparameters after tuning',study.best_params)\n",
        "print('Hyperparameters before tuning',config.DDPG_PARAMS)"
      ],
      "metadata": {
        "id": "gXzfenQTdqq_",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ad76d6e8-c0ff-4d10-d0f2-efd5b0972e3e"
      },
      "execution_count": 166,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Hyperparameters after tuning {'gamma': 0.98, 'tau': 0.04, 'train_freq': 1024, 'noise_type': 'normal', 'noise_std': 0.4, 'net_arch': 'big'}\n",
            "Hyperparameters before tuning {'batch_size': 128, 'buffer_size': 50000, 'learning_rate': 0.001}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "study.best_trial"
      ],
      "metadata": {
        "id": "VQcuj-yidu0O",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0496257d-abd8-4566-b571-8cecf333c4c7"
      },
      "execution_count": 167,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "FrozenTrial(number=4, state=TrialState.COMPLETE, values=[1.3926463459776786], datetime_start=datetime.datetime(2023, 5, 2, 14, 40, 26, 156017), datetime_complete=datetime.datetime(2023, 5, 2, 14, 40, 42, 543469), params={'gamma': 0.98, 'tau': 0.04, 'train_freq': 1024, 'noise_type': 'normal', 'noise_std': 0.4, 'net_arch': 'big'}, user_attrs={}, system_attrs={}, intermediate_values={}, distributions={'gamma': CategoricalDistribution(choices=(0.9, 0.92, 0.94, 0.96, 0.98)), 'tau': CategoricalDistribution(choices=(0.02, 0.04, 0.06, 0.08, 0.1, 0.12)), 'train_freq': CategoricalDistribution(choices=(512, 768, 1024)), 'noise_type': CategoricalDistribution(choices=('ornstein-uhlenbeck', 'normal', None)), 'noise_std': CategoricalDistribution(choices=(0.1, 0.2, 0.3, 0.4, 0.5)), 'net_arch': CategoricalDistribution(choices=('small', 'big'))}, trial_id=4, value=None)"
            ]
          },
          "metadata": {},
          "execution_count": 167
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from stable_baselines3 import DDPG\n",
        "tuned_model_ddpg = DDPG.load('models/ddpg_{}.pth'.format(study.best_trial.number),env=env_train)"
      ],
      "metadata": {
        "id": "ZjYe9lOIdw3i"
      },
      "execution_count": 168,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#Trading period account value with tuned model\n",
        "df_account_value_tuned, df_actions_tuned = DRLAgent.DRL_prediction(\n",
        "    model=tuned_model_ddpg, \n",
        "    environment = e_trade_gym)"
      ],
      "metadata": {
        "id": "3bnbx87vdzHZ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a84cd21e-3154-43e3-ba2e-db3271bb3de4"
      },
      "execution_count": 178,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "hit end!\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_account_value_tuned[105:106][\"account_value\"]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "G1Y1jjMLf0qF",
        "outputId": "6e205e9b-9567-445f-d6d9-63f680806e02"
      },
      "execution_count": 179,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "105    533043.809441\n",
              "Name: account_value, dtype: float64"
            ]
          },
          "metadata": {},
          "execution_count": 179
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Backtesting with our pruned model\n",
        "print(\"==============Get Backtest Results===========\")\n",
        "now = datetime.datetime.now().strftime('%Y%m%d-%Hh%M')\n",
        "\n",
        "perf_stats_all_tuned = backtest_stats(account_value=df_account_value_tuned)\n",
        "perf_stats_all_tuned = pd.DataFrame(perf_stats_all_tuned)\n",
        "perf_stats_all_tuned.to_csv(\"./\"+config.RESULTS_DIR+\"/perf_stats_all_tuned_\"+now+'.csv')"
      ],
      "metadata": {
        "id": "FJrv144od3G_",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5fd8aee3-2ce7-4267-ca58-96a66364eba5"
      },
      "execution_count": 180,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==============Get Backtest Results===========\n",
            "Annual return          0.164324\n",
            "Cumulative returns     0.066088\n",
            "Annual volatility      0.115015\n",
            "Sharpe ratio           1.392646\n",
            "Calmar ratio           5.177199\n",
            "Stability              0.778060\n",
            "Max drawdown          -0.031740\n",
            "Omega ratio            1.269228\n",
            "Sortino ratio          2.187981\n",
            "Skew                        NaN\n",
            "Kurtosis                    NaN\n",
            "Tail ratio             1.171742\n",
            "Daily value at risk   -0.013855\n",
            "dtype: float64\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Now train with not tuned hyperaparameters\n",
        "#Default config.ddpg_PARAMS\n",
        "non_tuned_model_ddpg = agent.get_model(\"ddpg\",model_kwargs = config.DDPG_PARAMS )\n",
        "trained_ddpg = agent.train_model(model=non_tuned_model_ddpg, \n",
        "                             tb_log_name='ddpg',\n",
        "                             total_timesteps=50000)"
      ],
      "metadata": {
        "id": "asWsWzemd5dk",
        "outputId": "78a3262d-e665-4d79-a47d-a1ccfd892d70",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "execution_count": 88,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'batch_size': 128, 'buffer_size': 50000, 'learning_rate': 0.001}\n",
            "Using cuda device\n",
            "day: 82, episode: 570\n",
            "begin_total_asset: 500000.00\n",
            "end_total_asset: 434938.17\n",
            "total_reward: -65061.83\n",
            "total_cost: 25569.22\n",
            "total_trades: 1611\n",
            "Sharpe: -0.514\n",
            "=================================\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-64-5018ead22eec>:354: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-64-5018ead22eec>:268: RuntimeWarning: invalid value encountered in subtract\n",
            "  actions = np.where(((current_price < (self.avg_price - self.avg_price * .1))& (self.avg_price >0)), self.total_stockss*-1,actions  )\n",
            "<ipython-input-64-5018ead22eec>:354: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-64-5018ead22eec>:268: RuntimeWarning: invalid value encountered in subtract\n",
            "  actions = np.where(((current_price < (self.avg_price - self.avg_price * .1))& (self.avg_price >0)), self.total_stockss*-1,actions  )\n",
            "<ipython-input-64-5018ead22eec>:354: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-64-5018ead22eec>:268: RuntimeWarning: invalid value encountered in subtract\n",
            "  actions = np.where(((current_price < (self.avg_price - self.avg_price * .1))& (self.avg_price >0)), self.total_stockss*-1,actions  )\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "-----------------------------------\n",
            "| time/              |            |\n",
            "|    episodes        | 4          |\n",
            "|    fps             | 233        |\n",
            "|    time_elapsed    | 1          |\n",
            "|    total_timesteps | 332        |\n",
            "| train/             |            |\n",
            "|    actor_loss      | -21.1      |\n",
            "|    critic_loss     | 2.07e+03   |\n",
            "|    learning_rate   | 0.001      |\n",
            "|    n_updates       | 166        |\n",
            "|    reward          | -0.8715689 |\n",
            "-----------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-64-5018ead22eec>:354: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-64-5018ead22eec>:268: RuntimeWarning: invalid value encountered in subtract\n",
            "  actions = np.where(((current_price < (self.avg_price - self.avg_price * .1))& (self.avg_price >0)), self.total_stockss*-1,actions  )\n",
            "<ipython-input-64-5018ead22eec>:354: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-64-5018ead22eec>:268: RuntimeWarning: invalid value encountered in subtract\n",
            "  actions = np.where(((current_price < (self.avg_price - self.avg_price * .1))& (self.avg_price >0)), self.total_stockss*-1,actions  )\n",
            "<ipython-input-64-5018ead22eec>:354: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-64-5018ead22eec>:268: RuntimeWarning: invalid value encountered in subtract\n",
            "  actions = np.where(((current_price < (self.avg_price - self.avg_price * .1))& (self.avg_price >0)), self.total_stockss*-1,actions  )\n",
            "<ipython-input-64-5018ead22eec>:354: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-64-5018ead22eec>:268: RuntimeWarning: invalid value encountered in subtract\n",
            "  actions = np.where(((current_price < (self.avg_price - self.avg_price * .1))& (self.avg_price >0)), self.total_stockss*-1,actions  )\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "-----------------------------------\n",
            "| time/              |            |\n",
            "|    episodes        | 8          |\n",
            "|    fps             | 176        |\n",
            "|    time_elapsed    | 3          |\n",
            "|    total_timesteps | 664        |\n",
            "| train/             |            |\n",
            "|    actor_loss      | -15.6      |\n",
            "|    critic_loss     | 3.43e+03   |\n",
            "|    learning_rate   | 0.001      |\n",
            "|    n_updates       | 498        |\n",
            "|    reward          | -0.8715689 |\n",
            "-----------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-64-5018ead22eec>:354: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-64-5018ead22eec>:268: RuntimeWarning: invalid value encountered in subtract\n",
            "  actions = np.where(((current_price < (self.avg_price - self.avg_price * .1))& (self.avg_price >0)), self.total_stockss*-1,actions  )\n",
            "<ipython-input-64-5018ead22eec>:354: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-64-5018ead22eec>:268: RuntimeWarning: invalid value encountered in subtract\n",
            "  actions = np.where(((current_price < (self.avg_price - self.avg_price * .1))& (self.avg_price >0)), self.total_stockss*-1,actions  )\n",
            "<ipython-input-64-5018ead22eec>:354: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-64-5018ead22eec>:268: RuntimeWarning: invalid value encountered in subtract\n",
            "  actions = np.where(((current_price < (self.avg_price - self.avg_price * .1))& (self.avg_price >0)), self.total_stockss*-1,actions  )\n",
            "<ipython-input-64-5018ead22eec>:354: RuntimeWarning: divide by zero encountered in true_divide\n",
            "  self.avg_price = np.divide(self.total_price,\n",
            "<ipython-input-64-5018ead22eec>:268: RuntimeWarning: invalid value encountered in subtract\n",
            "  actions = np.where(((current_price < (self.avg_price - self.avg_price * .1))& (self.avg_price >0)), self.total_stockss*-1,actions  )\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "day: 82, episode: 580\n",
            "begin_total_asset: 500000.00\n",
            "end_total_asset: 498030.54\n",
            "total_reward: -1969.46\n",
            "total_cost: 703.88\n",
            "total_trades: 1231\n",
            "Sharpe: 0.281\n",
            "=================================\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-88-3b1b0264cc27>\u001b[0m in \u001b[0;36m<cell line: 4>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;31m#Default config.ddpg_PARAMS\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mnon_tuned_model_ddpg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0magent\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"ddpg\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mmodel_kwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDDPG_PARAMS\u001b[0m \u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m trained_ddpg = agent.train_model(model=non_tuned_model_ddpg, \n\u001b[0m\u001b[1;32m      5\u001b[0m                              \u001b[0mtb_log_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'ddpg'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m                              total_timesteps=50000)\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/finrl/agents/stablebaselines3/models.py\u001b[0m in \u001b[0;36mtrain_model\u001b[0;34m(self, model, tb_log_name, total_timesteps)\u001b[0m\n\u001b[1;32m    101\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    102\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mtrain_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtb_log_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m5000\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 103\u001b[0;31m         model = model.learn(\n\u001b[0m\u001b[1;32m    104\u001b[0m             \u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    105\u001b[0m             \u001b[0mtb_log_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtb_log_name\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/stable_baselines3/ddpg/ddpg.py\u001b[0m in \u001b[0;36mlearn\u001b[0;34m(self, total_timesteps, callback, log_interval, tb_log_name, reset_num_timesteps, progress_bar)\u001b[0m\n\u001b[1;32m    121\u001b[0m         \u001b[0mprogress_bar\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mbool\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    122\u001b[0m     ) -> SelfDDPG:\n\u001b[0;32m--> 123\u001b[0;31m         return super().learn(\n\u001b[0m\u001b[1;32m    124\u001b[0m             \u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    125\u001b[0m             \u001b[0mcallback\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallback\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/stable_baselines3/td3/td3.py\u001b[0m in \u001b[0;36mlearn\u001b[0;34m(self, total_timesteps, callback, log_interval, tb_log_name, reset_num_timesteps, progress_bar)\u001b[0m\n\u001b[1;32m    214\u001b[0m         \u001b[0mprogress_bar\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mbool\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    215\u001b[0m     ) -> SelfTD3:\n\u001b[0;32m--> 216\u001b[0;31m         return super().learn(\n\u001b[0m\u001b[1;32m    217\u001b[0m             \u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    218\u001b[0m             \u001b[0mcallback\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallback\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/stable_baselines3/common/off_policy_algorithm.py\u001b[0m in \u001b[0;36mlearn\u001b[0;34m(self, total_timesteps, callback, log_interval, tb_log_name, reset_num_timesteps, progress_bar)\u001b[0m\n\u001b[1;32m    328\u001b[0m                 \u001b[0;31m# Special case when the user passes `gradient_steps=0`\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    329\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mgradient_steps\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 330\u001b[0;31m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient_steps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mgradient_steps\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    331\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    332\u001b[0m         \u001b[0mcallback\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_training_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/stable_baselines3/td3/td3.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self, gradient_steps, batch_size)\u001b[0m\n\u001b[1;32m    195\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    196\u001b[0m                 \u001b[0mpolyak_update\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcritic\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcritic_target\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtau\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 197\u001b[0;31m                 \u001b[0mpolyak_update\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mactor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mactor_target\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtau\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    198\u001b[0m                 \u001b[0;31m# Copy running stats, see GH issue #996\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    199\u001b[0m                 \u001b[0mpolyak_update\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcritic_batch_norm_stats\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcritic_batch_norm_stats_target\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1.0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/stable_baselines3/common/utils.py\u001b[0m in \u001b[0;36mpolyak_update\u001b[0;34m(params, target_params, tau)\u001b[0m\n\u001b[1;32m    466\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mth\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mno_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    467\u001b[0m         \u001b[0;31m# zip does not raise an exception if length of parameters does not match.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 468\u001b[0;31m         \u001b[0;32mfor\u001b[0m \u001b[0mparam\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_param\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip_strict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    469\u001b[0m             \u001b[0mtarget_param\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmul_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mtau\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    470\u001b[0m             \u001b[0mth\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtarget_param\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparam\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0malpha\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtau\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtarget_param\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/site-packages/stable_baselines3/common/utils.py\u001b[0m in \u001b[0;36mzip_strict\u001b[0;34m(*iterables)\u001b[0m\n\u001b[1;32m    439\u001b[0m     \u001b[0msentinel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mobject\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    440\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mcombo\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip_longest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0miterables\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfillvalue\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msentinel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 441\u001b[0;31m         \u001b[0;32mif\u001b[0m \u001b[0msentinel\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mcombo\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    442\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Iterables have different lengths\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    443\u001b[0m         \u001b[0;32myield\u001b[0m \u001b[0mcombo\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_account_value, df_actions = DRLAgent.DRL_prediction(\n",
        "    model=trained_ddpg, \n",
        "    environment = e_trade_gym)"
      ],
      "metadata": {
        "id": "mv1zgwOgd9FY",
        "outputId": "3fd9a94a-98d4-41c0-8870-74b8271b6494",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-35-38368194789e>\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m df_account_value, df_actions = DRLAgent.DRL_prediction(\n\u001b[0;32m----> 2\u001b[0;31m     \u001b[0mmodel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrained_ddpg\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m     environment = e_trade_gym)\n",
            "\u001b[0;31mNameError\u001b[0m: name 'trained_ddpg' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Backtesting for not tuned hyperparamters\n",
        "print(\"==============Get Backtest Results===========\")\n",
        "now = datetime.datetime.now().strftime('%Y%m%d-%Hh%M')\n",
        "\n",
        "perf_stats_all = backtest_stats(account_value=df_account_value)\n",
        "perf_stats_all = pd.DataFrame(perf_stats_all)\n",
        "# perf_stats_all.to_csv(\"./\"+config.RESULTS_DIR+\"/perf_stats_all_\"+now+'.csv')"
      ],
      "metadata": {
        "id": "DuZL52qDeEz6",
        "outputId": "f4e9cd19-27cd-4dba-956e-353df0dabffa",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==============Get Backtest Results===========\n",
            "Annual return          0.247826\n",
            "Cumulative returns     0.368416\n",
            "Annual volatility      0.167226\n",
            "Sharpe ratio           1.412093\n",
            "Calmar ratio           2.531845\n",
            "Stability              0.897149\n",
            "Max drawdown          -0.097883\n",
            "Omega ratio            1.280406\n",
            "Sortino ratio          2.010344\n",
            "Skew                        NaN\n",
            "Kurtosis                    NaN\n",
            "Tail ratio             1.033938\n",
            "Daily value at risk   -0.020131\n",
            "dtype: float64\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#You can see with trial, our sharpe ratio is increasing\n",
        "#Certainly you can afford more number of trials for further optimization\n",
        "from optuna.visualization import plot_optimization_history\n",
        "plot_optimization_history(study)"
      ],
      "metadata": {
        "id": "5aSsEmpZeI5C",
        "outputId": "cba8ec8c-7654-44d7-88cb-bd6e1bad4a5b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 542
        }
      },
      "execution_count": 172,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<html>\n",
              "<head><meta charset=\"utf-8\" /></head>\n",
              "<body>\n",
              "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
              "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.20.0.min.js\"></script>                <div id=\"565b1505-44ed-4588-989d-e42e52b774ca\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"565b1505-44ed-4588-989d-e42e52b774ca\")) {                    Plotly.newPlot(                        \"565b1505-44ed-4588-989d-e42e52b774ca\",                        [{\"mode\":\"markers\",\"name\":\"Objective Value\",\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21],\"y\":[-0.49080299778360714,-0.3950000677893763,1.3011854021414007,-0.8267805987990207,1.3926463459776786,0.26111723817257965,-0.9640129324129495,0.2819918104525999,1.0018988775173625,-0.2701595920840011,1.1243951293107357,-0.40659941768702645,0.25085630521807395,0.2654094534716406,1.3315265826000702,-0.7288319825005803,0.25514408327043286,-1.0674379403572838,-0.6986206079122625,0.3810367303898103,-0.471019724578347,0.748178567824751],\"type\":\"scatter\"},{\"name\":\"Best Value\",\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21],\"y\":[-0.49080299778360714,-0.3950000677893763,1.3011854021414007,1.3011854021414007,1.3926463459776786,1.3926463459776786,1.3926463459776786,1.3926463459776786,1.3926463459776786,1.3926463459776786,1.3926463459776786,1.3926463459776786,1.3926463459776786,1.3926463459776786,1.3926463459776786,1.3926463459776786,1.3926463459776786,1.3926463459776786,1.3926463459776786,1.3926463459776786,1.3926463459776786,1.3926463459776786],\"type\":\"scatter\"}],                        {\"title\":{\"text\":\"Optimization History Plot\"},\"xaxis\":{\"title\":{\"text\":\"Trial\"}},\"yaxis\":{\"title\":{\"text\":\"Objective Value\"}},\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}}},                        {\"responsive\": true}                    ).then(function(){\n",
              "                            \n",
              "var gd = document.getElementById('565b1505-44ed-4588-989d-e42e52b774ca');\n",
              "var x = new MutationObserver(function (mutations, observer) {{\n",
              "        var display = window.getComputedStyle(gd).display;\n",
              "        if (!display || display === 'none') {{\n",
              "            console.log([gd, 'removed!']);\n",
              "            Plotly.purge(gd);\n",
              "            observer.disconnect();\n",
              "        }}\n",
              "}});\n",
              "\n",
              "// Listen for the removal of the full notebook cells\n",
              "var notebookContainer = gd.closest('#notebook-container');\n",
              "if (notebookContainer) {{\n",
              "    x.observe(notebookContainer, {childList: true});\n",
              "}}\n",
              "\n",
              "// Listen for the clearing of the current output cell\n",
              "var outputEl = gd.closest('.output');\n",
              "if (outputEl) {{\n",
              "    x.observe(outputEl, {childList: true});\n",
              "}}\n",
              "\n",
              "                        })                };                            </script>        </div>\n",
              "</body>\n",
              "</html>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from optuna.visualization import plot_contour\n",
        "from optuna.visualization import plot_edf\n",
        "from optuna.visualization import plot_intermediate_values\n",
        "from optuna.visualization import plot_optimization_history\n",
        "from optuna.visualization import plot_parallel_coordinate\n",
        "from optuna.visualization import plot_param_importances\n",
        "from optuna.visualization import plot_slice"
      ],
      "metadata": {
        "id": "K9zcey_BeJMe"
      },
      "execution_count": 173,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#Hyperparamters importance\n",
        "#Ent_coef is the most important\n",
        "plot_param_importances(study)"
      ],
      "metadata": {
        "id": "J6WmB5DneOgh",
        "outputId": "2d916588-ae89-4970-f430-249ef6b5c81a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 542
        }
      },
      "execution_count": 174,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<html>\n",
              "<head><meta charset=\"utf-8\" /></head>\n",
              "<body>\n",
              "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
              "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.20.0.min.js\"></script>                <div id=\"65abb362-3b70-4093-bbd4-fd5b77123c9e\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"65abb362-3b70-4093-bbd4-fd5b77123c9e\")) {                    Plotly.newPlot(                        \"65abb362-3b70-4093-bbd4-fd5b77123c9e\",                        [{\"cliponaxis\":false,\"hovertemplate\":[\"net_arch (CategoricalDistribution): 0.029499975293024668<extra></extra>\",\"train_freq (CategoricalDistribution): 0.0539951123903899<extra></extra>\",\"noise_type (CategoricalDistribution): 0.1655824179588198<extra></extra>\",\"tau (CategoricalDistribution): 0.1773850893223912<extra></extra>\",\"noise_std (CategoricalDistribution): 0.19437196725757405<extra></extra>\",\"gamma (CategoricalDistribution): 0.37916543777780054<extra></extra>\"],\"marker\":{\"color\":\"rgb(66,146,198)\"},\"orientation\":\"h\",\"text\":[\"0.03\",\"0.05\",\"0.17\",\"0.18\",\"0.19\",\"0.38\"],\"textposition\":\"outside\",\"x\":[0.029499975293024668,0.0539951123903899,0.1655824179588198,0.1773850893223912,0.19437196725757405,0.37916543777780054],\"y\":[\"net_arch\",\"train_freq\",\"noise_type\",\"tau\",\"noise_std\",\"gamma\"],\"type\":\"bar\"}],                        {\"showlegend\":false,\"title\":{\"text\":\"Hyperparameter Importances\"},\"xaxis\":{\"title\":{\"text\":\"Importance for Objective Value\"}},\"yaxis\":{\"title\":{\"text\":\"Hyperparameter\"}},\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}}},                        {\"responsive\": true}                    ).then(function(){\n",
              "                            \n",
              "var gd = document.getElementById('65abb362-3b70-4093-bbd4-fd5b77123c9e');\n",
              "var x = new MutationObserver(function (mutations, observer) {{\n",
              "        var display = window.getComputedStyle(gd).display;\n",
              "        if (!display || display === 'none') {{\n",
              "            console.log([gd, 'removed!']);\n",
              "            Plotly.purge(gd);\n",
              "            observer.disconnect();\n",
              "        }}\n",
              "}});\n",
              "\n",
              "// Listen for the removal of the full notebook cells\n",
              "var notebookContainer = gd.closest('#notebook-container');\n",
              "if (notebookContainer) {{\n",
              "    x.observe(notebookContainer, {childList: true});\n",
              "}}\n",
              "\n",
              "// Listen for the clearing of the current output cell\n",
              "var outputEl = gd.closest('.output');\n",
              "if (outputEl) {{\n",
              "    x.observe(outputEl, {childList: true});\n",
              "}}\n",
              "\n",
              "                        })                };                            </script>        </div>\n",
              "</body>\n",
              "</html>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def main():\n",
        "    import warnings\n",
        "    warnings.filterwarnings(\"ignore\")\n",
        "    import pandas as pd\n",
        "    import numpy as np\n",
        "    import matplotlib\n",
        "    import matplotlib.pyplot as plt\n",
        "    # matplotlib.use('Agg')\n",
        "    import datetime\n",
        "\n",
        "\n",
        "    from finrl.config_tickers import DOW_30_TICKER\n",
        "    from finrl.meta.preprocessor.yahoodownloader import YahooDownloader\n",
        "    from finrl.meta.preprocessor.preprocessors import FeatureEngineer, data_split\n",
        "    from finrl.meta.env_stock_trading.env_stocktrading import StockTradingEnv\n",
        "    from finrl.agents.stablebaselines3.models import DRLAgent, DRLEnsembleAgent\n",
        "    from finrl.plot import backtest_stats, backtest_plot, get_daily_return, get_baseline\n",
        "\n",
        "    from pprint import pprint\n",
        "\n",
        "    import sys\n",
        "    sys.path.append(\"../FinRL-Library\")\n",
        "\n",
        "    import itertools\n",
        "\n",
        "    import os\n",
        "    from finrl.main import check_and_make_directories\n",
        "    from finrl.config import (\n",
        "        DATA_SAVE_DIR,\n",
        "        TRAINED_MODEL_DIR,\n",
        "        TENSORBOARD_LOG_DIR,\n",
        "        RESULTS_DIR,\n",
        "        INDICATORS,\n",
        "        TRAIN_START_DATE,\n",
        "        TRAIN_END_DATE,\n",
        "        TEST_START_DATE,\n",
        "        TEST_END_DATE,\n",
        "        TRADE_START_DATE,\n",
        "        TRADE_END_DATE,\n",
        "    )\n",
        "\n",
        "    check_and_make_directories([DATA_SAVE_DIR, TRAINED_MODEL_DIR, TENSORBOARD_LOG_DIR, RESULTS_DIR])\n",
        "    print(DOW_30_TICKER)\n",
        "    TRAIN_START_DATE = '2009-04-01'\n",
        "    TRAIN_END_DATE = '2021-01-01'\n",
        "    TEST_START_DATE = '2021-01-01'\n",
        "    TEST_END_DATE = '2022-06-01'\n",
        "\n",
        "    df = YahooDownloader(start_date=TRAIN_START_DATE,\n",
        "                         end_date=TEST_END_DATE,\n",
        "                         ticker_list=DOW_30_TICKER).fetch_data()\n",
        "\n",
        "    df.sort_values(['date', 'tic']).head()\n",
        "\n",
        "    fe = FeatureEngineer(use_technical_indicator=True,\n",
        "                         tech_indicator_list=INDICATORS,\n",
        "                         use_turbulence=True,\n",
        "                         user_defined_feature=False)\n",
        "\n",
        "    processed = fe.preprocess_data(df)\n",
        "    processed = processed.copy()\n",
        "    processed = processed.fillna(0)\n",
        "    processed = processed.replace(np.inf, 0)\n",
        "\n",
        "    stock_dimension = len(processed.tic.unique())\n",
        "    state_space = 1 + 2 * stock_dimension + len(INDICATORS) * stock_dimension\n",
        "    print(f\"Stock Dimension: {stock_dimension}, State Space: {state_space}\")\n",
        "\n",
        "    env_kwargs = {\n",
        "        \"hmax\": 100,\n",
        "        \"initial_amount\": 1000000,\n",
        "        \"buy_cost_pct\": 0.001,\n",
        "        \"sell_cost_pct\": 0.001,\n",
        "        \"state_space\": state_space,\n",
        "        \"stock_dim\": stock_dimension,\n",
        "        \"tech_indicator_list\": INDICATORS,\n",
        "        \"action_space\": stock_dimension,\n",
        "        \"reward_scaling\": 1e-4,\n",
        "        \"print_verbosity\": 5\n",
        "\n",
        "    }\n",
        "\n",
        "    rebalance_window = 63  # rebalance_window is the number of days to retrain the model\n",
        "    validation_window = 63  # validation_window is the number of days to do validation and trading (e.g. if validation_window=63, then both validation and trading period will be 63 days)\n",
        "\n",
        "    ensemble_agent = DRLEnsembleAgent(df=processed,\n",
        "                                      train_period=(TRAIN_START_DATE, TRAIN_END_DATE),\n",
        "                                      val_test_period=(TEST_START_DATE, TEST_END_DATE),\n",
        "                                      rebalance_window=rebalance_window,\n",
        "                                      validation_window=validation_window,\n",
        "                                      **env_kwargs)\n",
        "\n",
        "    A2C_model_kwargs = {\n",
        "        'n_steps': 5,\n",
        "        'ent_coef': 0.005,\n",
        "        'learning_rate': 0.0007\n",
        "    }\n",
        "\n",
        "    PPO_model_kwargs = {\n",
        "        \"ent_coef\": 0.01,\n",
        "        \"n_steps\": 2048,\n",
        "        \"learning_rate\": 0.00025,\n",
        "        \"batch_size\": 128\n",
        "    }\n",
        "\n",
        "    DDPG_model_kwargs = {\n",
        "        # \"action_noise\":\"ornstein_uhlenbeck\",\n",
        "        \"buffer_size\": 10_000,\n",
        "        \"learning_rate\": 0.0005,\n",
        "        \"batch_size\": 64\n",
        "    }\n",
        "\n",
        "    timesteps_dict = {'a2c': 10_000,\n",
        "                      'ppo': 10_000,\n",
        "                      'ddpg': 10_000\n",
        "                      }\n",
        "    df_summary = ensemble_agent.run_ensemble_strategy(A2C_model_kwargs,\n",
        "                                                      PPO_model_kwargs,\n",
        "                                                      DDPG_model_kwargs,\n",
        "                                                      timesteps_dict)\n",
        "\n",
        "    unique_trade_date = processed[(processed.date > TEST_START_DATE) & (processed.date <= TEST_END_DATE)].date.unique()\n",
        "\n",
        "    df_trade_date = pd.DataFrame({'datadate': unique_trade_date})\n",
        "\n",
        "    df_account_value = pd.DataFrame()\n",
        "    for i in range(rebalance_window + validation_window, len(unique_trade_date) + 1, rebalance_window):\n",
        "        temp = pd.read_csv('results/account_value_trade_{}_{}.csv'.format('ensemble', i))\n",
        "        df_account_value = df_account_value.append(temp, ignore_index=True)\n",
        "    sharpe = (252 ** 0.5) * df_account_value.account_value.pct_change(\n",
        "        1).mean() / df_account_value.account_value.pct_change(1).std()\n",
        "    print('Sharpe Ratio: ', sharpe)\n",
        "    df_account_value = df_account_value.join(df_trade_date[validation_window:].reset_index(drop=True))\n",
        "\n",
        "    df_account_value.account_value.plot()\n",
        "\n",
        "    print(\"==============Get Backtest Results===========\")\n",
        "    now = datetime.datetime.now().strftime('%Y%m%d-%Hh%M')\n",
        "\n",
        "    perf_stats_all = backtest_stats(account_value=df_account_value)\n",
        "    perf_stats_all = pd.DataFrame(perf_stats_all)\n",
        "\n",
        "    # baseline stats\n",
        "    print(\"==============Get Baseline Stats===========\")\n",
        "    baseline_df = get_baseline(\n",
        "        ticker=\"^DJI\",\n",
        "        start=df_account_value.loc[0, 'date'],\n",
        "        end=df_account_value.loc[len(df_account_value) - 1, 'date'])\n",
        "\n",
        "    stats = backtest_stats(baseline_df, value_col_name='close')\n",
        "\n",
        "    print(\"==============Compare to DJIA===========\")\n",
        "\n",
        "    # S&P 500: ^GSPC\n",
        "    # Dow Jones Index: ^DJI\n",
        "    # NASDAQ 100: ^NDX\n",
        "    backtest_plot(df_account_value,\n",
        "                  baseline_ticker='^DJI',\n",
        "                  baseline_start=df_account_value.loc[0, 'date'],\n",
        "                  baseline_end=df_account_value.loc[len(df_account_value) - 1, 'date'])\n",
        "\n",
        "\n",
        "if __name__ == '__main__':\n",
        "    main()"
      ],
      "metadata": {
        "id": "s_xcVpBSeOyE",
        "outputId": "52b3d82a-1f8d-4af4-a405-6b5b9778781d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['AXP', 'AMGN', 'AAPL', 'BA', 'CAT', 'CSCO', 'CVX', 'GS', 'HD', 'HON', 'IBM', 'INTC', 'JNJ', 'KO', 'JPM', 'MCD', 'MMM', 'MRK', 'MSFT', 'NKE', 'PG', 'TRV', 'UNH', 'CRM', 'VZ', 'V', 'WBA', 'WMT', 'DIS', 'DOW']\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "[*********************100%***********************]  1 of 1 completed\n",
            "Shape of DataFrame:  (96942, 8)\n",
            "Successfully added technical indicators\n",
            "Successfully added turbulence index\n",
            "Stock Dimension: 29, State Space: 291\n",
            "============Start Ensemble Strategy============\n",
            "============================================\n",
            "turbulence_threshold:  203.40117769841157\n",
            "======Model training from:  2009-04-01 to  2021-01-04\n",
            "======A2C Training========\n",
            "{'n_steps': 5, 'ent_coef': 0.005, 'learning_rate': 0.0007}\n",
            "Using cuda device\n",
            "Logging to tensorboard_log/a2c/a2c_126_1\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 100        |\n",
            "|    iterations         | 100        |\n",
            "|    time_elapsed       | 4          |\n",
            "|    total_timesteps    | 500        |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.1      |\n",
            "|    explained_variance | 0.00295    |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 99         |\n",
            "|    policy_loss        | -71.2      |\n",
            "|    reward             | 0.78650117 |\n",
            "|    std                | 1          |\n",
            "|    value_loss         | 12.2       |\n",
            "--------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 101        |\n",
            "|    iterations         | 200        |\n",
            "|    time_elapsed       | 9          |\n",
            "|    total_timesteps    | 1000       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.1      |\n",
            "|    explained_variance | 0.00111    |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 199        |\n",
            "|    policy_loss        | 6.59       |\n",
            "|    reward             | 0.66569686 |\n",
            "|    std                | 0.998      |\n",
            "|    value_loss         | 1.03       |\n",
            "--------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 102       |\n",
            "|    iterations         | 300       |\n",
            "|    time_elapsed       | 14        |\n",
            "|    total_timesteps    | 1500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.1     |\n",
            "|    explained_variance | 0         |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 299       |\n",
            "|    policy_loss        | -123      |\n",
            "|    reward             | 2.9206326 |\n",
            "|    std                | 0.998     |\n",
            "|    value_loss         | 15.8      |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 102        |\n",
            "|    iterations         | 400        |\n",
            "|    time_elapsed       | 19         |\n",
            "|    total_timesteps    | 2000       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.1      |\n",
            "|    explained_variance | -1.19e-07  |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 399        |\n",
            "|    policy_loss        | -6.35      |\n",
            "|    reward             | -1.8242201 |\n",
            "|    std                | 0.999      |\n",
            "|    value_loss         | 3.27       |\n",
            "--------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                 |               |\n",
            "|    fps                | 102           |\n",
            "|    iterations         | 500           |\n",
            "|    time_elapsed       | 24            |\n",
            "|    total_timesteps    | 2500          |\n",
            "| train/                |               |\n",
            "|    entropy_loss       | -41.2         |\n",
            "|    explained_variance | -1.19e-07     |\n",
            "|    learning_rate      | 0.0007        |\n",
            "|    n_updates          | 499           |\n",
            "|    policy_loss        | 16            |\n",
            "|    reward             | -0.0061751665 |\n",
            "|    std                | 1             |\n",
            "|    value_loss         | 0.657         |\n",
            "-----------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 102       |\n",
            "|    iterations         | 600       |\n",
            "|    time_elapsed       | 29        |\n",
            "|    total_timesteps    | 3000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.2     |\n",
            "|    explained_variance | -1.19e-07 |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 599       |\n",
            "|    policy_loss        | -33.6     |\n",
            "|    reward             | 0.5337091 |\n",
            "|    std                | 1         |\n",
            "|    value_loss         | 0.837     |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 102        |\n",
            "|    iterations         | 700        |\n",
            "|    time_elapsed       | 34         |\n",
            "|    total_timesteps    | 3500       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.2      |\n",
            "|    explained_variance | -1.97      |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 699        |\n",
            "|    policy_loss        | -76.4      |\n",
            "|    reward             | -1.3246601 |\n",
            "|    std                | 1          |\n",
            "|    value_loss         | 3.85       |\n",
            "--------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 101        |\n",
            "|    iterations         | 800        |\n",
            "|    time_elapsed       | 39         |\n",
            "|    total_timesteps    | 4000       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.2      |\n",
            "|    explained_variance | 0.0632     |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 799        |\n",
            "|    policy_loss        | 44.6       |\n",
            "|    reward             | -1.6959082 |\n",
            "|    std                | 1          |\n",
            "|    value_loss         | 1.53       |\n",
            "--------------------------------------\n",
            "------------------------------------\n",
            "| time/                 |          |\n",
            "|    fps                | 100      |\n",
            "|    iterations         | 900      |\n",
            "|    time_elapsed       | 44       |\n",
            "|    total_timesteps    | 4500     |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -41.2    |\n",
            "|    explained_variance | 0.0211   |\n",
            "|    learning_rate      | 0.0007   |\n",
            "|    n_updates          | 899      |\n",
            "|    policy_loss        | -33.2    |\n",
            "|    reward             | 3.994608 |\n",
            "|    std                | 1        |\n",
            "|    value_loss         | 3.53     |\n",
            "------------------------------------\n",
            "---------------------------------------\n",
            "| time/                 |             |\n",
            "|    fps                | 101         |\n",
            "|    iterations         | 1000        |\n",
            "|    time_elapsed       | 49          |\n",
            "|    total_timesteps    | 5000        |\n",
            "| train/                |             |\n",
            "|    entropy_loss       | -41.2       |\n",
            "|    explained_variance | 1.79e-07    |\n",
            "|    learning_rate      | 0.0007      |\n",
            "|    n_updates          | 999         |\n",
            "|    policy_loss        | 36.3        |\n",
            "|    reward             | -0.20298508 |\n",
            "|    std                | 1           |\n",
            "|    value_loss         | 1.68        |\n",
            "---------------------------------------\n",
            "---------------------------------------\n",
            "| time/                 |             |\n",
            "|    fps                | 101         |\n",
            "|    iterations         | 1100        |\n",
            "|    time_elapsed       | 54          |\n",
            "|    total_timesteps    | 5500        |\n",
            "| train/                |             |\n",
            "|    entropy_loss       | -41.1       |\n",
            "|    explained_variance | 0.0114      |\n",
            "|    learning_rate      | 0.0007      |\n",
            "|    n_updates          | 1099        |\n",
            "|    policy_loss        | 393         |\n",
            "|    reward             | -0.92543656 |\n",
            "|    std                | 1           |\n",
            "|    value_loss         | 120         |\n",
            "---------------------------------------\n",
            "----------------------------------------\n",
            "| time/                 |              |\n",
            "|    fps                | 101          |\n",
            "|    iterations         | 1200         |\n",
            "|    time_elapsed       | 59           |\n",
            "|    total_timesteps    | 6000         |\n",
            "| train/                |              |\n",
            "|    entropy_loss       | -41.2        |\n",
            "|    explained_variance | 0.0236       |\n",
            "|    learning_rate      | 0.0007       |\n",
            "|    n_updates          | 1199         |\n",
            "|    policy_loss        | 151          |\n",
            "|    reward             | -0.099366836 |\n",
            "|    std                | 1            |\n",
            "|    value_loss         | 19.4         |\n",
            "----------------------------------------\n",
            "---------------------------------------\n",
            "| time/                 |             |\n",
            "|    fps                | 101         |\n",
            "|    iterations         | 1300        |\n",
            "|    time_elapsed       | 64          |\n",
            "|    total_timesteps    | 6500        |\n",
            "| train/                |             |\n",
            "|    entropy_loss       | -41.2       |\n",
            "|    explained_variance | -0.000106   |\n",
            "|    learning_rate      | 0.0007      |\n",
            "|    n_updates          | 1299        |\n",
            "|    policy_loss        | -47.6       |\n",
            "|    reward             | -0.39146426 |\n",
            "|    std                | 1           |\n",
            "|    value_loss         | 3.15        |\n",
            "---------------------------------------\n",
            "---------------------------------------\n",
            "| time/                 |             |\n",
            "|    fps                | 101         |\n",
            "|    iterations         | 1400        |\n",
            "|    time_elapsed       | 69          |\n",
            "|    total_timesteps    | 7000        |\n",
            "| train/                |             |\n",
            "|    entropy_loss       | -41.2       |\n",
            "|    explained_variance | 0           |\n",
            "|    learning_rate      | 0.0007      |\n",
            "|    n_updates          | 1399        |\n",
            "|    policy_loss        | 31.8        |\n",
            "|    reward             | -0.25985184 |\n",
            "|    std                | 1           |\n",
            "|    value_loss         | 1.84        |\n",
            "---------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 101       |\n",
            "|    iterations         | 1500      |\n",
            "|    time_elapsed       | 74        |\n",
            "|    total_timesteps    | 7500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.2     |\n",
            "|    explained_variance | 0         |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1499      |\n",
            "|    policy_loss        | -30.8     |\n",
            "|    reward             | 3.1766434 |\n",
            "|    std                | 1         |\n",
            "|    value_loss         | 4.93      |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 101        |\n",
            "|    iterations         | 1600       |\n",
            "|    time_elapsed       | 78         |\n",
            "|    total_timesteps    | 8000       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.2      |\n",
            "|    explained_variance | 0.00673    |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 1599       |\n",
            "|    policy_loss        | 133        |\n",
            "|    reward             | -2.0720153 |\n",
            "|    std                | 1          |\n",
            "|    value_loss         | 10.9       |\n",
            "--------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 101       |\n",
            "|    iterations         | 1700      |\n",
            "|    time_elapsed       | 83        |\n",
            "|    total_timesteps    | 8500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.2     |\n",
            "|    explained_variance | 5.96e-08  |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1699      |\n",
            "|    policy_loss        | -84.2     |\n",
            "|    reward             | 3.8356218 |\n",
            "|    std                | 1         |\n",
            "|    value_loss         | 7         |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 101        |\n",
            "|    iterations         | 1800       |\n",
            "|    time_elapsed       | 88         |\n",
            "|    total_timesteps    | 9000       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.2      |\n",
            "|    explained_variance | 0.063      |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 1799       |\n",
            "|    policy_loss        | 42         |\n",
            "|    reward             | 0.97460705 |\n",
            "|    std                | 1          |\n",
            "|    value_loss         | 2.23       |\n",
            "--------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 101       |\n",
            "|    iterations         | 1900      |\n",
            "|    time_elapsed       | 93        |\n",
            "|    total_timesteps    | 9500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.2     |\n",
            "|    explained_variance | 0.0122    |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1899      |\n",
            "|    policy_loss        | -84.7     |\n",
            "|    reward             | 2.2624621 |\n",
            "|    std                | 1         |\n",
            "|    value_loss         | 8.34      |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 101        |\n",
            "|    iterations         | 2000       |\n",
            "|    time_elapsed       | 98         |\n",
            "|    total_timesteps    | 10000      |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.2      |\n",
            "|    explained_variance | 0          |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 1999       |\n",
            "|    policy_loss        | 45.9       |\n",
            "|    reward             | -1.0189797 |\n",
            "|    std                | 1          |\n",
            "|    value_loss         | 1.77       |\n",
            "--------------------------------------\n",
            "======A2C Validation from:  2021-01-04 to  2021-04-06\n",
            "A2C Sharpe Ratio:  0.26168708794170054\n",
            "======PPO Training========\n",
            "{'ent_coef': 0.01, 'n_steps': 2048, 'learning_rate': 0.00025, 'batch_size': 128}\n",
            "Using cuda device\n",
            "Logging to tensorboard_log/ppo/ppo_126_1\n",
            "----------------------------------\n",
            "| time/              |           |\n",
            "|    fps             | 111       |\n",
            "|    iterations      | 1         |\n",
            "|    time_elapsed    | 18        |\n",
            "|    total_timesteps | 2048      |\n",
            "| train/             |           |\n",
            "|    reward          | 1.8877938 |\n",
            "----------------------------------\n",
            "day: 2959, episode: 5\n",
            "begin_total_asset: 1000000.00\n",
            "end_total_asset: 3881161.61\n",
            "total_reward: 2881161.61\n",
            "total_cost: 376867.37\n",
            "total_trades: 82873\n",
            "Sharpe: 0.776\n",
            "=================================\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 109         |\n",
            "|    iterations           | 2           |\n",
            "|    time_elapsed         | 37          |\n",
            "|    total_timesteps      | 4096        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.016203282 |\n",
            "|    clip_fraction        | 0.192       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.2       |\n",
            "|    explained_variance   | -0.0104     |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 4           |\n",
            "|    n_updates            | 10          |\n",
            "|    policy_gradient_loss | -0.0191     |\n",
            "|    reward               | 1.0358981   |\n",
            "|    std                  | 1           |\n",
            "|    value_loss           | 11.3        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 109         |\n",
            "|    iterations           | 3           |\n",
            "|    time_elapsed         | 56          |\n",
            "|    total_timesteps      | 6144        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.012528071 |\n",
            "|    clip_fraction        | 0.126       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.2       |\n",
            "|    explained_variance   | -0.00217    |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 57          |\n",
            "|    n_updates            | 20          |\n",
            "|    policy_gradient_loss | -0.0176     |\n",
            "|    reward               | 0.07821791  |\n",
            "|    std                  | 1           |\n",
            "|    value_loss           | 55.7        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 107         |\n",
            "|    iterations           | 4           |\n",
            "|    time_elapsed         | 76          |\n",
            "|    total_timesteps      | 8192        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.012139827 |\n",
            "|    clip_fraction        | 0.146       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.2       |\n",
            "|    explained_variance   | -0.00454    |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 13.7        |\n",
            "|    n_updates            | 30          |\n",
            "|    policy_gradient_loss | -0.018      |\n",
            "|    reward               | -1.5925834  |\n",
            "|    std                  | 1           |\n",
            "|    value_loss           | 43.4        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 107         |\n",
            "|    iterations           | 5           |\n",
            "|    time_elapsed         | 94          |\n",
            "|    total_timesteps      | 10240       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.019309906 |\n",
            "|    clip_fraction        | 0.17        |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.3       |\n",
            "|    explained_variance   | -0.0161     |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 5.07        |\n",
            "|    n_updates            | 40          |\n",
            "|    policy_gradient_loss | -0.0225     |\n",
            "|    reward               | 0.0615912   |\n",
            "|    std                  | 1.01        |\n",
            "|    value_loss           | 11.3        |\n",
            "-----------------------------------------\n",
            "======PPO Validation from:  2021-01-04 to  2021-04-06\n",
            "PPO Sharpe Ratio:  0.3016310407987196\n",
            "======DDPG Training========\n",
            "{'buffer_size': 10000, 'learning_rate': 0.0005, 'batch_size': 64}\n",
            "Using cuda device\n",
            "Logging to tensorboard_log/ddpg/ddpg_126_1\n",
            "day: 2959, episode: 10\n",
            "begin_total_asset: 1000000.00\n",
            "end_total_asset: 4424085.63\n",
            "total_reward: 3424085.63\n",
            "total_cost: 1037.34\n",
            "total_trades: 50241\n",
            "Sharpe: 0.797\n",
            "=================================\n",
            "----------------------------------\n",
            "| time/              |           |\n",
            "|    episodes        | 4         |\n",
            "|    fps             | 78        |\n",
            "|    time_elapsed    | 151       |\n",
            "|    total_timesteps | 11840     |\n",
            "| train/             |           |\n",
            "|    actor_loss      | -60.3     |\n",
            "|    critic_loss     | 441       |\n",
            "|    learning_rate   | 0.0005    |\n",
            "|    n_updates       | 8880      |\n",
            "|    reward          | 3.0541244 |\n",
            "----------------------------------\n",
            "======DDPG Validation from:  2021-01-04 to  2021-04-06\n",
            "======Best Model Retraining from:  2009-04-01 to  2021-04-06\n",
            "======Trading from:  2021-04-06 to  2021-07-06\n",
            "============================================\n",
            "turbulence_threshold:  203.40117769841157\n",
            "======Model training from:  2009-04-01 to  2021-04-06\n",
            "======A2C Training========\n",
            "{'n_steps': 5, 'ent_coef': 0.005, 'learning_rate': 0.0007}\n",
            "Using cuda device\n",
            "Logging to tensorboard_log/a2c/a2c_189_1\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 99         |\n",
            "|    iterations         | 100        |\n",
            "|    time_elapsed       | 5          |\n",
            "|    total_timesteps    | 500        |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41        |\n",
            "|    explained_variance | -0.446     |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 99         |\n",
            "|    policy_loss        | -85.9      |\n",
            "|    reward             | -0.6058027 |\n",
            "|    std                | 0.996      |\n",
            "|    value_loss         | 9.16       |\n",
            "--------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 100       |\n",
            "|    iterations         | 200       |\n",
            "|    time_elapsed       | 9         |\n",
            "|    total_timesteps    | 1000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41       |\n",
            "|    explained_variance | 0         |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 199       |\n",
            "|    policy_loss        | -19.4     |\n",
            "|    reward             | 1.8647146 |\n",
            "|    std                | 0.995     |\n",
            "|    value_loss         | 0.253     |\n",
            "-------------------------------------\n",
            "------------------------------------\n",
            "| time/                 |          |\n",
            "|    fps                | 100      |\n",
            "|    iterations         | 300      |\n",
            "|    time_elapsed       | 14       |\n",
            "|    total_timesteps    | 1500     |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -41      |\n",
            "|    explained_variance | 0.0122   |\n",
            "|    learning_rate      | 0.0007   |\n",
            "|    n_updates          | 299      |\n",
            "|    policy_loss        | -179     |\n",
            "|    reward             | 3.291392 |\n",
            "|    std                | 0.996    |\n",
            "|    value_loss         | 22.1     |\n",
            "------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 100       |\n",
            "|    iterations         | 400       |\n",
            "|    time_elapsed       | 19        |\n",
            "|    total_timesteps    | 2000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41       |\n",
            "|    explained_variance | 0.0507    |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 399       |\n",
            "|    policy_loss        | -11.1     |\n",
            "|    reward             | 0.3743614 |\n",
            "|    std                | 0.996     |\n",
            "|    value_loss         | 1.09      |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 100        |\n",
            "|    iterations         | 500        |\n",
            "|    time_elapsed       | 24         |\n",
            "|    total_timesteps    | 2500       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41        |\n",
            "|    explained_variance | 0          |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 499        |\n",
            "|    policy_loss        | 169        |\n",
            "|    reward             | -0.3291204 |\n",
            "|    std                | 0.995      |\n",
            "|    value_loss         | 16         |\n",
            "--------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 100        |\n",
            "|    iterations         | 600        |\n",
            "|    time_elapsed       | 29         |\n",
            "|    total_timesteps    | 3000       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41        |\n",
            "|    explained_variance | 0.054      |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 599        |\n",
            "|    policy_loss        | 905        |\n",
            "|    reward             | -1.0711567 |\n",
            "|    std                | 0.994      |\n",
            "|    value_loss         | 521        |\n",
            "--------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 100        |\n",
            "|    iterations         | 700        |\n",
            "|    time_elapsed       | 34         |\n",
            "|    total_timesteps    | 3500       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41        |\n",
            "|    explained_variance | -0.0433    |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 699        |\n",
            "|    policy_loss        | -32.9      |\n",
            "|    reward             | -2.9666545 |\n",
            "|    std                | 0.995      |\n",
            "|    value_loss         | 0.946      |\n",
            "--------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 100        |\n",
            "|    iterations         | 800        |\n",
            "|    time_elapsed       | 39         |\n",
            "|    total_timesteps    | 4000       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41        |\n",
            "|    explained_variance | -1.19e-07  |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 799        |\n",
            "|    policy_loss        | -3.25      |\n",
            "|    reward             | -0.5585839 |\n",
            "|    std                | 0.994      |\n",
            "|    value_loss         | 1.05       |\n",
            "--------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 100       |\n",
            "|    iterations         | 900       |\n",
            "|    time_elapsed       | 44        |\n",
            "|    total_timesteps    | 4500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -40.9     |\n",
            "|    explained_variance | -0.66     |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 899       |\n",
            "|    policy_loss        | 306       |\n",
            "|    reward             | 2.3080251 |\n",
            "|    std                | 0.992     |\n",
            "|    value_loss         | 70.3      |\n",
            "-------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 100       |\n",
            "|    iterations         | 1000      |\n",
            "|    time_elapsed       | 49        |\n",
            "|    total_timesteps    | 5000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41       |\n",
            "|    explained_variance | 0         |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 999       |\n",
            "|    policy_loss        | -108      |\n",
            "|    reward             | 1.1761196 |\n",
            "|    std                | 0.994     |\n",
            "|    value_loss         | 10.8      |\n",
            "-------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 100       |\n",
            "|    iterations         | 1100      |\n",
            "|    time_elapsed       | 54        |\n",
            "|    total_timesteps    | 5500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41       |\n",
            "|    explained_variance | 0         |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1099      |\n",
            "|    policy_loss        | -117      |\n",
            "|    reward             | 3.2501192 |\n",
            "|    std                | 0.995     |\n",
            "|    value_loss         | 10.7      |\n",
            "-------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 100       |\n",
            "|    iterations         | 1200      |\n",
            "|    time_elapsed       | 59        |\n",
            "|    total_timesteps    | 6000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41       |\n",
            "|    explained_variance | -1.19e-07 |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1199      |\n",
            "|    policy_loss        | 362       |\n",
            "|    reward             | 6.932239  |\n",
            "|    std                | 0.994     |\n",
            "|    value_loss         | 84.6      |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 100        |\n",
            "|    iterations         | 1300       |\n",
            "|    time_elapsed       | 64         |\n",
            "|    total_timesteps    | 6500       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41        |\n",
            "|    explained_variance | 0          |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 1299       |\n",
            "|    policy_loss        | 44.8       |\n",
            "|    reward             | -1.5383602 |\n",
            "|    std                | 0.994      |\n",
            "|    value_loss         | 2.08       |\n",
            "--------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 100       |\n",
            "|    iterations         | 1400      |\n",
            "|    time_elapsed       | 69        |\n",
            "|    total_timesteps    | 7000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41       |\n",
            "|    explained_variance | 0         |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1399      |\n",
            "|    policy_loss        | -143      |\n",
            "|    reward             | 0.7461217 |\n",
            "|    std                | 0.994     |\n",
            "|    value_loss         | 17        |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 100        |\n",
            "|    iterations         | 1500       |\n",
            "|    time_elapsed       | 74         |\n",
            "|    total_timesteps    | 7500       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41        |\n",
            "|    explained_variance | 0          |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 1499       |\n",
            "|    policy_loss        | -145       |\n",
            "|    reward             | -1.6722031 |\n",
            "|    std                | 0.995      |\n",
            "|    value_loss         | 14.8       |\n",
            "--------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 100       |\n",
            "|    iterations         | 1600      |\n",
            "|    time_elapsed       | 79        |\n",
            "|    total_timesteps    | 8000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41       |\n",
            "|    explained_variance | 1.19e-07  |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1599      |\n",
            "|    policy_loss        | 4.35      |\n",
            "|    reward             | 3.1992574 |\n",
            "|    std                | 0.995     |\n",
            "|    value_loss         | 0.916     |\n",
            "-------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 100       |\n",
            "|    iterations         | 1700      |\n",
            "|    time_elapsed       | 84        |\n",
            "|    total_timesteps    | 8500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -40.9     |\n",
            "|    explained_variance | -0.00566  |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1699      |\n",
            "|    policy_loss        | -1.16e+03 |\n",
            "|    reward             | 11.774156 |\n",
            "|    std                | 0.993     |\n",
            "|    value_loss         | 936       |\n",
            "-------------------------------------\n",
            "------------------------------------\n",
            "| time/                 |          |\n",
            "|    fps                | 100      |\n",
            "|    iterations         | 1800     |\n",
            "|    time_elapsed       | 89       |\n",
            "|    total_timesteps    | 9000     |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -40.9    |\n",
            "|    explained_variance | -0.00464 |\n",
            "|    learning_rate      | 0.0007   |\n",
            "|    n_updates          | 1799     |\n",
            "|    policy_loss        | -185     |\n",
            "|    reward             | 4.000574 |\n",
            "|    std                | 0.993    |\n",
            "|    value_loss         | 41.6     |\n",
            "------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 100        |\n",
            "|    iterations         | 1900       |\n",
            "|    time_elapsed       | 94         |\n",
            "|    total_timesteps    | 9500       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -40.9      |\n",
            "|    explained_variance | 0          |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 1899       |\n",
            "|    policy_loss        | 12.7       |\n",
            "|    reward             | -1.3208354 |\n",
            "|    std                | 0.991      |\n",
            "|    value_loss         | 0.306      |\n",
            "--------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 100       |\n",
            "|    iterations         | 2000      |\n",
            "|    time_elapsed       | 99        |\n",
            "|    total_timesteps    | 10000     |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -40.9     |\n",
            "|    explained_variance | 0         |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1999      |\n",
            "|    policy_loss        | -10.6     |\n",
            "|    reward             | 0.7043803 |\n",
            "|    std                | 0.992     |\n",
            "|    value_loss         | 0.431     |\n",
            "-------------------------------------\n",
            "======A2C Validation from:  2021-04-06 to  2021-07-06\n",
            "A2C Sharpe Ratio:  0.23121777505990648\n",
            "======PPO Training========\n",
            "{'ent_coef': 0.01, 'n_steps': 2048, 'learning_rate': 0.00025, 'batch_size': 128}\n",
            "Using cuda device\n",
            "Logging to tensorboard_log/ppo/ppo_189_1\n",
            "----------------------------------\n",
            "| time/              |           |\n",
            "|    fps             | 104       |\n",
            "|    iterations      | 1         |\n",
            "|    time_elapsed    | 19        |\n",
            "|    total_timesteps | 2048      |\n",
            "| train/             |           |\n",
            "|    reward          | 1.1126131 |\n",
            "----------------------------------\n",
            "day: 3022, episode: 5\n",
            "begin_total_asset: 1000000.00\n",
            "end_total_asset: 3636477.12\n",
            "total_reward: 2636477.12\n",
            "total_cost: 386575.15\n",
            "total_trades: 84625\n",
            "Sharpe: 0.731\n",
            "=================================\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 105         |\n",
            "|    iterations           | 2           |\n",
            "|    time_elapsed         | 38          |\n",
            "|    total_timesteps      | 4096        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.015871149 |\n",
            "|    clip_fraction        | 0.234       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.2       |\n",
            "|    explained_variance   | -0.0129     |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 3.2         |\n",
            "|    n_updates            | 10          |\n",
            "|    policy_gradient_loss | -0.0282     |\n",
            "|    reward               | 0.88015693  |\n",
            "|    std                  | 1           |\n",
            "|    value_loss           | 10.1        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 105         |\n",
            "|    iterations           | 3           |\n",
            "|    time_elapsed         | 58          |\n",
            "|    total_timesteps      | 6144        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.019049045 |\n",
            "|    clip_fraction        | 0.166       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.2       |\n",
            "|    explained_variance   | 0.00262     |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 32.1        |\n",
            "|    n_updates            | 20          |\n",
            "|    policy_gradient_loss | -0.023      |\n",
            "|    reward               | 0.49640504  |\n",
            "|    std                  | 1           |\n",
            "|    value_loss           | 53.7        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 105         |\n",
            "|    iterations           | 4           |\n",
            "|    time_elapsed         | 77          |\n",
            "|    total_timesteps      | 8192        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.015561214 |\n",
            "|    clip_fraction        | 0.161       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.2       |\n",
            "|    explained_variance   | -0.0133     |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 31.9        |\n",
            "|    n_updates            | 30          |\n",
            "|    policy_gradient_loss | -0.0201     |\n",
            "|    reward               | -0.7624933  |\n",
            "|    std                  | 1           |\n",
            "|    value_loss           | 45.6        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 105         |\n",
            "|    iterations           | 5           |\n",
            "|    time_elapsed         | 96          |\n",
            "|    total_timesteps      | 10240       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.014756277 |\n",
            "|    clip_fraction        | 0.141       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.2       |\n",
            "|    explained_variance   | -0.00517    |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 6.56        |\n",
            "|    n_updates            | 40          |\n",
            "|    policy_gradient_loss | -0.0204     |\n",
            "|    reward               | 1.2296445   |\n",
            "|    std                  | 1           |\n",
            "|    value_loss           | 14.8        |\n",
            "-----------------------------------------\n",
            "======PPO Validation from:  2021-04-06 to  2021-07-06\n",
            "PPO Sharpe Ratio:  0.010011405217654536\n",
            "======DDPG Training========\n",
            "{'buffer_size': 10000, 'learning_rate': 0.0005, 'batch_size': 64}\n",
            "Using cuda device\n",
            "Logging to tensorboard_log/ddpg/ddpg_189_1\n",
            "day: 3022, episode: 10\n",
            "begin_total_asset: 1000000.00\n",
            "end_total_asset: 5009064.34\n",
            "total_reward: 4009064.34\n",
            "total_cost: 1282.47\n",
            "total_trades: 51375\n",
            "Sharpe: 0.880\n",
            "=================================\n",
            "---------------------------------\n",
            "| time/              |          |\n",
            "|    episodes        | 4        |\n",
            "|    fps             | 76       |\n",
            "|    time_elapsed    | 157      |\n",
            "|    total_timesteps | 12092    |\n",
            "| train/             |          |\n",
            "|    actor_loss      | -65.5    |\n",
            "|    critic_loss     | 3.14e+03 |\n",
            "|    learning_rate   | 0.0005   |\n",
            "|    n_updates       | 9069     |\n",
            "|    reward          | 8.301459 |\n",
            "---------------------------------\n",
            "======DDPG Validation from:  2021-04-06 to  2021-07-06\n",
            "======Best Model Retraining from:  2009-04-01 to  2021-07-06\n",
            "======Trading from:  2021-07-06 to  2021-10-04\n",
            "============================================\n",
            "turbulence_threshold:  203.40117769841157\n",
            "======Model training from:  2009-04-01 to  2021-07-06\n",
            "======A2C Training========\n",
            "{'n_steps': 5, 'ent_coef': 0.005, 'learning_rate': 0.0007}\n",
            "Using cuda device\n",
            "Logging to tensorboard_log/a2c/a2c_252_1\n",
            "---------------------------------------\n",
            "| time/                 |             |\n",
            "|    fps                | 98          |\n",
            "|    iterations         | 100         |\n",
            "|    time_elapsed       | 5           |\n",
            "|    total_timesteps    | 500         |\n",
            "| train/                |             |\n",
            "|    entropy_loss       | -41.1       |\n",
            "|    explained_variance | -0.0351     |\n",
            "|    learning_rate      | 0.0007      |\n",
            "|    n_updates          | 99          |\n",
            "|    policy_loss        | -28.6       |\n",
            "|    reward             | -0.15436654 |\n",
            "|    std                | 0.998       |\n",
            "|    value_loss         | 5.15        |\n",
            "---------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 97        |\n",
            "|    iterations         | 200       |\n",
            "|    time_elapsed       | 10        |\n",
            "|    total_timesteps    | 1000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41       |\n",
            "|    explained_variance | 0         |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 199       |\n",
            "|    policy_loss        | 50.5      |\n",
            "|    reward             | 1.3676119 |\n",
            "|    std                | 0.996     |\n",
            "|    value_loss         | 2.12      |\n",
            "-------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 98        |\n",
            "|    iterations         | 300       |\n",
            "|    time_elapsed       | 15        |\n",
            "|    total_timesteps    | 1500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41       |\n",
            "|    explained_variance | 0         |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 299       |\n",
            "|    policy_loss        | -179      |\n",
            "|    reward             | 5.1952586 |\n",
            "|    std                | 0.995     |\n",
            "|    value_loss         | 22.2      |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 400        |\n",
            "|    time_elapsed       | 20         |\n",
            "|    total_timesteps    | 2000       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41        |\n",
            "|    explained_variance | 0          |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 399        |\n",
            "|    policy_loss        | 79.1       |\n",
            "|    reward             | -0.3978011 |\n",
            "|    std                | 0.995      |\n",
            "|    value_loss         | 7.82       |\n",
            "--------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 500        |\n",
            "|    time_elapsed       | 25         |\n",
            "|    total_timesteps    | 2500       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -40.9      |\n",
            "|    explained_variance | 0.19       |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 499        |\n",
            "|    policy_loss        | -8.81      |\n",
            "|    reward             | 0.19138733 |\n",
            "|    std                | 0.993      |\n",
            "|    value_loss         | 6.38       |\n",
            "--------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 600        |\n",
            "|    time_elapsed       | 30         |\n",
            "|    total_timesteps    | 3000       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41        |\n",
            "|    explained_variance | -0.00925   |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 599        |\n",
            "|    policy_loss        | -498       |\n",
            "|    reward             | -11.956305 |\n",
            "|    std                | 0.996      |\n",
            "|    value_loss         | 222        |\n",
            "--------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 700        |\n",
            "|    time_elapsed       | 35         |\n",
            "|    total_timesteps    | 3500       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41        |\n",
            "|    explained_variance | -0.169     |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 699        |\n",
            "|    policy_loss        | -68.7      |\n",
            "|    reward             | 0.39965674 |\n",
            "|    std                | 0.996      |\n",
            "|    value_loss         | 3.86       |\n",
            "--------------------------------------\n",
            "---------------------------------------\n",
            "| time/                 |             |\n",
            "|    fps                | 98          |\n",
            "|    iterations         | 800         |\n",
            "|    time_elapsed       | 40          |\n",
            "|    total_timesteps    | 4000        |\n",
            "| train/                |             |\n",
            "|    entropy_loss       | -41         |\n",
            "|    explained_variance | -0.0865     |\n",
            "|    learning_rate      | 0.0007      |\n",
            "|    n_updates          | 799         |\n",
            "|    policy_loss        | -164        |\n",
            "|    reward             | -0.33571205 |\n",
            "|    std                | 0.996       |\n",
            "|    value_loss         | 16.1        |\n",
            "---------------------------------------\n",
            "---------------------------------------\n",
            "| time/                 |             |\n",
            "|    fps                | 98          |\n",
            "|    iterations         | 900         |\n",
            "|    time_elapsed       | 45          |\n",
            "|    total_timesteps    | 4500        |\n",
            "| train/                |             |\n",
            "|    entropy_loss       | -41         |\n",
            "|    explained_variance | 0.159       |\n",
            "|    learning_rate      | 0.0007      |\n",
            "|    n_updates          | 899         |\n",
            "|    policy_loss        | 46.5        |\n",
            "|    reward             | -0.19355251 |\n",
            "|    std                | 0.996       |\n",
            "|    value_loss         | 1.99        |\n",
            "---------------------------------------\n",
            "---------------------------------------\n",
            "| time/                 |             |\n",
            "|    fps                | 98          |\n",
            "|    iterations         | 1000        |\n",
            "|    time_elapsed       | 50          |\n",
            "|    total_timesteps    | 5000        |\n",
            "| train/                |             |\n",
            "|    entropy_loss       | -41.1       |\n",
            "|    explained_variance | 0.0618      |\n",
            "|    learning_rate      | 0.0007      |\n",
            "|    n_updates          | 999         |\n",
            "|    policy_loss        | -15.1       |\n",
            "|    reward             | -0.48996487 |\n",
            "|    std                | 0.997       |\n",
            "|    value_loss         | 0.677       |\n",
            "---------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 98        |\n",
            "|    iterations         | 1100      |\n",
            "|    time_elapsed       | 55        |\n",
            "|    total_timesteps    | 5500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41       |\n",
            "|    explained_variance | -0.192    |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1099      |\n",
            "|    policy_loss        | -233      |\n",
            "|    reward             | 1.1524396 |\n",
            "|    std                | 0.997     |\n",
            "|    value_loss         | 35.2      |\n",
            "-------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 98        |\n",
            "|    iterations         | 1200      |\n",
            "|    time_elapsed       | 60        |\n",
            "|    total_timesteps    | 6000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.1     |\n",
            "|    explained_variance | -0.214    |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1199      |\n",
            "|    policy_loss        | 14.9      |\n",
            "|    reward             | -2.838138 |\n",
            "|    std                | 0.998     |\n",
            "|    value_loss         | 3.95      |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 1300       |\n",
            "|    time_elapsed       | 65         |\n",
            "|    total_timesteps    | 6500       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.2      |\n",
            "|    explained_variance | 0.167      |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 1299       |\n",
            "|    policy_loss        | 43.7       |\n",
            "|    reward             | -1.0078373 |\n",
            "|    std                | 1          |\n",
            "|    value_loss         | 1.86       |\n",
            "--------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 98        |\n",
            "|    iterations         | 1400      |\n",
            "|    time_elapsed       | 70        |\n",
            "|    total_timesteps    | 7000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.2     |\n",
            "|    explained_variance | -0.0727   |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1399      |\n",
            "|    policy_loss        | -17.9     |\n",
            "|    reward             | 2.1590974 |\n",
            "|    std                | 1         |\n",
            "|    value_loss         | 1.71      |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 1500       |\n",
            "|    time_elapsed       | 75         |\n",
            "|    total_timesteps    | 7500       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.2      |\n",
            "|    explained_variance | 0          |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 1499       |\n",
            "|    policy_loss        | 42.5       |\n",
            "|    reward             | 0.47163484 |\n",
            "|    std                | 1          |\n",
            "|    value_loss         | 1.17       |\n",
            "--------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 98        |\n",
            "|    iterations         | 1600      |\n",
            "|    time_elapsed       | 80        |\n",
            "|    total_timesteps    | 8000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.2     |\n",
            "|    explained_variance | 0.000262  |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1599      |\n",
            "|    policy_loss        | -73.2     |\n",
            "|    reward             | 0.8027305 |\n",
            "|    std                | 1         |\n",
            "|    value_loss         | 19.5      |\n",
            "-------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 98        |\n",
            "|    iterations         | 1700      |\n",
            "|    time_elapsed       | 85        |\n",
            "|    total_timesteps    | 8500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.2     |\n",
            "|    explained_variance | 0         |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1699      |\n",
            "|    policy_loss        | -184      |\n",
            "|    reward             | 0.3686392 |\n",
            "|    std                | 1         |\n",
            "|    value_loss         | 29.3      |\n",
            "-------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 98        |\n",
            "|    iterations         | 1800      |\n",
            "|    time_elapsed       | 90        |\n",
            "|    total_timesteps    | 9000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.2     |\n",
            "|    explained_variance | -1.19e-07 |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1799      |\n",
            "|    policy_loss        | 116       |\n",
            "|    reward             | 1.6162095 |\n",
            "|    std                | 1         |\n",
            "|    value_loss         | 21.6      |\n",
            "-------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 98        |\n",
            "|    iterations         | 1900      |\n",
            "|    time_elapsed       | 96        |\n",
            "|    total_timesteps    | 9500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.2     |\n",
            "|    explained_variance | -0.00951  |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1899      |\n",
            "|    policy_loss        | -1.74     |\n",
            "|    reward             | 1.1433389 |\n",
            "|    std                | 1         |\n",
            "|    value_loss         | 0.195     |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 2000       |\n",
            "|    time_elapsed       | 101        |\n",
            "|    total_timesteps    | 10000      |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.2      |\n",
            "|    explained_variance | -0.0113    |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 1999       |\n",
            "|    policy_loss        | -38.9      |\n",
            "|    reward             | 0.97623837 |\n",
            "|    std                | 1          |\n",
            "|    value_loss         | 0.806      |\n",
            "--------------------------------------\n",
            "======A2C Validation from:  2021-07-06 to  2021-10-04\n",
            "A2C Sharpe Ratio:  -0.0528373614471615\n",
            "======PPO Training========\n",
            "{'ent_coef': 0.01, 'n_steps': 2048, 'learning_rate': 0.00025, 'batch_size': 128}\n",
            "Using cuda device\n",
            "Logging to tensorboard_log/ppo/ppo_252_1\n",
            "----------------------------------\n",
            "| time/              |           |\n",
            "|    fps             | 101       |\n",
            "|    iterations      | 1         |\n",
            "|    time_elapsed    | 20        |\n",
            "|    total_timesteps | 2048      |\n",
            "| train/             |           |\n",
            "|    reward          | 1.2801732 |\n",
            "----------------------------------\n",
            "day: 3085, episode: 5\n",
            "begin_total_asset: 1000000.00\n",
            "end_total_asset: 4550937.53\n",
            "total_reward: 3550937.53\n",
            "total_cost: 408098.28\n",
            "total_trades: 86397\n",
            "Sharpe: 0.873\n",
            "=================================\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 102         |\n",
            "|    iterations           | 2           |\n",
            "|    time_elapsed         | 39          |\n",
            "|    total_timesteps      | 4096        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.018978704 |\n",
            "|    clip_fraction        | 0.233       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.2       |\n",
            "|    explained_variance   | -0.00994    |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 5.48        |\n",
            "|    n_updates            | 10          |\n",
            "|    policy_gradient_loss | -0.0284     |\n",
            "|    reward               | 0.49718028  |\n",
            "|    std                  | 1           |\n",
            "|    value_loss           | 10.8        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 103         |\n",
            "|    iterations           | 3           |\n",
            "|    time_elapsed         | 59          |\n",
            "|    total_timesteps      | 6144        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.012346724 |\n",
            "|    clip_fraction        | 0.155       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.2       |\n",
            "|    explained_variance   | -0.00278    |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 19.6        |\n",
            "|    n_updates            | 20          |\n",
            "|    policy_gradient_loss | -0.0154     |\n",
            "|    reward               | -0.8982904  |\n",
            "|    std                  | 1           |\n",
            "|    value_loss           | 50.6        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 103         |\n",
            "|    iterations           | 4           |\n",
            "|    time_elapsed         | 78          |\n",
            "|    total_timesteps      | 8192        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.014507987 |\n",
            "|    clip_fraction        | 0.123       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.2       |\n",
            "|    explained_variance   | -0.012      |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 43.8        |\n",
            "|    n_updates            | 30          |\n",
            "|    policy_gradient_loss | -0.016      |\n",
            "|    reward               | -0.0212465  |\n",
            "|    std                  | 1           |\n",
            "|    value_loss           | 74.7        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 104         |\n",
            "|    iterations           | 5           |\n",
            "|    time_elapsed         | 98          |\n",
            "|    total_timesteps      | 10240       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.016703494 |\n",
            "|    clip_fraction        | 0.186       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.3       |\n",
            "|    explained_variance   | -0.0496     |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 4.09        |\n",
            "|    n_updates            | 40          |\n",
            "|    policy_gradient_loss | -0.0285     |\n",
            "|    reward               | 1.0109037   |\n",
            "|    std                  | 1.01        |\n",
            "|    value_loss           | 14.2        |\n",
            "-----------------------------------------\n",
            "======PPO Validation from:  2021-07-06 to  2021-10-04\n",
            "PPO Sharpe Ratio:  -0.05262682094979255\n",
            "======DDPG Training========\n",
            "{'buffer_size': 10000, 'learning_rate': 0.0005, 'batch_size': 64}\n",
            "Using cuda device\n",
            "Logging to tensorboard_log/ddpg/ddpg_252_1\n",
            "day: 3085, episode: 10\n",
            "begin_total_asset: 1000000.00\n",
            "end_total_asset: 4501430.61\n",
            "total_reward: 3501430.61\n",
            "total_cost: 1082.97\n",
            "total_trades: 61741\n",
            "Sharpe: 0.737\n",
            "=================================\n",
            "----------------------------------\n",
            "| time/              |           |\n",
            "|    episodes        | 4         |\n",
            "|    fps             | 76        |\n",
            "|    time_elapsed    | 161       |\n",
            "|    total_timesteps | 12344     |\n",
            "| train/             |           |\n",
            "|    actor_loss      | -3.66     |\n",
            "|    critic_loss     | 176       |\n",
            "|    learning_rate   | 0.0005    |\n",
            "|    n_updates       | 9258      |\n",
            "|    reward          | 2.1645079 |\n",
            "----------------------------------\n",
            "======DDPG Validation from:  2021-07-06 to  2021-10-04\n",
            "======Best Model Retraining from:  2009-04-01 to  2021-10-04\n",
            "======Trading from:  2021-10-04 to  2022-01-03\n",
            "============================================\n",
            "turbulence_threshold:  203.40117769841157\n",
            "======Model training from:  2009-04-01 to  2021-10-04\n",
            "======A2C Training========\n",
            "{'n_steps': 5, 'ent_coef': 0.005, 'learning_rate': 0.0007}\n",
            "Using cuda device\n",
            "Logging to tensorboard_log/a2c/a2c_315_1\n",
            "----------------------------------------\n",
            "| time/                 |              |\n",
            "|    fps                | 98           |\n",
            "|    iterations         | 100          |\n",
            "|    time_elapsed       | 5            |\n",
            "|    total_timesteps    | 500          |\n",
            "| train/                |              |\n",
            "|    entropy_loss       | -41.3        |\n",
            "|    explained_variance | 0            |\n",
            "|    learning_rate      | 0.0007       |\n",
            "|    n_updates          | 99           |\n",
            "|    policy_loss        | -51.2        |\n",
            "|    reward             | -0.096114956 |\n",
            "|    std                | 1.01         |\n",
            "|    value_loss         | 4.56         |\n",
            "----------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 98        |\n",
            "|    iterations         | 200       |\n",
            "|    time_elapsed       | 10        |\n",
            "|    total_timesteps    | 1000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.3     |\n",
            "|    explained_variance | 5.96e-08  |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 199       |\n",
            "|    policy_loss        | -36.9     |\n",
            "|    reward             | 2.9646184 |\n",
            "|    std                | 1         |\n",
            "|    value_loss         | 0.756     |\n",
            "-------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 97        |\n",
            "|    iterations         | 300       |\n",
            "|    time_elapsed       | 15        |\n",
            "|    total_timesteps    | 1500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.3     |\n",
            "|    explained_variance | 0         |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 299       |\n",
            "|    policy_loss        | -236      |\n",
            "|    reward             | 1.1201329 |\n",
            "|    std                | 1.01      |\n",
            "|    value_loss         | 36.4      |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 400        |\n",
            "|    time_elapsed       | 20         |\n",
            "|    total_timesteps    | 2000       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.4      |\n",
            "|    explained_variance | 0          |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 399        |\n",
            "|    policy_loss        | -53.9      |\n",
            "|    reward             | 0.03644575 |\n",
            "|    std                | 1.01       |\n",
            "|    value_loss         | 2.41       |\n",
            "--------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 98        |\n",
            "|    iterations         | 500       |\n",
            "|    time_elapsed       | 25        |\n",
            "|    total_timesteps    | 2500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.3     |\n",
            "|    explained_variance | 0         |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 499       |\n",
            "|    policy_loss        | 34.1      |\n",
            "|    reward             | 0.3948357 |\n",
            "|    std                | 1         |\n",
            "|    value_loss         | 1.08      |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 600        |\n",
            "|    time_elapsed       | 30         |\n",
            "|    total_timesteps    | 3000       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.3      |\n",
            "|    explained_variance | -1.19e-07  |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 599        |\n",
            "|    policy_loss        | -618       |\n",
            "|    reward             | -12.502208 |\n",
            "|    std                | 1          |\n",
            "|    value_loss         | 338        |\n",
            "--------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 700        |\n",
            "|    time_elapsed       | 35         |\n",
            "|    total_timesteps    | 3500       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.2      |\n",
            "|    explained_variance | -0.0164    |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 699        |\n",
            "|    policy_loss        | -105       |\n",
            "|    reward             | 0.30092302 |\n",
            "|    std                | 1          |\n",
            "|    value_loss         | 10.8       |\n",
            "--------------------------------------\n",
            "---------------------------------------\n",
            "| time/                 |             |\n",
            "|    fps                | 98          |\n",
            "|    iterations         | 800         |\n",
            "|    time_elapsed       | 40          |\n",
            "|    total_timesteps    | 4000        |\n",
            "| train/                |             |\n",
            "|    entropy_loss       | -41.3       |\n",
            "|    explained_variance | 1.19e-07    |\n",
            "|    learning_rate      | 0.0007      |\n",
            "|    n_updates          | 799         |\n",
            "|    policy_loss        | 74.6        |\n",
            "|    reward             | 0.009111182 |\n",
            "|    std                | 1           |\n",
            "|    value_loss         | 4.73        |\n",
            "---------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 98        |\n",
            "|    iterations         | 900       |\n",
            "|    time_elapsed       | 45        |\n",
            "|    total_timesteps    | 4500      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.3     |\n",
            "|    explained_variance | -0.0124   |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 899       |\n",
            "|    policy_loss        | -131      |\n",
            "|    reward             | 2.0478728 |\n",
            "|    std                | 1         |\n",
            "|    value_loss         | 16.5      |\n",
            "-------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 1000       |\n",
            "|    time_elapsed       | 50         |\n",
            "|    total_timesteps    | 5000       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.3      |\n",
            "|    explained_variance | 0          |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 999        |\n",
            "|    policy_loss        | -34.9      |\n",
            "|    reward             | 0.04952446 |\n",
            "|    std                | 1.01       |\n",
            "|    value_loss         | 2.24       |\n",
            "--------------------------------------\n",
            "---------------------------------------\n",
            "| time/                 |             |\n",
            "|    fps                | 98          |\n",
            "|    iterations         | 1100        |\n",
            "|    time_elapsed       | 55          |\n",
            "|    total_timesteps    | 5500        |\n",
            "| train/                |             |\n",
            "|    entropy_loss       | -41.3       |\n",
            "|    explained_variance | -0.0188     |\n",
            "|    learning_rate      | 0.0007      |\n",
            "|    n_updates          | 1099        |\n",
            "|    policy_loss        | 344         |\n",
            "|    reward             | -0.11844914 |\n",
            "|    std                | 1.01        |\n",
            "|    value_loss         | 64.7        |\n",
            "---------------------------------------\n",
            "---------------------------------------\n",
            "| time/                 |             |\n",
            "|    fps                | 98          |\n",
            "|    iterations         | 1200        |\n",
            "|    time_elapsed       | 60          |\n",
            "|    total_timesteps    | 6000        |\n",
            "| train/                |             |\n",
            "|    entropy_loss       | -41.4       |\n",
            "|    explained_variance | 0           |\n",
            "|    learning_rate      | 0.0007      |\n",
            "|    n_updates          | 1199        |\n",
            "|    policy_loss        | 58.9        |\n",
            "|    reward             | -0.28185675 |\n",
            "|    std                | 1.01        |\n",
            "|    value_loss         | 6.99        |\n",
            "---------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 1300       |\n",
            "|    time_elapsed       | 66         |\n",
            "|    total_timesteps    | 6500       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.4      |\n",
            "|    explained_variance | 0.549      |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 1299       |\n",
            "|    policy_loss        | -11.7      |\n",
            "|    reward             | -0.7957832 |\n",
            "|    std                | 1.01       |\n",
            "|    value_loss         | 0.135      |\n",
            "--------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 1400       |\n",
            "|    time_elapsed       | 71         |\n",
            "|    total_timesteps    | 7000       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.4      |\n",
            "|    explained_variance | 1.19e-07   |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 1399       |\n",
            "|    policy_loss        | -29.7      |\n",
            "|    reward             | 0.16117185 |\n",
            "|    std                | 1.01       |\n",
            "|    value_loss         | 0.859      |\n",
            "--------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 1500       |\n",
            "|    time_elapsed       | 76         |\n",
            "|    total_timesteps    | 7500       |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.4      |\n",
            "|    explained_variance | 1.79e-07   |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 1499       |\n",
            "|    policy_loss        | 15.7       |\n",
            "|    reward             | 0.17504984 |\n",
            "|    std                | 1.01       |\n",
            "|    value_loss         | 0.528      |\n",
            "--------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 98        |\n",
            "|    iterations         | 1600      |\n",
            "|    time_elapsed       | 81        |\n",
            "|    total_timesteps    | 8000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.4     |\n",
            "|    explained_variance | 0         |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1599      |\n",
            "|    policy_loss        | 48.7      |\n",
            "|    reward             | 3.1540215 |\n",
            "|    std                | 1.01      |\n",
            "|    value_loss         | 6.42      |\n",
            "-------------------------------------\n",
            "---------------------------------------\n",
            "| time/                 |             |\n",
            "|    fps                | 98          |\n",
            "|    iterations         | 1700        |\n",
            "|    time_elapsed       | 86          |\n",
            "|    total_timesteps    | 8500        |\n",
            "| train/                |             |\n",
            "|    entropy_loss       | -41.4       |\n",
            "|    explained_variance | 0           |\n",
            "|    learning_rate      | 0.0007      |\n",
            "|    n_updates          | 1699        |\n",
            "|    policy_loss        | 22.9        |\n",
            "|    reward             | -0.09754947 |\n",
            "|    std                | 1.01        |\n",
            "|    value_loss         | 0.677       |\n",
            "---------------------------------------\n",
            "-------------------------------------\n",
            "| time/                 |           |\n",
            "|    fps                | 98        |\n",
            "|    iterations         | 1800      |\n",
            "|    time_elapsed       | 91        |\n",
            "|    total_timesteps    | 9000      |\n",
            "| train/                |           |\n",
            "|    entropy_loss       | -41.4     |\n",
            "|    explained_variance | -0.171    |\n",
            "|    learning_rate      | 0.0007    |\n",
            "|    n_updates          | 1799      |\n",
            "|    policy_loss        | 99        |\n",
            "|    reward             | -1.175583 |\n",
            "|    std                | 1.01      |\n",
            "|    value_loss         | 9.98      |\n",
            "-------------------------------------\n",
            "------------------------------------\n",
            "| time/                 |          |\n",
            "|    fps                | 98       |\n",
            "|    iterations         | 1900     |\n",
            "|    time_elapsed       | 96       |\n",
            "|    total_timesteps    | 9500     |\n",
            "| train/                |          |\n",
            "|    entropy_loss       | -41.4    |\n",
            "|    explained_variance | 0        |\n",
            "|    learning_rate      | 0.0007   |\n",
            "|    n_updates          | 1899     |\n",
            "|    policy_loss        | -24.6    |\n",
            "|    reward             | 0.362812 |\n",
            "|    std                | 1.01     |\n",
            "|    value_loss         | 0.367    |\n",
            "------------------------------------\n",
            "--------------------------------------\n",
            "| time/                 |            |\n",
            "|    fps                | 98         |\n",
            "|    iterations         | 2000       |\n",
            "|    time_elapsed       | 101        |\n",
            "|    total_timesteps    | 10000      |\n",
            "| train/                |            |\n",
            "|    entropy_loss       | -41.4      |\n",
            "|    explained_variance | 0          |\n",
            "|    learning_rate      | 0.0007     |\n",
            "|    n_updates          | 1999       |\n",
            "|    policy_loss        | -146       |\n",
            "|    reward             | -1.7740997 |\n",
            "|    std                | 1.01       |\n",
            "|    value_loss         | 13.5       |\n",
            "--------------------------------------\n",
            "======A2C Validation from:  2021-10-04 to  2022-01-03\n",
            "A2C Sharpe Ratio:  0.3744625013790845\n",
            "======PPO Training========\n",
            "{'ent_coef': 0.01, 'n_steps': 2048, 'learning_rate': 0.00025, 'batch_size': 128}\n",
            "Using cuda device\n",
            "Logging to tensorboard_log/ppo/ppo_315_1\n",
            "----------------------------------\n",
            "| time/              |           |\n",
            "|    fps             | 106       |\n",
            "|    iterations      | 1         |\n",
            "|    time_elapsed    | 19        |\n",
            "|    total_timesteps | 2048      |\n",
            "| train/             |           |\n",
            "|    reward          | 0.7561091 |\n",
            "----------------------------------\n",
            "day: 3148, episode: 5\n",
            "begin_total_asset: 1000000.00\n",
            "end_total_asset: 4259003.47\n",
            "total_reward: 3259003.47\n",
            "total_cost: 426068.70\n",
            "total_trades: 87901\n",
            "Sharpe: 0.790\n",
            "=================================\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 101         |\n",
            "|    iterations           | 2           |\n",
            "|    time_elapsed         | 40          |\n",
            "|    total_timesteps      | 4096        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.016951367 |\n",
            "|    clip_fraction        | 0.229       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.2       |\n",
            "|    explained_variance   | -0.000697   |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 5.25        |\n",
            "|    n_updates            | 10          |\n",
            "|    policy_gradient_loss | -0.0307     |\n",
            "|    reward               | 0.9373541   |\n",
            "|    std                  | 1           |\n",
            "|    value_loss           | 9.91        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 102         |\n",
            "|    iterations           | 3           |\n",
            "|    time_elapsed         | 60          |\n",
            "|    total_timesteps      | 6144        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.017283382 |\n",
            "|    clip_fraction        | 0.176       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.3       |\n",
            "|    explained_variance   | 0.00633     |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 33.4        |\n",
            "|    n_updates            | 20          |\n",
            "|    policy_gradient_loss | -0.0207     |\n",
            "|    reward               | 7.712371    |\n",
            "|    std                  | 1           |\n",
            "|    value_loss           | 56.7        |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 102         |\n",
            "|    iterations           | 4           |\n",
            "|    time_elapsed         | 79          |\n",
            "|    total_timesteps      | 8192        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.010827523 |\n",
            "|    clip_fraction        | 0.0962      |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.3       |\n",
            "|    explained_variance   | 0.00682     |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 73.4        |\n",
            "|    n_updates            | 30          |\n",
            "|    policy_gradient_loss | -0.018      |\n",
            "|    reward               | -0.83698237 |\n",
            "|    std                  | 1.01        |\n",
            "|    value_loss           | 134         |\n",
            "-----------------------------------------\n",
            "-----------------------------------------\n",
            "| time/                   |             |\n",
            "|    fps                  | 102         |\n",
            "|    iterations           | 5           |\n",
            "|    time_elapsed         | 99          |\n",
            "|    total_timesteps      | 10240       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.013603406 |\n",
            "|    clip_fraction        | 0.172       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -41.4       |\n",
            "|    explained_variance   | 0.0243      |\n",
            "|    learning_rate        | 0.00025     |\n",
            "|    loss                 | 4.64        |\n",
            "|    n_updates            | 40          |\n",
            "|    policy_gradient_loss | -0.0243     |\n",
            "|    reward               | 0.5904208   |\n",
            "|    std                  | 1.01        |\n",
            "|    value_loss           | 12.7        |\n",
            "-----------------------------------------\n",
            "======PPO Validation from:  2021-10-04 to  2022-01-03\n",
            "PPO Sharpe Ratio:  0.0593551480877188\n",
            "======DDPG Training========\n",
            "{'buffer_size': 10000, 'learning_rate': 0.0005, 'batch_size': 64}\n",
            "Using cuda device\n",
            "Logging to tensorboard_log/ddpg/ddpg_315_1\n",
            "day: 3148, episode: 10\n",
            "begin_total_asset: 1000000.00\n",
            "end_total_asset: 4617783.42\n",
            "total_reward: 3617783.42\n",
            "total_cost: 1110.92\n",
            "total_trades: 49019\n",
            "Sharpe: 0.745\n",
            "=================================\n",
            "----------------------------------\n",
            "| time/              |           |\n",
            "|    episodes        | 4         |\n",
            "|    fps             | 75        |\n",
            "|    time_elapsed    | 167       |\n",
            "|    total_timesteps | 12596     |\n",
            "| train/             |           |\n",
            "|    actor_loss      | 36.1      |\n",
            "|    critic_loss     | 656       |\n",
            "|    learning_rate   | 0.0005    |\n",
            "|    n_updates       | 9447      |\n",
            "|    reward          | 5.4972343 |\n",
            "----------------------------------\n",
            "======DDPG Validation from:  2021-10-04 to  2022-01-03\n",
            "======Best Model Retraining from:  2009-04-01 to  2022-01-03\n",
            "======Trading from:  2022-01-03 to  2022-04-04\n",
            "Ensemble Strategy took:  25.210217595100403  minutes\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "AttributeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-30-4153cda81a48>\u001b[0m in \u001b[0;36m<cell line: 163>\u001b[0;34m()\u001b[0m\n\u001b[1;32m    162\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    163\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0m__name__\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'__main__'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 164\u001b[0;31m     \u001b[0mmain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-30-4153cda81a48>\u001b[0m in \u001b[0;36mmain\u001b[0;34m()\u001b[0m\n\u001b[1;32m    127\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrebalance_window\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mvalidation_window\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0munique_trade_date\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrebalance_window\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    128\u001b[0m         \u001b[0mtemp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'results/account_value_trade_{}_{}.csv'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'ensemble'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 129\u001b[0;31m         \u001b[0mdf_account_value\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdf_account_value\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtemp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mignore_index\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    130\u001b[0m     sharpe = (252 ** 0.5) * df_account_value.account_value.pct_change(\n\u001b[1;32m    131\u001b[0m         1).mean() / df_account_value.account_value.pct_change(1).std()\n",
            "\u001b[0;32m/usr/local/lib/python3.9/site-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36m__getattr__\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m   5987\u001b[0m         ):\n\u001b[1;32m   5988\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 5989\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mobject\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__getattribute__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   5990\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   5991\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__setattr__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mAttributeError\u001b[0m: 'DataFrame' object has no attribute 'append'"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "S15p3oh8smm1"
      },
      "execution_count": 401,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data = [(50, 126.8078079), (-50, 125.1344223), (59, 125.0349121), (-29, 122.90924835205078), (-26, 123.868042)]\n",
        "\n",
        "total_shares = 0\n",
        "total_price = 0\n",
        "\n",
        "for shares, price in data:\n",
        "    total_shares += shares\n",
        "    total_price += shares * price\n",
        "\n",
        "average_price = total_price / total_shares\n",
        "\n",
        "print(\"Average price of the stock: \", round(average_price, 2))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IJO0W8Uxsr2i",
        "outputId": "0397056e-3e73-4968-d87a-74aee73518f9"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Average price of the stock:  168.95\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data = [(50, 126.8078079), (-50, 125.1344223), (59, 125.0349121), (-29, 122.90924835205078), (-26, 123.868042)]\n",
        "\n",
        "total_shares = 0\n",
        "total_price = 0\n",
        "\n",
        "for shares, price in data:\n",
        "    total_shares += abs(shares)\n",
        "    total_price += shares * price\n",
        "\n",
        "average_price = total_price / total_shares\n",
        "\n",
        "print(\"Average price of the stock: \", round(average_price, 2))"
      ],
      "metadata": {
        "id": "diwuHZ3Q0-fA",
        "outputId": "2729289a-639b-4aea-9f47-2b7ebcd80a36",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Average price of the stock:  3.16\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "class Portfolio:\n",
        "    def __init__(self):\n",
        "        self.num_shares = 0\n",
        "        self.total_cost = 0.0\n",
        "        self.avg_cost = 0.0\n",
        "\n",
        "    def buy(self, num_shares, price):\n",
        "        self.total_cost += num_shares * price\n",
        "        self.num_shares += num_shares\n",
        "        self.avg_cost = self.total_cost / self.num_shares\n",
        "\n",
        "    def sell(self, num_shares):\n",
        "        self.total_cost -= self.avg_cost * num_shares\n",
        "        self.num_shares -= num_shares\n",
        "\n",
        "    def get_num_shares(self):\n",
        "        return self.num_shares\n",
        "\n",
        "    def get_total_cost(self):\n",
        "        return self.total_cost\n",
        "\n",
        "    def get_avg_cost(self):\n",
        "        return self.avg_cost\n",
        "\n",
        "# Example usage\n",
        "portfolio = Portfolio()\n",
        "\n",
        "# Buy 50 shares at $126.81\n",
        "portfolio.buy(50, 126.81)\n",
        "\n",
        "# Sell 25 shares\n",
        "portfolio.sell(25)\n",
        "\n",
        "# Buy 30 shares at $125.03\n",
        "portfolio.buy(30, 125.03)\n",
        "\n",
        "# Sell 10 shares\n",
        "portfolio.sell(10)\n",
        "\n",
        "\n",
        "print(\"Number of shares: \", portfolio.get_num_shares())\n",
        "print(\"Total cost: \", round(portfolio.get_total_cost(), 2))\n",
        "print(\"Average cost: \", round(portfolio.get_avg_cost(), 2))\n"
      ],
      "metadata": {
        "id": "EZuXe0Zo1Nig",
        "outputId": "2a4b1d3d-4f24-45d7-bb62-f781fa9f1df1",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Number of shares:  -25\n",
            "Total cost:  0.0\n",
            "Average cost:  0.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "\n",
        "class Portfolio:\n",
        "    def __init__(self):\n",
        "        self.num_shares = 0\n",
        "        self.total_cost = 0.0\n",
        "        self.avg_cost = 0.0\n",
        "\n",
        "    def buy(self, num_shares, price):\n",
        "        self.total_cost += num_shares * price\n",
        "        self.num_shares += num_shares\n",
        "        self.avg_cost = self.total_cost / self.num_shares\n",
        "\n",
        "    def sell(self, num_shares):\n",
        "        self.total_cost -= self.avg_cost * num_shares\n",
        "        self.num_shares -= num_shares\n",
        "\n",
        "    def get_num_shares(self):\n",
        "        return self.num_shares\n",
        "\n",
        "    def get_total_cost(self):\n",
        "        return self.total_cost\n",
        "\n",
        "    def get_avg_cost(self):\n",
        "        return self.avg_cost\n",
        "\n",
        "    @staticmethod\n",
        "    def calculate_portfolio(data):\n",
        "        shares, prices = np.array(data).T\n",
        "        abs_shares = np.abs(shares)\n",
        "\n",
        "        total_shares = np.cumsum(shares)\n",
        "        total_cost = np.cumsum(shares * prices)\n",
        "        avg_cost = total_cost / total_shares\n",
        "\n",
        "        sell_mask = shares < 0\n",
        "        num_sold = abs_shares[sell_mask]\n",
        "        cost_sold = num_sold * avg_cost[sell_mask]\n",
        "        total_cost[sell_mask] -= cost_sold\n",
        "        total_shares[sell_mask] -= num_sold\n",
        "\n",
        "        return total_shares[-1], total_cost[-1], avg_cost[-1]\n",
        "\n",
        "# Example usage\n",
        "data = [(50, 126.81), (-25, 0.0), (30, 125.03), (-10, 0.0)]\n",
        "num_shares, total_cost, avg_cost = Portfolio.calculate_portfolio(data)\n",
        "\n",
        "print(\"Number of shares: \", num_shares)\n",
        "print(\"Total cost: \", round(total_cost, 2))\n",
        "print(\"Average cost: \", round(avg_cost, 2))\n"
      ],
      "metadata": {
        "id": "bBd99tTE1z0o",
        "outputId": "a1300a6d-be53-498e-f931-e4518a8cd602",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Number of shares:  35.0\n",
            "Total cost:  7848.87\n",
            "Average cost:  224.25\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "yzHIcPMv2TSp"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "colab": {
      "name": "Making the most of your colab subscription",
      "provenance": [],
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "accelerator": "GPU",
    "gpuClass": "standard"
  },
  "nbformat": 4,
  "nbformat_minor": 0
}