vision3d/latent_nerf.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyO76b+tBCf/WBJbMUbIS5EN",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "gpuClass": "standard"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/softmurata/colab_notebooks/blob/main/vision3d/latent_nerf.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YIexV0_Up1Rr",
        "outputId": "9b99bfe6-6a2a-4480-ddb8-0727335a2e37"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'latent-nerf'...\n",
            "remote: Enumerating objects: 163, done.\u001b[K\n",
            "remote: Counting objects: 100% (150/150), done.\u001b[K\n",
            "remote: Compressing objects: 100% (123/123), done.\u001b[K\n",
            "remote: Total 163 (delta 35), reused 122 (delta 21), pack-reused 13\u001b[K\n",
            "Receiving objects: 100% (163/163), 78.87 MiB | 26.58 MiB/s, done.\n",
            "Resolving deltas: 100% (35/35), done.\n"
          ]
        }
      ],
      "source": [
        "!git clone https://github.com/eladrich/latent-nerf.git"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "os.chdir(\"/content/latent-nerf\")"
      ],
      "metadata": {
        "id": "ta3hBcVzqUgs"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install torch==1.12.1+cu116 torchvision==0.13.1+cu116 torchaudio==0.12.1 --extra-index-url https://download.pytorch.org/whl/cu116"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "N9fu0q51vDUG",
        "outputId": "3b3c1bbd-ef25-45d0-f569-fa628325a624"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/, https://download.pytorch.org/whl/cu116\n",
            "Collecting torch==1.12.1+cu116\n",
            "  Downloading https://download.pytorch.org/whl/cu116/torch-1.12.1%2Bcu116-cp38-cp38-linux_x86_64.whl (1904.8 MB)\n",
            "\u001b[2K     \u001b[91m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m \u001b[32m1.9/1.9 GB\u001b[0m \u001b[31m106.0 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0mtcmalloc: large alloc 1904771072 bytes == 0x3c08000 @  0x7fa76a43d1e7 0x4d30a0 0x4d312c 0x5d6f4c 0x51edd1 0x51ef5b 0x4f750a 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x5d8868 0x4997a2 0x55cd91 0x5d8941 0x49abe4 0x55cd91 0x5d8941 0x4997a2\n",
            "\u001b[2K     \u001b[91m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m \u001b[32m1.9/1.9 GB\u001b[0m \u001b[31m105.1 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0mtcmalloc: large alloc 2380963840 bytes == 0x75490000 @  0x7fa76a43e615 0x5d6f4c 0x51edd1 0x51ef5b 0x4f750a 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x5d8868 0x4997a2 0x55cd91 0x5d8941 0x49abe4 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941\n",
            "tcmalloc: large alloc 1904771072 bytes == 0x3c08000 @  0x7fa76a43d1e7 0x4d30a0 0x5dede2 0x6758aa 0x4f750a 0x4997a2 0x55cd91 0x5d8941 0x4997a2 0x55cd91 0x5d8941 0x4fe318 0x5da092 0x62042c 0x5d8d8c 0x561f80 0x4fd2db 0x4997c7 0x4fd8b5 0x4997c7 0x4fd8b5 0x49abe4 0x4f5fe9 0x55e146 0x4f5fe9 0x55e146 0x4f5fe9 0x55e146 0x5d8868 0x5da092 0x587116\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.9/1.9 GB\u001b[0m \u001b[31m911.2 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting torchvision==0.13.1+cu116\n",
            "  Downloading https://download.pytorch.org/whl/cu116/torchvision-0.13.1%2Bcu116-cp38-cp38-linux_x86_64.whl (23.5 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m23.5/23.5 MB\u001b[0m \u001b[31m39.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting torchaudio==0.12.1\n",
            "  Downloading https://download.pytorch.org/whl/cu116/torchaudio-0.12.1%2Bcu116-cp38-cp38-linux_x86_64.whl (3.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.8/3.8 MB\u001b[0m \u001b[31m30.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: typing-extensions in /usr/local/lib/python3.8/dist-packages (from torch==1.12.1+cu116) (4.4.0)\n",
            "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /usr/local/lib/python3.8/dist-packages (from torchvision==0.13.1+cu116) (7.1.2)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.8/dist-packages (from torchvision==0.13.1+cu116) (2.25.1)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.8/dist-packages (from torchvision==0.13.1+cu116) (1.21.6)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.8/dist-packages (from requests->torchvision==0.13.1+cu116) (2.10)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.8/dist-packages (from requests->torchvision==0.13.1+cu116) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.8/dist-packages (from requests->torchvision==0.13.1+cu116) (2022.12.7)\n",
            "Requirement already satisfied: chardet<5,>=3.0.2 in /usr/local/lib/python3.8/dist-packages (from requests->torchvision==0.13.1+cu116) (4.0.0)\n",
            "Installing collected packages: torch, torchvision, torchaudio\n",
            "  Attempting uninstall: torch\n",
            "    Found existing installation: torch 1.13.0+cu116\n",
            "    Uninstalling torch-1.13.0+cu116:\n",
            "      Successfully uninstalled torch-1.13.0+cu116\n",
            "  Attempting uninstall: torchvision\n",
            "    Found existing installation: torchvision 0.14.0+cu116\n",
            "    Uninstalling torchvision-0.14.0+cu116:\n",
            "      Successfully uninstalled torchvision-0.14.0+cu116\n",
            "  Attempting uninstall: torchaudio\n",
            "    Found existing installation: torchaudio 0.13.0+cu116\n",
            "    Uninstalling torchaudio-0.13.0+cu116:\n",
            "      Successfully uninstalled torchaudio-0.13.0+cu116\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "torchtext 0.14.0 requires torch==1.13.0, but you have torch 1.12.1+cu116 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0mSuccessfully installed torch-1.12.1+cu116 torchaudio-0.12.1+cu116 torchvision-0.13.1+cu116\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -r requirements.txt"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2AKyreZfqY3G",
        "outputId": "8e1da2f4-d637-4a39-fb00-d702c9d689a6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting transformers\n",
            "  Downloading transformers-4.25.1-py3-none-any.whl (5.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.8/5.8 MB\u001b[0m \u001b[31m23.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting diffusers\n",
            "  Downloading diffusers-0.11.1-py3-none-any.whl (524 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m524.9/524.9 KB\u001b[0m \u001b[31m50.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting accelerate\n",
            "  Downloading accelerate-0.15.0-py3-none-any.whl (191 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m191.5/191.5 KB\u001b[0m \u001b[31m25.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting huggingface-hub\n",
            "  Downloading huggingface_hub-0.11.1-py3-none-any.whl (182 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m182.4/182.4 KB\u001b[0m \u001b[31m26.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting ninja\n",
            "  Downloading ninja-1.11.1-py2.py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (145 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m146.0/146.0 KB\u001b[0m \u001b[31m20.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting xatlas\n",
            "  Downloading xatlas-0.0.7-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (229 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m229.2/229.2 KB\u001b[0m \u001b[31m23.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: imageio in /usr/local/lib/python3.8/dist-packages (from -r requirements.txt (line 9)) (2.9.0)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.8/dist-packages (from -r requirements.txt (line 10)) (3.2.2)\n",
            "Collecting pyrallis\n",
            "  Downloading pyrallis-0.3.1-py3-none-any.whl (33 kB)\n",
            "Collecting loguru\n",
            "  Downloading loguru-0.6.0-py3-none-any.whl (58 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 KB\u001b[0m \u001b[31m7.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: tqdm in /usr/local/lib/python3.8/dist-packages (from -r requirements.txt (line 13)) (4.64.1)\n",
            "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1\n",
            "  Downloading tokenizers-0.13.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.6/7.6 MB\u001b[0m \u001b[31m88.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.8/dist-packages (from transformers->-r requirements.txt (line 3)) (2.25.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.8/dist-packages (from transformers->-r requirements.txt (line 3)) (3.9.0)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.8/dist-packages (from transformers->-r requirements.txt (line 3)) (21.3)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.8/dist-packages (from transformers->-r requirements.txt (line 3)) (6.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.8/dist-packages (from transformers->-r requirements.txt (line 3)) (1.21.6)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.8/dist-packages (from transformers->-r requirements.txt (line 3)) (2022.6.2)\n",
            "Requirement already satisfied: Pillow in /usr/local/lib/python3.8/dist-packages (from diffusers->-r requirements.txt (line 4)) (7.1.2)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.8/dist-packages (from diffusers->-r requirements.txt (line 4)) (6.0.0)\n",
            "Requirement already satisfied: torch>=1.4.0 in /usr/local/lib/python3.8/dist-packages (from accelerate->-r requirements.txt (line 5)) (1.12.1+cu116)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.8/dist-packages (from accelerate->-r requirements.txt (line 5)) (5.4.8)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.8/dist-packages (from huggingface-hub->-r requirements.txt (line 6)) (4.4.0)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.8/dist-packages (from matplotlib->-r requirements.txt (line 10)) (3.0.9)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.8/dist-packages (from matplotlib->-r requirements.txt (line 10)) (1.4.4)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.8/dist-packages (from matplotlib->-r requirements.txt (line 10)) (0.11.0)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.8/dist-packages (from matplotlib->-r requirements.txt (line 10)) (2.8.2)\n",
            "Collecting typing-inspect\n",
            "  Downloading typing_inspect-0.8.0-py3-none-any.whl (8.7 kB)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.8/dist-packages (from python-dateutil>=2.1->matplotlib->-r requirements.txt (line 10)) (1.15.0)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.8/dist-packages (from importlib-metadata->diffusers->-r requirements.txt (line 4)) (3.11.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.8/dist-packages (from requests->transformers->-r requirements.txt (line 3)) (2022.12.7)\n",
            "Requirement already satisfied: chardet<5,>=3.0.2 in /usr/local/lib/python3.8/dist-packages (from requests->transformers->-r requirements.txt (line 3)) (4.0.0)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.8/dist-packages (from requests->transformers->-r requirements.txt (line 3)) (2.10)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.8/dist-packages (from requests->transformers->-r requirements.txt (line 3)) (1.24.3)\n",
            "Collecting mypy-extensions>=0.3.0\n",
            "  Downloading mypy_extensions-0.4.3-py2.py3-none-any.whl (4.5 kB)\n",
            "Installing collected packages: xatlas, tokenizers, ninja, mypy-extensions, typing-inspect, loguru, pyrallis, huggingface-hub, accelerate, transformers, diffusers\n",
            "Successfully installed accelerate-0.15.0 diffusers-0.11.1 huggingface-hub-0.11.1 loguru-0.6.0 mypy-extensions-0.4.3 ninja-1.11.1 pyrallis-0.3.1 tokenizers-0.13.2 transformers-4.25.1 typing-inspect-0.8.0 xatlas-0.0.7\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# kaolin install (https://kaolin.readthedocs.io/en/latest/notes/installation.html)\n",
        "!pip install git+https://github.com/NVIDIAGameWorks/kaolin.git"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "rFvXzcROq3Sm",
        "outputId": "ca6e183d-b2a7-433f-a43e-e1c42ec6cc68"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "PIL"
                ]
              }
            }
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/',\n",
              " 'Collecting git+https://github.com/NVIDIAGameWorks/kaolin.git',\n",
              " '  Cloning https://github.com/NVIDIAGameWorks/kaolin.git to /tmp/pip-req-build-cm51uk4w',\n",
              " '  Running command git clone --filter=blob:none --quiet https://github.com/NVIDIAGameWorks/kaolin.git /tmp/pip-req-build-cm51uk4w',\n",
              " '  Resolved https://github.com/NVIDIAGameWorks/kaolin.git to commit a50ed1324a3c84e1bee2af1087688461f4b98fe7',\n",
              " '  Running command git submodule update --init --recursive -q',\n",
              " '  Preparing metadata (setup.py) ... \\x1b[?25l\\x1b[?25hdone',\n",
              " 'Collecting Pillow>=8.0.0',\n",
              " '  Downloading Pillow-9.4.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.3 MB)',\n",
              " '\\x1b[?25l     \\x1b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\\x1b[0m \\x1b[32m0.0/3.3 MB\\x1b[0m \\x1b[31m?\\x1b[0m eta \\x1b[36m-:--:--\\x1b[0m',\n",
              " '\\x1b[2K     \\x1b[91m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\\x1b[0m\\x1b[91m╸\\x1b[0m \\x1b[32m3.2/3.3 MB\\x1b[0m \\x1b[31m107.8 MB/s\\x1b[0m eta \\x1b[36m0:00:01\\x1b[0m',\n",
              " '\\x1b[2K     \\x1b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\\x1b[0m \\x1b[32m3.3/3.3 MB\\x1b[0m \\x1b[31m59.0 MB/s\\x1b[0m eta \\x1b[36m0:00:00\\x1b[0m',\n",
              " '\\x1b[?25hRequirement already satisfied: tqdm>=4.51.0 in /usr/local/lib/python3.8/dist-packages (from kaolin==0.12.0) (4.64.1)',\n",
              " 'Requirement already satisfied: scipy>=1.2.0 in /usr/local/lib/python3.8/dist-packages (from kaolin==0.12.0) (1.7.3)',\n",
              " 'Collecting usd-core<22.8',\n",
              " '  Downloading usd_core-22.5.post1-cp38-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (24.3 MB)',\n",
              " '\\x1b[?25l     \\x1b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\\x1b[0m \\x1b[32m0.0/24.3 MB\\x1b[0m \\x1b[31m?\\x1b[0m eta \\x1b[36m-:--:--\\x1b[0m',\n",
              " '\\x1b[2K     \\x1b[91m━━━━━\\x1b[0m\\x1b[90m╺\\x1b[0m\\x1b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\\x1b[0m \\x1b[32m3.1/24.3 MB\\x1b[0m \\x1b[31m217.6 MB/s\\x1b[0m eta \\x1b[36m0:00:01\\x1b[0m',\n",
              " '\\x1b[2K     \\x1b[91m━━━━━\\x1b[0m\\x1b[90m╺\\x1b[0m\\x1b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\\x1b[0m \\x1b[32m3.1/24.3 MB\\x1b[0m \\x1b[31m217.6 MB/s\\x1b[0m eta \\x1b[36m0:00:01\\x1b[0m',\n",
              " '\\x1b[2K     \\x1b[91m━━━━━\\x1b[0m\\x1b[90m╺\\x1b[0m\\x1b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\\x1b[0m \\x1b[32m3.1/24.3 MB\\x1b[0m \\x1b[31m217.6 MB/s\\x1b[0m eta \\x1b[36m0:00:01\\x1b[0m',\n",
              " '\\x1b[2K     \\x1b[91m━━━━━\\x1b[0m\\x1b[90m╺\\x1b[0m\\x1b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\\x1b[0m \\x1b[32m3.1/24.3 MB\\x1b[0m \\x1b[31m217.6 MB/s\\x1b[0m eta \\x1b[36m0:00:01\\x1b[0m',\n",
              " '\\x1b[2K     \\x1b[91m━━━━━━\\x1b[0m\\x1b[90m╺\\x1b[0m\\x1b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\\x1b[0m \\x1b[32m3.8/24.3 MB\\x1b[0m \\x1b[31m22.0 MB/s\\x1b[0m eta \\x1b[36m0:00:01\\x1b[0m',\n",
              " '\\x1b[2K     \\x1b[91m━━━━━━━━━━━━━\\x1b[0m\\x1b[91m╸\\x1b[0m\\x1b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━\\x1b[0m \\x1b[32m8.4/24.3 MB\\x1b[0m \\x1b[31m42.9 MB/s\\x1b[0m eta \\x1b[36m0:00:01\\x1b[0m',\n",
              " '\\x1b[2K     \\x1b[91m━━━━━━━━━━━━━\\x1b[0m\\x1b[91m╸\\x1b[0m\\x1b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━\\x1b[0m \\x1b[32m8.4/24.3 MB\\x1b[0m \\x1b[31m42.9 MB/s\\x1b[0m eta \\x1b[36m0:00:01\\x1b[0m',\n",
              " '\\x1b[2K     \\x1b[91m━━━━━━━━━━━━━\\x1b[0m\\x1b[91m╸\\x1b[0m\\x1b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━\\x1b[0m \\x1b[32m8.4/24.3 MB\\x1b[0m \\x1b[31m42.9 MB/s\\x1b[0m eta \\x1b[36m0:00:01\\x1b[0m',\n",
              " '\\x1b[2K     \\x1b[91m━━━━━━━━━━━━━\\x1b[0m\\x1b[91m╸\\x1b[0m\\x1b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━\\x1b[0m \\x1b[32m8.4/24.3 MB\\x1b[0m \\x1b[31m42.9 MB/s\\x1b[0m eta \\x1b[36m0:00:01\\x1b[0m',\n",
              " '\\x1b[2K     \\x1b[91m━━━━━━━━━━━━━\\x1b[0m\\x1b[91m╸\\x1b[0m\\x1b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━\\x1b[0m \\x1b[32m8.4/24.3 MB\\x1b[0m \\x1b[31m42.9 MB/s\\x1b[0m eta \\x1b[36m0:00:01\\x1b[0m',\n",
              " '\\x1b[2K     \\x1b[91m━━━━━━━━━━━━━\\x1b[0m\\x1b[91m╸\\x1b[0m\\x1b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━\\x1b[0m \\x1b[32m8.4/24.3 MB\\x1b[0m \\x1b[31m42.9 MB/s\\x1b[0m eta \\x1b[36m0:00:01\\x1b[0m',\n",
              " '\\x1b[2K     \\x1b[91m━━━━━━━━━━━━━\\x1b[0m\\x1b[91m╸\\x1b[0m\\x1b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━\\x1b[0m \\x1b[32m8.4/24.3 MB\\x1b[0m \\x1b[31m42.9 MB/s\\x1b[0m eta \\x1b[36m0:00:01\\x1b[0m',\n",
              " '\\x1b[2K     \\x1b[91m━━━━━━━━━━━━━\\x1b[0m\\x1b[91m╸\\x1b[0m\\x1b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━\\x1b[0m \\x1b[32m8.4/24.3 MB\\x1b[0m \\x1b[31m42.9 MB/s\\x1b[0m eta \\x1b[36m0:00:01\\x1b[0m',\n",
              " '\\x1b[2K     \\x1b[91m━━━━━━━━━━━━━\\x1b[0m\\x1b[91m╸\\x1b[0m\\x1b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━\\x1b[0m \\x1b[32m8.4/24.3 MB\\x1b[0m \\x1b[31m42.9 MB/s\\x1b[0m eta \\x1b[36m0:00:01\\x1b[0m',\n",
              " '\\x1b[2K     \\x1b[91m━━━━━━━━━━━━━━━━━━━━━━\\x1b[0m\\x1b[91m╸\\x1b[0m\\x1b[90m━━━━━━━━━━━━━━━━━\\x1b[0m \\x1b[32m13.9/24.3 MB\\x1b[0m \\x1b[31m29.2 MB/s\\x1b[0m eta \\x1b[36m0:00:01\\x1b[0m',\n",
              " '\\x1b[2K     \\x1b[91m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\\x1b[0m\\x1b[91m╸\\x1b[0m\\x1b[90m━━━━━━\\x1b[0m \\x1b[32m20.5/24.3 MB\\x1b[0m \\x1b[31m192.8 MB/s\\x1b[0m eta \\x1b[36m0:00:01\\x1b[0m',\n",
              " '\\x1b[2K     \\x1b[91m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\\x1b[0m\\x1b[91m╸\\x1b[0m \\x1b[32m24.3/24.3 MB\\x1b[0m \\x1b[31m201.1 MB/s\\x1b[0m eta \\x1b[36m0:00:01\\x1b[0m',\n",
              " '\\x1b[2K     \\x1b[91m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\\x1b[0m\\x1b[91m╸\\x1b[0m \\x1b[32m24.3/24.3 MB\\x1b[0m \\x1b[31m201.1 MB/s\\x1b[0m eta \\x1b[36m0:00:01\\x1b[0m',\n",
              " '\\x1b[2K     \\x1b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\\x1b[0m \\x1b[32m24.3/24.3 MB\\x1b[0m \\x1b[31m68.9 MB/s\\x1b[0m eta \\x1b[36m0:00:00\\x1b[0m',\n",
              " '\\x1b[?25hRequirement already satisfied: numpy<1.23.0,>=1.16.5 in /usr/local/lib/python3.8/dist-packages (from scipy>=1.2.0->kaolin==0.12.0) (1.21.6)',\n",
              " 'Building wheels for collected packages: kaolin',\n",
              " '  Building wheel for kaolin (setup.py) ... \\x1b[?25l\\x1b[?25hdone',\n",
              " '  Created wheel for kaolin: filename=kaolin-0.12.0-cp38-cp38-linux_x86_64.whl size=8446255 sha256=f976f89227462c56e597eb4738f759b67b6729cc3dbe7a83d3ede6961a8fbd2f',\n",
              " '  Stored in directory: /tmp/pip-ephem-wheel-cache-lf1cil4j/wheels/b0/0c/b7/c21c4d2eb6360afe4d8728dfb21dbf88790f99fefa0885a0b5',\n",
              " 'Successfully built kaolin',\n",
              " 'Installing collected packages: usd-core, Pillow, kaolin',\n",
              " '  Attempting uninstall: Pillow',\n",
              " '    Found existing installation: Pillow 7.1.2',\n",
              " '    Uninstalling Pillow-7.1.2:',\n",
              " '      Successfully uninstalled Pillow-7.1.2',\n",
              " 'Successfully installed Pillow-9.4.0 kaolin-0.12.0 usd-core-22.5.post1']"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import kaolin as kal"
      ],
      "metadata": {
        "id": "5F7Ncal8ujoS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# huggingface login, Please get login tokens\n",
        "!huggingface-cli login"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DKlhG9uEyNiS",
        "outputId": "038fcc68-8015-4df4-e753-cf48de918131"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "    _|    _|  _|    _|    _|_|_|    _|_|_|  _|_|_|  _|      _|    _|_|_|      _|_|_|_|    _|_|      _|_|_|  _|_|_|_|\n",
            "    _|    _|  _|    _|  _|        _|          _|    _|_|    _|  _|            _|        _|    _|  _|        _|\n",
            "    _|_|_|_|  _|    _|  _|  _|_|  _|  _|_|    _|    _|  _|  _|  _|  _|_|      _|_|_|    _|_|_|_|  _|        _|_|_|\n",
            "    _|    _|  _|    _|  _|    _|  _|    _|    _|    _|    _|_|  _|    _|      _|        _|    _|  _|        _|\n",
            "    _|    _|    _|_|      _|_|_|    _|_|_|  _|_|_|  _|      _|    _|_|_|      _|        _|    _|    _|_|_|  _|_|_|_|\n",
            "\n",
            "    To login, `huggingface_hub` now requires a token generated from https://huggingface.co/settings/tokens .\n",
            "    \n",
            "Token: \n",
            "Add token as git credential? (Y/n) Y\n",
            "Token is valid.\n",
            "\u001b[1m\u001b[31mCannot authenticate through git-credential as no helper is defined on your machine.\n",
            "You might have to re-authenticate when pushing to the Hugging Face Hub.\n",
            "Run the following command in your terminal in case you want to set the 'store' credential helper as default.\n",
            "\n",
            "git config --global credential.helper store\n",
            "\n",
            "Read https://git-scm.com/book/en/v2/Git-Tools-Credential-Storage for more details.\u001b[0m\n",
            "Token has not been saved to git credential helper.\n",
            "Your token has been saved to /root/.huggingface/token\n",
            "Login successful\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!python3 -m scripts.train_latent_paint --config_path demo_configs/latent_paint/goldfish.yaml"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6guEV0c6qut3",
        "outputId": "b6acd3fe-9670-4caa-e9a4-14575e6fe688"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[32m2023-01-13 02:59:39\u001b[0m \u001b[1mLoaded texture-mesh Mesh, #parameters: 176128\u001b[0m\n",
            "\u001b[32m2023-01-13 02:59:39\u001b[0m \u001b[1mTexturedMeshModel()\u001b[0m\n",
            "\u001b[32m2023-01-13 02:59:39\u001b[0m \u001b[33m\u001b[1mtry to load hugging face access token from the default place, make sure you have run `huggingface-cli login`.\u001b[0m\n",
            "\u001b[32m2023-01-13 02:59:39\u001b[0m \u001b[1mloading stable diffusion with CompVis/stable-diffusion-v1-4...\u001b[0m\n",
            "\u001b[32m2023-01-13 03:00:09\u001b[0m \u001b[1m\t successfully loaded stable diffusion!\u001b[0m\n",
            "\u001b[32m2023-01-13 03:00:09\u001b[0m \u001b[1mSuccessfully initialized goldfish\u001b[0m\n",
            "\u001b[32m2023-01-13 03:00:09\u001b[0m \u001b[1mStarting training ^_^\u001b[0m\n",
            "\u001b[32m2023-01-13 03:00:09\u001b[0m \u001b[1mEvaluating and saving model, iteration #0...\u001b[0m\n",
            "\u001b[32m2023-01-13 03:00:34\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 03:01:32\u001b[0m \u001b[1mEvaluating and saving model, iteration #100...\u001b[0m\n",
            "\u001b[32m2023-01-13 03:01:56\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 03:02:56\u001b[0m \u001b[1mEvaluating and saving model, iteration #200...\u001b[0m\n",
            "\u001b[32m2023-01-13 03:03:19\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 03:04:20\u001b[0m \u001b[1mEvaluating and saving model, iteration #300...\u001b[0m\n",
            "\u001b[32m2023-01-13 03:04:42\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 03:05:43\u001b[0m \u001b[1mEvaluating and saving model, iteration #400...\u001b[0m\n",
            "\u001b[32m2023-01-13 03:06:05\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 03:07:05\u001b[0m \u001b[1mEvaluating and saving model, iteration #500...\u001b[0m\n",
            "\u001b[32m2023-01-13 03:07:28\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 03:08:31\u001b[0m \u001b[1mEvaluating and saving model, iteration #600...\u001b[0m\n",
            "\u001b[32m2023-01-13 03:08:54\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 03:09:53\u001b[0m \u001b[1mEvaluating and saving model, iteration #700...\u001b[0m\n",
            "\u001b[32m2023-01-13 03:10:16\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 03:11:16\u001b[0m \u001b[1mEvaluating and saving model, iteration #800...\u001b[0m\n",
            "\u001b[32m2023-01-13 03:11:39\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 03:12:38\u001b[0m \u001b[1mEvaluating and saving model, iteration #900...\u001b[0m\n",
            "\u001b[32m2023-01-13 03:13:01\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 03:14:00\u001b[0m \u001b[1mEvaluating and saving model, iteration #1000...\u001b[0m\n",
            "\u001b[32m2023-01-13 03:14:23\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 03:15:28\u001b[0m \u001b[1mEvaluating and saving model, iteration #1100...\u001b[0m\n",
            "\u001b[32m2023-01-13 03:15:51\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 03:16:51\u001b[0m \u001b[1mEvaluating and saving model, iteration #1200...\u001b[0m\n",
            "\u001b[32m2023-01-13 03:17:14\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 03:18:15\u001b[0m \u001b[1mEvaluating and saving model, iteration #1300...\u001b[0m\n",
            "\u001b[32m2023-01-13 03:18:37\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 03:19:38\u001b[0m \u001b[1mEvaluating and saving model, iteration #1400...\u001b[0m\n",
            "\u001b[32m2023-01-13 03:20:01\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 03:21:02\u001b[0m \u001b[1mEvaluating and saving model, iteration #1500...\u001b[0m\n",
            "\u001b[32m2023-01-13 03:21:25\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 03:22:25\u001b[0m \u001b[1mEvaluating and saving model, iteration #1600...\u001b[0m\n",
            "\u001b[32m2023-01-13 03:22:48\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 03:23:48\u001b[0m \u001b[1mEvaluating and saving model, iteration #1700...\u001b[0m\n",
            "\u001b[32m2023-01-13 03:24:11\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 03:25:11\u001b[0m \u001b[1mEvaluating and saving model, iteration #1800...\u001b[0m\n",
            "\u001b[32m2023-01-13 03:25:34\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 03:26:34\u001b[0m \u001b[1mEvaluating and saving model, iteration #1900...\u001b[0m\n",
            "\u001b[32m2023-01-13 03:26:57\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 03:27:59\u001b[0m \u001b[1mEvaluating and saving model, iteration #2000...\u001b[0m\n",
            "\u001b[32m2023-01-13 03:28:22\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 03:29:24\u001b[0m \u001b[1mEvaluating and saving model, iteration #2100...\u001b[0m\n",
            "\u001b[32m2023-01-13 03:29:47\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 03:30:50\u001b[0m \u001b[1mEvaluating and saving model, iteration #2200...\u001b[0m\n",
            "\u001b[32m2023-01-13 03:31:13\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 03:32:12\u001b[0m \u001b[1mEvaluating and saving model, iteration #2300...\u001b[0m\n",
            "\u001b[32m2023-01-13 03:32:35\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 03:33:33\u001b[0m \u001b[1mEvaluating and saving model, iteration #2400...\u001b[0m\n",
            "\u001b[32m2023-01-13 03:33:56\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 03:35:02\u001b[0m \u001b[1mEvaluating and saving model, iteration #2500...\u001b[0m\n",
            "\u001b[32m2023-01-13 03:35:25\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 03:36:22\u001b[0m \u001b[1mEvaluating and saving model, iteration #2600...\u001b[0m\n",
            "\u001b[32m2023-01-13 03:36:45\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 03:37:45\u001b[0m \u001b[1mEvaluating and saving model, iteration #2700...\u001b[0m\n",
            "\u001b[32m2023-01-13 03:38:08\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 03:39:05\u001b[0m \u001b[1mEvaluating and saving model, iteration #2800...\u001b[0m\n",
            "\u001b[32m2023-01-13 03:39:28\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 03:40:27\u001b[0m \u001b[1mEvaluating and saving model, iteration #2900...\u001b[0m\n",
            "\u001b[32m2023-01-13 03:40:50\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 03:41:54\u001b[0m \u001b[1mEvaluating and saving model, iteration #3000...\u001b[0m\n",
            "\u001b[32m2023-01-13 03:42:16\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 03:43:14\u001b[0m \u001b[1mEvaluating and saving model, iteration #3100...\u001b[0m\n",
            "\u001b[32m2023-01-13 03:43:37\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 03:44:36\u001b[0m \u001b[1mEvaluating and saving model, iteration #3200...\u001b[0m\n",
            "\u001b[32m2023-01-13 03:44:59\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 03:46:04\u001b[0m \u001b[1mEvaluating and saving model, iteration #3300...\u001b[0m\n",
            "\u001b[32m2023-01-13 03:46:27\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 03:47:26\u001b[0m \u001b[1mEvaluating and saving model, iteration #3400...\u001b[0m\n",
            "\u001b[32m2023-01-13 03:47:49\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 03:48:52\u001b[0m \u001b[1mEvaluating and saving model, iteration #3500...\u001b[0m\n",
            "\u001b[32m2023-01-13 03:49:15\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 03:50:16\u001b[0m \u001b[1mEvaluating and saving model, iteration #3600...\u001b[0m\n",
            "\u001b[32m2023-01-13 03:50:39\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 03:51:39\u001b[0m \u001b[1mEvaluating and saving model, iteration #3700...\u001b[0m\n",
            "\u001b[32m2023-01-13 03:52:02\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 03:53:03\u001b[0m \u001b[1mEvaluating and saving model, iteration #3800...\u001b[0m\n",
            "\u001b[32m2023-01-13 03:53:26\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 03:54:26\u001b[0m \u001b[1mEvaluating and saving model, iteration #3900...\u001b[0m\n",
            "\u001b[32m2023-01-13 03:54:49\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 03:55:51\u001b[0m \u001b[1mEvaluating and saving model, iteration #4000...\u001b[0m\n",
            "\u001b[32m2023-01-13 03:56:14\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 03:57:14\u001b[0m \u001b[1mEvaluating and saving model, iteration #4100...\u001b[0m\n",
            "\u001b[32m2023-01-13 03:57:37\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 03:58:36\u001b[0m \u001b[1mEvaluating and saving model, iteration #4200...\u001b[0m\n",
            "\u001b[32m2023-01-13 03:58:59\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 03:59:57\u001b[0m \u001b[1mEvaluating and saving model, iteration #4300...\u001b[0m\n",
            "\u001b[32m2023-01-13 04:00:20\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 04:01:23\u001b[0m \u001b[1mEvaluating and saving model, iteration #4400...\u001b[0m\n",
            "\u001b[32m2023-01-13 04:01:46\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 04:02:50\u001b[0m \u001b[1mEvaluating and saving model, iteration #4500...\u001b[0m\n",
            "\u001b[32m2023-01-13 04:03:13\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 04:04:14\u001b[0m \u001b[1mEvaluating and saving model, iteration #4600...\u001b[0m\n",
            "\u001b[32m2023-01-13 04:04:37\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 04:05:36\u001b[0m \u001b[1mEvaluating and saving model, iteration #4700...\u001b[0m\n",
            "\u001b[32m2023-01-13 04:05:59\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 04:07:03\u001b[0m \u001b[1mEvaluating and saving model, iteration #4800...\u001b[0m\n",
            "\u001b[32m2023-01-13 04:07:26\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 04:08:24\u001b[0m \u001b[1mEvaluating and saving model, iteration #4900...\u001b[0m\n",
            "\u001b[32m2023-01-13 04:08:47\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 04:09:48\u001b[0m \u001b[1mEvaluating and saving model, iteration #5000...\u001b[0m\n",
            "\u001b[32m2023-01-13 04:10:11\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 04:11:09\u001b[0m \u001b[1mEvaluating and saving model, iteration #5100...\u001b[0m\n",
            "\u001b[32m2023-01-13 04:11:32\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 04:12:35\u001b[0m \u001b[1mEvaluating and saving model, iteration #5200...\u001b[0m\n",
            "\u001b[32m2023-01-13 04:12:58\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 04:13:59\u001b[0m \u001b[1mEvaluating and saving model, iteration #5300...\u001b[0m\n",
            "\u001b[32m2023-01-13 04:14:22\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 04:15:22\u001b[0m \u001b[1mEvaluating and saving model, iteration #5400...\u001b[0m\n",
            "\u001b[32m2023-01-13 04:15:45\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 04:16:46\u001b[0m \u001b[1mEvaluating and saving model, iteration #5500...\u001b[0m\n",
            "\u001b[32m2023-01-13 04:17:09\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 04:18:09\u001b[0m \u001b[1mEvaluating and saving model, iteration #5600...\u001b[0m\n",
            "\u001b[32m2023-01-13 04:18:32\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 04:19:31\u001b[0m \u001b[1mEvaluating and saving model, iteration #5700...\u001b[0m\n",
            "\u001b[32m2023-01-13 04:19:54\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 04:20:57\u001b[0m \u001b[1mEvaluating and saving model, iteration #5800...\u001b[0m\n",
            "\u001b[32m2023-01-13 04:21:20\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 04:22:23\u001b[0m \u001b[1mEvaluating and saving model, iteration #5900...\u001b[0m\n",
            "\u001b[32m2023-01-13 04:22:46\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 04:23:45\u001b[0m \u001b[1mEvaluating and saving model, iteration #6000...\u001b[0m\n",
            "\u001b[32m2023-01-13 04:24:08\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 04:25:11\u001b[0m \u001b[1mEvaluating and saving model, iteration #6100...\u001b[0m\n",
            "\u001b[32m2023-01-13 04:25:34\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 04:26:34\u001b[0m \u001b[1mEvaluating and saving model, iteration #6200...\u001b[0m\n",
            "\u001b[32m2023-01-13 04:26:57\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 04:27:57\u001b[0m \u001b[1mEvaluating and saving model, iteration #6300...\u001b[0m\n",
            "\u001b[32m2023-01-13 04:28:20\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 04:29:18\u001b[0m \u001b[1mEvaluating and saving model, iteration #6400...\u001b[0m\n",
            "\u001b[32m2023-01-13 04:29:41\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 04:30:42\u001b[0m \u001b[1mEvaluating and saving model, iteration #6500...\u001b[0m\n",
            "\u001b[32m2023-01-13 04:31:05\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 04:32:05\u001b[0m \u001b[1mEvaluating and saving model, iteration #6600...\u001b[0m\n",
            "\u001b[32m2023-01-13 04:32:28\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 04:33:32\u001b[0m \u001b[1mEvaluating and saving model, iteration #6700...\u001b[0m\n",
            "\u001b[32m2023-01-13 04:33:55\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 04:34:56\u001b[0m \u001b[1mEvaluating and saving model, iteration #6800...\u001b[0m\n",
            "\u001b[32m2023-01-13 04:35:19\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 04:36:19\u001b[0m \u001b[1mEvaluating and saving model, iteration #6900...\u001b[0m\n",
            "\u001b[32m2023-01-13 04:36:42\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 04:37:43\u001b[0m \u001b[1mEvaluating and saving model, iteration #7000...\u001b[0m\n",
            "\u001b[32m2023-01-13 04:38:06\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 04:39:06\u001b[0m \u001b[1mEvaluating and saving model, iteration #7100...\u001b[0m\n",
            "\u001b[32m2023-01-13 04:39:29\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 04:40:31\u001b[0m \u001b[1mEvaluating and saving model, iteration #7200...\u001b[0m\n",
            "\u001b[32m2023-01-13 04:40:55\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 04:41:55\u001b[0m \u001b[1mEvaluating and saving model, iteration #7300...\u001b[0m\n",
            "\u001b[32m2023-01-13 04:42:18\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 04:43:17\u001b[0m \u001b[1mEvaluating and saving model, iteration #7400...\u001b[0m\n",
            "\u001b[32m2023-01-13 04:43:40\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 04:44:41\u001b[0m \u001b[1mEvaluating and saving model, iteration #7500...\u001b[0m\n",
            "\u001b[32m2023-01-13 04:45:04\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 04:46:06\u001b[0m \u001b[1mEvaluating and saving model, iteration #7600...\u001b[0m\n",
            "\u001b[32m2023-01-13 04:46:29\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 04:47:30\u001b[0m \u001b[1mEvaluating and saving model, iteration #7700...\u001b[0m\n",
            "\u001b[32m2023-01-13 04:47:53\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 04:49:00\u001b[0m \u001b[1mEvaluating and saving model, iteration #7800...\u001b[0m\n",
            "\u001b[32m2023-01-13 04:49:23\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 04:50:26\u001b[0m \u001b[1mEvaluating and saving model, iteration #7900...\u001b[0m\n",
            "\u001b[32m2023-01-13 04:50:49\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 04:51:50\u001b[0m \u001b[1mEvaluating and saving model, iteration #8000...\u001b[0m\n",
            "\u001b[32m2023-01-13 04:52:13\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 04:53:13\u001b[0m \u001b[1mEvaluating and saving model, iteration #8100...\u001b[0m\n",
            "\u001b[32m2023-01-13 04:53:36\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 04:54:37\u001b[0m \u001b[1mEvaluating and saving model, iteration #8200...\u001b[0m\n",
            "\u001b[32m2023-01-13 04:55:00\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 04:56:01\u001b[0m \u001b[1mEvaluating and saving model, iteration #8300...\u001b[0m\n",
            "\u001b[32m2023-01-13 04:56:24\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 04:57:25\u001b[0m \u001b[1mEvaluating and saving model, iteration #8400...\u001b[0m\n",
            "\u001b[32m2023-01-13 04:57:48\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 04:58:48\u001b[0m \u001b[1mEvaluating and saving model, iteration #8500...\u001b[0m\n",
            "\u001b[32m2023-01-13 04:59:11\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 05:00:11\u001b[0m \u001b[1mEvaluating and saving model, iteration #8600...\u001b[0m\n",
            "\u001b[32m2023-01-13 05:00:34\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 05:01:38\u001b[0m \u001b[1mEvaluating and saving model, iteration #8700...\u001b[0m\n",
            "\u001b[32m2023-01-13 05:02:01\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 05:03:00\u001b[0m \u001b[1mEvaluating and saving model, iteration #8800...\u001b[0m\n",
            "\u001b[32m2023-01-13 05:03:23\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 05:04:22\u001b[0m \u001b[1mEvaluating and saving model, iteration #8900...\u001b[0m\n",
            "\u001b[32m2023-01-13 05:04:45\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 05:05:43\u001b[0m \u001b[1mEvaluating and saving model, iteration #9000...\u001b[0m\n",
            "\u001b[32m2023-01-13 05:06:06\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 05:07:04\u001b[0m \u001b[1mEvaluating and saving model, iteration #9100...\u001b[0m\n",
            "\u001b[32m2023-01-13 05:07:27\u001b[0m \u001b[1mDone!\u001b[0m\n",
            "\u001b[32m2023-01-13 05:08:31\u001b[0m \u001b[1mEvaluating and saving model, iteration #9200...\u001b[0m\n",
            "\u001b[32m2023-01-13 05:08:54\u001b[0m \u001b[1mDone!\u001b[0m\n",
            " 92% 9232/10000 [2:08:40<07:45,  1.65it/s]"
          ]
        }
      ]
    }
  ]
}