quick_demo.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Hg8LXAe8mt-G"
      },
      "source": [
        "# **VideoCrafter：A Toolkit for Text-to-Video Generation and Editing**\n",
        "\n",
        "\n",
        "VideoCrafter is an open-source video generation and editing toolbox for crafting video content.\n",
        "\n",
        "More details can be founded in [![GitHub](https://img.shields.io/github/stars/VideoCrafter/VideoCrafter?style=social)](https://github.com/VideoCrafter/VideoCrafter)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ijIJmBFLebe3",
        "outputId": "eb2ef369-4a59-45f5-fdc8-ce52eaf6d951"
      },
      "outputs": [],
      "source": [
        "### make sure that CUDA is available in Edit -> Nootbook settings -> GPU\n",
        "!nvidia-smi --query-gpu=name,memory.total,memory.free --format=csv,noheader"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Jn_LkaeUmFJQ"
      },
      "source": [
        "## Installnation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "l4XypmLmfJNw",
        "outputId": "444a45f0-f366-464c-ea8b-bb4ae511f3e8"
      },
      "outputs": [],
      "source": [
        "!update-alternatives --install /usr/local/bin/python3 python3 /usr/bin/python3.8 2  \n",
        "!update-alternatives --install /usr/local/bin/python3 python3 /usr/bin/python3.9 1  \n",
        "!python --version  \n",
        "!apt-get update\n",
        "!apt install software-properties-common\n",
        "!sudo dpkg --remove --force-remove-reinstreq python3-pip python3-setuptools python3-wheel\n",
        "!apt-get install python3-pip\n",
        "\n",
        "print('Git clone project and install requirements...')\n",
        "!git clone https://github.com/VideoCrafter/VideoCrafter &> /dev/null\n",
        "%cd VideoCrafter \n",
        "!export PYTHONPATH=/content/VideoCrafter:$PYTHONPATH \n",
        "\n",
        "!python3.8 -m pip install torch==1.12.1+cu113 torchvision==0.13.1+cu113 torchaudio==0.12.1 --extra-index-url https://download.pytorch.org/whl/cu113\n",
        "!apt update\n",
        "!apt install ffmpeg &> /dev/null  \n",
        "!python3.8 -m pip install pytorch-lightning==1.8.3 omegaconf==2.1.1 einops==0.3.0 transformers==4.25.1\n",
        "!python3.8 -m pip install opencv-python==4.1.2.30 imageio==2.9.0 imageio-ffmpeg==0.4.2\n",
        "!python3.8 -m pip install av moviepy\n",
        "!python3.8 -m pip install -e ."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_y8F_z_-hd9w",
        "outputId": "e8e6da64-a1d2-43bd-cbe2-279b4a584c9e"
      },
      "outputs": [],
      "source": [
        "### download all model form hugging-face\n",
        "! rm -rf models/\n",
        "! git lfs install\n",
        "! git clone https://huggingface.co/VideoCrafter/t2v-version-1-1/\n",
        "! mv t2v-version-1-1/models ."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QOH7mJYFl8Y0"
      },
      "source": [
        "### Base T2V: Generic Text-to-video Generation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "T0djMBodhGw7",
        "outputId": "afcf63ee-9fb5-4f1c-8ca5-c36fd5e2411d"
      },
      "outputs": [],
      "source": [
        "PROMPT=\"astronaut riding a horse outer space\" #@param {type:\"string\"}\n",
        "OUTDIR=\"results/\"\n",
        "\n",
        "BASE_PATH=\"models/base_t2v/model.ckpt\"\n",
        "CONFIG_PATH=\"models/base_t2v/model_config.yaml\"\n",
        "\n",
        "! python scripts/sample_text2video.py \\\n",
        "    --ckpt_path $BASE_PATH \\\n",
        "    --config_path $CONFIG_PATH \\\n",
        "    --prompt \"$PROMPT\" \\\n",
        "    --save_dir $OUTDIR \\\n",
        "    --n_samples 1 \\\n",
        "    --batch_size 1 \\\n",
        "    --seed 1000 \\\n",
        "    --show_denoising_progress"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 294
        },
        "id": "e7Oo0taGkTJM",
        "outputId": "fa33e269-0c01-4bc9-dc2d-376d3709800e"
      },
      "outputs": [],
      "source": [
        "# visualize\n",
        "from IPython.display import HTML\n",
        "from base64 import b64encode\n",
        "import os, sys, glob\n",
        "\n",
        "# get the last from results\n",
        "\n",
        "mp4_name = sorted(os.listdir(OUTDIR+'/videos'))[0]\n",
        "\n",
        "mp4_name = os.path.join(OUTDIR+'/videos', mp4_name)\n",
        "\n",
        "mp4 = open('{}'.format(mp4_name),'rb').read()\n",
        "data_url = \"data:video/mp4;base64,\" + b64encode(mp4).decode()\n",
        "\n",
        "print('Display animation: {}'.format(mp4_name), file=sys.stderr)\n",
        "display(HTML(\"\"\"\n",
        "  <video width=256 controls>\n",
        "        <source src=\"%s\" type=\"video/mp4\">\n",
        "  </video>\n",
        "  \"\"\" % data_url))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8BzW_xqemQzH"
      },
      "source": [
        "## VideoLoRA: Personalized Text-to-Video Generation with LoRA"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HsdoxaYmmQYQ",
        "outputId": "c21eac9c-634f-4b37-cdd0-2eea319103fd"
      },
      "outputs": [],
      "source": [
        "PROMPT=\"astronaut riding a horse\" #@param {type:\"string\"}\n",
        "OUTDIR=\"results/videolora\"\n",
        "\n",
        "BASE_PATH=\"models/base_t2v/model.ckpt\"\n",
        "CONFIG_PATH=\"models/base_t2v/model_config.yaml\"\n",
        "\n",
        "LORA_PATH=\"models/videolora/lora_003_MakotoShinkaiYourName_style.ckpt\" #@param [\"models/videolora/lora_001_Loving_Vincent_style.ckpt\", \"models/videolora/lora_002_frozenmovie_style.ckpt\", \"models/videolora/lora_003_MakotoShinkaiYourName_style.ckpt\", \"models/videolora/lora_004_coco_style.ckpt\"]\n",
        "\n",
        "\n",
        "### get tags from lora model\n",
        "lora_dict = {\n",
        "    \"models/videolora/lora_001_Loving_Vincent_style.ckpt\": \", Loving Vincent style\", \n",
        "    \"models/videolora/lora_002_frozenmovie_style.ckpt\": \", frozenmovie style\",\n",
        "    \"models/videolora/lora_003_MakotoShinkaiYourName_style.ckpt\": \", MakotoShinkaiYourName style\", \n",
        "    \"models/videolora/lora_004_coco_style.ckpt\": \", coco style\"\n",
        "}\n",
        "\n",
        "TAG=lora_dict[LORA_PATH]\n",
        "\n",
        "! python scripts/sample_text2video.py \\\n",
        "    --ckpt_path $BASE_PATH \\\n",
        "    --config_path $CONFIG_PATH \\\n",
        "    --prompt \"$PROMPT\" \\\n",
        "    --save_dir $OUTDIR \\\n",
        "    --n_samples 1 \\\n",
        "    --batch_size 1 \\\n",
        "    --seed 1000 \\\n",
        "    --show_denoising_progress \\\n",
        "    --inject_lora \\\n",
        "    --lora_path $LORA_PATH \\\n",
        "    --lora_trigger_word \"$TAG\" \\\n",
        "    --lora_scale 1.0"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 295
        },
        "id": "Q2GlC3Hbmi68",
        "outputId": "b9f177be-9208-4485-c266-ba3017f1718e"
      },
      "outputs": [],
      "source": [
        "# visualize\n",
        "from IPython.display import HTML\n",
        "from base64 import b64encode\n",
        "import os, sys, glob\n",
        "\n",
        "# get the last from results\n",
        "\n",
        "mp4_name = sorted(os.listdir(OUTDIR+'/videos'))[0]\n",
        "\n",
        "mp4_name = os.path.join(OUTDIR+'/videos', mp4_name)\n",
        "\n",
        "mp4 = open('{}'.format(mp4_name),'rb').read()\n",
        "data_url = \"data:video/mp4;base64,\" + b64encode(mp4).decode()\n",
        "\n",
        "print('Display animation: {}'.format(mp4_name), file=sys.stderr)\n",
        "display(HTML(\"\"\"\n",
        "  <video width=256 controls>\n",
        "        <source src=\"%s\" type=\"video/mp4\">\n",
        "  </video>\n",
        "  \"\"\" % data_url))"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": []
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}