model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import load_dataset\n",
    "from torchvision.transforms import Compose, Resize, ToTensor, Lambda\n",
    "import torchvision.transforms.functional as TF\n",
    "import torch\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "import cv2\n",
    "import numpy as np\n",
    "from torch.nn.utils.rnn import pad_sequence\n",
    "from torchvision.transforms.functional import to_pil_image\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\belkh\\Code\\ml_final_project\\.conda\\Lib\\site-packages\\datasets\\load.py:1486: FutureWarning: The repository for jinmang2/ucf_crime contains custom code which must be executed to correctly load the dataset. You can inspect the repository content at https://hf.co/datasets/jinmang2/ucf_crime\n",
      "You can avoid this message in future by passing the argument `trust_remote_code=True`.\n",
      "Passing `trust_remote_code=True` will be mandatory to load this dataset from the next major release of `datasets`.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "datasets = load_dataset(\"jinmang2/ucf_crime\")\n",
    "datasets = datasets['train'].shuffle(seed=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_test_split = datasets.train_test_split(test_size=0.2, seed=42)\n",
    "train_dataset = train_test_split['train']\n",
    "test_dataset = train_test_split['test']\n",
    "\n",
    "train_val_split = train_dataset.train_test_split(test_size=0.2, seed=42)\n",
    "train_dataset = train_val_split['train']\n",
    "val_dataset = train_val_split['test']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torchvision.models as models\n",
    "import torchvision.transforms as transforms\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "import cv2\n",
    "\n",
    "class VideoDataset(Dataset):\n",
    "    def __init__(self, dataset, target_fps=1, transform=None):\n",
    "        self.dataset = dataset\n",
    "        self.target_fps = target_fps\n",
    "        self.transform = transform or transforms.Compose([\n",
    "            transforms.ToPILImage(),\n",
    "            transforms.Resize((112, 112)), \n",
    "            transforms.ToTensor(),\n",
    "            transforms.Normalize(mean=[0.485, 0.456, 0.406], \\\n",
    "                std=[0.229, 0.224, 0.225])\n",
    "        ])\n",
    "        self.feature_extractor = \\\n",
    "            models.mobilenet_v2(pretrained=True)\n",
    "\n",
    "        self.feature_extractor.fc = torch.nn.Identity()\n",
    "        self.feature_extractor.eval()\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.dataset)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        video_path = self.dataset[idx]['video_path']\n",
    "        frames = self.load_video(video_path, self.target_fps)\n",
    "        features = []\n",
    "        with torch.no_grad():\n",
    "            for frame in frames:\n",
    "                frame = self.transform(frame)\n",
    "                frame = frame.unsqueeze(0)  \n",
    "                feature = self.feature_extractor(frame)\n",
    "                features.append(feature.squeeze(0)) \n",
    "        features = torch.stack(features)\n",
    "        label = self.dataset[idx]['anomaly']\n",
    "        return features, label\n",
    "\n",
    "    def load_video(self, video_path, target_fps):\n",
    "        cap = cv2.VideoCapture(video_path)\n",
    "        frames = []\n",
    "        native_fps = cap.get(cv2.CAP_PROP_FPS)\n",
    "        frame_ratio = max(1, round(native_fps / target_fps))\n",
    "\n",
    "        frame_idx = 0\n",
    "        while True:\n",
    "            ret, frame = cap.read()\n",
    "            if not ret:\n",
    "                break\n",
    "            if frame_idx % frame_ratio == 0:\n",
    "                frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\n",
    "                frames.append(frame)\n",
    "            frame_idx += 1\n",
    "        cap.release()\n",
    "        return frames\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\belkh\\Code\\ml_final_project\\.conda\\Lib\\site-packages\\torchvision\\models\\_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
      "  warnings.warn(\n",
      "c:\\Users\\belkh\\Code\\ml_final_project\\.conda\\Lib\\site-packages\\torchvision\\models\\_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=MobileNet_V2_Weights.IMAGENET1K_V1`. You can also use `weights=MobileNet_V2_Weights.DEFAULT` to get the most up-to-date weights.\n",
      "  warnings.warn(msg)\n"
     ]
    }
   ],
   "source": [
    "from torch.utils.data import DataLoader\n",
    "\n",
    "train_dataset = VideoDataset(train_dataset)\n",
    "val_dataset = VideoDataset(val_dataset)\n",
    "\n",
    "train_loader = DataLoader(train_dataset, batch_size=1, shuffle=True)\n",
    "val_loader = DataLoader(val_dataset, batch_size=1, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "\n",
    "class LSTM(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(LSTM, self).__init__()\n",
    "        self.input_dim = 1000\n",
    "        self.h = 256 \n",
    "        self.numOfLayers = 1 \n",
    "        self.numOfClasses = 2 \n",
    "        self.W = nn.Linear(self.h, self.numOfClasses)\n",
    "        self.lstm = nn.LSTM(self.input_dim, self.h, self.numOfLayers, batch_first=True)\n",
    "\n",
    "\n",
    "    def forward(self, inputs):\n",
    "        # Forward pass through LSTM layer\n",
    "        # x of shape (batch, seq, feature)\n",
    "        output, (hidden, cn) = self.lstm(inputs)\n",
    "        # Assuming using the last hidden state\n",
    "        out = self.W(hidden[-1])\n",
    "        return out\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def validate(model, val_loader, device, return_misclassified=False):\n",
    "    model.eval()\n",
    "    correct_predictions = 0\n",
    "    total_samples = 0\n",
    "    misclassified_examples = []\n",
    "\n",
    "    with torch.no_grad():\n",
    "        for inputs, labels in val_loader:\n",
    "            inputs = inputs.to(device)\n",
    "            labels = labels.to(device)\n",
    "\n",
    "            outputs = model(inputs)\n",
    "            _, predicted = torch.max(outputs, 1)\n",
    "            correct_predictions += (predicted == labels).sum().item()\n",
    "            total_samples += labels.size(0)\n",
    "    accuracy = correct_predictions / total_samples\n",
    "    model.train()\n",
    "\n",
    "    if return_misclassified:\n",
    "        return accuracy, misclassified_examples\n",
    "    return accuracy\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "def train(model, val_loader, computeLoss, optimizer, num_epochs, device, save_path='best_model_RNN.pth'):\n",
    "    model = model.to(device)\n",
    "    previous_val_accuracy = 0\n",
    "    best_val_accuracy = 0\n",
    "\n",
    "    for epoch in range(num_epochs):\n",
    "        model.train()\n",
    "        total_correct = 0\n",
    "        total_samples = 0\n",
    "        batch_losses = []\n",
    "        batch_accuracies = []\n",
    "        train_loader = DataLoader(train_dataset, batch_size=1, shuffle=True)\n",
    "\n",
    "\n",
    "        for i, (inputs, labels) in enumerate(train_loader):\n",
    "            print(inputs.shape, labels)\n",
    "            inputs = inputs.to(device)\n",
    "            labels = labels.to(device)\n",
    "\n",
    "            # Forward pass\n",
    "            outputs = model(inputs)\n",
    "            loss = computeLoss(outputs, labels)\n",
    "            batch_losses.append(loss.item())\n",
    "\n",
    "            # Backward and optimize\n",
    "            optimizer.zero_grad()\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "\n",
    "            # Calculate batch accuracy\n",
    "            _, predicted = torch.max(outputs, 1)\n",
    "            correct = (predicted == labels).sum().item()\n",
    "            total_correct += correct\n",
    "            total_samples += labels.size(0)\n",
    "\n",
    "            if (i + 1) % 100 == 0:\n",
    "                batch_accuracy = 100.0 * total_correct / total_samples\n",
    "                batch_accuracies.append(batch_accuracy)\n",
    "                print(f'Epoch {epoch+1}, Step {i+1}, Loss: {sum(batch_losses) / len(batch_losses):.4f}, '\n",
    "                      f'Accuracy: {batch_accuracy:.2f}%')\n",
    "                total_correct = 0\n",
    "                total_samples = 0\n",
    "                batch_losses = []\n",
    "\n",
    "        val_accuracy = validate(model, val_loader, device)\n",
    "        print(f'Epoch {epoch+1}: Validation Accuracy: {val_accuracy:.4f}')\n",
    "\n",
    "\n",
    "        # Saving the model if it has the best validation loss\n",
    "        if val_accuracy < best_val_accuracy:\n",
    "            best_val_accuracy = val_accuracy\n",
    "            torch.save(model.state_dict(), save_path)\n",
    "            print(f'Saved best model to {save_path}')\n",
    "\n",
    "        if val_accuracy < 0.9 * previous_val_accuracy:\n",
    "            print(\"Stopping early due to less than 10% decrease in validation loss.\")\n",
    "            break\n",
    "        previous_val_accuracy = val_accuracy\n",
    "\n",
    "        # Plotting\n",
    "        plt.figure(figsize=(10, 5))\n",
    "        plt.subplot(1, 2, 1)\n",
    "        plt.plot(batch_accuracies, label='Accuracy per 100 examples')\n",
    "        plt.title('Accuracy per 100 examples')\n",
    "        plt.xlabel('Batch')\n",
    "        plt.ylabel('Accuracy')\n",
    "        plt.legend()\n",
    "\n",
    "        plt.show()\n",
    "\n",
    "    final_model_path = 'final_model_RNN.pth'\n",
    "    torch.save(model.state_dict(), final_model_path)\n",
    "    print(f'Saved final model state to {final_model_path}')\n",
    "\n",
    "model = LSTM()\n",
    "computeLoss = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=0.001)\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "print(device)\n",
    "\n",
    "train(model, val_loader, computeLoss, optimizer, num_epochs=1, device=device)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'VideoDataset3DCNN' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[5], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m test_dataset \u001b[38;5;241m=\u001b[39m \u001b[43mVideoDataset3DCNN\u001b[49m(test_dataset)\n\u001b[0;32m      3\u001b[0m test_loader \u001b[38;5;241m=\u001b[39m DataLoader(test_dataset, batch_size\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m, shuffle\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n",
      "\u001b[1;31mNameError\u001b[0m: name 'VideoDataset3DCNN' is not defined"
     ]
    }
   ],
   "source": [
    "test_dataset = VideoDataset3DCNN(test_dataset)\n",
    "\n",
    "test_loader = DataLoader(test_dataset, batch_size=1, shuffle=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "def test(model, data_loader, device):\n",
    "    model = model.to(device)\n",
    "    model.eval() \n",
    "    total_loss = 0\n",
    "    correct_predictions = 0\n",
    "    total_predictions = 0\n",
    "    counter = 0\n",
    "    with torch.no_grad(): \n",
    "            for inputs, labels in data_loader:\n",
    "                inputs = inputs.to(device)\n",
    "                labels = labels.to(device)\n",
    "                \n",
    "                outputs = model(inputs)\n",
    "                loss = criterion(outputs, labels)\n",
    "                total_loss += loss.item()\n",
    "\n",
    "                _, predicted = torch.max(outputs, 1)\n",
    "                correct_predictions += (predicted == labels).sum().item()\n",
    "                total_predictions += labels.size(0)\n",
    "                counter += 1\n",
    "                print(predicted, labels, correct_predictions, total_predictions)\n",
    "    \n",
    "    avg_loss = total_loss / len(data_loader)\n",
    "    accuracy = correct_predictions / total_predictions\n",
    "    return avg_loss, accuracy\n",
    "\n",
    "test_loss, test_accuracy = test(modelFinal, test_loader, device)\n",
    "print(f'Test Loss: {test_loss:.4f}, Test Accuracy: {test_accuracy:.4f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "import torchvision.transforms as transforms\n",
    "import cv2\n",
    "import numpy as np\n",
    "\n",
    "class VideoDataset3DCNN(Dataset):\n",
    "    def __init__(self, dataset, clip_length=240, transform=None):\n",
    "        self.dataset = dataset\n",
    "        self.clip_length = clip_length \n",
    "        self.transform = transform or transforms.Compose([\n",
    "            transforms.ToPILImage(),\n",
    "            transforms.Resize((224, 224)),\n",
    "            transforms.ToTensor(),\n",
    "            transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n",
    "        ])\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.dataset)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        video_path = self.dataset[idx]['video_path']\n",
    "        label = self.dataset[idx]['anomaly']\n",
    "        frames = self.load_video(video_path, self.clip_length)\n",
    "\n",
    "        if self.transform:\n",
    "            frames = [self.transform(frame) for frame in frames]\n",
    "\n",
    "        frames_tensor = torch.stack(frames, dim=0)\n",
    "        frames_tensor = frames_tensor.permute(1, 0, 2, 3)\n",
    "\n",
    "        return frames_tensor, label\n",
    "\n",
    "    def load_video(self, video_path, clip_length):\n",
    "        \"\"\"\n",
    "        Load a clip containing 'clip_length' frames from a video.\n",
    "        \"\"\"\n",
    "        cap = cv2.VideoCapture(video_path)\n",
    "        frames = []\n",
    "        fps = int(cap.get(cv2.CAP_PROP_FPS))\n",
    "        frame_indices = [min(int(fps * i), int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) - 1) for i in range(clip_length)]\n",
    "\n",
    "        for idx in frame_indices:\n",
    "            cap.set(cv2.CAP_PROP_POS_FRAMES, idx)\n",
    "            ret, frame = cap.read()\n",
    "            if ret:\n",
    "                frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)  # Convert BGR to RGB\n",
    "                frames.append(frame)\n",
    "            elif frames:\n",
    "                frames += [frames[-1]] * (clip_length - len(frames))\n",
    "                break\n",
    "            else:\n",
    "                break\n",
    "        cap.release()\n",
    "        return frames\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import DataLoader\n",
    "\n",
    "train_dataset = VideoDataset3DCNN(train_dataset)\n",
    "val_dataset = VideoDataset3DCNN(val_dataset)\n",
    "\n",
    "train_loader = DataLoader(train_dataset, batch_size=1, shuffle=True)\n",
    "val_loader = DataLoader(val_dataset, batch_size=1, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "\n",
    "class Conv3D(nn.Module):\n",
    "    def __init__(self, num_classes):\n",
    "        super(Conv3D, self).__init__()\n",
    "        \n",
    "        self.conv1 = nn.Conv3d(3, 32, kernel_size=(3, 3, 3), stride=(1, 2, 2), padding=1) \n",
    "        self.relu = nn.ReLU()\n",
    "        self.pool = nn.MaxPool3d(kernel_size=(2, 2, 2), stride=(2, 2, 2)) \n",
    "        self.conv2 = nn.Conv3d(32, 64, kernel_size=(3, 3, 3), stride=(1, 2, 2), padding=1)\n",
    "        \n",
    "        num_features = 752640 \n",
    "        \n",
    "        self.fc1 = nn.Linear(num_features, 256)\n",
    "        self.fc2 = nn.Linear(256, num_classes)\n",
    "\n",
    "    def forward(self, inputs):\n",
    "        # Conv layer operations\n",
    "        layer1 = self.pool(self.relu(self.conv1(inputs)))\n",
    "        layer2 = self.pool(self.relu(self.conv2(layer1)))\n",
    "        # Flatten the tensor for the fully connected layer\n",
    "        feature_vector = layer2.view(layer2.size(0), -1)\n",
    "        output1 = self.relu(self.fc1(feature_vector))\n",
    "        output2 = self.fc2(output1)\n",
    "        return output2\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def CNNvalidate(model, val_loader, device):\n",
    "    model.eval()\n",
    "    total_correct = 0\n",
    "    total_samples = 0\n",
    "\n",
    "    with torch.no_grad():\n",
    "        for inputs, labels in val_loader:\n",
    "            inputs = inputs.to(device)\n",
    "            labels = labels.to(device)\n",
    "\n",
    "            outputs = model(inputs)\n",
    "            _, predicted = torch.max(outputs, 1)  \n",
    "            total_correct += (predicted == labels).sum().item()\n",
    "            total_samples += labels.size(0)\n",
    "\n",
    "    accuracy = total_correct / total_samples\n",
    "    print(f'Validation Accuracy: {accuracy * 100:.2f}%')\n",
    "    return accuracy\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "def CNNtrain(model, val_loader, computeLoss, optimizer, num_epochs, device, save_path='best_model.pth'):\n",
    "    model = model.to(device)\n",
    "    best_val_accuracy = 0\n",
    "    error_examples = []\n",
    "\n",
    "    for epoch in range(num_epochs):\n",
    "        model.train()\n",
    "        total_correct = 0\n",
    "        total_samples = 0\n",
    "        batch_losses = []\n",
    "        batch_accuracies = []\n",
    "\n",
    "        for i, (inputs, labels) in enumerate(train_loader):\n",
    "            print(inputs.shape, labels)\n",
    "            inputs = inputs.to(device)\n",
    "            labels = labels.to(device)\n",
    "\n",
    "            # Forward pass\n",
    "            outputs = model(inputs)\n",
    "            loss = computeLoss(outputs, labels)\n",
    "            batch_losses.append(loss.item())\n",
    "\n",
    "            # Backward and optimize\n",
    "            optimizer.zero_grad()\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "\n",
    "            # Calculate batch accuracy\n",
    "            _, predicted = torch.max(outputs.data, 1)\n",
    "            correct = (predicted == labels).sum().item()\n",
    "            total_correct += correct\n",
    "            total_samples += labels.size(0)\n",
    "\n",
    "            if (i + 1) % 100 == 0:\n",
    "                batch_accuracy = 100.0 * total_correct / total_samples\n",
    "                batch_accuracies.append(batch_accuracy)\n",
    "                print(f'Epoch {epoch+1}, Step {i+1}, Loss: {sum(batch_losses) / len(batch_losses):.4f}, '\n",
    "                      f'Accuracy: {batch_accuracy:.2f}%')\n",
    "                total_correct = 0\n",
    "                total_samples = 0\n",
    "                batch_losses = []\n",
    "\n",
    "        # Validation after each epoch\n",
    "        val_accuracy = CNNvalidate(model, val_loader, device)\n",
    "        print(f'Epoch {epoch+1}: Validation Accuracy: {val_accuracy:.4f}')\n",
    "\n",
    "        if val_accuracy > best_val_accuracy:\n",
    "            best_val_accuracy = val_accuracy\n",
    "            torch.save(model.state_dict(), save_path)\n",
    "            print(f'Saved best model to {save_path}')\n",
    "\n",
    "    # Plotting\n",
    "    plt.figure(figsize=(10, 5))\n",
    "    plt.plot(batch_accuracies, label='Accuracy per 100 examples')\n",
    "    plt.title('Accuracy per 100 examples')\n",
    "    plt.xlabel('Batch')\n",
    "    plt.ylabel('Accuracy')\n",
    "    plt.legend()\n",
    "    plt.show()\n",
    "\n",
    "    final_model_path = 'final_model.pth'\n",
    "    torch.save(model.state_dict(), final_model_path)\n",
    "    print(f'Saved final model state to {final_model_path}')\n",
    "\n",
    "\n",
    "model = Conv3D(2)\n",
    "computeLoss = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=0.001)\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "print(device)\n",
    "\n",
    "CNNtrain(model, val_loader, computeLoss, optimizer, num_epochs=5, device=device)\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "myenv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}