torch-examples.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.nn import functional as F\n",
    "import numpy as np\n",
    "import time\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.00969601\n",
      "CPU times: total: 0 ns\n",
      "Wall time: 10.2 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "start_time = time.time()\n",
    "# matrix operations here\n",
    "zeros = torch.zeros(1, 1)\n",
    "end_time = time.time()\n",
    "\n",
    "elapsed_time = end_time - start_time\n",
    "print(f\"{elapsed_time:.8f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "using the time library to record execution time\n",
    "%%time to record the time taken for the cell to execute"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.83151722\n",
      "0.07781816\n"
     ]
    }
   ],
   "source": [
    "torch_rand1 = torch.rand(100, 100, 100, 100).to(device)\n",
    "torch_rand2 = torch.rand(100, 100, 100, 100).to(device)\n",
    "np_rand1 = torch.rand(100, 100, 100, 100)\n",
    "np_rand2 = torch.rand(100, 100, 100, 100)\n",
    "\n",
    "start_time = time.time()\n",
    "\n",
    "rand = (torch_rand1 @ torch_rand2)\n",
    "\n",
    "end_time = time.time()\n",
    "\n",
    "elapsed_time = end_time - start_time\n",
    "print(f\"{elapsed_time:.8f}\")\n",
    "\n",
    "\n",
    "start_time = time.time()\n",
    "\n",
    "rand = np.multiply(np_rand1, np_rand2)\n",
    "end_time = time.time()\n",
    "elapsed_time = end_time - start_time\n",
    "print(f\"{elapsed_time:.8f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1])\n"
     ]
    }
   ],
   "source": [
    "# embeddings, torch.stack, torch.multinomial, torch.tril, torch.triu, input.T / input.transpose, nn.Linear, torch.cat, F.softmax (show all the examples of functions/methods with pytorch docs)\n",
    "\n",
    "\n",
    "# Define a probability tensor\n",
    "probabilities = torch.tensor([0.1, 0.9])\n",
    "# 10% or 0.1 => 0, 90% or 0.9 => 1. each probability points to the index of the probability in the tensor\n",
    "# Draw 5 samples from the multinomial distribution\n",
    "samples = torch.multinomial(probabilities, num_samples=10, replacement=True)\n",
    "print(samples)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([1, 2, 3, 4, 5])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tensor = torch.tensor([1, 2, 3, 4])\n",
    "out = torch.cat((tensor, torch.tensor([5])), dim=0)\n",
    "out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1., 0., 0., 0., 0.],\n",
       "        [1., 1., 0., 0., 0.],\n",
       "        [1., 1., 1., 0., 0.],\n",
       "        [1., 1., 1., 1., 0.],\n",
       "        [1., 1., 1., 1., 1.]])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "out = torch.tril(torch.ones(5, 5))\n",
    "out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1., 1., 1., 1., 1.],\n",
       "        [0., 1., 1., 1., 1.],\n",
       "        [0., 0., 1., 1., 1.],\n",
       "        [0., 0., 0., 1., 1.],\n",
       "        [0., 0., 0., 0., 1.]])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "out = torch.triu(torch.ones(5, 5))\n",
    "out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0., -inf, -inf, -inf, -inf],\n",
       "        [0., 0., -inf, -inf, -inf],\n",
       "        [0., 0., 0., -inf, -inf],\n",
       "        [0., 0., 0., 0., -inf],\n",
       "        [0., 0., 0., 0., 0.]])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "out = torch.zeros(5, 5).masked_fill(torch.tril(torch.ones(5, 5)) == 0, float('-inf'))\n",
    "out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1., 0., 0., 0., 0.],\n",
       "        [1., 1., 0., 0., 0.],\n",
       "        [1., 1., 1., 0., 0.],\n",
       "        [1., 1., 1., 1., 0.],\n",
       "        [1., 1., 1., 1., 1.]])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.exp(out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([3, 2, 4])\n",
      "torch.Size([2, 4, 3])\n"
     ]
    }
   ],
   "source": [
    "input = torch.zeros(2, 3, 4)\n",
    "out1 = input.transpose(0, 1)\n",
    "out2 = input.transpose(-2,-1)\n",
    "print(out1.shape)\n",
    "print(out2.shape)\n",
    "# torch.permute works the same but you provide the new order of dimensions instead of the dimensions you'd like to swap."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1, 2, 3],\n",
       "        [4, 5, 6],\n",
       "        [7, 8, 9]])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tensor1 = torch.tensor([1, 2, 3])\n",
    "tensor2 = torch.tensor([4, 5, 6])\n",
    "tensor3 = torch.tensor([7, 8, 9])\n",
    "\n",
    "# Stack the tensors along a new dimension\n",
    "stacked_tensor = torch.stack([tensor1, tensor2, tensor3])\n",
    "stacked_tensor\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([ 0.7475, -2.3593, -3.4591], grad_fn=<SqueezeBackward4>)\n"
     ]
    }
   ],
   "source": [
    "import torch.nn as nn\n",
    "sample = torch.tensor([10.,10.,10.])\n",
    "linear = nn.Linear(3, 3, bias=False)\n",
    "print(linear(sample))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([0.0900, 0.2447, 0.6652])\n"
     ]
    }
   ],
   "source": [
    "import torch.nn.functional as F\n",
    "\n",
    "# Create a tensor\n",
    "tensor1 = torch.tensor([1.0, 2.0, 3.0])\n",
    "\n",
    "# Apply softmax using torch.nn.functional.softmax()\n",
    "softmax_output = F.softmax(tensor1, dim=0)\n",
    "\n",
    "print(softmax_output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([4, 6])\n",
      "tensor([[-0.1182, -0.0551,  2.0661, -0.1224,  0.8691, -0.2577],\n",
      "        [-1.9364,  0.1501,  0.0839, -1.7630,  0.7601,  0.5639],\n",
      "        [ 0.3985,  0.8211, -1.7631, -0.2576, -0.7638,  0.1854],\n",
      "        [ 1.1235, -0.2815, -0.2051,  0.3126, -0.2904,  2.0155]],\n",
      "       grad_fn=<EmbeddingBackward0>)\n"
     ]
    }
   ],
   "source": [
    "# Initialize an embedding layer\n",
    "vocab_size = 80\n",
    "embedding_dim = 6\n",
    "embedding = nn.Embedding(vocab_size, embedding_dim)\n",
    "\n",
    "# Create some input indices\n",
    "input_indices = torch.LongTensor([1, 5, 3, 2])\n",
    "\n",
    "# Apply the embedding layer\n",
    "embedded_output = embedding(input_indices)\n",
    "\n",
    "# The output will be a tensor of shape (4, 100), where 4 is the number of inputs\n",
    "# and 100 is the dimensionality of the embedding vectors\n",
    "print(embedded_output.shape)\n",
    "print(embedded_output)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 27,  30,  33],\n",
      "        [ 61,  68,  75],\n",
      "        [ 95, 106, 117]])\n"
     ]
    }
   ],
   "source": [
    "a = torch.tensor([[1,2],[3,4],[5,6]])\n",
    "b = torch.tensor([[7,8,9],[10,11,12]])\n",
    "# print(a @ b)\n",
    "print(torch.matmul(a, b))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0., 0., 0.],\n",
      "        [0., 0., 0.],\n",
      "        [0., 0., 0.]])\n"
     ]
    }
   ],
   "source": [
    "int_64 = torch.randint(1, (3, 2)).float()\n",
    "#type int64\n",
    "float_32 = torch.rand(2,3)\n",
    "#type float32\n",
    "# print(int_64.dtype, float_32.dtype)\n",
    "result = torch.matmul(int_64, float_32)\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([2, 3, 5])\n",
      "torch.Size([2, 3, 5])\n"
     ]
    }
   ],
   "source": [
    "a = torch.rand(2, 3, 5)\n",
    "print(a.shape)\n",
    "x, y, z = a.shape\n",
    "a = a.view(x,y,z)\n",
    "# print(x, y, z)\n",
    "print(a.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[8.4784e-01, 7.1257e-01, 5.8318e-02, 1.1662e-01, 5.6686e-01, 1.9303e-01,\n",
      "         6.6632e-01, 6.3045e-01, 1.1654e-01, 2.2591e-01],\n",
      "        [4.4540e-01, 1.8662e-01, 1.6590e-01, 7.8310e-01, 2.8952e-01, 3.6106e-01,\n",
      "         2.4187e-01, 5.7216e-01, 5.1018e-01, 8.8388e-01],\n",
      "        [2.1610e-01, 4.2283e-01, 2.7590e-01, 1.3142e-02, 2.8317e-02, 9.7419e-01,\n",
      "         9.3365e-01, 6.6609e-01, 5.2375e-01, 8.5033e-02],\n",
      "        [7.0206e-01, 5.1114e-01, 7.0352e-03, 2.6985e-01, 9.7451e-01, 1.1618e-01,\n",
      "         5.5879e-01, 8.5950e-01, 3.5737e-01, 6.6099e-01],\n",
      "        [8.6752e-01, 1.4063e-01, 1.3130e-01, 4.8753e-01, 1.3561e-01, 2.6779e-01,\n",
      "         4.9860e-01, 9.8521e-01, 2.7027e-01, 8.7181e-01],\n",
      "        [9.0724e-04, 9.4394e-01, 5.9852e-01, 2.3980e-01, 2.1402e-02, 2.2790e-01,\n",
      "         9.2466e-01, 1.5136e-01, 3.5337e-01, 2.2062e-01],\n",
      "        [8.0447e-01, 4.4472e-01, 5.8952e-01, 4.1328e-01, 8.5890e-01, 6.8109e-01,\n",
      "         4.5790e-01, 6.8573e-01, 7.8548e-01, 2.3513e-01],\n",
      "        [1.1809e-01, 5.6395e-01, 1.0018e-01, 4.3966e-01, 6.3728e-01, 8.3437e-01,\n",
      "         8.9655e-01, 3.1252e-01, 8.8037e-02, 1.1689e-01],\n",
      "        [9.9730e-01, 1.8197e-01, 5.5324e-01, 4.2488e-01, 2.3790e-03, 2.5658e-01,\n",
      "         4.1206e-01, 8.6530e-01, 6.9251e-01, 8.8614e-01],\n",
      "        [1.1549e-02, 9.5455e-01, 3.8725e-01, 8.5537e-01, 7.4989e-01, 8.2071e-01,\n",
      "         5.9832e-01, 3.0360e-01, 5.4594e-01, 8.4376e-01],\n",
      "        [1.3777e-01, 4.1883e-01, 5.6526e-01, 7.4036e-02, 8.3878e-01, 1.6556e-02,\n",
      "         6.9582e-01, 7.0584e-01, 4.3799e-01, 3.5505e-02],\n",
      "        [9.8989e-01, 5.1932e-01, 7.0095e-01, 7.2378e-01, 6.6765e-01, 9.8325e-01,\n",
      "         2.9214e-01, 7.4497e-01, 2.8286e-01, 9.9305e-01],\n",
      "        [8.8807e-01, 8.8728e-01, 7.7809e-01, 2.0169e-01, 8.1256e-01, 3.0491e-01,\n",
      "         6.3829e-01, 4.2424e-01, 3.8111e-01, 3.7713e-01],\n",
      "        [8.6794e-01, 2.4137e-01, 9.4477e-01, 3.7884e-01, 6.0674e-01, 2.4725e-02,\n",
      "         1.2078e-01, 5.0054e-01, 8.3786e-01, 4.7303e-01],\n",
      "        [6.4304e-01, 8.3795e-02, 8.6081e-03, 9.3872e-01, 1.3191e-01, 9.0981e-01,\n",
      "         2.6003e-01, 1.5305e-02, 3.7633e-01, 5.9801e-02],\n",
      "        [9.0000e-01, 5.5200e-01, 5.3571e-02, 8.6615e-01, 5.3864e-01, 1.9017e-01,\n",
      "         1.0007e-01, 6.0474e-01, 5.8185e-02, 1.3151e-01],\n",
      "        [6.5226e-01, 5.3695e-01, 8.5617e-01, 9.6370e-01, 5.5281e-01, 5.8211e-01,\n",
      "         6.1119e-01, 7.7745e-01, 3.3720e-01, 3.5373e-01],\n",
      "        [8.7052e-01, 2.7166e-01, 6.5231e-01, 1.0057e-01, 1.0628e-01, 6.8225e-01,\n",
      "         9.5398e-01, 5.2178e-01, 8.5180e-01, 5.2549e-01],\n",
      "        [2.2479e-01, 4.1928e-01, 8.0073e-01, 8.4228e-01, 3.8563e-01, 1.5114e-01,\n",
      "         6.8264e-01, 4.5133e-01, 4.8565e-01, 8.7312e-01],\n",
      "        [3.7959e-02, 5.0993e-01, 8.9903e-01, 6.8165e-01, 4.8601e-01, 8.6840e-01,\n",
      "         1.0306e-01, 1.1990e-02, 1.5555e-01, 6.9926e-01],\n",
      "        [6.9057e-02, 7.6252e-01, 6.3782e-01, 8.0156e-01, 7.3988e-01, 7.5775e-01,\n",
      "         4.8732e-01, 4.1785e-01, 4.5084e-01, 8.3595e-01],\n",
      "        [6.1156e-01, 8.5702e-01, 9.8226e-02, 7.8539e-02, 4.8247e-01, 2.7219e-01,\n",
      "         2.4549e-01, 9.6610e-01, 3.2557e-01, 5.5234e-01],\n",
      "        [3.4030e-01, 6.7352e-01, 7.8608e-01, 6.7252e-01, 7.0807e-01, 5.0895e-01,\n",
      "         4.2008e-01, 4.1877e-01, 3.6420e-01, 3.8804e-01],\n",
      "        [7.5602e-01, 8.6862e-01, 1.7415e-01, 9.5588e-01, 8.8406e-01, 1.9198e-01,\n",
      "         4.5605e-01, 7.4624e-01, 9.9847e-02, 7.9370e-01],\n",
      "        [4.2986e-01, 3.7908e-01, 9.7233e-01, 5.2798e-01, 7.7853e-01, 6.7777e-01,\n",
      "         5.0770e-01, 8.4769e-01, 6.7054e-01, 2.4795e-01],\n",
      "        [3.4168e-01, 1.1087e-01, 9.6107e-01, 4.6941e-01, 6.6239e-01, 1.4574e-01,\n",
      "         6.6885e-01, 9.4786e-01, 8.0510e-02, 1.1269e-01],\n",
      "        [3.4333e-02, 9.2557e-01, 5.1897e-01, 9.7018e-01, 7.0402e-01, 9.8951e-01,\n",
      "         1.1039e-01, 4.1054e-01, 9.5528e-01, 7.6245e-01],\n",
      "        [9.5671e-01, 1.8762e-02, 3.0689e-01, 4.4409e-01, 2.1237e-01, 7.5092e-02,\n",
      "         5.7327e-01, 3.9093e-01, 5.5734e-01, 6.3038e-01],\n",
      "        [6.8145e-01, 1.6966e-02, 4.5854e-01, 5.3712e-01, 3.0549e-01, 5.4741e-02,\n",
      "         9.9189e-01, 6.1542e-01, 5.1386e-01, 3.4015e-01],\n",
      "        [1.9131e-01, 5.8434e-01, 7.5250e-01, 6.0344e-01, 1.4139e-01, 5.0211e-01,\n",
      "         9.6055e-01, 3.6290e-01, 6.8361e-01, 5.4033e-01],\n",
      "        [8.7327e-01, 1.2722e-01, 5.5897e-01, 5.0354e-01, 4.5040e-01, 6.8135e-01,\n",
      "         8.6287e-01, 6.9475e-01, 7.9137e-01, 3.5860e-01],\n",
      "        [7.9182e-01, 1.8288e-01, 2.6567e-01, 3.1575e-01, 8.3716e-01, 3.6981e-01,\n",
      "         7.7873e-03, 8.2700e-01, 5.3342e-01, 3.9319e-01]])\n"
     ]
    },
    {
     "ename": "IndexError",
     "evalue": "too many indices for tensor of dimension 2",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[10], line 6\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[38;5;28mprint\u001b[39m(output)\n\u001b[0;32m      5\u001b[0m \u001b[38;5;66;03m# print(input)\u001b[39;00m\n\u001b[1;32m----> 6\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[43moutput\u001b[49m\u001b[43m[\u001b[49m\u001b[43m:\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m-\u001b[39;49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m:\u001b[49m\u001b[43m]\u001b[49m)\n",
      "\u001b[1;31mIndexError\u001b[0m: too many indices for tensor of dimension 2"
     ]
    }
   ],
   "source": [
    "input = torch.rand((4, 8, 10))\n",
    "B, T, C = input.shape\n",
    "output = input.view(B*T, C)\n",
    "print(output)\n",
    "# print(input)\n",
    "print(output[:, -1, :])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([1.])\n"
     ]
    }
   ],
   "source": [
    "x = torch.tensor([10], dtype=torch.float32)\n",
    "y = F.tanh(x)\n",
    "print(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "cuda-gpt",
   "language": "python",
   "name": "cuda"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}