Recurrent Neural Network.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Modeling RNN by using time-series data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import library\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# number of nodes of each neurons\n",
    "n_hidden = 35\n",
    "lr = 0.01\n",
    "epochs = 1000\n",
    "\n",
    "string = 'hello pytorch. how long can a rnn cell remember?'\n",
    "chars = 'abcedfghijklmnopqrstuvwxyz ?!.,:;01'\n",
    "char_list = [i for i in chars]\n",
    "n_letters = len(char_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# convert string to one-hot vector\n",
    "def string_to_onehot(string):\n",
    "    start = np.zeros(shape = len(char_list), dtype = int)\n",
    "    end = np.zeros(shape = len(char_list), dtype = int)\n",
    "    start[-2] = 1\n",
    "    end [-1] = 1\n",
    "    \n",
    "    for i in string:\n",
    "        idx = char_list.index(i)\n",
    "        zero = np.zeros(shape = n_letters, dtype = int)\n",
    "        zero[idx] = 1\n",
    "        start = np.vstack([start, zero])\n",
    "    output = np.vstack([start, end])\n",
    "    return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# convert one-hot vector into string\n",
    "def onehot_to_word(onehot_1):\n",
    "    onehot = torch.Tensor.numpy(onehot_1)\n",
    "    return char_list[onehot.argmax()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(2.5712, grad_fn=<AddBackward0>)\n",
      "tensor(0.9538, grad_fn=<AddBackward0>)\n",
      "tensor(0.6199, grad_fn=<AddBackward0>)\n",
      "tensor(0.4200, grad_fn=<AddBackward0>)\n",
      "tensor(0.2871, grad_fn=<AddBackward0>)\n",
      "tensor(0.2167, grad_fn=<AddBackward0>)\n",
      "tensor(0.1608, grad_fn=<AddBackward0>)\n",
      "tensor(0.1299, grad_fn=<AddBackward0>)\n",
      "tensor(0.1027, grad_fn=<AddBackward0>)\n",
      "tensor(0.1030, grad_fn=<AddBackward0>)\n",
      "tensor(0.0777, grad_fn=<AddBackward0>)\n",
      "tensor(0.0661, grad_fn=<AddBackward0>)\n",
      "tensor(0.0762, grad_fn=<AddBackward0>)\n",
      "tensor(0.0563, grad_fn=<AddBackward0>)\n",
      "tensor(0.0505, grad_fn=<AddBackward0>)\n",
      "tensor(0.0528, grad_fn=<AddBackward0>)\n",
      "tensor(0.0475, grad_fn=<AddBackward0>)\n",
      "tensor(0.0416, grad_fn=<AddBackward0>)\n",
      "tensor(0.0562, grad_fn=<AddBackward0>)\n",
      "tensor(0.0371, grad_fn=<AddBackward0>)\n",
      "tensor(0.0327, grad_fn=<AddBackward0>)\n",
      "tensor(0.0302, grad_fn=<AddBackward0>)\n",
      "tensor(0.0330, grad_fn=<AddBackward0>)\n",
      "tensor(0.0298, grad_fn=<AddBackward0>)\n",
      "tensor(0.0249, grad_fn=<AddBackward0>)\n",
      "tensor(0.0231, grad_fn=<AddBackward0>)\n",
      "tensor(0.0235, grad_fn=<AddBackward0>)\n",
      "tensor(0.0280, grad_fn=<AddBackward0>)\n",
      "tensor(0.0217, grad_fn=<AddBackward0>)\n",
      "tensor(0.0193, grad_fn=<AddBackward0>)\n",
      "tensor(0.0176, grad_fn=<AddBackward0>)\n",
      "tensor(0.0165, grad_fn=<AddBackward0>)\n",
      "tensor(0.0177, grad_fn=<AddBackward0>)\n",
      "tensor(0.0175, grad_fn=<AddBackward0>)\n",
      "tensor(0.0157, grad_fn=<AddBackward0>)\n",
      "tensor(0.0137, grad_fn=<AddBackward0>)\n",
      "tensor(0.0129, grad_fn=<AddBackward0>)\n",
      "tensor(0.0144, grad_fn=<AddBackward0>)\n",
      "tensor(0.0161, grad_fn=<AddBackward0>)\n",
      "tensor(0.0131, grad_fn=<AddBackward0>)\n",
      "tensor(0.0149, grad_fn=<AddBackward0>)\n",
      "tensor(0.0142, grad_fn=<AddBackward0>)\n",
      "tensor(0.0108, grad_fn=<AddBackward0>)\n",
      "tensor(0.0102, grad_fn=<AddBackward0>)\n",
      "tensor(0.0097, grad_fn=<AddBackward0>)\n",
      "tensor(0.0260, grad_fn=<AddBackward0>)\n",
      "tensor(0.0157, grad_fn=<AddBackward0>)\n",
      "tensor(0.0104, grad_fn=<AddBackward0>)\n",
      "tensor(0.0101, grad_fn=<AddBackward0>)\n",
      "tensor(0.0098, grad_fn=<AddBackward0>)\n",
      "tensor(0.0111, grad_fn=<AddBackward0>)\n",
      "tensor(0.0087, grad_fn=<AddBackward0>)\n",
      "tensor(0.0101, grad_fn=<AddBackward0>)\n",
      "tensor(0.0090, grad_fn=<AddBackward0>)\n",
      "tensor(0.0076, grad_fn=<AddBackward0>)\n",
      "tensor(0.0075, grad_fn=<AddBackward0>)\n",
      "tensor(0.0133, grad_fn=<AddBackward0>)\n",
      "tensor(0.0080, grad_fn=<AddBackward0>)\n",
      "tensor(0.0072, grad_fn=<AddBackward0>)\n",
      "tensor(0.0067, grad_fn=<AddBackward0>)\n",
      "tensor(0.0229, grad_fn=<AddBackward0>)\n",
      "tensor(0.0116, grad_fn=<AddBackward0>)\n",
      "tensor(0.0085, grad_fn=<AddBackward0>)\n",
      "tensor(0.0066, grad_fn=<AddBackward0>)\n",
      "tensor(0.0063, grad_fn=<AddBackward0>)\n",
      "tensor(0.0178, grad_fn=<AddBackward0>)\n",
      "tensor(0.0086, grad_fn=<AddBackward0>)\n",
      "tensor(0.0068, grad_fn=<AddBackward0>)\n",
      "tensor(0.0058, grad_fn=<AddBackward0>)\n",
      "tensor(0.0057, grad_fn=<AddBackward0>)\n",
      "tensor(0.0142, grad_fn=<AddBackward0>)\n",
      "tensor(0.0063, grad_fn=<AddBackward0>)\n",
      "tensor(0.0055, grad_fn=<AddBackward0>)\n",
      "tensor(0.0050, grad_fn=<AddBackward0>)\n",
      "tensor(0.0048, grad_fn=<AddBackward0>)\n",
      "tensor(0.0045, grad_fn=<AddBackward0>)\n",
      "tensor(0.0086, grad_fn=<AddBackward0>)\n",
      "tensor(0.0094, grad_fn=<AddBackward0>)\n",
      "tensor(0.0056, grad_fn=<AddBackward0>)\n",
      "tensor(0.0047, grad_fn=<AddBackward0>)\n",
      "tensor(0.0044, grad_fn=<AddBackward0>)\n",
      "tensor(0.0108, grad_fn=<AddBackward0>)\n",
      "tensor(0.0063, grad_fn=<AddBackward0>)\n",
      "tensor(0.0052, grad_fn=<AddBackward0>)\n",
      "tensor(0.0044, grad_fn=<AddBackward0>)\n",
      "tensor(0.0043, grad_fn=<AddBackward0>)\n",
      "tensor(0.0111, grad_fn=<AddBackward0>)\n",
      "tensor(0.0040, grad_fn=<AddBackward0>)\n",
      "tensor(0.0077, grad_fn=<AddBackward0>)\n",
      "tensor(0.0048, grad_fn=<AddBackward0>)\n",
      "tensor(0.0039, grad_fn=<AddBackward0>)\n",
      "tensor(0.0035, grad_fn=<AddBackward0>)\n",
      "tensor(0.0034, grad_fn=<AddBackward0>)\n",
      "tensor(0.0114, grad_fn=<AddBackward0>)\n",
      "tensor(0.0046, grad_fn=<AddBackward0>)\n",
      "tensor(0.0038, grad_fn=<AddBackward0>)\n",
      "tensor(0.0031, grad_fn=<AddBackward0>)\n",
      "tensor(0.0030, grad_fn=<AddBackward0>)\n",
      "tensor(0.0148, grad_fn=<AddBackward0>)\n",
      "tensor(0.0089, grad_fn=<AddBackward0>)\n"
     ]
    }
   ],
   "source": [
    "# Recurrent Neural Network\n",
    "class RNN(nn.Module):\n",
    "    def __init__(self, input_size, hidden_size, output_size):\n",
    "        super(RNN, self).__init__()\n",
    "        \n",
    "        self.input_size = input_size\n",
    "        self.hidden_size = hidden_size\n",
    "        self.output_size = output_size\n",
    "        \n",
    "        self.i2h = nn.Linear(input_size, hidden_size)\n",
    "        self.h2h = nn.Linear(hidden_size, output_size)\n",
    "        self.i2o = nn.Linear(hidden_size, output_size)\n",
    "        self.act_fn = nn.Tanh()\n",
    "        \n",
    "    def forward(self, input, hidden):\n",
    "        hidden = self.act_fn(self.i2h(input)+self.h2h(hidden))\n",
    "        output = self.i2o(hidden)\n",
    "        return output, hidden\n",
    "    \n",
    "    def init_hidden(self):\n",
    "        return torch.zeros(1, self.hidden_size)\n",
    "    \n",
    "rnn = RNN(n_letters, n_hidden, n_letters)\n",
    "\n",
    "# define loss function and optimization function\n",
    "loss_func = nn.MSELoss()\n",
    "optimizer = torch.optim.Adam(rnn.parameters(), lr = lr)\n",
    "\n",
    "# training\n",
    "one_hot = torch.from_numpy(string_to_onehot(string)).type_as(torch.FloatTensor())\n",
    "\n",
    "for i in range(epochs):\n",
    "    rnn.zero_grad()\n",
    "    total_loss = 0\n",
    "    hidden = rnn.init_hidden()\n",
    "    \n",
    "    for j in range(one_hot.size()[0]-1):\n",
    "        input_ = one_hot[j:j+1, :]\n",
    "        target = one_hot[j+1]\n",
    "        \n",
    "        output, hidden = rnn.forward(input_, hidden)\n",
    "        loss = loss_func(output.view(-1), target.view(-1))\n",
    "        total_loss += loss\n",
    "        \n",
    "    total_loss.backward()\n",
    "    optimizer.step()\n",
    "    \n",
    "    if i%10 == 0:\n",
    "        print(total_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hello pelorwel? welog elmeebepwelwemlmellmeelmem\n"
     ]
    }
   ],
   "source": [
    "# validate whether model trained or not\n",
    "start = torch.zeros(1, len(char_list))\n",
    "start[:, -2] = 1\n",
    "\n",
    "with torch.no_grad():\n",
    "    hidden = rnn.init_hidden()  # initialize output of the hidden vector\n",
    "    input_ = start\n",
    "    output_string = \"\"\n",
    "    for i in range(len(string)):\n",
    "        output, hidden = rnn.forward(input_, hidden)\n",
    "        output_string += onehot_to_word(output.data)\n",
    "        input_ = output\n",
    "        \n",
    "    print(output_string)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Embedding + RNN, LSTM, GRU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2020-12-07 21:08:34--  https://raw.githubusercontent.com/GunhoChoi/PyTorch-FastCampus/master/05_RNN/2_Char_RNN/data/linux.txt\n",
      "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.76.133\n",
      "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.76.133|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 33756 (33K) [text/plain]\n",
      "Saving to: ‘./data/linux.txt.1’\n",
      "\n",
      "linux.txt.1         100%[===================>]  32.96K  --.-KB/s    in 0.03s   \n",
      "\n",
      "2020-12-07 21:08:35 (1.18 MB/s) - ‘./data/linux.txt.1’ saved [33756/33756]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import os \n",
    "\n",
    "try:\n",
    "  os.mkdir(\"./data\")\n",
    "except:\n",
    "  pass\n",
    "\n",
    "!wget https://raw.githubusercontent.com/GunhoChoi/PyTorch-FastCampus/master/05_RNN/2_Char_RNN/data/linux.txt -P ./data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import unidecode\n",
    "import string\n",
    "import random\n",
    "import re\n",
    "import time, math"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_epochs = 2000\n",
    "print_every = 100\n",
    "plot_every = 10\n",
    "chunk_len = 200\n",
    "hidden_size = 100\n",
    "batch_size = 1\n",
    "num_layers = 1\n",
    "embedding_size = 70\n",
    "lr = 0.002"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~ \t\n",
      "\r",
      "\u000b",
      "\f",
      "\n",
      "num_chars =  100\n"
     ]
    }
   ],
   "source": [
    "# Commonly used part\n",
    "all_characters = string.printable\n",
    "n_characters = len(all_characters)\n",
    "print(all_characters)\n",
    "print('num_chars = ', n_characters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "file_len =  33756\n"
     ]
    }
   ],
   "source": [
    "# import data\n",
    "file = unidecode.unidecode(open('./data/linux.txt').read())\n",
    "file_len = len(file)\n",
    "print('file_len = ', file_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([36, 37, 38, 13, 14, 15])\n"
     ]
    }
   ],
   "source": [
    "# Since all files can't be read at once, we need to divide it in certain length.\n",
    "def random_chunk():\n",
    "    start_index = random.randint(0, file_len - chunk_len)\n",
    "    end_index = start_index + chunk_len + 1\n",
    "    return file[start_index : end_index]\n",
    "\n",
    "# convert string list into index\n",
    "def char_tensor(string):\n",
    "    tensor = torch.zeros(len(string)).long()\n",
    "    for c in range(len(string)):\n",
    "        tensor[c] = all_characters.index(string[c])\n",
    "    return tensor\n",
    "\n",
    "print(char_tensor(\"ABCdef\"))\n",
    "\n",
    "# split indexed string into input and target\n",
    "def random_training_set():\n",
    "    chunk = random_chunk()\n",
    "    inp = char_tensor(chunk[:-1])\n",
    "    target = char_tensor(chunk[1:])\n",
    "    return inp, target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " tensor([4.6298], grad_fn=<DivBackward0>) \n",
      "\n",
      "ns.!nc;\"v:`=K 3d^8W)(AB;dp*\n",
      "lSu?>*jt]i!3i9\\;O\n",
      "rx_su^nB5##a{lp~TX}tIkoza<4/[876b}ok:Ii7d3#!S\f",
      "DiqyVyOi80pv~@HcP}CI8l??-s'=*h\n",
      " tensor([2.5853], grad_fn=<DivBackward0>) \n",
      "\n",
      "ber.\n",
      "beU erdan sat loue iil] thread on areele\n",
      "By saes noris os and sine so o cis;\n",
      "Thenite to bans phald faive sawet thite het ine wen.:\n",
      "\n",
      "Th ye he sacos foret te hareantt ine anin\n",
      "Afs ;oud be lein yalh \n",
      " tensor([2.1391], grad_fn=<DivBackward0>) \n",
      "\n",
      "bomertertm be wid,\n",
      " meand the tuere brerorcell,\n",
      "Do I and my go ther! lort, to.\n",
      "kuch our my Marte?\n",
      "\n",
      "Uun mobe hit ard you'd mesos he hee the hitel\n",
      "\n",
      "\n",
      "CIANWANRRSRIS:\n",
      "C tham! brovent chom yene mowr th\n",
      " tensor([2.2910], grad_fn=<DivBackward0>) \n",
      "\n",
      "bert\n",
      "Howind of a ifeide of to the lor taresou lare-\n",
      "\n",
      "LUCELOANTE:\n",
      "Ans my arecs the deve of fice mingj nut home wat\n",
      "The gust dor thid wak is ir nor word'd be strour st the mat hatipe sime how me the mant\n",
      " tensor([2.1721], grad_fn=<DivBackward0>) \n",
      "\n",
      "bros: besther, secliargbe wards me to for with these get\n",
      "Bith and hatith then and couch your ichrus fave you I to louro,\n",
      "And,\n",
      "I lay,\n",
      "Stald sord and antry.\n",
      "\n",
      "QLTEOON:\n",
      "Fut aul. Gurded in wat the tard\n",
      "Tien\n",
      " tensor([2.1085], grad_fn=<DivBackward0>) \n",
      "\n",
      "bely, nather cave bray, and and the danlo herentat, wond to enow seme say'd be.\n",
      "\n",
      "NUOIL:\n",
      "Whe day with breacho\n",
      "That be tilre.\n",
      "\n",
      "TARDES:\n",
      "Co pid fid ussat! efear of sulless Ronge concowy bried'd thencient; \n",
      " tensor([1.9670], grad_fn=<DivBackward0>) \n",
      "\n",
      "by the shere hele feat theathing lomcour tlou fait cive fowbele my het my that beat me.\n",
      "\n",
      "PENROTEN:\n",
      "I you\n",
      "EREO:\n",
      "I the heed doath we to and lord I wind at the cove the dantoulfon beay, brothander sustare\n",
      " tensor([1.9869], grad_fn=<DivBackward0>) \n",
      "\n",
      "by,\n",
      "That dave is loiteam seif chall bat seeto, evell courner.\n",
      "\n",
      "BORENTIO:\n",
      "Thersel\n",
      "beve will ama day,\n",
      "ID some, and congeir that you'ld thou theel I pother'd my in my wich theret thou herperoner that mell\n",
      " tensor([2.0846], grad_fn=<DivBackward0>) \n",
      "\n",
      "byal.\n",
      "\n",
      "KENVOLHARD:\n",
      "onte with of do have martus.\n",
      "\n",
      "CORTIS:\n",
      "To untence man-wan proak here proper, what that shourd in that gren, the singeswand our me a condy, a farint, the spored, where in, why frown so\n",
      " tensor([1.8029], grad_fn=<DivBackward0>) \n",
      "\n",
      "by.\n",
      "\n",
      "LADY ICARD:\n",
      "A dowises with ye, mest rom: brow be for bathen me beemer,\n",
      "If lather, a heed,\n",
      "O, a speat me forgerery the heanty what me'le\n",
      "That torem of Jurdeat hath a mice,\n",
      "Be?\n",
      "\n",
      "ANINTENIUS:\n",
      "No your \n",
      " tensor([2.0825], grad_fn=<DivBackward0>) \n",
      "\n",
      "but,\n",
      "Them that strear are if and feverswent moment mine, that have, my fathers love where whonesce, faiths there lord, and with all entle him waevall the was ink sich soge not for\n",
      "Is who knied and you.\n",
      " tensor([1.9184], grad_fn=<DivBackward0>) \n",
      "\n",
      "borit my thou hink as be tong that is gost the me, lyen of Hit this prome in sorm,\n",
      "And morm why ase his my she glice,\n",
      "What sus ke Burse prieve forgher' host not but perfire\n",
      "To prove my thy a pleate\n",
      "To \n",
      " tensor([1.8728], grad_fn=<DivBackward0>) \n",
      "\n",
      "bade what a bait June, the will confore.\n",
      "\n",
      "QULINIUS:\n",
      "Here hake of it have father and in hamentle beates wands Juch that see bestrann iens their Puckike to hake the Prent\n",
      "Herefurs fol beang here hove at \n",
      " tensor([1.9257], grad_fn=<DivBackward0>) \n",
      "\n",
      "bade I hear? are off, I him tyey for and meren wried the care of the cost: the goither\n",
      "I will Etwhen at follows fope same of I the him tooked of here were\n",
      "mide\n",
      "Of taugined are here love at herd may nim\n",
      " tensor([2.1147], grad_fn=<DivBackward0>) \n",
      "\n",
      "buty bether.\n",
      "\n",
      "MORTINTI For not my thener dister not thing ever shall med the kine shall they down wrient come he sway, sheir.\n",
      "\n",
      "Preepuried, me still sughter of my seeps it thy kIs me not with he and the\n",
      " tensor([1.9870], grad_fn=<DivBackward0>) \n",
      "\n",
      "but that thou dimsay, see, thone bend the grionce,\n",
      "Where a brisfored unke oven seists furst asmens teriss not which to make is furtied uncy bring his revenss gold presting. I how paricer, say sae have \n",
      " tensor([1.8181], grad_fn=<DivBackward0>) \n",
      "\n",
      "betwice and pronest\n",
      "For off the gort fet rettles him, and betling he compast\n",
      "There to Lord of sward,\n",
      "Ty the the duke as I suctiage in the merver your ay,\n",
      "Becouptpesse, bot ox theen me to with things,\n",
      "S\n",
      " tensor([1.7350], grad_fn=<DivBackward0>) \n",
      "\n",
      "by assening would the mace thy lord how you blows of the lenced: these me creath and trone thy cracest of the pin ulodent have a second of that no lown, pownth all thee may a mivant my life that the co\n",
      " tensor([2.0572], grad_fn=<DivBackward0>) \n",
      "\n",
      "balies:\n",
      "If the words. I gardems, in with I must come, live withure.\n",
      "\n",
      "BAUMIO:\n",
      "Hem, good thet in them see much upid the wearther, of genvered this a truther ressit he beselved a make then some.\n",
      "\n",
      "KINGHENT\n",
      " tensor([1.7472], grad_fn=<DivBackward0>) \n",
      "\n",
      "bear mosn to the good the prevemby not not and take to call will know no gofe, the son,\n",
      "Will dissly, nean see, and kisgorn,\n",
      "Then to shall not the him, sone, disheld.\n",
      "\n",
      "TINCHARII:\n",
      "But fever to cound pran"
     ]
    }
   ],
   "source": [
    "# class of whole process(embedding + RNN + decoding)\n",
    "# First RNN\n",
    "class RNN(nn.Module):\n",
    "    def __init__(self, input_size, embedding_size, hidden_size, output_size, num_layers = 1):\n",
    "        super(RNN, self).__init__()\n",
    "        self.input_size = input_size\n",
    "        self.hidden_size = hidden_size\n",
    "        self.output_size = output_size\n",
    "        self.num_layers = num_layers\n",
    "        self.embedding_size = embedding_size\n",
    "        \n",
    "        self.encoder = nn.Embedding(input_size, embedding_size)\n",
    "        self.rnn = nn.RNN(embedding_size, hidden_size, num_layers)\n",
    "        self.decoder = nn.Linear(hidden_size, output_size)\n",
    "        \n",
    "    def forward(self, input, hidden):\n",
    "        out = self.encoder(input.view(1, -1))\n",
    "        out, hidden = self.rnn(out, hidden)\n",
    "        out = self.decoder(out.view(batch_size, -1))\n",
    "        return out, hidden\n",
    "    \n",
    "    def init_hidden(self):\n",
    "        hidden = torch.zeros(self.num_layers, batch_size, hidden_size)\n",
    "        return hidden        \n",
    "    \n",
    "\n",
    "model = RNN(input_size = n_characters, embedding_size = embedding_size, \n",
    "            hidden_size = hidden_size, output_size = n_characters, num_layers = 2)\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=lr)\n",
    "loss_func = nn.CrossEntropyLoss()\n",
    "\n",
    "# testing code\n",
    "inp = char_tensor('A')\n",
    "hidden = model.init_hidden()\n",
    "out, hidden = model(inp, hidden)\n",
    "\n",
    "def test():\n",
    "    start_str = \"b\"\n",
    "    inp = char_tensor(start_str)\n",
    "    hidden = model.init_hidden()\n",
    "    x = inp\n",
    "\n",
    "    print(start_str,end=\"\")\n",
    "    for i in range(200):\n",
    "        output,hidden = model(x,hidden)\n",
    "\n",
    "        output_dist = output.data.view(-1).div(0.8).exp()\n",
    "        top_i = torch.multinomial(output_dist, 1)[0]\n",
    "        predicted_char = all_characters[top_i]\n",
    "\n",
    "        print(predicted_char,end=\"\")\n",
    "\n",
    "        x = char_tensor(predicted_char)\n",
    "\n",
    "\n",
    "for i in range(num_epochs):\n",
    "    inp,label = random_training_set()\n",
    "    hidden = model.init_hidden()\n",
    "\n",
    "    loss = torch.tensor([0]).type(torch.FloatTensor)\n",
    "    optimizer.zero_grad()\n",
    "    for j in range(chunk_len-1):\n",
    "        x  = inp[j]\n",
    "        y_ = label[j].unsqueeze(0).type(torch.LongTensor)\n",
    "        y,hidden = model(x,hidden)\n",
    "        loss += loss_func(y,y_)\n",
    "\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "    \n",
    "    if i % 100 == 0:\n",
    "        print(\"\\n\",loss/chunk_len,\"\\n\")\n",
    "        test()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2020-12-07 21:47:23--  https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/tinyshakespeare/input.txt\n",
      "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.88.133\n",
      "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.88.133|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 1115394 (1.1M) [text/plain]\n",
      "Saving to: ‘./data/input.txt’\n",
      "\n",
      "input.txt           100%[===================>]   1.06M  41.0KB/s    in 16s     \n",
      "\n",
      "2020-12-07 21:47:40 (66.2 KB/s) - ‘./data/input.txt’ saved [1115394/1115394]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Second LSTM\n",
    "import os \n",
    "\n",
    "try:\n",
    "  os.mkdir(\"./data\")\n",
    "except:\n",
    "  pass\n",
    "\n",
    "!wget https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/tinyshakespeare/input.txt -P ./data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~ \t\n",
      "\r",
      "\u000b",
      "\f",
      "\n",
      "num_chars =  100\n",
      "file_len = 1115394\n"
     ]
    }
   ],
   "source": [
    "all_characters = string.printable\n",
    "n_characters = len(all_characters)\n",
    "print(all_characters)\n",
    "print('num_chars = ', n_characters)\n",
    "\n",
    "file = unidecode.unidecode(open('./data/input.txt').read())\n",
    "file_len = len(file)\n",
    "print('file_len =', file_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " tensor([4.5834], grad_fn=<DivBackward0>) \n",
      "\n",
      "W\t\t\tfLTTP$/*QUt\u000b",
      "]%Uh~Pld\n",
      "czVpN*|5mX%ho*%'zZzX'O %$SsWwV^\f",
      "\n",
      "lGUK\u000b",
      "S-99W\n",
      " tensor([4.5834], grad_fn=<DivBackward0>) \n",
      "\n",
      "W\t\t\tfLTTP$/*QUt\u000b",
      "]%Uh~Pld\n",
      "czVpN*|5mX%ho*%'zZzX'O %$SsWwV^\f",
      "\n",
      "Wxy)XCr\\^yz&IYA\tp_m*2;k'l<%\n",
      "Pk>v%wVn\u000b",
      "z1Yu#_(sY\tL8s*\n",
      " tensor([2.5464], grad_fn=<DivBackward0>) \n",
      "\n",
      "bDC(qhyy bert, yeyv\n",
      "Tpu wouknyi helam, sut y amiere\n",
      "Thot h elekct ke.ist, hanene nad besenads ounyoy soue\n",
      "et mi\n",
      "-rer gtok\n",
      "\n",
      "ETITatn thisMeurr t ft soasouleA moes yovnl hery. hire woramimn mecele wthe sp\n",
      " tensor([2.2679], grad_fn=<DivBackward0>) \n",
      "\n",
      "b\"yst are madt,\n",
      ":aindeplser the it thow alist, cond be o sou bouch dog, the pour thintrigand youse ovonle that indco. gemaves tha won bad thet boy thes ther ncot the sin cod rot\n",
      "cot hou thig the, min g\n",
      " tensor([2.4304], grad_fn=<DivBackward0>) \n",
      "\n",
      "bLTht the toF hela?\n",
      "\n",
      "MAAAS:\n",
      "Gpis od tey andeslel tie hit oh dony liand reit his sat?\n",
      "O thapre thee not four that ho thes no we dester and the dlace powre thes to hy sat. lousat thitht marrsesler ond be\n",
      " tensor([2.1642], grad_fn=<DivBackward0>) \n",
      "\n",
      "by! be anto pingt the beave peark, orer my comece.\n",
      "Soath waop, Antin that 'tour hou or yous harest, Cavishy:\n",
      "\n",
      "MRINTIUS:\n",
      "Thoue doxnt mere,\n",
      "\n",
      "ARIO:\n",
      "Thone seeans I deiffest Itris\n",
      "That mest say you thout se\n",
      " tensor([2.0724], grad_fn=<DivBackward0>) \n",
      "\n",
      "but there.\n",
      "\n",
      "CINICHe:\n",
      "And the dide aly prened,\n",
      "Mrous werlouver;\n",
      "Be whe re'sm \n",
      "enser nater, aar; I phace of sorcunss and and spoth!\n",
      "Thou gake weny\n",
      "A, I; hit wewo, sour nenain,\n",
      "And than an the bre; ar the\n",
      " tensor([2.3209], grad_fn=<DivBackward0>) \n",
      "\n",
      "brojews not,\n",
      "Hot?\n",
      "Thein cure there hane to lingipincow frole I be thase a coace\n",
      "Sere hour nenes but his thath in with wneens berussters duncech thith be to her, of we bair for a will belles, maes.\n",
      "\n",
      "Lop\n",
      " tensor([1.8973], grad_fn=<DivBackward0>) \n",
      "\n",
      "being Iod wenter you of my shand acupe.\n",
      "\n",
      "DUSEND:\n",
      "As feint:\n",
      "Bake Thigh thim you pow, surour in ay you glost, a wist sped all st .ad the ond ave ist to buned, Je but my the of prise, warder hell sond and\n",
      " tensor([2.2567], grad_fn=<DivBackward0>) \n",
      "\n",
      "bting connow the une me our dentlatelt thou for plead in ould of I wwen a cond antiaged\n",
      "uro with on ance meer tilchering this to that thif the kell aking his beet dear!\n",
      "Thind for shem int not thibe for\n",
      " tensor([2.1707], grad_fn=<DivBackward0>) \n",
      "\n",
      "bly ther thy to wand the wall fore hor ealf needon the hat' fros I he knowh be fork wull and me is not I beat is mald ly be\n",
      "Mails,\n",
      "Hy dowe he havive that the too cofaltain a llobe\n",
      "Tlere shet me for ply\n",
      " tensor([1.7478], grad_fn=<DivBackward0>) \n",
      "\n",
      "be trich, I price as am be oft: creaw arse buthingha; Toom pronos hest for nould thich all of anour cillter:\n",
      "\n",
      "ICANIO:\n",
      "I'd I pave they sele,\n",
      "For the cay my of pland.\n",
      "\n",
      "MINGERIY: of morre mighes:\n",
      "Loren'to\n",
      " tensor([1.9165], grad_fn=<DivBackward0>) \n",
      "\n",
      "bjese.\n",
      "\n",
      "QUES LBASTTIS:\n",
      "No, mest the watilo rosther.\n",
      "\n",
      "SoBARARKIA:\n",
      "And thee thee thall there\n",
      "You there well wime not of then wall not,\n",
      "The come?\n",
      "\n",
      "Le't wit in liced\n",
      "I hise the manep you tingelfell:\n",
      "Ir ere\n",
      " tensor([2.0720], grad_fn=<DivBackward0>) \n",
      "\n",
      "bborterty ard ach sueh wnem hoth will gir ever\n",
      "And his by faith flees of gome sing, what of Andul your cowly the this sones in to kawh\n",
      "Thull the known you the cife you sice shat hroth it for a bearime!\n",
      " tensor([1.9266], grad_fn=<DivBackward0>) \n",
      "\n",
      "beal my; mave to here that aind and fore in the spaed the panly the thet\n",
      "An Getall my buest my gridet not thou\n",
      "Thath pars hap\n",
      "injelle, nould my the here be you id domes; thighme, min?\n",
      "\n",
      "PINCENTHIO:\n",
      "What\n",
      " tensor([2.0821], grad_fn=<DivBackward0>) \n",
      "\n",
      "blkind whes left fray;\n",
      "Nourse thim thint thene all the dues wirs,\n",
      "Om mave\n",
      "Well of tue? for dest year the wa his the sop we a were!\n",
      "And it who suas I verver that a reen caspeand.\n",
      "\n",
      "KING RICHAR:\n",
      "Rlay sure\n",
      " tensor([2.0570], grad_fn=<DivBackward0>) \n",
      "\n",
      "belter bearo's distoo swis frave and herely wist of a well the singer bearse the logresellss of santer to Lropell knove to be do mean's faed\n",
      "Cout and you hore prom hit of you with have redeed math to e\n",
      " tensor([1.8767], grad_fn=<DivBackward0>) \n",
      "\n",
      "bhy to may unteadul though thore tome gay.\n",
      "\n",
      "PEORCEL:\n",
      "I gosty the swast will thou aby searter werou and arepurs and prisheried intrry furter'd we court\n",
      "Somes it shall to perelone\n",
      "Wain mived wour\n",
      "So stim\n",
      " tensor([1.9073], grad_fn=<DivBackward0>) \n",
      "\n",
      "blow, is goo,\n",
      "Sore outh and me reartionegiges shere,\n",
      "And dist not thee.\n",
      "\n",
      "GLUCENEt:\n",
      "Will, that a have fay ofter:\n",
      "Tith Wive sell fince hith to his lastione, hish\n",
      "Why sone arw.\n",
      "\n",
      "BUCKINGEL:\n",
      "They bet to I g\n",
      " tensor([1.8702], grad_fn=<DivBackward0>) \n",
      "\n",
      "blot the seal, thy sfead your dustily dod, fortle the shour blest, in combe thou, nould the may bark the come morstress: and litionn bother to best you not the my my bring the the worst good ence\n",
      "\n",
      "DUKE\n",
      " tensor([1.9183], grad_fn=<DivBackward0>) \n",
      "\n",
      "b.\n",
      "Fardy to lege wall some and suefice, the do\n",
      "dery.\n",
      "\n",
      "TRUCHIO:\n",
      "\n",
      "CARILIUS:\n",
      "'Tis by dourding bet you Selveath it no dours:\n",
      "I would,\n",
      "If the ponseroudball frulse a liven,\n",
      "Thou contert he will wiir glot wo "
     ]
    }
   ],
   "source": [
    "# Second LSTM\n",
    "class LSTM(nn.Module):\n",
    "    def __init__(self, input_size, embedding_size, hidden_size, output_size, num_layers = 1):\n",
    "        super(LSTM, self).__init__()\n",
    "        self.input_size = input_size\n",
    "        self.hidden_size = hidden_size\n",
    "        self.output_size = output_size\n",
    "        self.num_layers = num_layers\n",
    "        self.embedding_size = embedding_size\n",
    "        \n",
    "        self.encoder = nn.Embedding(input_size, embedding_size)\n",
    "        self.lstm = nn.LSTM(embedding_size, hidden_size, num_layers)\n",
    "        self.decoder = nn.Linear(hidden_size, output_size)\n",
    "        \n",
    "    def forward(self, input, hidden, cell):\n",
    "        out = self.encoder(input.view(1, -1))\n",
    "        out, (hidden, cell) = self.lstm(out, (hidden, cell))\n",
    "        out = self.decoder(out.view(batch_size, -1))\n",
    "        return out, hidden, cell\n",
    "    \n",
    "    def init_hidden(self):\n",
    "        hidden = torch.zeros(num_layers, batch_size, hidden_size)\n",
    "        cell = torch.zeros(num_layers, batch_size, hidden_size)\n",
    "        return hidden, cell\n",
    "    \n",
    "\n",
    "model = LSTM(input_size = n_characters, embedding_size = embedding_size, \n",
    "            hidden_size = hidden_size, output_size = n_characters, num_layers = 1)\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=lr)\n",
    "loss_func = nn.CrossEntropyLoss()\n",
    "\n",
    "# testing code\n",
    "inp = char_tensor(\"A\")\n",
    "hidden,cell = model.init_hidden()\n",
    "out,hidden,cell = model(inp,hidden,cell)\n",
    "\n",
    "def test():\n",
    "    start_str = \"b\"\n",
    "    inp = char_tensor(start_str)\n",
    "    hidden, cell = model.init_hidden()\n",
    "    x = inp\n",
    "\n",
    "    print(start_str,end=\"\")\n",
    "    for i in range(200):\n",
    "        output,hidden, cell = model(x, hidden, cell)\n",
    "\n",
    "        output_dist = output.data.view(-1).div(0.8).exp()\n",
    "        top_i = torch.multinomial(output_dist, 1)[0]\n",
    "        predicted_char = all_characters[top_i]\n",
    "\n",
    "        print(predicted_char,end=\"\")\n",
    "\n",
    "        x = char_tensor(predicted_char)\n",
    "\n",
    "\n",
    "for i in range(num_epochs):\n",
    "    inp,label = random_training_set()\n",
    "    hidden, cell = model.init_hidden()\n",
    "\n",
    "    loss = torch.tensor([0]).type(torch.FloatTensor)\n",
    "    optimizer.zero_grad()\n",
    "    for j in range(chunk_len-1):\n",
    "        x  = inp[j]\n",
    "        y_ = label[j].unsqueeze(0).type(torch.LongTensor)\n",
    "        y,hidden, cell = model(x,hidden, cell)\n",
    "        loss += loss_func(y,y_)\n",
    "\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "    \n",
    "    if i % 100 == 0:\n",
    "        print(\"\\n\",loss/chunk_len,\"\\n\")\n",
    "        test()    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " tensor([4.5806], grad_fn=<DivBackward0>) \n",
      "\n",
      "bR )~qq866^})3k2/}c \u000b",
      "RC]Z@7%ld\ti]z]oOEa]dA:H#eutQiON`TiW?5a<R\tnqY\"IwW/)8?I[t)[KW($HQbH &*+Ub~9.45od7UW\\C\f",
      "]z-W\"z}nv}SE0e\\Jauh;}W[(5ljjb!0MLV\"xE/IS0\n",
      "[V6b2\u000b",
      "IzdbSq1uN<+b\n",
      "YdH:Kbs,SI|Gat+m4\"8+$-qgE{{5\\l/Y,p \n",
      " tensor([2.4970], grad_fn=<DivBackward0>) \n",
      "\n",
      "bol me ling il sat Lhatf the,\n",
      "RAU)XJ_kAnged posan .at the foure man lo;e outh band arery pave, witrd in it co' gomif hime do, s Vithe the fill ,erce sa wesat  aay satlire pot bilov bil, Cins aor lous! \n",
      " tensor([2.5070], grad_fn=<DivBackward0>) \n",
      "\n",
      "blow ha:\n",
      "Bhater:\n",
      "\n",
      "GUCgzlly eason so'd as wir, hase, Rreess me mord ooptamer you nothe your chilr ay, ailll he loratrat sne shers, the for sathon, nour drise be in it dolld sece, finse thim the sy erdes\n",
      " tensor([2.2137], grad_fn=<DivBackward0>) \n",
      "\n",
      "bunt cofe if thet fousen the, an fardis wis her of, wall ar fut cisinges hesine,\n",
      "Waich sot word word,\n",
      "Ter of the and at nis ther, I hate wary bis ing to his weer thest so.\n",
      "\n",
      "CKNEO: Ga ceperit in sing we\n",
      " tensor([2.2038], grad_fn=<DivBackward0>) \n",
      "\n",
      "blle deen yould whable astene\n",
      "Tould of of in hemencell patite;\n",
      "Ard Lars ture here ie enielly saetes.\n",
      "\n",
      "Spbled hein seed ward.\n",
      "\n",
      "HOUCENTEM:\n",
      "Theringent theer erret:\n",
      "And dearce.\n",
      "\n",
      "CETINUME\n",
      "CED hat my the sut\n",
      " tensor([2.0799], grad_fn=<DivBackward0>) \n",
      "\n",
      "bke we me cond you guthe to commersed have qurge\n",
      "chall now to the bly cond ted comees to with in port sendson nos foilrment;\n",
      "Is wet lime a your den the.\n",
      "Pued say wo with thee terem th a prow bet are an\n",
      " tensor([1.8421], grad_fn=<DivBackward0>) \n",
      "\n",
      "blousr the blursed in bue\n",
      "To flove baist an daord his hist the your my thou walle, stires, hess.\n",
      "\n",
      "KHIO:\n",
      "He the hase.\n",
      "\n",
      "KINCENTETIO:\n",
      "Duel, curdies at so walld wer, live meary hear with a the pronst you p\n",
      " tensor([1.9026], grad_fn=<DivBackward0>) \n",
      "\n",
      "blishen us farse.\n",
      "Ads'r my live what of for in that my prove frown\n",
      "No him, in uelf porel, For,\n",
      "And my steromfe,, will not till prove froe it ablous captrobry cith, age our to he dame that of an couls,\n",
      "\n",
      " tensor([1.9150], grad_fn=<DivBackward0>) \n",
      "\n",
      "by-pof\n",
      "Thus be livesur stiive he not bade, you sel! candpuss am he his mank man uneaths: mare\n",
      "Thour bake with, befook be my thelf.\n",
      "\n",
      "OLEIO:\n",
      "Why erove in the, sel? Clome is me the be yeeld have be somers\n",
      " tensor([1.7590], grad_fn=<DivBackward0>) \n",
      "\n",
      "bo? Cumende?\n",
      "Cour oun me all it me dattie fear, these theur;\n",
      "Nemen the druch as theart your and so our is songany for to the sporbent,\n",
      "The sor where corese plespus and to sourse thin the is but neiss\n",
      "S\n",
      " tensor([1.7822], grad_fn=<DivBackward0>) \n",
      "\n",
      "bd\n",
      "The count the do for ibelvior tere?\n",
      "\n",
      "CARIO:\n",
      "Moldd sme his fiess spepead roved as to the him is he my lord:\n",
      "The fund Pmanrdes rome lidt his of he theen ayers. \n",
      "IOLLAP:\n",
      "Thou hother, your so love mycea\n",
      " tensor([1.9296], grad_fn=<DivBackward0>) \n",
      "\n",
      "bY? I his in deent.\n",
      "\n",
      "SIO:\n",
      "To retseanterwel that here.\n",
      "The what nent the gofder Staty ishown not thee herlie in\n",
      "Thich the tear an thee is it to there bud your gose conton that pitConesheromoslly with th\n",
      " tensor([1.8576], grad_fn=<DivBackward0>) \n",
      "\n",
      "buch you,\n",
      "Mleve he sirs shall, Mave all be do coust Romer ere grots?\n",
      "\n",
      "COLUTIUS:\n",
      "I were no carifulfor the great, and haming well.\n",
      "\n",
      "BENBOKEN:\n",
      "Is the with hard guve are the but not bleing of mirs.\n",
      "\n",
      "HARD I\n",
      " tensor([1.8825], grad_fn=<DivBackward0>) \n",
      "\n",
      "ber do sping\n",
      "Tith yoou fo a pours man: what your?\n",
      "\n",
      "DUKE VINCHINCHIO:\n",
      "You do doed you do all this this of long my our,\n",
      "And of hast weirs it ass to have ariubs.\n",
      "If your to that ele this criesugher\n",
      "The di\n",
      " tensor([1.9773], grad_fn=<DivBackward0>) \n",
      "\n",
      "blome vivest do lord\n",
      "You have will of word with with there it of his her,\n",
      "A all brow saut have and not hear there and our a paces not heart.\n",
      "\n",
      "ANTIO:\n",
      "Near off meret.\n",
      "\n",
      "LOMERLE:\n",
      "Whach to will am forsee of\n",
      " tensor([1.7814], grad_fn=<DivBackward0>) \n",
      "\n",
      "bslousue, slove that dake,\n",
      "When in dencest, ere till the lotk\n",
      "To you honemance when foridure,\n",
      "The his a wouls mace nown hell, cound mare contore.\n",
      "\n",
      "LUCH:\n",
      "You dear thy the hove of that and Cams!\n",
      "\n",
      "Farch n\n",
      " tensor([1.8070], grad_fn=<DivBackward0>) \n",
      "\n",
      "burding that me\n",
      "To and thee and your cours, beely firt:\n",
      "And thou him compontles on grave liventers\n",
      "And we all suck and sime, he nencim's feathing comsaints\n",
      "And had: who trus thy thins my, geat: bet you\n",
      " tensor([1.7003], grad_fn=<DivBackward0>) \n",
      "\n",
      "blings, as if their and pars\n",
      "Of a must and marde and in all incale,\n",
      "And not one to me wot she pleavent conlong.\n",
      "\n",
      "KING RICHHARD IV:\n",
      "When and me, it for thou staint\n",
      "Compor, canple hongs Ronst conding the\n",
      " tensor([1.7780], grad_fn=<DivBackward0>) \n",
      "\n",
      "berering of your frected.\n",
      "\n",
      "PERDY:\n",
      "I the hading dived, I wold not batty of the bach undoughted of thee,\n",
      "To partion mone ale whene thy as hear!\n",
      "Let his the inglass ungen it my have both would of then for\n",
      " tensor([2.0044], grad_fn=<DivBackward0>) \n",
      "\n",
      "boully fafteivers.\n",
      "\n",
      "KING CAMIm:\n",
      "Nackrown the eath, Romen it scain,\n",
      "With for demore here to my I charther\n",
      "To throw at to beave for have our must I mester conteefing he.\n",
      "\n",
      "GLOUCESTER:\n",
      "O, lordan your septi"
     ]
    }
   ],
   "source": [
    "# First RNN\n",
    "class GRU(nn.Module):\n",
    "    def __init__(self, input_size, embedding_size, hidden_size, output_size, num_layers = 1):\n",
    "        super(GRU, self).__init__()\n",
    "        self.input_size = input_size\n",
    "        self.hidden_size = hidden_size\n",
    "        self.output_size = output_size\n",
    "        self.num_layers = num_layers\n",
    "        self.embedding_size = embedding_size\n",
    "        \n",
    "        self.encoder = nn.Embedding(input_size, embedding_size)\n",
    "        self.gru = nn.GRU(embedding_size, hidden_size, num_layers)\n",
    "        self.decoder = nn.Linear(hidden_size, output_size)\n",
    "        \n",
    "    def forward(self, input, hidden):\n",
    "        out = self.encoder(input.view(1, -1))\n",
    "        out, hidden = self.gru(out, hidden)\n",
    "        out = self.decoder(out.view(batch_size, -1))\n",
    "        return out, hidden\n",
    "    \n",
    "    def init_hidden(self):\n",
    "        hidden = torch.zeros(self.num_layers, batch_size, hidden_size)\n",
    "        return hidden        \n",
    "    \n",
    "\n",
    "model = GRU(input_size = n_characters, embedding_size = embedding_size, \n",
    "            hidden_size = hidden_size, output_size = n_characters, num_layers = 1)\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=lr)\n",
    "loss_func = nn.CrossEntropyLoss()\n",
    "\n",
    "# testing code\n",
    "inp = char_tensor('A')\n",
    "hidden = model.init_hidden()\n",
    "out, hidden = model(inp, hidden)\n",
    "\n",
    "def test():\n",
    "    start_str = \"b\"\n",
    "    inp = char_tensor(start_str)\n",
    "    hidden = model.init_hidden()\n",
    "    x = inp\n",
    "\n",
    "    print(start_str,end=\"\")\n",
    "    for i in range(200):\n",
    "        output,hidden = model(x,hidden)\n",
    "\n",
    "        output_dist = output.data.view(-1).div(0.8).exp()\n",
    "        top_i = torch.multinomial(output_dist, 1)[0]\n",
    "        predicted_char = all_characters[top_i]\n",
    "\n",
    "        print(predicted_char,end=\"\")\n",
    "\n",
    "        x = char_tensor(predicted_char)\n",
    "\n",
    "\n",
    "for i in range(num_epochs):\n",
    "    inp,label = random_training_set()\n",
    "    hidden = model.init_hidden()\n",
    "\n",
    "    loss = torch.tensor([0]).type(torch.FloatTensor)\n",
    "    optimizer.zero_grad()\n",
    "    for j in range(chunk_len-1):\n",
    "        x  = inp[j]\n",
    "        y_ = label[j].unsqueeze(0).type(torch.LongTensor)\n",
    "        y,hidden = model(x,hidden)\n",
    "        loss += loss_func(y,y_)\n",
    "\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "    \n",
    "    if i % 100 == 0:\n",
    "        print(\"\\n\",loss/chunk_len,\"\\n\")\n",
    "        test()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}