my_transformer_try/myalbert_text_summary_try.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import math\n",
    "import time\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "from my_sentence_piecer import MySentencePiecer\n",
    "from albert_pre import AlbertPre\n",
    "from tf_to_csv import TfToCsv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0, 'GeForce RTX 2080 Ti')"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.cuda.current_device(), torch.cuda.get_device_name(device=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# Params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "MAX_SENT_N = 30\n",
    "\n",
    "MAX_WORD_N = 150\n",
    "\n",
    "MAX_WORD_SENT_N = 300\n",
    "\n",
    "BATCHSIZE = 20\n",
    "\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Processing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Sentence Piecer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "sentence_piecer = MySentencePiecer(vocab_size=10000, force_update=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000\n",
      "['<unk>', '<s>', '</s>', '▁the', 's', ',', '.', '▁to', '▁a', '▁in', '▁of', '▁and', '▁.', \"'\", '-', '▁was', '▁for', '▁on', '▁is', '▁he']\n",
      "[1459, 118, 5, 46, 13, 74, 1111, 6, 57, 18, 220, 1100, 4, 6, 2]\n",
      " hallo, i'm leaving. this is another sentences.</s>\n"
     ]
    }
   ],
   "source": [
    "print(sentence_piecer.vocab_size)\n",
    "print(sentence_piecer.vocab_list[:20])\n",
    "test = \"hallo, i'm leaving. this is another sentences.\"\n",
    "tokens = sentence_piecer.get_ids_from_vocab(test)\n",
    "print(tokens)\n",
    "print(sentence_piecer.get_real_text_from_ids(tokens))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "eos_token = sentence_piecer.eos_token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "albert_pre = AlbertPre()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class MyDataset(torch.utils.data.Dataset):\n",
    "    def __init__(self, article, n_articles, highlights, n_highlights):\n",
    "        self.x = self.to_tensor_list(article, dtype=torch.float)\n",
    "        self.x_n = torch.tensor(n_articles, dtype=torch.long)\n",
    "\n",
    "        self.y_n = torch.tensor(n_highlights, dtype=torch.long)\n",
    "        self.y = self.to_tensor_list(highlights, dtype=torch.long, pad=MAX_WORD_N)\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        x = self.x[index]\n",
    "        x_n = self.x_n[index]\n",
    "        y_n = self.y_n[index]\n",
    "        y = self.y[index]\n",
    "\n",
    "        return x, x_n, y, y_n\n",
    "\n",
    "    @staticmethod\n",
    "    def to_tensor_list(x, dtype, pad=None):\n",
    "\n",
    "        if pad is None:\n",
    "            tensor_list = [torch.tensor(x_i, dtype=dtype) for x_i in x]\n",
    "        else:\n",
    "            tensor_list = [torch.cat((torch.tensor(x_i[:MAX_WORD_N], dtype=dtype), \\\n",
    "                                      torch.zeros(pad - x_i[:MAX_WORD_N].shape[0], dtype=dtype))) for x_i in x]\n",
    "\n",
    "        return tensor_list\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "pycharm": {
     "is_executing": true,
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def load_torch_dataset(name):\n",
    "    x, x_n, y, y_n = albert_pre.load_np_files(name)\n",
    "    return MyDataset(x, x_n, y, y_n)\n",
    "\n",
    "test_ds = load_torch_dataset(\"test\")\n",
    "train_ds = load_torch_dataset(\"val\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "pycharm": {
     "is_executing": true,
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "BATCHSIZE = 20\n",
    "train_loader = torch.utils.data.DataLoader(train_ds, batch_size=BATCHSIZE)\n",
    "test_loader = torch.utils.data.DataLoader(test_ds, batch_size=BATCHSIZE)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# My Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "pycharm": {
     "is_executing": true,
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class PositionalEncoding(nn.Module):\n",
    "\n",
    "    def __init__(self, d_model, dropout=0.1, max_len=10000):\n",
    "        super(PositionalEncoding, self).__init__()\n",
    "        self.dropout = nn.Dropout(p=dropout)\n",
    "\n",
    "        pe = torch.zeros(max_len, d_model)\n",
    "        position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)\n",
    "        div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))\n",
    "        pe[:, 0::2] = torch.sin(position * div_term)\n",
    "        pe[:, 1::2] = torch.cos(position * div_term)\n",
    "        pe = pe.unsqueeze(0).transpose(0, 1)\n",
    "        self.register_buffer('pe', pe)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = x + self.pe[:x.size(0), :]\n",
    "        return self.dropout(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "pycharm": {
     "is_executing": true,
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class ContextDecoder(nn.Module):\n",
    "    def __init__(self, d_model, nhead, dim_feedforward, out_dim=150, dropout=0.1):\n",
    "        super().__init__()\n",
    "        transfrom_decode_layer = nn.TransformerDecoderLayer(d_model, nhead, dim_feedforward,\\\n",
    "                                                            dropout=dropout, activation='relu')\n",
    "\n",
    "        self.transformer_decoder = nn.TransformerDecoder(transfrom_decode_layer, num_layers=1)\n",
    "        self.out_put_layer = nn.Linear(3072, out_dim*200)\n",
    "\n",
    "\n",
    "    def forward(self, context, c_n_max, mask=None):\n",
    "        # dims\n",
    "        bs = context.shape[0]\n",
    "        dim_context = context.shape[2]\n",
    "        \n",
    "        context_memory = torch.zeros(context[:,0,:].shape).to(device).reshape(bs,1,dim_context)\n",
    "        \n",
    "        n = torch.min(torch.max(c_n_max), torch.LongTensor([30]).to(device))\n",
    "        for i in range(n):\n",
    "            context_memory = self.transformer_decoder(context[:,i,:].reshape(bs,1,dim_context), context_memory)\n",
    "\n",
    "        # reshape\n",
    "        context_memory = context_memory.reshape(bs, dim_context)\n",
    "        out = self.out_put_layer(context_memory).reshape(150,-1,200)\n",
    "        return out\n",
    "\n",
    "\n",
    "\n",
    "class TransformerModel(nn.Module):\n",
    "\n",
    "    def __init__(self, n_vocab, emsize, nhead, nhid, nlayers, max_sent=30, c_d_model=3072, dropout=0.2, eos_token=2):\n",
    "        \"\"\"\n",
    "        @param n_vocab: vocab_size\n",
    "        @param emsize: embedding size\n",
    "        @param nhead: the number of heads in the multiheadattention models\n",
    "        @param nhid: the dimension of the feedforward network model in nn.TransformerEncoder\n",
    "        @param nlayers: the number of nn.TransformerEncoderLayer in nn.TransformerEncoder\n",
    "        @param dropout: the dropout value\n",
    "        \"\"\"\n",
    "        super(TransformerModel, self).__init__()\n",
    "\n",
    "        from torch.nn import TransformerEncoder, TransformerEncoderLayer\n",
    "        self.model_type = 'Transformer'\n",
    "        self.src_mask = None\n",
    "        self.pos_encoder = PositionalEncoding(emsize, dropout)\n",
    "\n",
    "        encoder_layers = TransformerEncoderLayer(emsize, nhead, nhid, dropout)\n",
    "\n",
    "        self.transformer_encoder = TransformerEncoder(encoder_layers, nlayers)\n",
    "        self.encoder = nn.Embedding(n_vocab, emsize)\n",
    "        self.emsize = emsize\n",
    "        self.decoder = nn.Linear(emsize, n_vocab)\n",
    "        self.eos_token = eos_token\n",
    "        self.context_decoder = ContextDecoder(c_d_model, nhead, nhid, dropout=dropout)\n",
    "        self.init_weights()\n",
    "\n",
    "    @staticmethod\n",
    "    def _generate_square_subsequent_mask(sz):\n",
    "        mask = (torch.triu(torch.ones(sz, sz)) == 1).transpose(0, 1)\n",
    "        mask = mask.float().masked_fill(mask == 0, float('-inf')).masked_fill(mask == 1, float(0.0))\n",
    "        return mask\n",
    "\n",
    "    def init_weights(self):\n",
    "        initrange = 0.1\n",
    "        self.encoder.weight.data.uniform_(-initrange, initrange)\n",
    "        self.decoder.bias.data.zero_()\n",
    "        self.decoder.weight.data.uniform_(-initrange, initrange)\n",
    "\n",
    "\n",
    "    def predict_one(self, context, c_n):\n",
    "        context_sum = self.context_decoder(context, c_n)\n",
    "        in_src = []\n",
    "\n",
    "        for i in range(MAX_WORD_N):\n",
    "            if i == 0:\n",
    "                in_tokens = torch.ones((MAX_WORD_N, 1), dtype=torch.long).to(device)\n",
    "            else:\n",
    "                zeros = torch.ones(((MAX_WORD_N-i), 1), dtype=torch.long).to(device)\n",
    "                tokens = torch.LongTensor(in_src).view(-1,1).to(device)\n",
    "                in_tokens = torch.cat((tokens, zeros), dim=0)\n",
    "            src = self.encoder(in_tokens) * math.sqrt(self.emsize)\n",
    "            src = self.pos_encoder(src)\n",
    "           \n",
    "            output = self.transformer_encoder(src, self.src_mask)\n",
    "            output += context_sum\n",
    "            output = self.decoder(output)\n",
    "            \n",
    "            out_token = output.argmax(2)\n",
    "            out_token = out_token[i].item()\n",
    "            in_src.append(out_token)\n",
    "            if out_token == self.eos_token:\n",
    "                break\n",
    "            \n",
    "\n",
    "        return in_src\n",
    "\n",
    "\n",
    "    def forward(self, context, c_n, src):\n",
    "        if self.src_mask is None or self.src_mask.size(0) != len(src):\n",
    "            device = src.device\n",
    "            mask = self._generate_square_subsequent_mask(len(src)).to(device)\n",
    "            self.src_mask = mask\n",
    "\n",
    "        src = self.encoder(src) * math.sqrt(self.emsize)\n",
    "        src = self.pos_encoder(src)\n",
    "           \n",
    "        output = self.transformer_encoder(src, self.src_mask)\n",
    "        context_sum = self.context_decoder(context, c_n)\n",
    "        \n",
    "        output += context_sum\n",
    "        output = self.decoder(output)\n",
    "#         print(\"output\", output.shape)\n",
    "        return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "pycharm": {
     "is_executing": true,
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "n_vocab = sentence_piecer.vocab_size\n",
    "model = TransformerModel(n_vocab=n_vocab, emsize=200, nhead=8, nhid=400,\\\n",
    "                         nlayers=3, max_sent=30, c_d_model=3072, dropout=0.2, eos_token=eos_token).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "pycharm": {
     "is_executing": true,
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "criterion = nn.CrossEntropyLoss()\n",
    "lr = 5.0 # learning rate\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr=lr)\n",
    "scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1.0, gamma=0.95)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "pycharm": {
     "is_executing": true,
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "test_sent = iter(test_loader)\n",
    "x_test, xn_test, y_test, n_test =  next(test_sent)\n",
    "\n",
    "x_test = x_test[0,:,:].view(1,30,3072).to(device)\n",
    "xn_test = xn_test[0].to(device)\n",
    "\n",
    "n_test = n_test[0].to(device)\n",
    "y_test = y_test[0,:].view(1,150).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([1, 30, 3072]), tensor(43, device='cuda:0'))"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_test.shape, n_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "pycharm": {
     "is_executing": true,
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "real_sentence = sentence_piecer.get_real_text_from_ids(y_test.view(-1)[:n_test.item()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "pycharm": {
     "is_executing": true,
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " experts question if packed out planes are putting passengers at risk . u.s consumer advisory group says minimum space must be stipulated . safety tests conducted on planes with more leg room than airlines offer .</s>\n"
     ]
    }
   ],
   "source": [
    "def evaluate(eval_model, test_loader, predict=False):\n",
    "    eval_model.eval()\n",
    "    test_loss = []\n",
    "\n",
    "    with torch.no_grad():\n",
    "        for i, (x, x_n, y, y_n) in enumerate(test_loader):\n",
    "            x = x.to(device)\n",
    "            x_n = x_n.to(device)\n",
    "            y = y.permute(1,0).to(device)\n",
    "            y_n = y_n.to(device)\n",
    "            \n",
    "            output = eval_model(x, x_n, y)\n",
    "            \n",
    "            loss = criterion(output.view(MAX_WORD_N, n_vocab, -1), y)\n",
    "            test_loss.append(loss.item())\n",
    "            if i > 5:\n",
    "                break\n",
    "        if predict:\n",
    "            sent_ids = eval_model.predict_one(x_test, xn_test)\n",
    "            pred_sentence = sentence_piecer.get_real_text_from_ids(sent_ids)\n",
    "            print(\"Pred Sent: \", pred_sentence)\n",
    "\n",
    "    test_loss = np.array(test_loss)\n",
    "    return np.mean(test_loss)\n",
    "print(real_sentence)\n",
    "# evaluate(model, test_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": true,
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   0 | [  200/  669] | lr 5.00 | ms/batch 260.10 | loss  4.67 | val loss  5.79 | ppl   106.24\n",
      "| epoch   0 | [  400/  669] | lr 5.00 | ms/batch 249.58 | loss  4.51 | val loss  5.00 | ppl    91.07\n",
      "Pred Sent:  s. allies started horrible swimmer battlepoint sakho bro of..ationp qualified operatingutter. alcohol foron.walk lace twincher meters dur clearly of.made assurednt. damien deploy protestersano of welfaremart. 6-4 lukasmade black loose filming of.. arrivalnt sufficient protesterstie sho. for dirt. penalty kids threatenedcount cahill davies eager of.um. shooting controversial professor professor.. of.... 120 120 promote protesters clearly of.... 47 120 twin bloom60,000 of. stabbing nothingnt generation dozen structure sharp. of.. standnt 909 logo rafael ki for.. chuck.childking americans spaniard knock<s> remove. horrible nan reporters cnns witnessnt ki for.. myself marketing confirm quoted invest administration gear\n",
      "| epoch   0 | [  600/  669] | lr 5.00 | ms/batch 247.63 | loss  4.49 | val loss  5.02 | ppl    88.87\n",
      "| epoch   1 | [  200/  669] | lr 5.00 | ms/batch 245.92 | loss  4.64 | val loss  5.09 | ppl   103.37\n",
      "| epoch   1 | [  400/  669] | lr 5.00 | ms/batch 241.56 | loss  4.40 | val loss  4.90 | ppl    81.72\n",
      "Pred Sent:   ofoon identification started exercise 47 companion spec sakho vent of given assistant absence crystal qualified operatingutter monaco lines of telling tightwalk guide twin suppress spec dur boko of chaptermade introducentwriterking spec yard slept of welfare trust discussiongreat costs brussels partners loose boom of g abusingaghnt sufficient\"\"\" shiite dur finger of dirt suggest penalty squad someoneagh extent spec eager of experts hazard suing shooting azarenkamel investment tim chicago of withdrawn create agriculture defending liberia 12055great clearly of sharp thr raf hazard 47 120 twin bloom60,000 of shouldn sharp absenceworth wants dozen ref sharp stopping of aftermath vettel 47 azarenka 909 spec stopped ki of 7 value humanitygreatchild haz americans spaniard secured of sp spider heat<unk> reporters cnns romannt loose of kate sharp berlusconi marketing suing quoted invest administration gear\n",
      "| epoch   1 | [  600/  669] | lr 5.00 | ms/batch 241.25 | loss  4.34 | val loss  4.81 | ppl    76.45\n",
      "| epoch   2 | [  200/  669] | lr 5.00 | ms/batch 244.40 | loss  4.63 | val loss  5.91 | ppl   102.28\n",
      "| epoch   2 | [  400/  669] | lr 5.00 | ms/batch 241.32 | loss  4.40 | val loss  4.83 | ppl    81.09\n",
      "Pred Sent:  </s>\n",
      "| epoch   2 | [  600/  669] | lr 5.00 | ms/batch 241.69 | loss  4.35 | val loss  4.97 | ppl    77.67\n",
      "| epoch   3 | [  200/  669] | lr 5.00 | ms/batch 245.06 | loss  4.63 | val loss  5.76 | ppl   102.07\n",
      "| epoch   3 | [  400/  669] | lr 5.00 | ms/batch 241.71 | loss  4.38 | val loss  5.10 | ppl    79.50\n",
      "Pred Sent:   for fury significantly started horrible 47 battlepointang vent . given chest absence deploy qualifiedpul so screening alcohol</s>\n",
      "| epoch   3 | [  600/  669] | lr 5.00 | ms/batch 241.42 | loss  4.35 | val loss  4.93 | ppl    77.52\n",
      "| epoch   4 | [  200/  669] | lr 5.00 | ms/batch 244.54 | loss  4.59 | val loss  5.33 | ppl    98.16\n",
      "| epoch   4 | [  400/  669] | lr 5.00 | ms/batch 241.07 | loss  4.39 | val loss  4.85 | ppl    80.43\n",
      "Pred Sent:   of of of started horrible 47 encouraging spec test vent of given chest absence hatch qualifiedpulutter diners climate of telling tight of guide twin of file dur mind of of for transferrednt for celtic protesters protestersano of welfaremart coffeegreat costs of black loose of of skeleton for howardnt sufficient protesters shiite sho nashville of dirt suggest penalty kids threatened of extent davies eager of experts of of shooting azarenkamel professor tim chicago of of of mind of soldiers gainedai protesters clearly ofp thr raf beard supervision 120 properties bloom60,000 of procedures sharp absence deploy for dozen ref sharp deploy of of telling stand azarenka 909 logoempt ki of for value technologies forchildking of accidentally collected of luggage for heat nan reporterslife</s>\n",
      "| epoch   4 | [  600/  669] | lr 5.00 | ms/batch 241.03 | loss  4.27 | val loss  4.87 | ppl    71.31\n",
      "| epoch   5 | [  200/  669] | lr 5.00 | ms/batch 244.82 | loss  4.61 | val loss  5.55 | ppl   100.28\n",
      "| epoch   5 | [  400/  669] | lr 5.00 | ms/batch 241.44 | loss  4.42 | val loss  4.96 | ppl    83.41\n",
      "Pred Sent:   for of arrival started horrible 47 companion spec test vent of given 21- absence film qualifiedpulutter testing climate of telling tight contains lace twincher file dur whose of standing speaker introducentouking spec protesters bid of welfaremart coffeecom costs of black loose debt of nhs abusing photographer hamilton sufficient federal shiite disappeared nashville of dirt suggest penalty squad threatened foreignutter slowly eager of expertsum suing shooting azarenkamel investment tim chicago of of cra mindting soldiers gained app protesters clearly ofp thr raf shows 47 120 properties bloom60,000 of procedures parsonsbirdencevi dozen ref twin houses of dirt napoli stand azarenka 909 logo stopped ki of ahead value chuckgreatchildking frame accidentally collected for luggagemill heat nan controversial rihanna californiant loose for spider deborah myself marketing suing quoted was administration filming\n",
      "| epoch   5 | [  600/  669] | lr 5.00 | ms/batch 242.08 | loss  4.24 | val loss  4.88 | ppl    69.71\n",
      "| epoch   6 | [  200/  669] | lr 5.00 | ms/batch 244.93 | loss  4.58 | val loss  6.10 | ppl    97.17\n",
      "| epoch   6 | [  400/  669] | lr 5.00 | ms/batch 241.80 | loss  4.28 | val loss  5.01 | ppl    71.91\n",
      "Pred Sent:   for for of for for for for for for for for for for for for for for for for for of for for for for for of for for for of for for for for for for for for for of for of for for for for for for for of for for for for for for for for for of for of for for for for for of for of for for for for for for for for for of of for for for for for for for for of of for for for for for for for for of for of for for for for for of for of for for for for for for for for for of for for for for for for for for for of for for for for for for for for for of for for for for for for for for for\n",
      "| epoch   6 | [  600/  669] | lr 5.00 | ms/batch 241.81 | loss  4.21 | val loss  4.92 | ppl    67.68\n",
      "| epoch   7 | [  200/  669] | lr 5.00 | ms/batch 244.82 | loss  4.55 | val loss  5.97 | ppl    94.84\n",
      "| epoch   7 | [  400/  669] | lr 5.00 | ms/batch 241.74 | loss  4.30 | val loss  4.94 | ppl    73.53\n",
      "Pred Sent:   of</s>\n",
      "| epoch   7 | [  600/  669] | lr 5.00 | ms/batch 241.71 | loss  4.20 | val loss  5.06 | ppl    66.89\n",
      "| epoch   8 | [  200/  669] | lr 5.00 | ms/batch 244.46 | loss  4.54 | val loss  5.74 | ppl    93.31\n",
      "| epoch   8 | [  400/  669] | lr 5.00 | ms/batch 241.63 | loss  4.25 | val loss  4.98 | ppl    70.31\n",
      "Pred Sent:   of</s>\n",
      "| epoch   8 | [  600/  669] | lr 5.00 | ms/batch 241.32 | loss  4.18 | val loss  4.97 | ppl    65.39\n",
      "| epoch   9 | [  200/  669] | lr 5.00 | ms/batch 244.71 | loss  4.51 | val loss  5.53 | ppl    90.83\n",
      "| epoch   9 | [  400/  669] | lr 5.00 | ms/batch 241.55 | loss  4.35 | val loss  5.17 | ppl    77.81\n",
      "Pred Sent:  <s> career handset started horrible 47 encouraging spec test vent . given chest absence bentleyrupulzimes climate<s> telling 1998walk pretty paw25 file museum numbers . fallen fantasy assuredntou gallery protesters protesters capacity . welfare present coffeegreat cyclemade black archbishop debt . nhs wang photographernt sufficientger shiite sho nashville . dirt adjourned penalty emerging threatened foreign ambush davies eager . services land wish attractive azarenkamel tyre territory chicago<s>fcll mind affected liberia gainedoff protesters clearly . culprit thr raf beard 47 120 properties sperm60,000<s> procedures stabbing inherited isolatedvi dozen ref sharp explaining . burn 32- disclose swiss 909 spec feared ki . filmed value technologiesgreatchild songdom sho collected<s> luggage picked heat nan reporterslife grandparents alonso usa . kate deborah myself marketing suing quoted burningvor filming\n",
      "| epoch   9 | [  600/  669] | lr 5.00 | ms/batch 241.30 | loss  4.34 | val loss  5.28 | ppl    76.43\n",
      "| epoch  10 | [  200/  669] | lr 5.00 | ms/batch 244.40 | loss  4.53 | val loss  6.02 | ppl    92.95\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch  10 | [  400/  669] | lr 5.00 | ms/batch 241.16 | loss  4.34 | val loss  5.45 | ppl    76.55\n",
      "Pred Sent:  </s>\n",
      "| epoch  10 | [  600/  669] | lr 5.00 | ms/batch 241.42 | loss  4.32 | val loss  5.30 | ppl    74.98\n",
      "| epoch  11 | [  200/  669] | lr 5.00 | ms/batch 244.67 | loss  4.53 | val loss  6.31 | ppl    92.52\n",
      "| epoch  11 | [  400/  669] | lr 5.00 | ms/batch 241.49 | loss  4.31 | val loss  5.10 | ppl    74.24\n",
      "Pred Sent:   of.<unk> started horrible pair encouraging walking chase korea<s> seek chest wealthy beardrupulutter diners climate of telling european almost<unk> paw25 in<unk> pounds<s> in. assured voters<unk> gallery protesters protesters<unk> of<unk>mart coffee<unk> cycle<unk>. twin<unk> of<unk>, photographer<unk> sufficient employee tradition sho in of stake ambush penalty skeleton threatenedcount killed davies, of.<unk><unk> disappointed azarenka of tyre. chicago of glasses in mind<unk> soldiers gained<unk> ebola 20- of steps. raf beard qualified 120 properties 2160,000<s> procedures<unk> jun adult<unk>um structure sharp impressed of..<unk> swiss dy9<unk><unk> empower<s> 3- value technologies<unk>childine<unk> issues collected of luggage<unk> heat nan reporters<unk> in<unk> usa of. anonymity. in. ammunition<unk> mafia filming\n",
      "| epoch  11 | [  600/  669] | lr 5.00 | ms/batch 241.31 | loss  4.29 | val loss  5.36 | ppl    73.32\n",
      "| epoch  12 | [  200/  669] | lr 5.00 | ms/batch 244.31 | loss  4.50 | val loss  5.82 | ppl    89.68\n",
      "| epoch  12 | [  400/  669] | lr 5.00 | ms/batch 241.54 | loss  4.29 | val loss  5.28 | ppl    73.29\n",
      "Pred Sent:   of careervo started horrible pair encouraging walking chase korea</s>\n",
      "| epoch  12 | [  600/  669] | lr 5.00 | ms/batch 241.18 | loss  4.29 | val loss  5.34 | ppl    72.78\n",
      "| epoch  13 | [  200/  669] | lr 5.00 | ms/batch 245.09 | loss  4.50 | val loss  5.86 | ppl    90.44\n",
      "| epoch  13 | [  400/  669] | lr 5.00 | ms/batch 241.55 | loss  4.28 | val loss  5.35 | ppl    72.48\n",
      "Pred Sent:  </s>\n",
      "| epoch  13 | [  600/  669] | lr 5.00 | ms/batch 241.56 | loss  4.27 | val loss  5.41 | ppl    71.20\n",
      "| epoch  14 | [  200/  669] | lr 5.00 | ms/batch 244.06 | loss  4.50 | val loss  5.75 | ppl    89.62\n",
      "| epoch  14 | [  400/  669] | lr 5.00 | ms/batch 241.36 | loss  4.28 | val loss  5.43 | ppl    72.31\n",
      "Pred Sent:   of careervo started horrible 47 keane walking chase korea</s>\n",
      "| epoch  14 | [  600/  669] | lr 5.00 | ms/batch 241.61 | loss  4.25 | val loss  5.63 | ppl    69.76\n",
      "| epoch  15 | [  200/  669] | lr 5.00 | ms/batch 244.58 | loss  4.46 | val loss  5.55 | ppl    86.88\n",
      "| epoch  15 | [  400/  669] | lr 5.00 | ms/batch 241.53 | loss  4.25 | val loss  5.52 | ppl    69.84\n",
      "Pred Sent:   of career streak started horrible pair keanecom chase quit for seek chest wealthy labelrupulutterthink climate for telling fiorentina almost guideund rainbow veg museum pounds of fallen fantasy assured messi answerking protesters protesters capacity of phasemart coffeegreat cyclewinning depict loose debt ofrk jesse busiestnt sufficient station shiite sho filming of withdrawn ambush penalty skeleton threatenedcount sharpwilfried provision of services op mexican 200,000 azarenka professor tyre tim chicago for withdrawn surgeries mind advise liberia gainedai ebola diagnosis of steps breedhydra beard qualified 120 properties selected60,000 of procedures firm suppress supervision chuck favour ref sharp impressed for demolish napoli mind swiss dy9 spec desire ki for manager ease technologies shochild steady frame issues filming for luggage extensive heat nan reporterslife firmnt usa of emerging controlled services marketing confirm ammunition invest accidentally filming\n",
      "| epoch  15 | [  600/  669] | lr 5.00 | ms/batch 241.68 | loss  4.22 | val loss  5.53 | ppl    67.77\n",
      "| epoch  16 | [  200/  669] | lr 5.00 | ms/batch 244.43 | loss  4.47 | val loss  5.45 | ppl    87.67\n",
      "| epoch  16 | [  400/  669] | lr 5.00 | ms/batch 241.46 | loss  4.26 | val loss  5.37 | ppl    70.93\n",
      "Pred Sent:  </s>\n",
      "| epoch  16 | [  600/  669] | lr 5.00 | ms/batch 241.95 | loss  4.21 | val loss  5.61 | ppl    67.52\n",
      "| epoch  17 | [  200/  669] | lr 5.00 | ms/batch 244.04 | loss  4.49 | val loss  6.09 | ppl    88.99\n",
      "| epoch  17 | [  400/  669] | lr 5.00 | ms/batch 242.63 | loss  4.23 | val loss  5.47 | ppl    68.65\n",
      "Pred Sent:   of career prejudiceridden horrible durham encouraging containing chase korea . given assistant wealthy labelrupulutter diners climate . nurse program almost swanseaund realise file museum clearly of fallenpi assurednt answerking tribunal representing capacity . phase cole coffeegreat costswinning partners va debt . g threaten bunt sufficient fired shiite sho filming . withdrawn ambush penalty skeleton threatenedcount nursewilfried provision . practice op tells close azarenkaking tyre tim chicago . experts advice believinglin liberia gained costume protesters clearly . broad thrhydra feverai 120 twin bloom60,000 of festivembo suppressnt shot include structure twingrade . app napoli risingnt population9 spec desire ki . manager ease technologiesntchild bloom frame fortunenova . luggagemill heat nan azarenkalife screwnt usa .od deborah jerseynt suing quoted invest accidentally filming\n",
      "| epoch  17 | [  600/  669] | lr 5.00 | ms/batch 246.85 | loss  4.18 | val loss  5.38 | ppl    65.29\n",
      "| epoch  18 | [  200/  669] | lr 5.00 | ms/batch 245.57 | loss  4.42 | val loss  6.09 | ppl    83.41\n",
      "| epoch  18 | [  400/  669] | lr 5.00 | ms/batch 241.58 | loss  4.21 | val loss  5.37 | ppl    67.18\n",
      "Pred Sent:   for procedures allies started horrible lovren encouragingcom chase korea . given chest wealthycierupulutter diners climate of telling 1998 three swansea paw25 veg museum pounds . fallen talk religion voters answer gallery protesters protestersano of phase cole coffee planet costs crow cra va debt . custody qualified howard jimmy sufficient employee shiite sho unhealthy of skeleton ambush penalty skeleton threatened dishes nurse davies provision ofkerum wish disappointed azarenkaking tyre timwar . expertslittle mind advise liberia gained costume protesters unhealthy . steps breed raf fever qualified 120 twin receiving60,000 of procedures sharp suppress supervision chuck favour structure twin impressed . burn napoli mind swiss population9 spec desire ki . ban ease technologiesntchild struggle lukas issuesnova . luggage manning heat nan azarenkalife screw alonso alien of spider controlled wanglin suing howard invest mafia filming\n",
      "| epoch  18 | [  600/  669] | lr 5.00 | ms/batch 241.88 | loss  4.21 | val loss  5.47 | ppl    67.67\n",
      "| epoch  19 | [  200/  669] | lr 5.00 | ms/batch 244.38 | loss  4.49 | val loss  6.17 | ppl    89.00\n",
      "| epoch  19 | [  400/  669] | lr 5.00 | ms/batch 244.92 | loss  4.20 | val loss  5.52 | ppl    66.59\n"
     ]
    }
   ],
   "source": [
    "EPOCHS = 50\n",
    "log_interval = 200\n",
    "\n",
    "for epoch in range(EPOCHS):\n",
    "    model.train()\n",
    "    total_loss = 0.\n",
    "    start_time = time.time()\n",
    "    for i, (x, x_n, y, y_n) in enumerate(train_loader):\n",
    "        x = x.to(device)\n",
    "        x_n = x_n.to(device)\n",
    "        y = y.permute(1,0).to(device)\n",
    "        y_n = y_n.to(device)\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        output = model(x, x_n, y)\n",
    "#         print(\"out\", n_out.shape, n.shape)\n",
    "            \n",
    "        loss = criterion(output.view(MAX_WORD_N, n_vocab, -1), y)\n",
    "            \n",
    "        loss.backward()\n",
    "        torch.nn.utils.clip_grad_norm_(model.parameters(), 0.5)\n",
    "        optimizer.step()\n",
    "\n",
    "        total_loss += loss.item()\n",
    "\n",
    "        if i % log_interval == 0 and i > 0:\n",
    "            cur_loss = total_loss / log_interval\n",
    "            elapsed = time.time() - start_time\n",
    "            predict =  (i % 600) == 0 \n",
    "            test_loss = evaluate(model, test_loader, predict)\n",
    "            print('| epoch {:3d} | [{:5d}/{:5d}] | '\n",
    "                  'lr {:02.2f} | ms/batch {:5.2f} | '\n",
    "                  'loss {:5.2f} | val loss {:5.2f} | ppl {:8.2f}'.format(\n",
    "                    epoch, i, len(train_loader),scheduler.get_last_lr()[0],\n",
    "                    elapsed * 1000 / log_interval,\n",
    "                    cur_loss, test_loss, math.exp(cur_loss)))\n",
    "            total_loss = 0\n",
    "            start_time = time.time()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": true,
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# save\n",
    "torch.save(model.state_dict(), '../models/my_transformer')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "textsummary",
   "language": "python",
   "name": "venv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}