task_3-NER.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "02ad34c2-be1e-4413-8f98-c9055b38afc0",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "import string\n",
    "import gensim\n",
    "import time \n",
    "import json\n",
    "\n",
    "import nltk\n",
    "from nltk.stem.wordnet import WordNetLemmatizer\n",
    "from nltk.stem import PorterStemmer\n",
    "from nltk import word_tokenize\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from gensim.corpora import Dictionary\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "import seaborn as sns\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torch.nn import functional as F\n",
    "import math\n",
    "from argparse import Namespace\n",
    "\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from collections import Counter\n",
    "from tqdm import tqdm_notebook\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.metrics import ConfusionMatrixDisplay\n",
    "import codecs\n",
    "from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence\n",
    "\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import ConfusionMatrixDisplay\n",
    "from tabulate import tabulate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7d28c9fa-1b25-4740-81dc-bbf9187e000f",
   "metadata": {},
   "outputs": [],
   "source": [
    "args = Namespace(\n",
    "    # Data and Path hyper parameters\n",
    "    data_file='./data/ner.csv',\n",
    "    lable_file='labels.txt',\n",
    "    vectorizer_file=\"nervectorizer.json\",\n",
    "    model_state_file=\"NERteachingForce.h5\",\n",
    "    lr = 0.0001,\n",
    "    seed=666,\n",
    "    dropout_p=0.1,\n",
    "    batch_train=1024,\n",
    "    batch_valid=512,\n",
    "    batch_test=1024,\n",
    "    num_epochs=200,\n",
    "    cuda=True,\n",
    "    reload_from_files=False,\n",
    "    expand_filepaths_to_save_dir=True,\n",
    "    source_embedding_size=128, \n",
    "    target_embedding_size=128, \n",
    "    encoding_size=128,\n",
    ")\n",
    "\n",
    "np.random.seed(666)\n",
    "torch.manual_seed(666)\n",
    "if args.cuda:\n",
    "    torch.cuda.manual_seed_all(666)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b73b8a2d-a103-4073-b26c-a45c80567dc2",
   "metadata": {},
   "source": [
    "# Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "79d92ce5-d05a-4863-82f3-acdbb2f38315",
   "metadata": {},
   "outputs": [],
   "source": [
    "def update_train_state(args, model, train_state):\n",
    "    # Save one model at least\n",
    "    if train_state['epoch_index'] == 0:\n",
    "        torch.save(model.state_dict(), train_state['model_filename'])\n",
    "        train_state['stop_early'] = False\n",
    "\n",
    "    # Save model if performance improved\n",
    "    elif train_state['epoch_index'] >= 1:\n",
    "        loss_tm1, loss_t = train_state['val_loss'][-2:]\n",
    "\n",
    "        # If loss worsened\n",
    "        if loss_t >= train_state['early_stopping_best_val']:\n",
    "            # Update step\n",
    "            train_state['early_stopping_step'] += 1\n",
    "        # Loss decreased\n",
    "        else:\n",
    "            # Save the best model\n",
    "            if loss_t < train_state['early_stopping_best_val']:\n",
    "                torch.save(model.state_dict(), train_state['model_filename'])\n",
    "\n",
    "            # Reset early stopping step\n",
    "            train_state['early_stopping_step'] = 0\n",
    "\n",
    "        # Stop early ?\n",
    "        train_state['stop_early'] = \\\n",
    "            train_state['early_stopping_step'] >= args.early_stopping_criteria\n",
    "\n",
    "    return train_state\n",
    "\n",
    "def generate_batches(dataset, batch_size, shuffle=True, drop_last=True, device=\"cpu\"):\n",
    "    dataloader = DataLoader(dataset=dataset, batch_size=batch_size, shuffle=shuffle, drop_last=drop_last)\n",
    "    for data_dict in dataloader:\n",
    "        lengths = data_dict['x_length'].numpy()\n",
    "        sorted_length_indices = lengths.argsort()[::-1].tolist()\n",
    "        out_data_dict = {}\n",
    "        for name, tensor in data_dict.items():\n",
    "            out_data_dict[name] = data_dict[name][sorted_length_indices].to(device)\n",
    "        yield out_data_dict\n",
    "\n",
    "def sequence_loss(y_pred, y_true, mask_index, weight=None):\n",
    "    y_pred, y_true = normalize_sizes(y_pred, y_true)\n",
    "    indexs = []\n",
    "    for i in range(len(y_true)):\n",
    "        if y_true[i].item() != 179 or i%4 ==0:\n",
    "            indexs.append(i)\n",
    "    return F.cross_entropy(y_pred[indexs], y_true[indexs], ignore_index=mask_index, weight=weight)\n",
    "\n",
    "def compute_accuracy(y_pred, y_true, mask_index):\n",
    "    y_pred, y_true = normalize_sizes(y_pred, y_true)\n",
    "\n",
    "    _, y_pred_indices = y_pred.max(dim=1)\n",
    "    \n",
    "    correct_indices = torch.eq(y_pred_indices, y_true).float()\n",
    "    valid_indices = torch.ne(y_true, mask_index).float()\n",
    "    \n",
    "    n_correct = (correct_indices * valid_indices).sum().item()\n",
    "    n_valid = valid_indices.sum().item()\n",
    "\n",
    "    return n_correct / n_valid * 100\n",
    "\n",
    "def normalize_sizes(y_pred, y_true):\n",
    "    if len(y_pred.size()) == 3:\n",
    "        y_pred = y_pred.contiguous().view(-1, y_pred.size(2))\n",
    "    if len(y_true.size()) == 2:\n",
    "        y_true = y_true.contiguous().view(-1)\n",
    "    return y_pred, y_true\n",
    "\n",
    "def set_seed_everywhere(seed, cuda):\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    if cuda:\n",
    "        torch.cuda.manual_seed_all(seed)\n",
    "\n",
    "def make_train_state(args):\n",
    "    return {'stop_early': False, 'early_stopping_step': 0, 'early_stopping_best_val': 1e8, 'learning_rate': args.lr, 'epoch_index': 0, 'train_loss': [], 'train_acc': [], 'val_loss': [],\n",
    "            'val_acc': [], 'test_loss': -1, 'test_acc': -1, 'model_filename': args.model_state_file}\n",
    "\n",
    "def pretty_print(epoch, t_loss, t_acc, v_loss, v_acc):\n",
    "    det = str(epoch)\n",
    "    if epoch < 100:\n",
    "        det = '0' + det\n",
    "    \n",
    "    if epoch < 10:\n",
    "        det = '0' + det\n",
    "    print(\"Epochs:\", det, '| Train loss', str(t_loss)[: 6], 'Train acc', str(t_acc)[: 6], 'Val loss', str(v_loss)[: 6], 'Val acc', str(v_acc)[: 6])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c7811b37-8f5e-4eb6-a01a-462bbdebd0a8",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c4ecf86b-1bd8-41c4-aaaf-e69f822df5b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Vocabulary(object):\n",
    "    def __init__(self, token_to_idx=None):\n",
    "        if token_to_idx is None:\n",
    "            token_to_idx = {}\n",
    "        self._token_to_idx = token_to_idx\n",
    "        self._idx_to_token = {idx: token for token, idx in self._token_to_idx.items()}\n",
    "\n",
    "    def to_serializable(self):\n",
    "        return {'token_to_idx': self._token_to_idx}\n",
    "\n",
    "    @classmethod\n",
    "    def from_serializable(cls, contents):\n",
    "        return cls(**contents)\n",
    "\n",
    "    def add_token(self, token):\n",
    "        if token in self._token_to_idx:\n",
    "            index = self._token_to_idx[token]\n",
    "        else:\n",
    "            index = len(self._token_to_idx)\n",
    "            self._token_to_idx[token] = index\n",
    "            self._idx_to_token[index] = token\n",
    "        return index\n",
    "\n",
    "    def add_many(self, tokens):\n",
    "        return [self.add_token(token) for token in tokens]\n",
    "\n",
    "    def lookup_token(self, token):\n",
    "        return self._token_to_idx[token]\n",
    "\n",
    "    def lookup_index(self, index):\n",
    "        if index not in self._idx_to_token:\n",
    "            raise KeyError(\"the index (%d) is not in the Vocabulary\" % index)\n",
    "        return self._idx_to_token[index]\n",
    "\n",
    "    def __str__(self):\n",
    "        return \"<Vocabulary(size=%d)>\" % len(self)\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self._token_to_idx)\n",
    "\n",
    "class SequenceVocabulary(Vocabulary):\n",
    "    def __init__(self, token_to_idx=None, unk_token=\"<UNK>\", mask_token=\"<MASK>\", begin_seq_token=\"<BEGIN>\", end_seq_token=\"<END>\"):\n",
    "\n",
    "        super(SequenceVocabulary, self).__init__(token_to_idx)\n",
    "        self._mask_token = mask_token\n",
    "        self._unk_token = unk_token\n",
    "        self._begin_seq_token = begin_seq_token\n",
    "        self._end_seq_token = end_seq_token\n",
    "\n",
    "        self.mask_index = self.add_token(self._mask_token)\n",
    "        self.unk_index = self.add_token(self._unk_token)\n",
    "        self.begin_seq_index = self.add_token(self._begin_seq_token)\n",
    "        self.end_seq_index = self.add_token(self._end_seq_token)\n",
    "\n",
    "    def to_serializable(self):\n",
    "        contents = super(SequenceVocabulary, self).to_serializable()\n",
    "        contents.update({'unk_token': self._unk_token, 'mask_token': self._mask_token, 'begin_seq_token': self._begin_seq_token, 'end_seq_token': self._end_seq_token})\n",
    "        return contents\n",
    "\n",
    "    def lookup_token(self, token):\n",
    "        if self.unk_index >= 0:\n",
    "            return self._token_to_idx.get(token, self.unk_index)\n",
    "        else:\n",
    "            return self._token_to_idx[token]\n",
    "\n",
    "class NERVectorizer(object):\n",
    "    def __init__(self, NARRATIVE_vocab, target_vocab):\n",
    "        self.NARRATIVE_vocab = NARRATIVE_vocab\n",
    "        self.target_vocab = target_vocab\n",
    "\n",
    "    def vectorize(self, NARRATIVE, target, vector_length=-1, target_vector_length=-1):\n",
    "        indices = [self.NARRATIVE_vocab.begin_seq_index]\n",
    "        indices.extend(self.NARRATIVE_vocab.lookup_token(token) for token in NARRATIVE.split(\"|\"))\n",
    "        indices.append(self.NARRATIVE_vocab.end_seq_index)\n",
    "\n",
    "        if vector_length < 0:\n",
    "            vector_length = len(indices)\n",
    "\n",
    "        x_source = np.zeros(vector_length, dtype=np.int64)\n",
    "        x_source[:len(indices)] = indices\n",
    "        x_source[len(indices):] = self.NARRATIVE_vocab.mask_index\n",
    "        \n",
    "        out_indices = [self.target_vocab.begin_seq_index]\n",
    "        out_indices.extend(self.target_vocab.lookup_token(token) for token in target.split(\"|\"))\n",
    "        out_indices.append(self.target_vocab.end_seq_index)\n",
    "        \n",
    "        if target_vector_length < 0:\n",
    "            target_vector_length = len(out_indices)\n",
    "        \n",
    "        x_target = np.zeros(target_vector_length, dtype=np.int64)\n",
    "        x_target[:len(out_indices[:-1])] = out_indices[:-1]\n",
    "        x_target[len(out_indices[:-1]):] = self.target_vocab.mask_index\n",
    "        \n",
    "        y_target = np.zeros(target_vector_length-1, dtype=np.int64)\n",
    "        y_target[:len(out_indices[1:])] = out_indices[1:]\n",
    "        y_target[len(out_indices[1:]):] = self.target_vocab.mask_index\n",
    "        return x_source, x_target, y_target, len(indices), len(out_indices)\n",
    "\n",
    "    @classmethod\n",
    "    def from_dataframe(cls, df, targets):\n",
    "        target_vocab = SequenceVocabulary()        \n",
    "        for target in sorted(targets):\n",
    "            target_vocab.add_token(target)\n",
    "\n",
    "        word_counts = Counter()\n",
    "        for NARRATIVE in df.NARRATIVE:\n",
    "            for token in NARRATIVE.split(\"|\"):\n",
    "                word_counts[token] += 1\n",
    "\n",
    "        NARRATIVE_vocab = SequenceVocabulary()\n",
    "        for word, word_count in word_counts.items():\n",
    "            NARRATIVE_vocab.add_token(word)\n",
    "        return cls(NARRATIVE_vocab, target_vocab)\n",
    "\n",
    "    @classmethod\n",
    "    def from_serializable(cls, contents):\n",
    "        title_vocab = SequenceVocabulary.from_serializable(contents['NARRATIVE_vocab'])\n",
    "        category_vocab = Vocabulary.from_serializable(contents['target_vocab'])\n",
    "        return cls(NARRATIVE_vocab=NARRATIVE_vocab, target_vocab=target_vocab)\n",
    "\n",
    "    def to_serializable(self):\n",
    "        return {'NARRATIVE_vocab': self.NARRATIVE_vocab.to_serializable(), 'target_vocab': self.target_vocab.to_serializable()}\n",
    "\n",
    "class NERDataset(Dataset):\n",
    "    def __init__(self, df, vectorizer, labels):\n",
    "        self.df = df\n",
    "        self._vectorizer = vectorizer\n",
    "        self.labels=labels\n",
    "\n",
    "        measure_len = lambda context: len(context.split(\"|\"))\n",
    "        self._max_seq_length = max(map(measure_len, df.NARRATIVE)) + 2\n",
    "        \n",
    "        self.train_df = self.df[self.df.split=='train']\n",
    "        self.train_size = len(self.train_df)\n",
    "\n",
    "        self.val_df = self.df[self.df.split=='val']\n",
    "        self.validation_size = len(self.val_df)\n",
    "\n",
    "        self.test_df = self.df[self.df.split=='test']\n",
    "        self.test_size = len(self.test_df)\n",
    "\n",
    "        self._lookup_dict = {'train': (self.train_df, self.train_size), 'val': (self.val_df, self.validation_size), 'test': (self.test_df, self.test_size)}\n",
    "        self.set_split('train')\n",
    "\n",
    "    @classmethod\n",
    "    def load_dataset_and_make_vectorizer(cls, news_csv, label_txt):\n",
    "        labels=[]\n",
    "        with codecs.open('labels.txt', 'r', encoding='utf-8', errors='ignore') as f:\n",
    "            for line in f:\n",
    "                if len(line)>0:\n",
    "                    labels.append(line.strip())\n",
    "        \n",
    "        df = pd.read_csv(news_csv)\n",
    "        return cls(df, NERVectorizer.from_dataframe(df, labels), labels)\n",
    "\n",
    "    @classmethod\n",
    "    def load_dataset_and_load_vectorizer(cls, news_csv, vectorizer_filepath):\n",
    "        df = pd.read_csv(news_csv)\n",
    "        vectorizer = cls.load_vectorizer_only(vectorizer_filepath)\n",
    "        return cls(news_csv, vectorizer)\n",
    "\n",
    "    @staticmethod\n",
    "    def load_vectorizer_only(vectorizer_filepath):\n",
    "        with open(vectorizer_filepath) as fp:\n",
    "            return NameVectorizer.from_serializable(json.load(fp))\n",
    "\n",
    "    def save_vectorizer(self, vectorizer_filepath):\n",
    "        with open(vectorizer_filepath, \"w\") as fp:\n",
    "            json.dump(self._vectorizer.to_serializable(), fp)\n",
    "\n",
    "    def get_vectorizer(self):\n",
    "        return self._vectorizer\n",
    "\n",
    "    def set_split(self, split=\"train\"):\n",
    "        self._target_split = split\n",
    "        self._target_df, self._target_size = self._lookup_dict[split]\n",
    "\n",
    "    def __len__(self):\n",
    "        return self._target_size\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        row = self._target_df.iloc[index]\n",
    "        NARRATIVE_vector, x_target, y_target, length1, length2 = self._vectorizer.vectorize(row.NARRATIVE, row.target, self._max_seq_length, self._max_seq_length)\n",
    "        return {\"x_source\": NARRATIVE_vector, \n",
    "                \"x_target\": x_target,\n",
    "                \"y_target\": y_target, \n",
    "                'x_length': length1,\n",
    "                'y_length': length2}\n",
    "\n",
    "    def get_num_batches(self, batch_size):\n",
    "        return len(self) // batch_size"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1114d895-7cbb-49b7-9230-3a5712930844",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "4796842f-9234-49bc-97ff-be5972c65d4c",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = NERDataset.load_dataset_and_make_vectorizer(args.data_file, args.lable_file)\n",
    "dataset.save_vectorizer(args.vectorizer_file)\n",
    "vectorizer = dataset.get_vectorizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e9c214b8-e8a2-4c7e-8661-302e54f8a2ce",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 3)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "7de230da-bb71-4787-9dab-3cbf5b23420f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'x_source': array([ 2,  4,  5,  6,  7,  8,  9, 10,  5, 11, 12, 13, 14,  5, 15,  8, 16,\n",
       "        17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 13, 27, 23, 28,  5, 29, 30,\n",
       "        31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 23, 37,  9,\n",
       "         6,  3,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "         0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0]),\n",
       " 'x_target': array([  2,   5, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179,\n",
       "        179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179,\n",
       "        179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179,\n",
       "        179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179,\n",
       "          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0]),\n",
       " 'y_target': array([  5, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179,\n",
       "        179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179,\n",
       "        179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179,\n",
       "        179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179, 179,   3,\n",
       "          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0]),\n",
       " 'x_length': 53,\n",
       " 'y_length': 53}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d9221bda-6be2-4bcd-b284-805eb27b9f12",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Teaching Force"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ccb26d63-ecb9-4b82-a2e7-14d5cff410d7",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "fe05ff0f-a947-4362-8e5e-12418475f8a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "class SeqToSeq(nn.Module):\n",
    "    def __init__(self, source_vocab_size, source_embedding_size,  target_vocab_size, target_embedding_size, encoding_size,  target_bos_index):\n",
    "        super(SeqToSeq, self).__init__()\n",
    "        self.encoder = Encoder(num_embeddings=source_vocab_size,  embedding_size=source_embedding_size, rnn_hidden_size=encoding_size)\n",
    "        decoding_size = encoding_size * 2\n",
    "        self.decoder = Decoder(num_embeddings=target_vocab_size,  embedding_size=target_embedding_size,  rnn_hidden_size=decoding_size, bos_index=target_bos_index)\n",
    "    \n",
    "    def forward(self, x_source, x_source_lengths, target_sequence, _train):\n",
    "        encoder_state, final_hidden_states = self.encoder(x_source, x_source_lengths)\n",
    "        decoded_states = self.decoder(encoder_state=encoder_state,  initial_hidden_state=final_hidden_states,  target_sequence=target_sequence, _train=_train)\n",
    "        return decoded_states\n",
    "\n",
    "class Decoder(nn.Module):\n",
    "    def __init__(self, num_embeddings, embedding_size, rnn_hidden_size, bos_index):\n",
    "        super(Decoder, self).__init__()\n",
    "        self._rnn_hidden_size = rnn_hidden_size\n",
    "        self.target_embedding = nn.Embedding(num_embeddings=num_embeddings,  embedding_dim=embedding_size,  padding_idx=0)\n",
    "        self.rnn = nn.GRUCell(embedding_size,  rnn_hidden_size)\n",
    "        self.classifier1 = nn.Linear(rnn_hidden_size, num_embeddings)\n",
    "        self.classifier2 = nn.Linear(num_embeddings, num_embeddings)\n",
    "        self.bos_index = bos_index\n",
    "    \n",
    "    def _init_indices(self, batch_size):\n",
    "        return torch.ones(batch_size, dtype=torch.int64) * self.bos_index\n",
    "    \n",
    "    def forward(self, encoder_state, initial_hidden_state, target_sequence, _train):\n",
    "\n",
    "        target_sequence = target_sequence.permute(1, 0)\n",
    "        output_sequence_size = target_sequence.size(0)\n",
    "        \n",
    "        h_t = initial_hidden_state\n",
    "        batch_size = encoder_state.size(0)\n",
    "\n",
    "\n",
    "        y_t_index = self._init_indices(batch_size)\n",
    "        \n",
    "        h_t = h_t.to(encoder_state.device)\n",
    "        y_t_index = y_t_index.to(encoder_state.device)\n",
    "\n",
    "        output_vectors = []\n",
    "        y_t_index = target_sequence[0]\n",
    "        ''' Lab example is from 0-output_sequence_size'''\n",
    "        for i in range(1, output_sequence_size):\n",
    "            '''Lab example use target_sequence[i] not correct'''\n",
    "            y_input_vector = self.target_embedding(y_t_index)\n",
    "            h_t = self.rnn(y_input_vector, h_t)\n",
    "\n",
    "            score_for_y_t_index = self.classifier1(h_t)\n",
    "            score_for_y_t_index = self.classifier2(score_for_y_t_index)\n",
    "            output_vectors.append(score_for_y_t_index)\n",
    "            \n",
    "            _, y_t_index = score_for_y_t_index.max(dim=1)\n",
    "#             y_t_index = score_for_y_t_index.argmax(1)\n",
    "            if self.training and torch.rand(1) < 0.2:\n",
    "                y_t_index = target_sequence[i]\n",
    "            \n",
    "        output_vectors = torch.stack(output_vectors).permute(1, 0, 2)\n",
    "        return output_vectors\n",
    "\n",
    "class Encoder(nn.Module):\n",
    "    def __init__(self, num_embeddings, embedding_size, rnn_hidden_size):\n",
    "        super(Encoder, self).__init__()\n",
    "    \n",
    "        self.source_embedding = nn.Embedding(num_embeddings, embedding_size, padding_idx=0)\n",
    "        self.rnn = nn.GRU(embedding_size, rnn_hidden_size, bidirectional=True, batch_first=True)\n",
    "    \n",
    "    def forward(self, x_source, x_lengths):\n",
    "        x_embedded = self.source_embedding(x_source)\n",
    "        x_packed = pack_padded_sequence(x_embedded, x_lengths.detach().cpu().numpy(), batch_first=True)\n",
    "\n",
    "        out, hidden  = self.rnn(x_packed)\n",
    "        hidden = hidden.permute(1, 0, 2)\n",
    "        hidden = hidden.contiguous().view(hidden.size(0), -1)\n",
    "        out, _ = pad_packed_sequence(out, batch_first=True)\n",
    "        \n",
    "        return out, hidden"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "20f7cb9a-88af-44d9-99f7-0e7b5f0f8539",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Build model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "80ff51c5-7418-4789-8191-9165ed8ffcab",
   "metadata": {},
   "outputs": [],
   "source": [
    "NERmodel = SeqToSeq(source_vocab_size=len(vectorizer.NARRATIVE_vocab), \n",
    "                 source_embedding_size=args.source_embedding_size, \n",
    "                 target_vocab_size=len(vectorizer.target_vocab),\n",
    "                 target_embedding_size=args.target_embedding_size, \n",
    "                 encoding_size=args.encoding_size,\n",
    "                 target_bos_index=vectorizer.target_vocab.begin_seq_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "4d0cc527-f62f-4548-aa9c-95c29b4ab0ba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SeqToSeq(\n",
       "  (encoder): Encoder(\n",
       "    (source_embedding): Embedding(13225, 128, padding_idx=0)\n",
       "    (rnn): GRU(128, 128, batch_first=True, bidirectional=True)\n",
       "  )\n",
       "  (decoder): Decoder(\n",
       "    (target_embedding): Embedding(180, 128, padding_idx=0)\n",
       "    (rnn): GRUCell(128, 256)\n",
       "    (classifier1): Linear(in_features=256, out_features=180, bias=True)\n",
       "    (classifier2): Linear(in_features=180, out_features=180, bias=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "NERmodel"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fd19c658-441d-4dc5-8918-50fae62f577b",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Set up"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "215652cb-fb43-4bc4-9cc5-5edda280edc7",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_state = make_train_state(args)\n",
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "args.device = torch.device(\"cuda\" if args.cuda else \"cpu\")\n",
    "NERmodel = NERmodel.to(args.device)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "06b9fb0f-2510-4314-aee5-5de8ced46a30",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Train\n",
    "\n",
    "Try to run if model dosent saved, then reset this kernel to free up GPU or CPU memory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "33fb4fd2-1463-4a35-ab6a-38398aee2acf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epochs: 000 | Train loss 5.1436 Train acc 16.288 Val loss 5.0210 Val acc 45.027\n",
      "Epochs: 001 | Train loss 4.7757 Train acc 75.596 Val loss 4.5663 Val acc 72.631\n",
      "Epochs: 002 | Train loss 4.2888 Train acc 81.388 Val loss 3.9805 Val acc 81.383\n",
      "Epochs: 003 | Train loss 3.5968 Train acc 87.895 Val loss 3.1260 Val acc 88.324\n",
      "Epochs: 004 | Train loss 2.8501 Train acc 87.760 Val loss 2.4846 Val acc 87.610\n",
      "Epochs: 005 | Train loss 2.4187 Train acc 87.601 Val loss 2.4147 Val acc 87.432\n",
      "Epochs: 006 | Train loss 2.3595 Train acc 87.588 Val loss 2.2832 Val acc 87.757\n",
      "Epochs: 007 | Train loss 2.2349 Train acc 87.587 Val loss 2.1087 Val acc 87.979\n",
      "Epochs: 008 | Train loss 2.1013 Train acc 87.577 Val loss 2.0230 Val acc 87.660\n",
      "Epochs: 009 | Train loss 2.0399 Train acc 88.142 Val loss 2.0103 Val acc 88.449\n",
      "Epochs: 010 | Train loss 1.9745 Train acc 88.104 Val loss 1.9361 Val acc 88.009\n",
      "Epochs: 011 | Train loss 1.9082 Train acc 87.785 Val loss 1.9178 Val acc 87.588\n",
      "Epochs: 012 | Train loss 1.8652 Train acc 87.781 Val loss 1.8555 Val acc 87.855\n",
      "Epochs: 013 | Train loss 1.8403 Train acc 87.785 Val loss 1.8156 Val acc 87.955\n",
      "Epochs: 014 | Train loss 1.8182 Train acc 87.788 Val loss 1.8350 Val acc 87.809\n",
      "Epochs: 015 | Train loss 1.8000 Train acc 87.783 Val loss 1.7799 Val acc 87.937\n",
      "Epochs: 016 | Train loss 1.7882 Train acc 87.786 Val loss 1.7698 Val acc 88.209\n",
      "Epochs: 017 | Train loss 1.7813 Train acc 87.785 Val loss 1.8154 Val acc 87.627\n",
      "Epochs: 018 | Train loss 1.7722 Train acc 87.788 Val loss 1.7682 Val acc 87.923\n",
      "Epochs: 019 | Train loss 1.7680 Train acc 87.789 Val loss 1.7629 Val acc 87.901\n",
      "Epochs: 020 | Train loss 1.7626 Train acc 87.789 Val loss 1.7630 Val acc 87.926\n",
      "Epochs: 021 | Train loss 1.7593 Train acc 87.781 Val loss 1.7461 Val acc 87.948\n",
      "Epochs: 022 | Train loss 1.7548 Train acc 87.789 Val loss 1.7385 Val acc 87.921\n",
      "Epochs: 023 | Train loss 1.7544 Train acc 87.778 Val loss 1.7673 Val acc 87.885\n",
      "Epochs: 024 | Train loss 1.7496 Train acc 87.792 Val loss 1.7279 Val acc 88.208\n",
      "Epochs: 025 | Train loss 1.7468 Train acc 87.791 Val loss 1.7730 Val acc 87.779\n",
      "Epochs: 026 | Train loss 1.7455 Train acc 87.791 Val loss 1.7475 Val acc 87.776\n",
      "Epochs: 027 | Train loss 1.7446 Train acc 87.789 Val loss 1.7118 Val acc 88.146\n",
      "Epochs: 028 | Train loss 1.7402 Train acc 87.789 Val loss 1.7728 Val acc 87.736\n",
      "Epochs: 029 | Train loss 1.7352 Train acc 87.794 Val loss 1.7407 Val acc 87.911\n",
      "Epochs: 030 | Train loss 1.7342 Train acc 87.799 Val loss 1.7298 Val acc 88.050\n",
      "Epochs: 031 | Train loss 1.7333 Train acc 87.895 Val loss 1.7320 Val acc 88.041\n",
      "Epochs: 032 | Train loss 1.7291 Train acc 87.974 Val loss 1.7391 Val acc 87.986\n",
      "Epochs: 033 | Train loss 1.7286 Train acc 87.997 Val loss 1.7309 Val acc 88.076\n",
      "Epochs: 034 | Train loss 1.7281 Train acc 88.009 Val loss 1.7593 Val acc 88.032\n",
      "Epochs: 035 | Train loss 1.7233 Train acc 88.070 Val loss 1.7255 Val acc 88.272\n",
      "Epochs: 036 | Train loss 1.7226 Train acc 88.131 Val loss 1.7587 Val acc 88.085\n",
      "Epochs: 037 | Train loss 1.7165 Train acc 88.218 Val loss 1.7285 Val acc 88.298\n",
      "Epochs: 038 | Train loss 1.7147 Train acc 88.305 Val loss 1.7635 Val acc 88.127\n",
      "Epochs: 039 | Train loss 1.7138 Train acc 88.381 Val loss 1.7132 Val acc 88.608\n",
      "Epochs: 040 | Train loss 1.7086 Train acc 88.445 Val loss 1.7326 Val acc 88.487\n",
      "Epochs: 041 | Train loss 1.7072 Train acc 88.525 Val loss 1.7507 Val acc 88.281\n",
      "Epochs: 042 | Train loss 1.7003 Train acc 88.567 Val loss 1.6899 Val acc 88.739\n",
      "Epochs: 043 | Train loss 1.6985 Train acc 88.596 Val loss 1.6884 Val acc 88.824\n",
      "Epochs: 044 | Train loss 1.6952 Train acc 88.608 Val loss 1.7154 Val acc 88.576\n",
      "Epochs: 045 | Train loss 1.6864 Train acc 88.607 Val loss 1.6925 Val acc 88.652\n",
      "Epochs: 046 | Train loss 1.6810 Train acc 88.583 Val loss 1.6939 Val acc 88.637\n",
      "Epochs: 047 | Train loss 1.6757 Train acc 88.590 Val loss 1.6527 Val acc 88.891\n",
      "Epochs: 048 | Train loss 1.6676 Train acc 88.566 Val loss 1.6395 Val acc 88.889\n",
      "Epochs: 049 | Train loss 1.6611 Train acc 88.552 Val loss 1.6405 Val acc 88.825\n",
      "Epochs: 050 | Train loss 1.6547 Train acc 88.521 Val loss 1.7003 Val acc 88.358\n",
      "Epochs: 051 | Train loss 1.6486 Train acc 88.536 Val loss 1.6476 Val acc 88.751\n",
      "Epochs: 052 | Train loss 1.6399 Train acc 88.510 Val loss 1.6682 Val acc 88.456\n",
      "Epochs: 053 | Train loss 1.6323 Train acc 88.464 Val loss 1.6572 Val acc 88.417\n",
      "Epochs: 054 | Train loss 1.6258 Train acc 88.452 Val loss 1.6543 Val acc 88.517\n",
      "Epochs: 055 | Train loss 1.6203 Train acc 88.375 Val loss 1.6521 Val acc 88.330\n",
      "Epochs: 056 | Train loss 1.6093 Train acc 88.333 Val loss 1.6385 Val acc 88.450\n",
      "Epochs: 057 | Train loss 1.6013 Train acc 88.342 Val loss 1.6334 Val acc 88.348\n",
      "Epochs: 058 | Train loss 1.5959 Train acc 88.198 Val loss 1.6222 Val acc 88.598\n",
      "Epochs: 059 | Train loss 1.5828 Train acc 88.206 Val loss 1.5964 Val acc 88.534\n",
      "Epochs: 060 | Train loss 1.5747 Train acc 88.151 Val loss 1.6048 Val acc 88.089\n",
      "Epochs: 061 | Train loss 1.5677 Train acc 88.090 Val loss 1.5704 Val acc 88.496\n",
      "Epochs: 062 | Train loss 1.5573 Train acc 88.095 Val loss 1.5655 Val acc 88.061\n",
      "Epochs: 063 | Train loss 1.5510 Train acc 88.016 Val loss 1.5870 Val acc 86.950\n",
      "Epochs: 064 | Train loss 1.5447 Train acc 87.778 Val loss 1.5974 Val acc 88.268\n",
      "Epochs: 065 | Train loss 1.5383 Train acc 87.816 Val loss 1.5795 Val acc 87.765\n",
      "Epochs: 066 | Train loss 1.5289 Train acc 88.043 Val loss 1.5832 Val acc 88.054\n",
      "Epochs: 067 | Train loss 1.5207 Train acc 87.856 Val loss 1.5810 Val acc 88.360\n",
      "Epochs: 068 | Train loss 1.5163 Train acc 87.918 Val loss 1.5440 Val acc 88.493\n",
      "Epochs: 069 | Train loss 1.5099 Train acc 88.011 Val loss 1.5208 Val acc 88.377\n",
      "Epochs: 070 | Train loss 1.5051 Train acc 88.101 Val loss 1.5581 Val acc 86.904\n",
      "Epochs: 071 | Train loss 1.4998 Train acc 88.055 Val loss 1.5401 Val acc 87.622\n",
      "Epochs: 072 | Train loss 1.4957 Train acc 87.956 Val loss 1.5088 Val acc 87.924\n",
      "Epochs: 073 | Train loss 1.4912 Train acc 87.971 Val loss 1.5383 Val acc 88.636\n",
      "Epochs: 074 | Train loss 1.4878 Train acc 88.024 Val loss 1.5485 Val acc 88.301\n",
      "Epochs: 075 | Train loss 1.4846 Train acc 88.195 Val loss 1.5006 Val acc 87.984\n",
      "Epochs: 076 | Train loss 1.4773 Train acc 88.422 Val loss 1.4951 Val acc 87.211\n",
      "Epochs: 077 | Train loss 1.4722 Train acc 88.251 Val loss 1.5133 Val acc 88.344\n",
      "Epochs: 078 | Train loss 1.4704 Train acc 88.254 Val loss 1.5293 Val acc 88.417\n",
      "Epochs: 079 | Train loss 1.4671 Train acc 88.267 Val loss 1.5275 Val acc 88.467\n",
      "Epochs: 080 | Train loss 1.4632 Train acc 88.380 Val loss 1.4918 Val acc 88.424\n",
      "Epochs: 081 | Train loss 1.4600 Train acc 88.433 Val loss 1.5047 Val acc 87.941\n",
      "Epochs: 082 | Train loss 1.4547 Train acc 88.517 Val loss 1.4959 Val acc 88.490\n",
      "Epochs: 083 | Train loss 1.4546 Train acc 88.547 Val loss 1.4903 Val acc 88.645\n",
      "Epochs: 084 | Train loss 1.4510 Train acc 88.605 Val loss 1.4916 Val acc 88.578\n",
      "Epochs: 085 | Train loss 1.4479 Train acc 88.612 Val loss 1.4906 Val acc 88.579\n",
      "Epochs: 086 | Train loss 1.4442 Train acc 88.693 Val loss 1.4988 Val acc 88.608\n",
      "Epochs: 087 | Train loss 1.4411 Train acc 88.785 Val loss 1.4603 Val acc 88.687\n",
      "Epochs: 088 | Train loss 1.4385 Train acc 88.873 Val loss 1.4900 Val acc 87.057\n",
      "Epochs: 090 | Train loss 1.4370 Train acc 88.639 Val loss 1.4839 Val acc 87.292\n",
      "Epochs: 091 | Train loss 1.4344 Train acc 88.641 Val loss 1.4851 Val acc 87.717\n",
      "Epochs: 092 | Train loss 1.4338 Train acc 88.792 Val loss 1.4748 Val acc 87.827\n",
      "Epochs: 093 | Train loss 1.4275 Train acc 88.916 Val loss 1.4752 Val acc 88.218\n",
      "Epochs: 094 | Train loss 1.4250 Train acc 88.959 Val loss 1.4632 Val acc 88.336\n",
      "Epochs: 095 | Train loss 1.4245 Train acc 88.998 Val loss 1.4682 Val acc 88.168\n",
      "Epochs: 096 | Train loss 1.4219 Train acc 89.010 Val loss 1.4666 Val acc 88.872\n",
      "Epochs: 097 | Train loss 1.4175 Train acc 89.090 Val loss 1.4951 Val acc 88.315\n",
      "Epochs: 098 | Train loss 1.4162 Train acc 89.132 Val loss 1.4878 Val acc 88.431\n",
      "Epochs: 099 | Train loss 1.4142 Train acc 89.150 Val loss 1.4606 Val acc 88.421\n",
      "Epochs: 100 | Train loss 1.4103 Train acc 89.077 Val loss 1.4674 Val acc 88.728\n",
      "Epochs: 102 | Train loss 1.4084 Train acc 89.055 Val loss 1.4647 Val acc 88.771\n",
      "Epochs: 105 | Train loss 1.4111 Train acc 89.101 Val loss 1.5050 Val acc 88.571\n",
      "Epochs: 106 | Train loss 1.4010 Train acc 89.287 Val loss 1.4570 Val acc 88.596\n",
      "Epochs: 107 | Train loss 1.4009 Train acc 89.362 Val loss 1.4600 Val acc 88.174\n",
      "Epochs: 108 | Train loss 1.3980 Train acc 89.218 Val loss 1.4650 Val acc 88.530\n",
      "Epochs: 112 | Train loss 1.3923 Train acc 89.319 Val loss 1.4950 Val acc 88.473\n",
      "Epochs: 114 | Train loss 1.3907 Train acc 89.449 Val loss 1.4317 Val acc 88.780\n",
      "Epochs: 115 | Train loss 1.3844 Train acc 89.501 Val loss 1.4698 Val acc 88.457\n",
      "Epochs: 118 | Train loss 1.3833 Train acc 89.605 Val loss 1.4543 Val acc 88.087\n",
      "Epochs: 120 | Train loss 1.3850 Train acc 89.563 Val loss 1.4851 Val acc 87.778\n",
      "Epochs: 122 | Train loss 1.3782 Train acc 89.454 Val loss 1.4267 Val acc 89.002\n",
      "Epochs: 123 | Train loss 1.3753 Train acc 89.721 Val loss 1.4300 Val acc 89.204\n",
      "Epochs: 124 | Train loss 1.3737 Train acc 89.822 Val loss 1.4415 Val acc 88.823\n",
      "Epochs: 126 | Train loss 1.3721 Train acc 89.781 Val loss 1.4516 Val acc 89.094\n",
      "Epochs: 127 | Train loss 1.3698 Train acc 89.775 Val loss 1.4546 Val acc 88.707\n",
      "Epochs: 128 | Train loss 1.3680 Train acc 89.825 Val loss 1.4168 Val acc 89.017\n",
      "Epochs: 129 | Train loss 1.3627 Train acc 89.822 Val loss 1.4594 Val acc 89.084\n",
      "Epochs: 131 | Train loss 1.3624 Train acc 89.891 Val loss 1.4669 Val acc 88.692\n",
      "Epochs: 133 | Train loss 1.3671 Train acc 89.705 Val loss 1.4698 Val acc 88.845\n",
      "Epochs: 134 | Train loss 1.3635 Train acc 89.831 Val loss 1.4496 Val acc 89.220\n",
      "Epochs: 135 | Train loss 1.3572 Train acc 89.941 Val loss 1.4216 Val acc 89.428\n",
      "Epochs: 136 | Train loss 1.3553 Train acc 90.026 Val loss 1.4691 Val acc 88.936\n",
      "Epochs: 139 | Train loss 1.3578 Train acc 89.953 Val loss 1.4376 Val acc 89.094\n",
      "Epochs: 140 | Train loss 1.3534 Train acc 90.036 Val loss 1.4340 Val acc 89.162\n",
      "Epochs: 141 | Train loss 1.3508 Train acc 90.030 Val loss 1.4291 Val acc 89.205\n",
      "Epochs: 142 | Train loss 1.3479 Train acc 90.121 Val loss 1.4517 Val acc 89.136\n",
      "Epochs: 143 | Train loss 1.3470 Train acc 90.136 Val loss 1.4725 Val acc 88.504\n",
      "Epochs: 145 | Train loss 1.3473 Train acc 90.026 Val loss 1.4685 Val acc 88.944\n",
      "Epochs: 146 | Train loss 1.3432 Train acc 90.085 Val loss 1.4605 Val acc 88.952\n",
      "Epochs: 149 | Train loss 1.3408 Train acc 90.260 Val loss 1.4523 Val acc 88.810\n",
      "Epochs: 150 | Train loss 1.3395 Train acc 90.122 Val loss 1.4074 Val acc 89.063\n",
      "Epochs: 151 | Train loss 1.3373 Train acc 90.259 Val loss 1.4255 Val acc 88.965\n",
      "Epochs: 153 | Train loss 1.3404 Train acc 90.098 Val loss 1.4568 Val acc 88.961\n",
      "Epochs: 154 | Train loss 1.3376 Train acc 90.147 Val loss 1.4713 Val acc 88.389\n",
      "Epochs: 155 | Train loss 1.3346 Train acc 90.181 Val loss 1.4558 Val acc 88.553\n",
      "Epochs: 156 | Train loss 1.3342 Train acc 90.165 Val loss 1.4215 Val acc 89.057\n",
      "Epochs: 157 | Train loss 1.3340 Train acc 90.237 Val loss 1.4377 Val acc 88.894\n",
      "Epochs: 158 | Train loss 1.3302 Train acc 90.270 Val loss 1.4448 Val acc 89.218\n",
      "Epochs: 159 | Train loss 1.3300 Train acc 90.306 Val loss 1.4239 Val acc 88.978\n",
      "Epochs: 161 | Train loss 1.3256 Train acc 90.368 Val loss 1.4526 Val acc 89.114\n",
      "Epochs: 162 | Train loss 1.3218 Train acc 90.428 Val loss 1.4202 Val acc 89.334\n",
      "Epochs: 164 | Train loss 1.3246 Train acc 90.219 Val loss 1.4417 Val acc 88.917\n",
      "Epochs: 167 | Train loss 1.3275 Train acc 90.114 Val loss 1.4350 Val acc 89.108\n",
      "Epochs: 168 | Train loss 1.3194 Train acc 90.315 Val loss 1.4516 Val acc 89.286\n",
      "Epochs: 169 | Train loss 1.3192 Train acc 90.440 Val loss 1.4418 Val acc 88.877\n",
      "Epochs: 172 | Train loss 1.3300 Train acc 90.133 Val loss 1.4390 Val acc 88.859\n",
      "Epochs: 173 | Train loss 1.3152 Train acc 90.331 Val loss 1.4639 Val acc 88.802\n",
      "Epochs: 174 | Train loss 1.3134 Train acc 90.397 Val loss 1.4438 Val acc 89.417\n",
      "Epochs: 175 | Train loss 1.3127 Train acc 90.495 Val loss 1.4227 Val acc 89.142\n",
      "Epochs: 176 | Train loss 1.3121 Train acc 90.280 Val loss 1.4368 Val acc 89.176\n",
      "Epochs: 179 | Train loss 1.3178 Train acc 90.123 Val loss 1.4162 Val acc 89.041\n",
      "Epochs: 180 | Train loss 1.3083 Train acc 90.382 Val loss 1.4351 Val acc 89.301\n",
      "Epochs: 181 | Train loss 1.3051 Train acc 90.493 Val loss 1.4661 Val acc 89.201\n",
      "Epochs: 183 | Train loss 1.3072 Train acc 90.417 Val loss 1.4214 Val acc 89.347\n",
      "Epochs: 186 | Train loss 1.3117 Train acc 90.400 Val loss 1.4413 Val acc 88.840\n",
      "Epochs: 187 | Train loss 1.3100 Train acc 90.264 Val loss 1.4339 Val acc 89.093\n",
      "Epochs: 188 | Train loss 1.3031 Train acc 90.293 Val loss 1.4017 Val acc 89.691\n",
      "Epochs: 190 | Train loss 1.2998 Train acc 90.444 Val loss 1.4181 Val acc 89.260\n",
      "Epochs: 191 | Train loss 1.2954 Train acc 90.500 Val loss 1.4223 Val acc 89.636\n",
      "Epochs: 195 | Train loss 1.3004 Train acc 90.495 Val loss 1.4399 Val acc 89.529\n",
      "Epochs: 196 | Train loss 1.2990 Train acc 90.569 Val loss 1.4161 Val acc 89.501\n",
      "Epochs: 197 | Train loss 1.2908 Train acc 90.627 Val loss 1.4026 Val acc 89.409\n",
      "Epochs: 199 | Train loss 1.2941 Train acc 90.403 Val loss 1.4443 Val acc 89.624\n"
     ]
    }
   ],
   "source": [
    "optimizer = optim.Adam(NERmodel.parameters(), lr=args.lr)\n",
    "mask_index = vectorizer.target_vocab.mask_index\n",
    "\n",
    "for epoch_index in range(args.num_epochs):\n",
    "    dataset.set_split('train')\n",
    "    batch_generator = generate_batches(dataset, batch_size=args.batch_train, device=args.device)\n",
    "    running_loss = 0.0\n",
    "    running_acc = 0.0\n",
    "    NERmodel.train()\n",
    "\n",
    "    for batch_index, batch_dict in enumerate(batch_generator):  \n",
    "        # step 1. zero the gradients\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        # step 2. compute the output\n",
    "        y_pred = NERmodel(batch_dict['x_source'],  batch_dict['x_length'], batch_dict['x_target'], _train=True)\n",
    "\n",
    "        # step 3. compute the loss\n",
    "        loss = sequence_loss(y_pred, batch_dict['y_target'], mask_index)\n",
    "\n",
    "        # step 4. use loss to produce gradients\n",
    "        loss.backward()\n",
    "\n",
    "        # step 5. use optimizer to take gradient step\n",
    "        optimizer.step()\n",
    "\n",
    "        # compute the running loss and running accuracy\n",
    "        running_loss += (loss.item() - running_loss) / (batch_index + 1)\n",
    "\n",
    "        acc_t = compute_accuracy(y_pred, batch_dict['y_target'], mask_index)\n",
    "        running_acc += (acc_t - running_acc) / (batch_index + 1)\n",
    "    \n",
    "    train_state['train_loss'].append(running_loss)\n",
    "    train_state['train_acc'].append(running_acc)\n",
    "    \n",
    "    dataset.set_split('val')\n",
    "    args.batch_valid = args.batch_valid if args.batch_valid else len(dataset)\n",
    "    batch_generator = generate_batches(dataset,  batch_size=args.batch_valid,  device=args.device)\n",
    "    \n",
    "    running_loss = 0.\n",
    "    running_acc = 0.\n",
    "    NERmodel.eval()\n",
    "\n",
    "    for batch_index, batch_dict in enumerate(batch_generator):\n",
    "        # compute the output\n",
    "        y_pred = NERmodel(batch_dict['x_source'],  batch_dict['x_length'],  batch_dict['x_target'], _train=False)\n",
    "\n",
    "        # step 1. compute the loss\n",
    "        loss = sequence_loss(y_pred, batch_dict['y_target'], mask_index)\n",
    "\n",
    "        # compute the running loss and accuracy\n",
    "        running_loss += (loss.item() - running_loss) / (batch_index + 1)\n",
    "\n",
    "        acc_t = compute_accuracy(y_pred, batch_dict['y_target'], mask_index)\n",
    "        running_acc += (acc_t - running_acc) / (batch_index + 1)\n",
    "    \n",
    "    train_state['val_loss'].append(running_loss)\n",
    "    train_state['val_acc'].append(running_acc)\n",
    "    \n",
    "    try:\n",
    "        if train_state['train_loss'][-1]< train_state['train_loss'][-2]:\n",
    "            pretty_print(epoch_index, train_state['train_loss'][-1], train_state['train_acc'][-1], train_state['val_loss'][-1], train_state['val_acc'][-1])\n",
    "    except:\n",
    "        pretty_print(epoch_index, train_state['train_loss'][-1], train_state['train_acc'][-1], train_state['val_loss'][-1], train_state['val_acc'][-1])\n",
    "    train_state = update_train_state(args=args, model=NERmodel, train_state=train_state)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eaf47299-ec7a-4611-ab02-75ef008fbced",
   "metadata": {},
   "source": [
    "## Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "70496693-aad1-4fd4-808c-8a4428e6ba68",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test loss 1.4316 Test acc 89.613\n"
     ]
    }
   ],
   "source": [
    "args.device = torch.device(\"cpu\")\n",
    "# checkpoint = torch.load(args.model_state_file)\n",
    "# NERmodel.load_state_dict(checkpoint['model_state_dict'])\n",
    "NERmodel.load_state_dict(torch.load(train_state['model_filename']))\n",
    "NERmodel = NERmodel.to(args.device)\n",
    "\n",
    "dataset.set_split('test')\n",
    "args.batch_test = args.batch_test if args.batch_test else len(dataset)\n",
    "args.batch_test = len(dataset)\n",
    "batch_generator = generate_batches(dataset,  batch_size=args.batch_test, shuffle=False, device=args.device)\n",
    "\n",
    "running_loss = 0.\n",
    "running_acc = 0.\n",
    "NERmodel.eval()\n",
    "mask_index = vectorizer.target_vocab.mask_index\n",
    "for batch_index, batch_dict in enumerate(batch_generator):\n",
    "    # compute the output\n",
    "    y_pred = NERmodel(batch_dict['x_source'],  batch_dict['x_length'],  batch_dict['x_target'], _train=False)\n",
    "    y_true = batch_dict['y_target']\n",
    "\n",
    "    # step 1. compute the loss\n",
    "    loss = sequence_loss(y_pred, batch_dict['y_target'], mask_index)\n",
    "\n",
    "    # compute the running loss and accuracy\n",
    "    running_loss += (loss.item() - running_loss) / (batch_index + 1)\n",
    "\n",
    "    acc_t = compute_accuracy(y_pred, batch_dict['y_target'], mask_index)\n",
    "    running_acc += (acc_t - running_acc) / (batch_index + 1)\n",
    "\n",
    "train_state['test_loss'] = running_loss\n",
    "train_state['test_acc'] = running_acc\n",
    "\n",
    "print('Test loss', str(running_loss)[: 6], 'Test acc', str(running_acc)[: 6])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "01210315-192f-42ce-8046-8e3f44933ebc",
   "metadata": {},
   "outputs": [],
   "source": [
    "View accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "ac762271-205d-4b14-9bc9-cabcd8ed78bd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.2773091297383876"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred, true = normalize_sizes(y_pred, y_true)\n",
    "true = true.cpu().detach().numpy()\n",
    "_, y_pred_indices = pred.max(dim=1)\n",
    "\n",
    "y_pred_indices = y_pred_indices.cpu().detach().numpy()\n",
    "df = pd.DataFrame({'y_true':true, 'y_pred':y_pred_indices})\n",
    "df = df[df['y_true'] != 0]\n",
    "df = df[df['y_true'] != 179]\n",
    "accuracy_score(df['y_true'], df['y_pred'])"
   ]
  },
  {
   "cell_type": "raw",
   "id": "3a2c8adc-6027-4cf8-9e3d-46aca07549c2",
   "metadata": {},
   "source": [
    "Out of 179 classes (exclude \"O\" tag) the performance of the mode is not good in terms of accuracy, But as it have 179 classess the performance is better than baseline (0.56%   1/179)\n",
    "\n",
    "\n",
    "To view the confusion matrix comment out code below - long time to run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "5959efc7-3862-47ca-a4a7-f66351ec2848",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plot is too big\n",
    "\n",
    "# from matplotlib.pyplot import figure\n",
    "# cmp = ConfusionMatrixDisplay.from_predictions(df['y_true'], df['y_pred'], normalize='pred')\n",
    "# fig, ax = plt.subplots(figsize=(80,80))\n",
    "# cmp.plot(ax=ax)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cae02843-14a7-468c-80ae-b75a78415c5e",
   "metadata": {
    "tags": []
   },
   "source": [
    "## View the entity\n",
    "By just look at the accuracy of prediciton will not make sense, as some labels in the sentinces may misclassified by human or some entites are similar, to check the actual performance we need to look at it result "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9a2cd98d-6173-4533-8e63-be31c02a51ef",
   "metadata": {},
   "source": [
    "Most entities it did not successfully predicted, only tag it can successfulyy predict is \n",
    "```\n",
    "employee                        B-person/employee                         B-person/employee\n",
    "```\n",
    "Methdos below using slef-attention is much better"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "06220fc2-3964-4ba2-a15c-cd7979c6e553",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## Print prediciton"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "867eda62-80d0-4315-a464-1cc74136a3e2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------  ----------------------------------------  -----------------\n",
      "employee                        B-person/employee                         B-person/employee\n",
      "when                            O                                         O\n",
      "work                            O                                         O\n",
      "in                              O                                         O\n",
      "his                             O                                         O\n",
      "area                            O                                         O\n",
      "cone                            O                                         O\n",
      "crusher                         B-equipment/mechanical_equipment/crusher  O\n",
      "he                              O                                         O\n",
      "be                              O                                         O\n",
      "wear                            O                                         O\n",
      "his                             O                                         O\n",
      "personal-protective-equipment-  O                                         O\n",
      "and                             O                                         O\n",
      "harness                         O                                         O\n",
      "and                             O                                         O\n",
      "he                              O                                         O\n",
      "slip                            O                                         O\n",
      "and                             O                                         O\n",
      "fell                            O                                         O\n",
      "ft-                             O                                         O\n",
      "from                            O                                         O\n",
      "the                             O                                         O\n",
      "ground                          O                                         O\n",
      "never                           O                                         O\n",
      "lose                            O                                         O\n",
      "consciousness                   O                                         O\n",
      "he                              O                                         O\n",
      "be                              O                                         O\n",
      "complain                        O                                         O\n",
      "about                           O                                         O\n",
      "his                             O                                         O\n",
      "left                            B-body_part/arm/hand/left_hand            O\n",
      "hand                            I-body_part/arm/hand/left_hand            O\n",
      "and                             O                                         O\n",
      "his                             O                                         O\n",
      "neck                            B-body_part/neck                          O\n",
      "when                            O                                         O\n",
      "em                              O                                         O\n",
      "arrive                          O                                         O\n",
      "he                              O                                         O\n",
      "be                              O                                         O\n",
      "triage                          O                                         O\n",
      "he                              O                                         O\n",
      "refuse                          O                                         O\n",
      "transportation                  O                                         O\n",
      "next                            O                                         O\n",
      "day                             O                                         O\n",
      "go                              O                                         O\n",
      "to                              O                                         O\n",
      "see                             O                                         O\n",
      "doctor                          O                                         O\n",
      "could                           O                                         O\n",
      "not                             O                                         O\n",
      "stand                           O                                         O\n",
      "the                             O                                         O\n",
      "pain                            O                                         O\n",
      "<END>                           <END>                                     O\n",
      "------------------------------  ----------------------------------------  -----------------\n",
      "-------------------------------  -----------------  -----------------\n",
      "employee                         B-person/employee  B-person/employee\n",
      "be                               O                  O\n",
      "instal                           O                  O\n",
      "roof                             O                  O\n",
      "bolt                             O                  O\n",
      "in                               O                  O\n",
      "the                              O                  O\n",
      "number                           O                  O\n",
      "entry                            O                  O\n",
      "face                             O                  O\n",
      "on                               O                  O\n",
      "mobile-mining-unit-number-9003-  O                  O\n",
      "section                          O                  O\n",
      "when                             O                  O\n",
      "the                              O                  O\n",
      "left                             O                  O\n",
      "rib                              O                  O\n",
      "roll                             O                  O\n",
      "off                              O                  O\n",
      "hit                              O                  O\n",
      "the                              O                  O\n",
      "employee                         B-person/employee  O\n",
      "from                             O                  O\n",
      "behind                           O                  O\n",
      "the                              O                  O\n",
      "rib                              O                  O\n",
      "injured                          O                  O\n",
      "the                              O                  O\n",
      "low                              O                  O\n",
      "part                             O                  O\n",
      "of                               O                  O\n",
      "the                              O                  O\n",
      "employee's                       O                  O\n",
      "leg                              O                  O\n",
      "<END>                            <END>              <END>\n",
      "-------------------------------  -----------------  -----------------\n",
      "------------  ---------------------------  -----\n",
      "while         O                            O\n",
      "backing       O                            O\n",
      "loader        O                            O\n",
      "up            O                            O\n",
      "from          O                            O\n",
      "get           O                            O\n",
      "bucket        O                            O\n",
      "of            O                            O\n",
      "coal          O                            O\n",
      "right         O                            O\n",
      "rear          O                            O\n",
      "wheel         O                            O\n",
      "drop          O                            O\n",
      "down          O                            O\n",
      "in            O                            O\n",
      "hole-bounce-  O                            O\n",
      "operator      B-person/operator            O\n",
      "and           O                            O\n",
      "injured       O                            O\n",
      "his           O                            O\n",
      "low           B-body_part/back/lower_back  O\n",
      "back          I-body_part/back/lower_back  O\n",
      "from          O                            O\n",
      "shock         O                            O\n",
      "above         O                            O\n",
      "and           O                            O\n",
      "bounce        O                            <END>\n",
      "<END>         <END>                        <END>\n",
      "------------  ---------------------------  -----\n",
      "------------------------------  ----------------------  -----\n",
      "miner                           O                       O\n",
      "suffer                          O                       O\n",
      "low                             O                       O\n",
      "left                            O                       O\n",
      "back                            B-body_part/back        O\n",
      "muscle                          B-injury/muscle         O\n",
      "strain                          B-injury/muscle/strain  O\n",
      "miner                           O                       O\n",
      "be                              O                       O\n",
      "on                              O                       O\n",
      "knee                            O                       O\n",
      "move                            O                       O\n",
      "cam                             O                       O\n",
      "to                              O                       O\n",
      "set                             O                       O\n",
      "screen                          O                       O\n",
      "deck                            O                       O\n",
      "follow                          O                       O\n",
      "normal                          O                       O\n",
      "procedure                       O                       O\n",
      "on                              O                       O\n",
      "rotex                           O                       O\n",
      "size                            O                       O\n",
      "machine                         O                       O\n",
      "and                             O                       O\n",
      "report                          O                       O\n",
      "sudden                          O                       O\n",
      "back                            B-body_part/back        O\n",
      "pain                            O                       O\n",
      "miner                           O                       O\n",
      "be                              O                       O\n",
      "experienced                     O                       O\n",
      "in                              O                       O\n",
      "this                            O                       O\n",
      "task                            O                       O\n",
      "miner                           O                       O\n",
      "be                              O                       O\n",
      "wear                            O                       O\n",
      "assign                          O                       O\n",
      "personal-protective-equipment-  O                       O\n",
      "possible                        O                       O\n",
      "factor                          O                       O\n",
      "include                         O                       O\n",
      "muscle                          B-injury/muscle         O\n",
      "fatigue                         O                       O\n",
      "from                            O                       O\n",
      "screen                          O                       O\n",
      "change                          O                       O\n",
      "inadequate                      O                       O\n",
      "rest                            O                       O\n",
      "period                          O                       O\n",
      "or                              O                       O\n",
      "stress                          O                       O\n",
      "of                              O                       O\n",
      "come                            O                       O\n",
      "erect                           O                       <END>\n",
      "from                            O                       <END>\n",
      "kneel                           O                       <END>\n",
      "<END>                           <END>                   <END>\n",
      "------------------------------  ----------------------  -----\n",
      "----------  ----------------------------------------  -----\n",
      "instal      O                                         O\n",
      "bit         O                                         O\n",
      "sleeve      O                                         O\n",
      "on          O                                         O\n",
      "continuous  B-vehicle/heavy_vehicle/continuous_miner  O\n",
      "miner       I-vehicle/heavy_vehicle/continuous_miner  O\n",
      "with        O                                         O\n",
      "two         O                                         O\n",
      "pound       O                                         O\n",
      "hammer      O                                         O\n",
      "when        O                                         O\n",
      "sliver      O                                         O\n",
      "of          O                                         O\n",
      "metal       O                                         O\n",
      "broke       O                                         O\n",
      "off         O                                         O\n",
      "of          O                                         O\n",
      "the         O                                         O\n",
      "sleeve      O                                         O\n",
      "go          O                                         O\n",
      "through     O                                         O\n",
      "mechanic    O                                         O\n",
      "glove       O                                         O\n",
      "and         O                                         O\n",
      "cut         O                                         O\n",
      "knuckle     O                                         O\n",
      "of          O                                         O\n",
      "left        O                                         O\n",
      "index       O                                         O\n",
      "finger      O                                         O\n",
      "stitch      O                                         O\n",
      "be          O                                         O\n",
      "require     O                                         O\n",
      "to          O                                         O\n",
      "close       O                                         O\n",
      "the         O                                         O\n",
      "wound       O                                         O\n",
      "he          O                                         O\n",
      "be          O                                         O\n",
      "release     O                                         O\n",
      "and         O                                         O\n",
      "return      O                                         O\n",
      "to          O                                         O\n",
      "work        O                                         O\n",
      "the         O                                         O\n",
      "next        O                                         O\n",
      "day         O                                         O\n",
      "<END>       <END>                                     <END>\n",
      "----------  ----------------------------------------  -----\n",
      "---------  --------------------  -\n",
      "three      O                     O\n",
      "miner      O                     O\n",
      "be         O                     O\n",
      "cleaning   O                     O\n",
      "up         O                     O\n",
      "one        O                     O\n",
      "be         O                     O\n",
      "operating  O                     O\n",
      "wheel      O                     O\n",
      "loader     O                     O\n",
      "and        O                     O\n",
      "the        O                     O\n",
      "other      O                     O\n",
      "two        O                     O\n",
      "be         O                     O\n",
      "throw      O                     O\n",
      "scrap      O                     O\n",
      "steel      O                     O\n",
      "into       O                     O\n",
      "the        O                     O\n",
      "bucket     O                     O\n",
      "to         O                     O\n",
      "be         O                     O\n",
      "take       O                     O\n",
      "to         O                     O\n",
      "dumpster   O                     O\n",
      "the        O                     O\n",
      "loader     O                     O\n",
      "operator   B-person/operator     O\n",
      "thought    O                     O\n",
      "one        O                     O\n",
      "of         O                     O\n",
      "the        O                     O\n",
      "other      O                     O\n",
      "miner      O                     O\n",
      "signal     O                     O\n",
      "operator   B-person/operator     O\n",
      "to         O                     O\n",
      "low        O                     O\n",
      "the        O                     O\n",
      "bucket     O                     O\n",
      "operator   B-person/operator     O\n",
      "then       O                     O\n",
      "lower      O                     O\n",
      "the        O                     O\n",
      "bucket     O                     O\n",
      "onto       O                     O\n",
      "one        O                     O\n",
      "of         O                     O\n",
      "the        O                     O\n",
      "miner      O                     O\n",
      "foot       B-body_part/leg/foot  O\n",
      "<END>      <END>                 O\n",
      "---------  --------------------  -\n",
      "------------  -------------------------------  -----\n",
      "fan           O                                O\n",
      "belt          O                                O\n",
      "be            O                                O\n",
      "instal        O                                O\n",
      "on            O                                O\n",
      "the           O                                O\n",
      "compressor    O                                O\n",
      "for           O                                O\n",
      "the           O                                O\n",
      "on            O                                O\n",
      "brookville    O                                O\n",
      "motor         O                                O\n",
      "the           O                                O\n",
      "fan           O                                O\n",
      "belt          O                                O\n",
      "have          O                                O\n",
      "twist         O                                O\n",
      "in            O                                O\n",
      "it            O                                O\n",
      "after         O                                O\n",
      "installation  O                                O\n",
      "the           O                                O\n",
      "victim        O                                O\n",
      "be            O                                O\n",
      "not           O                                O\n",
      "involve       O                                O\n",
      "in            O                                O\n",
      "the           O                                O\n",
      "installation  O                                O\n",
      "of            O                                O\n",
      "the           O                                O\n",
      "belt          O                                O\n",
      "but           O                                O\n",
      "want          O                                O\n",
      "to            O                                O\n",
      "fix           O                                O\n",
      "the           O                                O\n",
      "twist         O                                O\n",
      "at            O                                O\n",
      "this          O                                O\n",
      "time          O                                O\n",
      "the           O                                O\n",
      "motorman      O                                O\n",
      "set           O                                O\n",
      "the           O                                O\n",
      "breaker       O                                O\n",
      "on            O                                O\n",
      "the           O                                O\n",
      "motor         O                                O\n",
      "the           O                                O\n",
      "victim        O                                O\n",
      "reach         O                                O\n",
      "for           O                                O\n",
      "the           O                                O\n",
      "belt          O                                O\n",
      "a             O                                O\n",
      "the           O                                O\n",
      "breaker       O                                O\n",
      "be            O                                O\n",
      "set           O                                O\n",
      "compressor    O                                O\n",
      "start         O                                O\n",
      "hand          B-body_part/arm/hand             O\n",
      "be            O                                O\n",
      "caught        B-accident_cause/caught_between  O\n",
      "between       I-accident_cause/caught_between  O\n",
      "belt          O                                O\n",
      "and           O                                <END>\n",
      "pull          O                                <END>\n",
      "<END>         <END>                            <END>\n",
      "------------  -------------------------------  -----\n",
      "-------------------  ---------------------  ---------------------\n",
      "north                O                      O\n",
      "to                   O                      B-accident_cause/fall\n",
      "block                O                      O\n",
      "entry                O                      O\n",
      "mobile-mining-unit-  O                      O\n",
      "004.                 O                      O\n",
      "support              O                      O\n",
      "ft                   O                      O\n",
      "3/4                  O                      O\n",
      "inch                 O                      O\n",
      "resin                O                      O\n",
      "bolt                 O                      O\n",
      "pan                  O                      O\n",
      "brow                 O                      O\n",
      "strap                O                      O\n",
      "screen               O                      O\n",
      "17                   O                      O\n",
      "ft                   O                      O\n",
      "wide                 O                      O\n",
      "70                   O                      O\n",
      "ft                   O                      O\n",
      "long                 O                      O\n",
      "ft                   O                      O\n",
      "high                 O                      O\n",
      "fall                 B-accident_cause/fall  O\n",
      "dangered             O                      O\n",
      "off                  O                      O\n",
      "additional           O                      O\n",
      "roof                 O                      O\n",
      "support              O                      O\n",
      "instal               O                      O\n",
      "around               O                      O\n",
      "fall                 B-accident_cause/fall  O\n",
      "and                  O                      O\n",
      "clean                O                      O\n",
      "up                   O                      O\n",
      "plan                 O                      O\n",
      "post                 O                      O\n",
      "<END>                <END>                  <END>\n",
      "-------------------  ---------------------  ---------------------\n",
      "--------  --------------------------------  -----------------\n",
      "employee  B-person/employee                 B-person/employee\n",
      "have      O                                 O\n",
      "just      O                                 O\n",
      "park      O                                 O\n",
      "the       O                                 O\n",
      "tractor   O                                 O\n",
      "by        O                                 O\n",
      "the       O                                 O\n",
      "ore       O                                 O\n",
      "muck      O                                 O\n",
      "bay       O                                 O\n",
      "he        O                                 O\n",
      "step      O                                 O\n",
      "off       O                                 O\n",
      "of        O                                 O\n",
      "the       O                                 O\n",
      "tractor   O                                 O\n",
      "and       O                                 O\n",
      "his       O                                 O\n",
      "left      B-body_part/leg/foot/left_foot    O\n",
      "foot      I-body_part/leg/foot/left_foot    O\n",
      "come      O                                 O\n",
      "down      O                                 O\n",
      "on        O                                 O\n",
      "small     O                                 O\n",
      "rock      O                                 O\n",
      "he        O                                 O\n",
      "twist     O                                 O\n",
      "his       O                                 O\n",
      "left      B-body_part/leg/ankle/left_ankle  O\n",
      "ankle     I-body_part/leg/ankle/left_ankle  O\n",
      "<END>     <END>                             <END>\n",
      "--------  --------------------------------  -----------------\n",
      "----------  --------------------  -----------------\n",
      "employee    B-person/employee     B-person/employee\n",
      "be          O                     O\n",
      "change      O                     O\n",
      "feed        O                     O\n",
      "chute       B-equipment/chute     O\n",
      "rubber      O                     O\n",
      "skirt       O                     O\n",
      "while       O                     O\n",
      "reposition  O                     O\n",
      "himself     O                     O\n",
      "employee    B-person/employee     O\n",
      "grabbed     O                     O\n",
      "skirt       O                     O\n",
      "clamp       O                     O\n",
      "that        O                     O\n",
      "be          O                     O\n",
      "hang        O                     O\n",
      "knock       O                     O\n",
      "it          O                     O\n",
      "down        O                     O\n",
      "on          O                     O\n",
      "his         O                     O\n",
      "hand        B-body_part/arm/hand  O\n",
      "break       O                     O\n",
      "the         O                     O\n",
      "tip         O                     O\n",
      "of          O                     O\n",
      "his         O                     O\n",
      "finger      O                     O\n",
      "<END>       <END>                 <END>\n",
      "----------  --------------------  -----------------\n",
      "--------  ------------------------  -----------------\n",
      "employee  B-person/employee         B-person/employee\n",
      "remove    O                         O\n",
      "rock      O                         O\n",
      "stuck     O                         O\n",
      "between   O                         O\n",
      "the       O                         O\n",
      "tire      O                         O\n",
      "of        O                         O\n",
      "water     O                         O\n",
      "truck     O                         O\n",
      "and       O                         O\n",
      "strain    O                         O\n",
      "his       O                         O\n",
      "shoulder  B-body_part/arm/shoulder  O\n",
      "torn      O                         O\n",
      "right     O                         O\n",
      "bicep     O                         O\n",
      "be        O                         O\n",
      "diagnose  O                         O\n",
      "<END>     <END>                     <END>\n",
      "--------  ------------------------  -----------------\n",
      "--------  -----------------------------  -----------------\n",
      "employee  B-person/employee              B-person/employee\n",
      "have      O                              O\n",
      "change    O                              O\n",
      "light     O                              O\n",
      "on        O                              O\n",
      "cab       O                              O\n",
      "of        O                              O\n",
      "d10n      O                              O\n",
      "dozer     B-vehicle/heavy_vehicle/dozer  O\n",
      "a         O                              O\n",
      "he        O                              O\n",
      "change    O                              O\n",
      "body      B-body_part/body               O\n",
      "position  O                              O\n",
      "he        O                              O\n",
      "either    O                              O\n",
      "slip      O                              O\n",
      "or        O                              O\n",
      "tripped   O                              O\n",
      "and       O                              O\n",
      "fell      O                              O\n",
      "off       O                              O\n",
      "dozer     B-vehicle/heavy_vehicle/dozer  O\n",
      "to        O                              O\n",
      "the       O                              O\n",
      "ground    O                              <END>\n",
      "<END>     <END>                          <END>\n",
      "--------  -----------------------------  -----------------\n",
      "--------  ---------------------------  -----------------\n",
      "employee  B-person/employee            B-person/employee\n",
      "drive     B-activity/driving           O\n",
      "rock      O                            O\n",
      "truck     O                            O\n",
      "up        O                            O\n",
      "out       O                            O\n",
      "of        O                            O\n",
      "steep     O                            O\n",
      "pit       O                            O\n",
      "when      O                            O\n",
      "the       O                            O\n",
      "load      O                            O\n",
      "shift     O                            O\n",
      "to        O                            O\n",
      "the       O                            O\n",
      "rear      O                            O\n",
      "cause     O                            O\n",
      "the       O                            O\n",
      "truck     O                            O\n",
      "front     O                            O\n",
      "end       O                            O\n",
      "to        O                            O\n",
      "raise     O                            O\n",
      "up        O                            O\n",
      "4-5       O                            O\n",
      "foot      O                            O\n",
      "and       O                            O\n",
      "then      O                            O\n",
      "slam      O                            O\n",
      "down      O                            O\n",
      "jarring   O                            O\n",
      "the       O                            O\n",
      "truck     O                            O\n",
      "cause     O                            O\n",
      "pain      O                            O\n",
      "in        O                            O\n",
      "low       B-body_part/back/lower_back  O\n",
      "back      I-body_part/back/lower_back  <END>\n",
      "<END>     <END>                        <END>\n",
      "--------  ---------------------------  -----------------\n",
      "--------  -----  -\n",
      "highwall  O      O\n",
      "failure   O      O\n",
      "measure   O      O\n",
      "40        O      O\n",
      "high      O      O\n",
      "thick     O      O\n",
      "30        O      O\n",
      "wide      O      O\n",
      "<END>     <END>  O\n",
      "--------  -----  -\n",
      "--------  ------------------------  -----------------\n",
      "employee  B-person/employee         B-person/employee\n",
      "be        O                         O\n",
      "injured   O                         O\n",
      "on        O                         O\n",
      "7/27/06   O                         O\n",
      "he        O                         O\n",
      "return    O                         O\n",
      "to        O                         O\n",
      "work      O                         O\n",
      "on        O                         O\n",
      "8/9/06    O                         O\n",
      "on        O                         O\n",
      "regular   O                         O\n",
      "duty      O                         O\n",
      "he        O                         O\n",
      "state     O                         O\n",
      "that      O                         O\n",
      "his       O                         O\n",
      "shoulder  B-body_part/arm/shoulder  O\n",
      "start     O                         O\n",
      "hurt      O                         O\n",
      "again     O                         O\n",
      "on        O                         O\n",
      "8/25/06   O                         <END>\n",
      "<END>     <END>                     <END>\n",
      "--------  ------------------------  -----------------\n",
      "----------  ---------------------------------  -----------------\n",
      "employee    B-person/employee                  B-person/employee\n",
      "be          O                                  O\n",
      "enlarge     O                                  O\n",
      "pilot       O                                  O\n",
      "hole        O                                  O\n",
      "in          O                                  O\n",
      "guard       O                                  O\n",
      "he          O                                  O\n",
      "be          O                                  O\n",
      "building    O                                  O\n",
      "with        O                                  O\n",
      "electric    O                                  O\n",
      "drill       O                                  O\n",
      "when        O                                  O\n",
      "bit         O                                  O\n",
      "caught      O                                  O\n",
      "and         O                                  O\n",
      "force       O                                  O\n",
      "drill       O                                  O\n",
      "from        O                                  O\n",
      "his         O                                  O\n",
      "hand        B-body_part/arm/hand               O\n",
      "the         O                                  O\n",
      "drill       O                                  O\n",
      "spun        O                                  O\n",
      "around      O                                  O\n",
      "and         O                                  O\n",
      "the         O                                  O\n",
      "handle      O                                  O\n",
      "struck      O                                  O\n",
      "his         O                                  O\n",
      "right       B-body_part/arm/wrist/right_wrist  O\n",
      "wrist       I-body_part/arm/wrist/right_wrist  O\n",
      "he          O                                  O\n",
      "broke       O                                  O\n",
      "his         O                                  O\n",
      "4th         O                                  O\n",
      "metatarsal  O                                  O\n",
      "bone        O                                  O\n",
      "in          O                                  O\n",
      "his         O                                  O\n",
      "right       B-body_part/arm/wrist/right_wrist  O\n",
      "wrist       I-body_part/arm/wrist/right_wrist  <END>\n",
      "<END>       <END>                              <END>\n",
      "----------  ---------------------------------  -----------------\n",
      "--------  -----------------  -\n",
      "bolter    O                  O\n",
      "operator  B-person/operator  O\n",
      "be        O                  O\n",
      "bolting   O                  O\n",
      "roof      O                  O\n",
      "in        O                  O\n",
      "number    O                  O\n",
      "entry     O                  O\n",
      "on        O                  O\n",
      "1st       O                  O\n",
      "right     O                  O\n",
      "section   O                  O\n",
      "top       O                  O\n",
      "be        O                  O\n",
      "real      O                  O\n",
      "hard      O                  O\n",
      "and       O                  O\n",
      "bit       O                  O\n",
      "tore      O                  O\n",
      "up        O                  O\n",
      "he        O                  O\n",
      "be        O                  O\n",
      "knock     O                  O\n",
      "the       O                  O\n",
      "bit       O                  O\n",
      "off       O                  O\n",
      "the       O                  O\n",
      "drill     O                  O\n",
      "steel     O                  O\n",
      "to        O                  O\n",
      "change    O                  O\n",
      "it        O                  O\n",
      "when      O                  O\n",
      "he        O                  O\n",
      "struck    O                  O\n",
      "his       O                  O\n",
      "left      O                  O\n",
      "index     O                  O\n",
      "finger    O                  O\n",
      "between   O                  O\n",
      "bolter    O                  O\n",
      "and       O                  O\n",
      "drill     O                  O\n",
      "steel     O                  O\n",
      "bruising  O                  O\n",
      "finger    O                  O\n",
      "he        O                  O\n",
      "be        O                  O\n",
      "off       O                  O\n",
      "day       O                  O\n",
      "<END>     <END>              O\n",
      "--------  -----------------  -\n",
      "----------  ----------------------------------------  -----------------\n",
      "on          O                                         O\n",
      "november    O                                         B-person/employee\n",
      "21          O                                         O\n",
      "02          O                                         O\n",
      "the         O                                         O\n",
      "employee    B-person/employee                         O\n",
      "be          O                                         O\n",
      "told        O                                         O\n",
      "he          O                                         O\n",
      "be          O                                         O\n",
      "go          O                                         O\n",
      "to          O                                         O\n",
      "be          O                                         O\n",
      "laid        O                                         O\n",
      "off         O                                         O\n",
      "on          O                                         O\n",
      "11/22/02    O                                         O\n",
      "he          O                                         O\n",
      "then        O                                         O\n",
      "state       O                                         O\n",
      "that        O                                         O\n",
      "he          O                                         O\n",
      "have        O                                         O\n",
      "injured     O                                         O\n",
      "his         O                                         O\n",
      "shoulder    B-body_part/arm/shoulder                  O\n",
      "shovel      O                                         O\n",
      "rock        O                                         O\n",
      "at          O                                         O\n",
      "the         O                                         O\n",
      "crusher     B-equipment/mechanical_equipment/crusher  O\n",
      "on          O                                         O\n",
      "10/31/02    O                                         O\n",
      "and         O                                         O\n",
      "have        O                                         O\n",
      "not         O                                         O\n",
      "report      O                                         O\n",
      "it          O                                         <END>\n",
      "previously  O                                         <END>\n",
      "<END>       <END>                                     <END>\n",
      "----------  ----------------------------------------  -----------------\n",
      "------------  -----------------  -----------------\n",
      "employee      B-person/employee  B-person/employee\n",
      "be            O                  O\n",
      "cleaning      O                  O\n",
      "breakroom     O                  O\n",
      "which         O                  O\n",
      "involve       O                  O\n",
      "move          O                  O\n",
      "furniture     O                  O\n",
      "to            O                  O\n",
      "clean         O                  O\n",
      "behind        O                  O\n",
      "it            O                  O\n",
      "when          O                  O\n",
      "the           O                  O\n",
      "refrigerator  O                  O\n",
      "be            O                  O\n",
      "move          O                  O\n",
      "the           O                  O\n",
      "employee      B-person/employee  O\n",
      "felt          O                  O\n",
      "pain          O                  O\n",
      "in            O                  O\n",
      "the           O                  O\n",
      "middle        O                  O\n",
      "right         O                  O\n",
      "side          O                  O\n",
      "of            O                  O\n",
      "her           O                  O\n",
      "back          B-body_part/back   O\n",
      "<END>         <END>              <END>\n",
      "------------  -----------------  -----------------\n",
      "-------------  -----------------  -----\n",
      "injured        O                  O\n",
      "be             O                  O\n",
      "use            O                  O\n",
      "wash           O                  O\n",
      "down           O                  O\n",
      "hose           O                  O\n",
      "to             O                  O\n",
      "clean          O                  O\n",
      "on             O                  O\n",
      "and            O                  O\n",
      "around         O                  O\n",
      "the            O                  O\n",
      "east           O                  O\n",
      "number-        O                  O\n",
      "belt           O                  O\n",
      "drive          O                  O\n",
      "after          O                  O\n",
      "employee's     O                  O\n",
      "work           O                  O\n",
      "be             O                  O\n",
      "complete       O                  O\n",
      "employee       B-person/employee  O\n",
      "be             O                  O\n",
      "go             O                  O\n",
      "to             O                  O\n",
      "cut            O                  O\n",
      "the            O                  O\n",
      "wash           O                  O\n",
      "down           O                  O\n",
      "hose           O                  O\n",
      "off            O                  O\n",
      "the            O                  O\n",
      "injured        O                  O\n",
      "slipped/fell   O                  O\n",
      "while          O                  O\n",
      "cut            O                  O\n",
      "the            O                  O\n",
      "handle         O                  O\n",
      "off            O                  O\n",
      "on             O                  O\n",
      "the            O                  O\n",
      "hose           O                  O\n",
      "employee       B-person/employee  O\n",
      "then           O                  O\n",
      "inadvertently  O                  O\n",
      "pressurize     O                  O\n",
      "the            O                  O\n",
      "hose           O                  O\n",
      "the            O                  O\n",
      "hose           O                  O\n",
      "come           O                  O\n",
      "loose          O                  O\n",
      "from           O                  O\n",
      "employee's     O                  O\n",
      "hand           O                  O\n",
      "and            O                  O\n",
      "struck         O                  O\n",
      "employee       B-person/employee  O\n",
      "in             O                  O\n",
      "the            O                  O\n",
      "left           O                  O\n",
      "side           O                  O\n",
      "of             O                  O\n",
      "employee's     O                  O\n",
      "forehead       O                  O\n",
      "near           O                  O\n",
      "hairline       O                  O\n",
      "cause          O                  O\n",
      "cut            O                  <END>\n",
      "<END>          <END>              <END>\n",
      "-------------  -----------------  -----\n",
      "----------------  ----------------------------  -----------------\n",
      "employee          B-person/employee             B-person/employee\n",
      "attempt           O                             O\n",
      "to                O                             O\n",
      "reinstall         O                             O\n",
      "an                O                             O\n",
      "air-conditioner-  O                             O\n",
      "vent              O                             O\n",
      "louver            O                             O\n",
      "employee          B-person/employee             O\n",
      "place             O                             O\n",
      "finger            B-body_part/arm/hand/fingers  O\n",
      "beside            O                             O\n",
      "and               O                             O\n",
      "over              O                             O\n",
      "the               O                             O\n",
      "edge              O                             O\n",
      "of                O                             O\n",
      "the               O                             O\n",
      "louver            O                             O\n",
      "try               O                             O\n",
      "to                O                             O\n",
      "hold              O                             O\n",
      "the               O                             O\n",
      "screw             O                             O\n",
      "so                O                             O\n",
      "employee          B-person/employee             O\n",
      "could             O                             O\n",
      "screw             O                             O\n",
      "the               O                             O\n",
      "louver            O                             O\n",
      "back              B-body_part/back              O\n",
      "on                O                             O\n",
      "the               O                             O\n",
      "louver            O                             O\n",
      "abruptly          O                             O\n",
      "turn              O                             O\n",
      "result            O                             O\n",
      "in                O                             O\n",
      "right             O                             O\n",
      "middle            O                             O\n",
      "finger            O                             O\n",
      "and               O                             O\n",
      "right             O                             O\n",
      "ring              O                             O\n",
      "finger            O                             O\n",
      "to                O                             O\n",
      "make              O                             O\n",
      "contact           O                             O\n",
      "with              O                             O\n",
      "the               O                             O\n",
      "squirrel          O                             O\n",
      "cage              B-equipment/cage              O\n",
      "fan               O                             O\n",
      "employee          B-person/employee             O\n",
      "suffer            O                             O\n",
      "laceration        B-injury/laceration           O\n",
      "and               O                             O\n",
      "receive           O                             O\n",
      "stitch            O                             O\n",
      "to                O                             O\n",
      "the               O                             O\n",
      "right             O                             O\n",
      "3rd               O                             O\n",
      "finger            O                             <END>\n",
      "<END>             <END>                         <END>\n",
      "----------------  ----------------------------  -----------------\n",
      "---------  ---------------------------  -----------------\n",
      "employee   B-person/employee            B-person/employee\n",
      "be         O                            O\n",
      "drive      B-activity/driving           O\n",
      "cu106      O                            O\n",
      "boom       O                            O\n",
      "rig        O                            O\n",
      "around     O                            O\n",
      "rib        O                            O\n",
      "at         O                            O\n",
      "main       O                            O\n",
      "shop       O                            O\n",
      "he         O                            O\n",
      "hit        O                            O\n",
      "pothole    O                            O\n",
      "which      O                            O\n",
      "cause      O                            O\n",
      "the        O                            O\n",
      "air        O                            O\n",
      "ride       O                            O\n",
      "seat       O                            O\n",
      "to         O                            O\n",
      "hit        O                            O\n",
      "bottom     O                            O\n",
      "and        O                            O\n",
      "jar        O                            O\n",
      "his        O                            O\n",
      "low        B-body_part/back/lower_back  O\n",
      "back       I-body_part/back/lower_back  O\n",
      "he         O                            O\n",
      "say        O                            O\n",
      "he         O                            O\n",
      "felt       O                            O\n",
      "no         O                            O\n",
      "immediate  O                            O\n",
      "pain       O                            O\n",
      "later      O                            O\n",
      "in         O                            O\n",
      "the        O                            O\n",
      "day        O                            O\n",
      "the        O                            O\n",
      "pain       O                            O\n",
      "increase   O                            O\n",
      "when       O                            O\n",
      "he         O                            O\n",
      "go         O                            O\n",
      "home       O                            O\n",
      "for        O                            O\n",
      "the        O                            O\n",
      "weekend    O                            O\n",
      "he         O                            O\n",
      "start      O                            O\n",
      "take       O                            O\n",
      "oxycontin  O                            O\n",
      "injury     B-injury                     O\n",
      "report     O                            O\n",
      "on         O                            O\n",
      "1/10/11    O                            O\n",
      "<END>      <END>                        <END>\n",
      "---------  ---------------------------  -----------------\n",
      "--------  --------------------------------------  -----------------\n",
      "employee  B-person/employee                       B-person/employee\n",
      "be        O                                       O\n",
      "use       O                                       O\n",
      "prybar    O                                       O\n",
      "to        O                                       O\n",
      "move      O                                       O\n",
      "heavy     O                                       O\n",
      "object    O                                       O\n",
      "when      O                                       O\n",
      "the       O                                       O\n",
      "prybar    O                                       O\n",
      "slip      O                                       O\n",
      "and       O                                       O\n",
      "pinch     O                                       O\n",
      "his       O                                       O\n",
      "right     B-body_part/arm/hand/thumb/right_thumb  O\n",
      "thumb     I-body_part/arm/hand/thumb/right_thumb  O\n",
      "against   O                                       O\n",
      "concrete  O                                       O\n",
      "wall      O                                       O\n",
      "directly  O                                       O\n",
      "behind    O                                       O\n",
      "him       O                                       <END>\n",
      "<END>     <END>                                   <END>\n",
      "--------  --------------------------------------  -----------------\n",
      "------------  ---------------------------------------  -----------------\n",
      "employee      B-person/employee                        B-person/employee\n",
      "board         O                                        O\n",
      "rubber        O                                        O\n",
      "tire          O                                        O\n",
      "dozer         B-vehicle/heavy_vehicle/dozer            O\n",
      "without       O                                        O\n",
      "incident      O                                        O\n",
      "at-start-     O                                        O\n",
      "of            O                                        O\n",
      "shift         O                                        O\n",
      "during        O                                        O\n",
      "shift         O                                        O\n",
      "experienced   O                                        O\n",
      "pain          O                                        O\n",
      "in-area-      O                                        O\n",
      "of            O                                        O\n",
      "right         B-body_part/arm/shoulder/right_shoulder  O\n",
      "shoulder      I-body_part/arm/shoulder/right_shoulder  O\n",
      "blade         O                                        O\n",
      "saw           O                                        O\n",
      "physician     O                                        O\n",
      "on            O                                        O\n",
      "2-5           O                                        O\n",
      "be            O                                        O\n",
      "diagnose      O                                        O\n",
      "with          O                                        O\n",
      "right         O                                        O\n",
      "shoulder      B-body_part/arm/shoulder                 O\n",
      "tendonitis    O                                        O\n",
      "be            O                                        O\n",
      "take          O                                        O\n",
      "off           O                                        O\n",
      "work          O                                        O\n",
      "on            O                                        O\n",
      "2-20-03.      O                                        O\n",
      "receive       O                                        O\n",
      "chart         O                                        O\n",
      "note          O                                        O\n",
      "on            O                                        O\n",
      "3-21-03.      O                                        O\n",
      "lose          O                                        O\n",
      "time          O                                        O\n",
      "restrict      O                                        O\n",
      "work          O                                        O\n",
      "occupational  O                                        <END>\n",
      "illness       O                                        <END>\n",
      "<END>         <END>                                    <END>\n",
      "------------  ---------------------------------------  -----------------\n",
      "--------  -----------------------------------------  ------------------\n",
      "employee  B-person/employee                          B-person/employee\n",
      "walk      B-activity/walking                         O\n",
      "along     O                                          B-activity/walking\n",
      "flat      O                                          O\n",
      "surface   O                                          O\n",
      "number    O                                          O\n",
      "101       O                                          O\n",
      "conveyor  B-equipment/mechanical_equipment/conveyor  O\n",
      "catwalk   O                                          O\n",
      "step      O                                          O\n",
      "on        O                                          O\n",
      "fist      O                                          O\n",
      "size      O                                          O\n",
      "rock      O                                          O\n",
      "twist     O                                          O\n",
      "left      B-body_part/leg/knee/left_knee             O\n",
      "knee      I-body_part/leg/knee/left_knee             O\n",
      "fell      O                                          O\n",
      "down      O                                          O\n",
      "come      O                                          O\n",
      "down      O                                          O\n",
      "on        O                                          O\n",
      "left      B-body_part/leg/knee/left_knee             O\n",
      "knee      I-body_part/leg/knee/left_knee             <END>\n",
      "<END>     <END>                                      <END>\n",
      "--------  -----------------------------------------  ------------------\n",
      "--------  ------------------------------  -\n",
      "he        O                               O\n",
      "be        O                               O\n",
      "shovel    O                               O\n",
      "at        O                               O\n",
      "the       O                               O\n",
      "headgate  O                               O\n",
      "drive     O                               O\n",
      "area      O                               O\n",
      "on        O                               O\n",
      "the       O                               O\n",
      "longwall  O                               O\n",
      "face      O                               O\n",
      "he        O                               O\n",
      "stood     O                               O\n",
      "up        O                               O\n",
      "and       O                               O\n",
      "cut       O                               O\n",
      "his       O                               O\n",
      "right     B-body_part/head/ear/right_ear  O\n",
      "ear       I-body_part/head/ear/right_ear  O\n",
      "on        O                               O\n",
      "roof      O                               O\n",
      "strap     O                               O\n",
      "the       O                               O\n",
      "result    O                               O\n",
      "be        O                               O\n",
      "suture    O                               O\n",
      "<END>     <END>                           O\n",
      "--------  ------------------------------  -\n",
      "-------------  ---------------------  ---------------------\n",
      "an             O                      O\n",
      "unplanned      O                      B-accident_cause/fall\n",
      "roof           O                      O\n",
      "fall           B-accident_cause/fall  O\n",
      "have           O                      O\n",
      "occur          O                      O\n",
      "above          O                      O\n",
      "permanent      O                      O\n",
      "anchorage      O                      O\n",
      "on             O                      O\n",
      "the            O                      O\n",
      "11             O                      O\n",
      "north          O                      O\n",
      "miner          O                      O\n",
      "section        O                      O\n",
      "this           O                      O\n",
      "fall           B-accident_cause/fall  O\n",
      "exists         O                      O\n",
      "in             O                      O\n",
      "the            O                      O\n",
      "number         O                      O\n",
      "belt           O                      O\n",
      "entry          O                      O\n",
      "one            O                      O\n",
      "cross-cut      O                      O\n",
      "outby          O                      O\n",
      "spad           O                      O\n",
      "number         O                      O\n",
      "566            O                      O\n",
      "and            O                      O\n",
      "measure        O                      O\n",
      "approximately  O                      O\n",
      "50'-           O                      O\n",
      "20             O                      O\n",
      "''             O                      O\n",
      "10             O                      O\n",
      "high           O                      O\n",
      "<END>          <END>                  O\n",
      "-------------  ---------------------  ---------------------\n",
      "------------  --------------------  -----\n",
      "maintenance   O                     O\n",
      "worker        B-person/worker       O\n",
      "have          O                     O\n",
      "be            O                     O\n",
      "cut           O                     O\n",
      "weld          O                     O\n",
      "and           O                     O\n",
      "preform       O                     O\n",
      "regular       O                     O\n",
      "maintenance   O                     O\n",
      "duty          O                     O\n",
      "throughout    O                     O\n",
      "the           O                     O\n",
      "day           O                     O\n",
      "employee      B-person/employee     O\n",
      "be            O                     O\n",
      "perspire      O                     O\n",
      "and           O                     O\n",
      "use           O                     O\n",
      "sleeve        O                     O\n",
      "to            O                     O\n",
      "wipe          O                     O\n",
      "perspiration  O                     O\n",
      "from          O                     O\n",
      "cheek         O                     O\n",
      "few           O                     O\n",
      "minute        O                     O\n",
      "later         O                     O\n",
      "employee      B-person/employee     O\n",
      "felt          O                     O\n",
      "like          O                     O\n",
      "eye           B-body_part/head/eye  O\n",
      "have          O                     O\n",
      "something     O                     O\n",
      "in            O                     O\n",
      "it            O                     O\n",
      "employee      B-person/employee     O\n",
      "wash          O                     O\n",
      "eye           B-body_part/head/eye  O\n",
      "out           O                     O\n",
      "but           O                     O\n",
      "eye           B-body_part/head/eye  O\n",
      "be            O                     O\n",
      "still         O                     O\n",
      "irritated     O                     O\n",
      "take          O                     O\n",
      "employee      B-person/employee     O\n",
      "to            O                     O\n",
      "doctor        O                     O\n",
      "eye           B-body_part/head/eye  O\n",
      "be            O                     O\n",
      "scratch       O                     O\n",
      "prescription  O                     O\n",
      "give          O                     O\n",
      "to            O                     O\n",
      "prevent       O                     O\n",
      "infection     O                     O\n",
      "<END>         <END>                 <END>\n",
      "------------  --------------------  -----\n",
      "--------  -------------------------------  -----------------\n",
      "employee  B-person/employee                B-person/employee\n",
      "be        O                                O\n",
      "building  O                                O\n",
      "stop      O                                O\n",
      "when      O                                O\n",
      "he        O                                O\n",
      "mash      O                                O\n",
      "finger    O                                O\n",
      "on        O                                O\n",
      "right     B-body_part/arm/hand/right_hand  O\n",
      "hand      I-body_part/arm/hand/right_hand  O\n",
      "<END>     <END>                            <END>\n",
      "--------  -------------------------------  -----------------\n",
      "---------  ------------------------  -----------------\n",
      "employee   B-person/employee         B-person/employee\n",
      "be         O                         O\n",
      "attend     O                         O\n",
      "annual     O                         O\n",
      "safety     O                         O\n",
      "retrain    O                         O\n",
      "during     O                         O\n",
      "lunch      O                         O\n",
      "break      B-injury/fracture/break   O\n",
      "two        O                         O\n",
      "employee   O                         O\n",
      "be         O                         O\n",
      "``         O                         O\n",
      "horse      O                         O\n",
      "play       O                         O\n",
      "''         O                         O\n",
      "employee   B-person/employee         O\n",
      "attempt    O                         O\n",
      "to         O                         O\n",
      "pick       O                         O\n",
      "up         O                         O\n",
      "another    O                         O\n",
      "employee   B-person/employee         O\n",
      "and        O                         O\n",
      "throw      O                         O\n",
      "him        O                         O\n",
      "over       O                         O\n",
      "his        O                         O\n",
      "shoulder   B-body_part/arm/shoulder  O\n",
      "both       O                         O\n",
      "employee   O                         O\n",
      "fell       O                         O\n",
      "to         O                         O\n",
      "the        O                         O\n",
      "ground     O                         O\n",
      "cause      O                         O\n",
      "employee   B-person/employee         O\n",
      "to         O                         O\n",
      "dislocate  O                         O\n",
      "his        O                         O\n",
      "hip        O                         <END>\n",
      "<END>      <END>                     <END>\n",
      "---------  ------------------------  -----------------\n",
      "--------------------------------------  ---------------------  ---------------------\n",
      "roof                                    O                      O\n",
      "fall                                    B-accident_cause/fall  B-accident_cause/fall\n",
      "be                                      O                      O\n",
      "discover                                O                      O\n",
      "in                                      O                      O\n",
      "the                                     O                      O\n",
      "primary                                 O                      O\n",
      "escape                                  O                      O\n",
      "way                                     O                      O\n",
      "cross-cuts                              O                      O\n",
      "inby                                    O                      O\n",
      "number                                  O                      O\n",
      "belt                                    O                      O\n",
      "drive                                   O                      O\n",
      "at                                      O                      O\n",
      "spad                                    O                      O\n",
      "number                                  O                      O\n",
      "1203.                                   O                      O\n",
      "the                                     O                      O\n",
      "fall                                    B-accident_cause/fall  O\n",
      "be                                      O                      O\n",
      "approximately                           O                      O\n",
      "30                                      O                      O\n",
      "long                                    O                      O\n",
      "19                                      O                      O\n",
      "wide-and-                               O                      O\n",
      "have                                    O                      O\n",
      "fall                                    O                      O\n",
      "above                                   O                      O\n",
      "permanent                               O                      O\n",
      "roof                                    O                      O\n",
      "support                                 O                      O\n",
      "the                                     O                      O\n",
      "fall                                    B-accident_cause/fall  O\n",
      "be                                      O                      O\n",
      "discover                                O                      O\n",
      "and                                     O                      O\n",
      "report                                  O                      O\n",
      "to                                      O                      O\n",
      "mine-safety-and-health-administration-  O                      O\n",
      "<END>                                   <END>                  O\n",
      "--------------------------------------  ---------------------  ---------------------\n",
      "-------------  -----------------  -\n",
      "the            O                  O\n",
      "number         O                  O\n",
      "rock           O                  O\n",
      "come           O                  O\n",
      "from           O                  O\n",
      "the            O                  O\n",
      "shot           O                  O\n",
      "in             O                  O\n",
      "big            O                  O\n",
      "piece          O                  O\n",
      "we             O                  O\n",
      "be             O                  O\n",
      "building       O                  O\n",
      "ramp           O                  O\n",
      "with           O                  O\n",
      "this           O                  O\n",
      "material       O                  O\n",
      "without        O                  O\n",
      "any            O                  O\n",
      "problem        O                  O\n",
      "it             O                  O\n",
      "start          O                  O\n",
      "to             O                  O\n",
      "rain           O                  O\n",
      "make           O                  O\n",
      "the            O                  O\n",
      "boulder        O                  O\n",
      "slippery       O                  O\n",
      "employee       B-person/employee  O\n",
      "climbed        O                  O\n",
      "his            O                  O\n",
      "front          O                  O\n",
      "tire           O                  O\n",
      "up             O                  O\n",
      "on             O                  O\n",
      "boulder        O                  O\n",
      "and            O                  O\n",
      "it             O                  O\n",
      "slip           O                  O\n",
      "off            O                  O\n",
      "the            O                  O\n",
      "other          O                  O\n",
      "side           O                  O\n",
      "bring          O                  O\n",
      "the            O                  O\n",
      "truck          O                  O\n",
      "down           O                  O\n",
      "fast           O                  O\n",
      "and            O                  O\n",
      "hard           O                  O\n",
      "jarring        O                  O\n",
      "and            O                  O\n",
      "rock           O                  O\n",
      "him            O                  O\n",
      "suddenly       O                  O\n",
      "this           O                  O\n",
      "motion         O                  O\n",
      "be             O                  O\n",
      "what           O                  O\n",
      "be             O                  O\n",
      "suspect        O                  O\n",
      "to             O                  O\n",
      "have           O                  O\n",
      "cause          O                  O\n",
      "the            O                  O\n",
      "cervical       O                  O\n",
      "radiculopathy  O                  O\n",
      "<END>          <END>              O\n",
      "-------------  -----------------  -\n",
      "---------  ------------------------------  -----\n",
      "driver     B-person/driver                 O\n",
      "and        O                               O\n",
      "passenger  O                               O\n",
      "where      O                               O\n",
      "drive      B-activity/driving              O\n",
      "down       O                               O\n",
      "haul       O                               O\n",
      "road       O                               O\n",
      "to         O                               O\n",
      "pete       O                               O\n",
      "bajo       O                               O\n",
      "pit        O                               O\n",
      "from       O                               O\n",
      "leeville   O                               O\n",
      "when       O                               O\n",
      "try        O                               O\n",
      "to         O                               O\n",
      "make       O                               O\n",
      "left       B-body_part/arm/hand/left_hand  O\n",
      "hand       I-body_part/arm/hand/left_hand  O\n",
      "turn       O                               O\n",
      "the        O                               O\n",
      "truck      O                               O\n",
      "roll       O                               O\n",
      "onto       O                               O\n",
      "the        O                               O\n",
      "passenger  O                               <END>\n",
      "side       O                               <END>\n",
      "<END>      <END>                           <END>\n",
      "---------  ------------------------------  -----\n",
      "--------------  ---------------------------  -----------------\n",
      "employee        B-person/employee            B-person/employee\n",
      "state           O                            O\n",
      "while           O                            O\n",
      "loosening       O                            O\n",
      "pack            O                            O\n",
      "gland           O                            O\n",
      "on              O                            O\n",
      "shuttle         O                            O\n",
      "car             B-vehicle/light_vehicle/car  O\n",
      "during          O                            O\n",
      "repair          O                            O\n",
      "struck          O                            O\n",
      "hand            B-body_part/arm/hand         O\n",
      "against-sharp-  O                            O\n",
      "metal           O                            O\n",
      "<END>           <END>                        <END>\n",
      "--------------  ---------------------------  -----------------\n",
      "----------  -------------------  -----------------\n",
      "employee    B-person/employee    B-person/employee\n",
      "left        O                    O\n",
      "the         O                    O\n",
      "section     O                    O\n",
      "and         O                    O\n",
      "go          O                    O\n",
      "outside     O                    O\n",
      "without     O                    O\n",
      "tell        O                    O\n",
      "his         O                    O\n",
      "supervisor  B-person/supervisor  O\n",
      "supervisor  B-person/supervisor  O\n",
      "call        O                    O\n",
      "outside     O                    O\n",
      "and         O                    O\n",
      "told        O                    O\n",
      "employee    B-person/employee    O\n",
      "to          O                    O\n",
      "return      O                    O\n",
      "to          O                    O\n",
      "the         O                    O\n",
      "section     O                    O\n",
      "or          O                    O\n",
      "he          O                    O\n",
      "consider    O                    O\n",
      "him         O                    O\n",
      "quit        O                    O\n",
      "for         O                    O\n",
      "walk        B-activity/walking   O\n",
      "off         O                    O\n",
      "the         O                    O\n",
      "job         O                    O\n",
      "employee    B-person/employee    O\n",
      "left        O                    O\n",
      "mine        O                    O\n",
      "site        O                    O\n",
      "and         O                    O\n",
      "go          O                    O\n",
      "home        O                    O\n",
      "after       O                    O\n",
      "he          O                    O\n",
      "get         O                    O\n",
      "home        O                    O\n",
      "employee    B-person/employee    O\n",
      "call        O                    O\n",
      "back        B-body_part/back     O\n",
      "to          O                    O\n",
      "the         O                    O\n",
      "mine        O                    O\n",
      "and         O                    O\n",
      "told        O                    O\n",
      "the         O                    O\n",
      "guard       O                    O\n",
      "that        O                    O\n",
      "he          O                    O\n",
      "hurt        O                    O\n",
      "his         O                    O\n",
      "back        B-body_part/back     O\n",
      "by          O                    O\n",
      "pull        O                    O\n",
      "piece       O                    O\n",
      "of          O                    O\n",
      "brattice    O                    O\n",
      "cloth       O                    O\n",
      "<END>       <END>                O\n",
      "----------  -------------------  -----------------\n",
      "--------------------------------------  ------------------  -----\n",
      "number                                  O                   O\n",
      "34                                      O                   O\n",
      "service                                 O                   O\n",
      "hoist                                   O                   O\n",
      "go                                      O                   O\n",
      "down                                    O                   O\n",
      "at                                      O                   O\n",
      "6:55                                    O                   O\n",
      "be                                      O                   O\n",
      "contact                                 O                   O\n",
      "electrician                             O                   O\n",
      "at                                      O                   O\n",
      "6:55                                    O                   O\n",
      "be                                      O                   O\n",
      "call                                    O                   O\n",
      "mine-safety-and-health-administration-  O                   O\n",
      "at                                      O                   O\n",
      "7:25                                    O                   O\n",
      "be                                      O                   O\n",
      "electrician                             O                   O\n",
      "found                                   O                   O\n",
      "bad                                     O                   O\n",
      "contact                                 O                   O\n",
      "hoist                                   O                   O\n",
      "be                                      O                   O\n",
      "back                                    B-body_part/back    O\n",
      "up                                      O                   O\n",
      "and                                     O                   O\n",
      "run                                     B-activity/running  O\n",
      "again                                   O                   O\n",
      "at                                      O                   O\n",
      "10:20                                   O                   O\n",
      "be                                      O                   O\n",
      "call                                    O                   O\n",
      "mine-safety-and-health-administration-  O                   O\n",
      "at                                      O                   O\n",
      "10:36                                   O                   O\n",
      "to                                      O                   O\n",
      "report                                  O                   O\n",
      "hoist                                   O                   <END>\n",
      "be                                      O                   <END>\n",
      "in                                      O                   <END>\n",
      "service                                 O                   <END>\n",
      "<END>                                   <END>               <END>\n",
      "--------------------------------------  ------------------  -----\n",
      "------------  -----------------  -\n",
      "negligence    O                  O\n",
      "of            O                  O\n",
      "employee-he-  O                  O\n",
      "be            O                  O\n",
      "stand         O                  O\n",
      "where         O                  O\n",
      "he            O                  O\n",
      "should        O                  O\n",
      "not           O                  O\n",
      "have          O                  O\n",
      "be            O                  O\n",
      "he            O                  O\n",
      "be            O                  O\n",
      "in            O                  O\n",
      "blind         O                  O\n",
      "spot          O                  O\n",
      "of            O                  O\n",
      "operator      B-person/operator  O\n",
      "<END>         <END>              O\n",
      "------------  -----------------  -\n",
      "-------------  ---------------------  ---------------------\n",
      "roof           O                      O\n",
      "fall           B-accident_cause/fall  B-accident_cause/fall\n",
      "be             O                      O\n",
      "found          O                      O\n",
      "in             O                      O\n",
      "entry          O                      O\n",
      "cross-cut      O                      O\n",
      "70-71          O                      O\n",
      "west           O                      O\n",
      "intake         O                      O\n",
      "area           O                      O\n",
      "be             O                      O\n",
      "mine           O                      O\n",
      "in             O                      O\n",
      "august         O                      O\n",
      "of             O                      O\n",
      "2009           O                      O\n",
      "and            O                      O\n",
      "have           O                      O\n",
      "be             O                      O\n",
      "support        O                      O\n",
      "with           O                      O\n",
      "4-6            O                      O\n",
      "resin          O                      O\n",
      "pin            O                      O\n",
      "no             O                      O\n",
      "injury         B-injury               O\n",
      "or             O                      O\n",
      "property       O                      O\n",
      "loss           O                      O\n",
      "occur          O                      O\n",
      "there          O                      O\n",
      "be             O                      O\n",
      "no             O                      O\n",
      "plan           O                      O\n",
      "to             O                      O\n",
      "clean          O                      O\n",
      "up             O                      O\n",
      "fall           B-accident_cause/fall  O\n",
      "travelway      O                      O\n",
      "be             O                      O\n",
      "be             O                      O\n",
      "rehabilitate   O                      O\n",
      "in             O                      O\n",
      "entry          O                      O\n",
      "and            O                      O\n",
      "will           O                      O\n",
      "serve          O                      O\n",
      "a              O                      O\n",
      "the            O                      O\n",
      "new            O                      O\n",
      "primary        O                      O\n",
      "escapeway      O                      O\n",
      "fall           B-accident_cause/fall  O\n",
      "be             O                      O\n",
      "approximately  O                      O\n",
      "35             O                      O\n",
      "long           O                      O\n",
      "20             O                      O\n",
      "wide           O                      O\n",
      "high           O                      <END>\n",
      "<END>          <END>                  <END>\n",
      "-------------  ---------------------  ---------------------\n",
      "-----------  -------------------  -----------------\n",
      "employee     B-person/employee    B-person/employee\n",
      "be           O                    O\n",
      "drive        B-activity/driving   O\n",
      "up           O                    O\n",
      "the          O                    O\n",
      "ramp         O                    O\n",
      "and          O                    O\n",
      "begin        O                    O\n",
      "to           O                    O\n",
      "feel         O                    O\n",
      "lightheaded  O                    O\n",
      "and          O                    O\n",
      "dizzy        O                    O\n",
      "employee     B-person/employee    O\n",
      "stop         O                    O\n",
      "truck        O                    O\n",
      "and          O                    O\n",
      "call         O                    O\n",
      "supervisor   B-person/supervisor  O\n",
      "and          O                    O\n",
      "get          O                    O\n",
      "out          O                    O\n",
      "of           O                    O\n",
      "truck        O                    O\n",
      "until        O                    O\n",
      "help         O                    O\n",
      "could        O                    O\n",
      "arrive       O                    O\n",
      "<END>        <END>                O\n",
      "-----------  -------------------  -----------------\n",
      "--------  -----------------  -----------------\n",
      "the       O                  O\n",
      "foreman   O                  B-person/employee\n",
      "employee  B-person/employee  B-person/employee\n",
      "be        O                  O\n",
      "carry     O                  O\n",
      "pipe      B-equipment/pipe   O\n",
      "from      O                  O\n",
      "the       O                  O\n",
      "return    O                  O\n",
      "to        O                  O\n",
      "number    O                  O\n",
      "belt      O                  O\n",
      "he        O                  O\n",
      "slip      O                  O\n",
      "on        O                  O\n",
      "the       O                  O\n",
      "ice       O                  O\n",
      "and       O                  O\n",
      "fell      O                  O\n",
      "<END>     <END>              <END>\n",
      "--------  -----------------  -----------------\n",
      "--------  ---------------------------  -----------------\n",
      "employee  B-person/employee            B-person/employee\n",
      "be        O                            O\n",
      "tend      O                            O\n",
      "miner     O                            O\n",
      "cable     O                            O\n",
      "in        O                            O\n",
      "the       O                            O\n",
      "number    O                            O\n",
      "entry     O                            O\n",
      "of        O                            O\n",
      "11th      O                            O\n",
      "west      O                            O\n",
      "when      O                            O\n",
      "piece     O                            O\n",
      "of        O                            O\n",
      "the       O                            O\n",
      "rib       O                            O\n",
      "be        O                            O\n",
      "dislodge  O                            O\n",
      "and       O                            O\n",
      "struck    O                            O\n",
      "the       O                            O\n",
      "employee  B-person/employee            O\n",
      "on        O                            O\n",
      "the       O                            O\n",
      "right     O                            O\n",
      "side      O                            O\n",
      "of        O                            O\n",
      "his       O                            O\n",
      "body      B-body_part/body             O\n",
      "and       O                            O\n",
      "knock     O                            O\n",
      "him       O                            O\n",
      "into      O                            O\n",
      "the       O                            O\n",
      "shuttle   O                            O\n",
      "car       B-vehicle/light_vehicle/car  O\n",
      "with      O                            O\n",
      "the       O                            O\n",
      "left      O                            O\n",
      "side      O                            O\n",
      "of        O                            O\n",
      "his       O                            O\n",
      "body      B-body_part/body             O\n",
      "<END>     <END>                        O\n",
      "--------  ---------------------------  -----------------\n",
      "----------  -------------------  -----------------\n",
      "employee    B-person/employee    B-person/employee\n",
      "be          O                    O\n",
      "tighten     O                    O\n",
      "ratchet     O                    O\n",
      "strap       O                    O\n",
      "the         O                    O\n",
      "strap       O                    O\n",
      "slip        O                    O\n",
      "off         O                    O\n",
      "and         O                    O\n",
      "hit         O                    O\n",
      "employee    B-person/employee    O\n",
      "in          O                    O\n",
      "the         O                    O\n",
      "face        O                    O\n",
      "cause       O                    O\n",
      "laceration  B-injury/laceration  O\n",
      "to          O                    O\n",
      "employee's  O                    O\n",
      "chin        O                    O\n",
      "which       O                    O\n",
      "require     O                    O\n",
      "several     O                    O\n",
      "stitch      O                    O\n",
      "<END>       <END>                O\n",
      "----------  -------------------  -----------------\n",
      "------------  -----------------  -----------------\n",
      "employee      B-person/employee  B-person/employee\n",
      "be            O                  O\n",
      "rid           O                  O\n",
      "in            O                  O\n",
      "deisel        O                  O\n",
      "tractor       O                  O\n",
      "a             O                  O\n",
      "the           O                  O\n",
      "tractor       O                  O\n",
      "tram          O                  O\n",
      "down          O                  O\n",
      "the           O                  O\n",
      "supply        O                  O\n",
      "road          O                  O\n",
      "it            O                  O\n",
      "hit           O                  O\n",
      "some          O                  O\n",
      "hole          O                  O\n",
      "in            O                  O\n",
      "the           O                  O\n",
      "roadway       O                  O\n",
      "the           O                  O\n",
      "employee's    O                  O\n",
      "right         O                  O\n",
      "hip           O                  O\n",
      "start         O                  O\n",
      "to            O                  O\n",
      "hurt          O                  O\n",
      "he            O                  O\n",
      "go            O                  O\n",
      "to            O                  O\n",
      "the           O                  O\n",
      "chiropractor  O                  O\n",
      "on            O                  O\n",
      "1/2/04        O                  O\n",
      "for           O                  O\n",
      "an            O                  <END>\n",
      "adjustment    O                  <END>\n",
      "<END>         <END>              <END>\n",
      "------------  -----------------  -----------------\n",
      "---------------  ------------------------  -----------------\n",
      "employee         B-person/employee         B-person/employee\n",
      "state            O                         O\n",
      "strain           O                         O\n",
      "back             B-body_part/back          O\n",
      "while            O                         O\n",
      "work             O                         O\n",
      "on               O                         O\n",
      "pipeline         O                         O\n",
      "for              O                         O\n",
      "the              O                         O\n",
      "dredge           O                         O\n",
      "state            O                         O\n",
      "work             O                         O\n",
      "out              O                         O\n",
      "of               O                         O\n",
      "designate        O                         O\n",
      "work             O                         O\n",
      "boat             O                         O\n",
      "and              O                         O\n",
      "gust             O                         O\n",
      "of               O                         O\n",
      "wind             O                         O\n",
      "blew             O                         O\n",
      "and              O                         O\n",
      "start            O                         O\n",
      "to               O                         O\n",
      "separate         O                         O\n",
      "the              O                         O\n",
      "pipeline         O                         O\n",
      "employee         B-person/employee         O\n",
      "state            O                         O\n",
      "shoulder         B-body_part/arm/shoulder  O\n",
      "popped           O                         O\n",
      "and              O                         O\n",
      "he               O                         O\n",
      "stop             O                         O\n",
      "work             O                         O\n",
      "and              O                         O\n",
      "stretch          O                         O\n",
      "he               O                         O\n",
      "go               O                         O\n",
      "back             B-body_part/back          O\n",
      "to               O                         O\n",
      "work             O                         O\n",
      "for              O                         O\n",
      "the              O                         O\n",
      "rest             O                         O\n",
      "of               O                         O\n",
      "the              O                         O\n",
      "day              O                         O\n",
      "with             O                         O\n",
      "no               O                         O\n",
      "problem          O                         O\n",
      "on               O                         O\n",
      "monday           O                         O\n",
      "6/13/05          O                         O\n",
      "he               O                         O\n",
      "report           O                         O\n",
      "to               O                         O\n",
      "the              O                         O\n",
      "emergency-room-  O                         O\n",
      "with             O                         O\n",
      "back             B-body_part/back          O\n",
      "pain             O                         O\n",
      "and              O                         O\n",
      "say              O                         O\n",
      "it               O                         O\n",
      "be               O                         O\n",
      "work             O                         O\n",
      "related          O                         O\n",
      "<END>            <END>                     <END>\n",
      "---------------  ------------------------  -----------------\n",
      "--------  ------------------------------  -----------------\n",
      "employee  B-person/employee               B-person/employee\n",
      "be        O                               O\n",
      "attempt   O                               O\n",
      "to        O                               O\n",
      "rerail    O                               O\n",
      "mantrip   O                               O\n",
      "when      O                               O\n",
      "the       O                               O\n",
      "jack      O                               O\n",
      "fell      O                               O\n",
      "over      O                               O\n",
      "and       O                               O\n",
      "struck    O                               O\n",
      "employee  B-person/employee               O\n",
      "in        O                               O\n",
      "the       O                               O\n",
      "left      B-body_part/leg/knee/left_knee  O\n",
      "knee      I-body_part/leg/knee/left_knee  O\n",
      "<END>     <END>                           <END>\n",
      "--------  ------------------------------  -----------------\n",
      "-----------  ---------------------  -----------------\n",
      "during       O                      O\n",
      "an           O                      O\n",
      "altercation  O                      B-person/employee\n",
      "with         O                      O\n",
      "another      O                      O\n",
      "employee     B-person/employee      O\n",
      "injured      O                      O\n",
      "employee     B-person/employee      O\n",
      "be           O                      O\n",
      "struck       O                      O\n",
      "in           O                      O\n",
      "nose         B-body_part/head/nose  O\n",
      "both         O                      O\n",
      "sent         O                      O\n",
      "home         O                      O\n",
      "until        O                      O\n",
      "incident     O                      O\n",
      "investigate  O                      O\n",
      "injured      O                      O\n",
      "employee     B-person/employee      O\n",
      "do           O                      O\n",
      "not          O                      O\n",
      "seek         O                      O\n",
      "medical      O                      O\n",
      "attention    O                      O\n",
      "for          O                      O\n",
      "11/2         O                      O\n",
      "week         O                      O\n",
      "safety       O                      O\n",
      "dept         O                      O\n",
      "be           O                      O\n",
      "notify       O                      O\n",
      "employee     B-person/employee      O\n",
      "have         O                      O\n",
      "miss         O                      O\n",
      "day          O                      O\n",
      "to           O                      O\n",
      "see          O                      O\n",
      "doctor       O                      O\n",
      "and          O                      O\n",
      "day          O                      O\n",
      "for          O                      O\n",
      "x-ray        O                      O\n",
      "all          O                      O\n",
      "negative     O                      O\n",
      "<END>        <END>                  O\n",
      "-----------  ---------------------  -----------------\n",
      "----------  -----------------------------  -----------------\n",
      "employee    B-person/employee              B-person/employee\n",
      "be          O                              O\n",
      "use         O                              O\n",
      "dilute      O                              O\n",
      "caustic     O                              O\n",
      "solution    O                              O\n",
      "to          O                              O\n",
      "hose-down-  O                              O\n",
      "mud         O                              O\n",
      "and         O                              O\n",
      "scale       O                              O\n",
      "on-side-    O                              O\n",
      "of          O                              O\n",
      "tank        O                              O\n",
      "some        O                              O\n",
      "of          O                              O\n",
      "the         O                              O\n",
      "caustic     O                              O\n",
      "solution    O                              O\n",
      "splash      O                              O\n",
      "back        B-body_part/back               O\n",
      "and         O                              O\n",
      "get         O                              O\n",
      "into        O                              O\n",
      "the         O                              O\n",
      "employee's  O                              O\n",
      "left        B-body_part/head/eye/left_eye  O\n",
      "eye         I-body_part/head/eye/left_eye  O\n",
      "employee    B-person/employee              O\n",
      "be          O                              O\n",
      "see         O                              O\n",
      "by          O                              O\n",
      "doctor      O                              O\n",
      "who         O                              O\n",
      "prescribed  O                              O\n",
      "antibiotic  O                              O\n",
      "to          O                              O\n",
      "treat       O                              O\n",
      "small       O                              O\n",
      "corneal     O                              O\n",
      "abrasion    O                              O\n",
      "there       O                              O\n",
      "be          O                              O\n",
      "no          O                              O\n",
      "lose        O                              O\n",
      "time        O                              O\n",
      "and         O                              O\n",
      "only        O                              O\n",
      "restrict    O                              O\n",
      "duty        O                              O\n",
      "<END>       <END>                          O\n",
      "----------  -----------------------------  -----------------\n",
      "--------------------  ---------------------------------  -----------------\n",
      "employee              B-person/employee                  B-person/employee\n",
      "be                    O                                  O\n",
      "replace               O                                  O\n",
      "the                   O                                  O\n",
      "rear                  O                                  O\n",
      "suspension            O                                  O\n",
      "cylinder              O                                  O\n",
      "on                    O                                  O\n",
      "cat                   O                                  O\n",
      "793                   O                                  O\n",
      "truck                 O                                  O\n",
      "a                     O                                  O\n",
      "he                    O                                  O\n",
      "be                    O                                  O\n",
      "use                   O                                  O\n",
      "sledge                O                                  O\n",
      "hammer                O                                  O\n",
      "to                    O                                  O\n",
      "drive                 O                                  O\n",
      "the                   O                                  O\n",
      "top                   O                                  O\n",
      "strut                 O                                  O\n",
      "pin                   O                                  O\n",
      "from                  O                                  O\n",
      "it                    O                                  O\n",
      "bore                  O                                  O\n",
      "he                    O                                  O\n",
      "felt                  O                                  O\n",
      "pain                  O                                  O\n",
      "in                    O                                  O\n",
      "his                   O                                  O\n",
      "right                 B-body_part/arm/wrist/right_wrist  O\n",
      "wrist                 I-body_part/arm/wrist/right_wrist  O\n",
      "he                    O                                  O\n",
      "do                    O                                  O\n",
      "not                   O                                  O\n",
      "report                O                                  O\n",
      "the                   O                                  O\n",
      "incident              O                                  O\n",
      "at                    O                                  O\n",
      "that                  O                                  O\n",
      "time                  O                                  O\n",
      "on                    O                                  O\n",
      "9/22/06               O                                  O\n",
      "a                     O                                  O\n",
      "he                    O                                  O\n",
      "be                    O                                  O\n",
      "loosening/tightening  O                                  O\n",
      "the                   O                                  O\n",
      "wear                  O                                  O\n",
      "plate                 O                                  O\n",
      "retain                O                                  O\n",
      "bolt                  O                                  O\n",
      "on                    O                                  O\n",
      "24m                   O                                  O\n",
      "motor                 O                                  O\n",
      "grader                B-vehicle/heavy_vehicle/grader     O\n",
      "use                   O                                  O\n",
      "3/4                   O                                  O\n",
      "''                    O                                  O\n",
      "impact                O                                  O\n",
      "wrench                O                                  O\n",
      "he                    O                                  O\n",
      "again                 O                                  O\n",
      "experienced           O                                  O\n",
      "pain                  O                                  <END>\n",
      "in                    O                                  <END>\n",
      "wrist                 B-body_part/arm/wrist              <END>\n",
      "<END>                 <END>                              <END>\n",
      "--------------------  ---------------------------------  -----------------\n",
      "--------  ---------------------------  -----------------\n",
      "employee  B-person/employee            B-person/employee\n",
      "be        O                            O\n",
      "replace   O                            O\n",
      "bag       O                            O\n",
      "in        O                            O\n",
      "baghouse  O                            O\n",
      "bending   O                            O\n",
      "over      O                            O\n",
      "and       O                            O\n",
      "pull      O                            O\n",
      "bag       O                            O\n",
      "strain    O                            O\n",
      "low       B-body_part/back/lower_back  O\n",
      "back      I-body_part/back/lower_back  <END>\n",
      "<END>     <END>                        <END>\n",
      "--------  ---------------------------  -----------------\n",
      "--------  -----------------  -----------------\n",
      "employee  B-person/employee  B-person/employee\n",
      "state     O                  O\n",
      "employee  B-person/employee  B-person/employee\n",
      "be        O                  O\n",
      "exit      O                  O\n",
      "the       O                  O\n",
      "connex    O                  O\n",
      "box       O                  O\n",
      "storage   O                  O\n",
      "trailer   O                  O\n",
      "when      O                  O\n",
      "employee  B-person/employee  B-person/employee\n",
      "slip      O                  O\n",
      "on        O                  O\n",
      "snow/ice  O                  O\n",
      "cause     O                  O\n",
      "employee  B-person/employee  O\n",
      "to        O                  O\n",
      "catch     O                  O\n",
      "self      O                  O\n",
      "in        O                  O\n",
      "split     O                  O\n",
      "<END>     <END>              <END>\n",
      "--------  -----------------  -----------------\n"
     ]
    }
   ],
   "source": [
    "args.device = torch.device(\"cpu\")\n",
    "# checkpoint = torch.load(args.model_state_file)\n",
    "# NERmodel.load_state_dict(checkpoint['model_state_dict'])\n",
    "NERmodel.load_state_dict(torch.load(train_state['model_filename']))\n",
    "NERmodel = NERmodel.to(args.device)\n",
    "\n",
    "dataset.set_split('test')\n",
    "batch_generator = generate_batches(dataset,  batch_size=1, shuffle=False, device=args.device)\n",
    "\n",
    "running_loss = 0.\n",
    "running_acc = 0.\n",
    "NERmodel.eval()\n",
    "mask_index = vectorizer.target_vocab.mask_index\n",
    "i = 0\n",
    "for batch_index, batch_dict in enumerate(batch_generator):\n",
    "    y_pred = NERmodel(batch_dict['x_source'],  batch_dict['x_length'],  batch_dict['x_target'], _train=False)\n",
    "    origin = batch_dict['x_source'][0]\n",
    "    y_true = batch_dict['y_target']\n",
    "    pred, true = normalize_sizes(y_pred, y_true)\n",
    "    _, y_pred_indices = pred.max(dim=1)\n",
    "    \n",
    "    final = []\n",
    "    for j in range(len(true)):\n",
    "        if true[j].item() != 0:\n",
    "            tmp = []\n",
    "            tmp.append(vectorizer.NARRATIVE_vocab.lookup_index(origin[j+1].item()))\n",
    "            tmp.append(vectorizer.target_vocab.lookup_index(true[j].item()))\n",
    "            tmp.append(vectorizer.target_vocab.lookup_index(y_pred_indices[j].item()))\n",
    "            \n",
    "            final.append(tmp)\n",
    "\n",
    "    print(tabulate(final))\n",
    "\n",
    "    i += 1\n",
    "    if i == 50:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6ba56601-9045-4f64-8f2e-8e14239438b4",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "dbc6212b-ce00-49e0-b322-ab95d3626175",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Self attention"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "395c4ad4-6884-4d4e-ba67-0911e62171a3",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Functions"
   ]
  },
  {
   "cell_type": "raw",
   "id": "6688225b-a3d3-481f-b618-5c9a3a7f86a3",
   "metadata": {},
   "source": [
    "We use "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "811894e0-116a-4dac-be01-5d1e430e2ac8",
   "metadata": {},
   "outputs": [],
   "source": [
    "def verbose_attention(encoder_state_vectors, query_vector):\n",
    "    batch_size, num_vectors, vector_size = encoder_state_vectors.size()\n",
    "    vector_scores = torch.sum(encoder_state_vectors * query_vector.view(batch_size, 1, vector_size),  dim=2) # Apply dot product\n",
    "    vector_probabilities = F.softmax(vector_scores, dim=1) # Apply softmax to ensure row sums to 1\n",
    "    weighted_vectors = encoder_state_vectors * vector_probabilities.view(batch_size, num_vectors, 1)\n",
    "    context_vectors = torch.sum(weighted_vectors, dim=1)\n",
    "    return context_vectors, vector_probabilities, vector_scores\n",
    "\n",
    "class Attention(nn.Module):\n",
    "    def __init__(self, dim):\n",
    "        super(Attention, self).__init__()\n",
    "\n",
    "    def set_mask(self, mask=0):\n",
    "        self.mask = mask\n",
    "\n",
    "    def forward(self, encoder_state_vectors, query_vector):\n",
    "        batch_size, num_vectors, vector_size = encoder_state_vectors.size()\n",
    "        \n",
    "        vector_scores = torch.sum(encoder_state_vectors * query_vector.view(batch_size, 1, vector_size),  dim=2) # Apply dot product\n",
    "        vector_probabilities = F.softmax(vector_scores, dim=1) # Apply softmax to make sure row sums to 1\n",
    "        \n",
    "        weighted_vectors = encoder_state_vectors * vector_probabilities.view(batch_size, num_vectors, 1) # Get Y^T\n",
    "        context_vectors = torch.sum(weighted_vectors, dim=1)  # Get y sum\n",
    "        \n",
    "        prediction_vector = torch.cat((context_vectors, query_vector), dim=1)\n",
    "\n",
    "        return prediction_vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "fa952801-2e20-420d-8a96-f18ee60d2d5a",
   "metadata": {},
   "outputs": [],
   "source": [
    "args = Namespace(\n",
    "    # Data and Path hyper parameters\n",
    "    data_file='./data/ner.csv',\n",
    "    lable_file='labels.txt',\n",
    "    vectorizer_file=\"nervectorizer.json\",\n",
    "    model_state_file=\"NERSelfAttention.h5\",\n",
    "    lr = 0.001,\n",
    "    seed=666,\n",
    "    dropout_p=0.1,\n",
    "    batch_train=1024,\n",
    "    batch_valid=512,\n",
    "    batch_test=1024,\n",
    "    num_epochs=200,\n",
    "    cuda=True,\n",
    "    reload_from_files=False,\n",
    "    expand_filepaths_to_save_dir=True,\n",
    "    source_embedding_size=128, \n",
    "    target_embedding_size=128, \n",
    "    encoding_size=128,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2533e2e4-6223-496d-acfc-cf43a4d3ee46",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "f33f834d-51fe-40f3-9ea9-42a8ad0df79f",
   "metadata": {},
   "outputs": [],
   "source": [
    "class SeqToSeq(nn.Module):\n",
    "    def __init__(self, source_vocab_size, source_embedding_size,  target_vocab_size, target_embedding_size, encoding_size,  target_bos_index):\n",
    "        super(SeqToSeq, self).__init__()\n",
    "        self.encoder = Encoder(num_embeddings=source_vocab_size,  embedding_size=source_embedding_size, rnn_hidden_size=encoding_size)\n",
    "        decoding_size = encoding_size * 2\n",
    "        self.decoder = Decoder(num_embeddings=target_vocab_size,  embedding_size=target_embedding_size,  rnn_hidden_size=decoding_size, bos_index=target_bos_index)\n",
    "    \n",
    "    def forward(self, x_source, x_source_lengths, target_sequence):\n",
    "        encoder_state, final_hidden_states = self.encoder(x_source, x_source_lengths)\n",
    "        decoded_states = self.decoder(encoder_state=encoder_state,  initial_hidden_state=final_hidden_states,  target_sequence=target_sequence)\n",
    "        return decoded_states\n",
    "\n",
    "class Decoder(nn.Module):\n",
    "    def __init__(self, num_embeddings, embedding_size, rnn_hidden_size, bos_index):\n",
    "        super(Decoder, self).__init__()\n",
    "        self._rnn_hidden_size = rnn_hidden_size\n",
    "        self.target_embedding = nn.Embedding(num_embeddings=num_embeddings,  embedding_dim=embedding_size,  padding_idx=0)\n",
    "        self.rnn = nn.GRUCell(embedding_size + rnn_hidden_size,  rnn_hidden_size)\n",
    "        self.hidden_map = nn.Linear(rnn_hidden_size, rnn_hidden_size)\n",
    "        self.classifier1 = nn.Linear(rnn_hidden_size * 2, num_embeddings*2)\n",
    "        self.classifier2 = nn.Linear(num_embeddings * 2, num_embeddings)\n",
    "        self.bos_index = bos_index\n",
    "        self.relu = nn.ReLU()\n",
    "    \n",
    "    def _init_indices(self, batch_size):\n",
    "        return torch.ones(batch_size, dtype=torch.int64) * self.bos_index\n",
    "    \n",
    "    def _init_context_vectors(self, batch_size):\n",
    "        return torch.zeros(batch_size, self._rnn_hidden_size)\n",
    "            \n",
    "    def forward(self, encoder_state, initial_hidden_state, target_sequence):\n",
    "        target_sequence = target_sequence.permute(1, 0)\n",
    "        output_sequence_size = target_sequence.size(0)\n",
    "\n",
    "        h_t = self.hidden_map(initial_hidden_state)\n",
    "        batch_size = encoder_state.size(0)\n",
    "        context_vectors = self._init_context_vectors(batch_size)\n",
    "\n",
    "        y_t_index = self._init_indices(batch_size)\n",
    "        h_t = h_t.to(encoder_state.device)\n",
    "        y_t_index = y_t_index.to(encoder_state.device)\n",
    "        context_vectors = context_vectors.to(encoder_state.device)\n",
    "        y_t_index = target_sequence[0]\n",
    "\n",
    "        output_vectors = []\n",
    "        for i in range(1, output_sequence_size):\n",
    "#             y_t_index = target_sequence[i] # Lab example use this as prediction\n",
    "                \n",
    "            # Step 1: Embed word and concat with previous context\n",
    "            y_input_vector = self.target_embedding(y_t_index)\n",
    "            rnn_input = torch.cat([y_input_vector, context_vectors], dim=1)\n",
    "            \n",
    "            # Step 2: Make a GRU step, getting a new hidden vector\n",
    "            h_t = self.rnn(rnn_input, h_t)\n",
    "            \n",
    "            # Step 3: Use the current hidden to attend to the encoder state\n",
    "            context_vectors, p_attn, _ = verbose_attention(encoder_state_vectors=encoder_state, query_vector=h_t)\n",
    "            \n",
    "            # Step 4: Use the current hidden and context vectors to make a prediction to the next word\n",
    "            prediction_vector = torch.cat((context_vectors, h_t), dim=1)\n",
    "            score_for_y_t_index = self.classifier1(F.dropout(prediction_vector, 0.1))\n",
    "            score_for_y_t_index = self.classifier2(F.dropout(score_for_y_t_index, 0.1))\n",
    "            '''\n",
    "            I switch to this one\n",
    "            due the understanding of Seq2Seq\n",
    "            '''\n",
    "            _, y_t_index = score_for_y_t_index.max(dim=1)\n",
    "            if self.training and torch.rand(1) < 0.2: \n",
    "                y_t_index = target_sequence[i]\n",
    "            output_vectors.append(score_for_y_t_index)\n",
    "        return torch.stack(output_vectors).permute(1, 0, 2)\n",
    "\n",
    "class Encoder(nn.Module):\n",
    "    def __init__(self, num_embeddings, embedding_size, rnn_hidden_size):\n",
    "        super(Encoder, self).__init__()\n",
    "    \n",
    "        self.source_embedding = nn.Embedding(num_embeddings, embedding_size, padding_idx=0)\n",
    "        self.rnn = nn.GRU(embedding_size, rnn_hidden_size, bidirectional=True, batch_first=True)\n",
    "    \n",
    "    def forward(self, x_source, x_lengths):\n",
    "        x_embedded = self.source_embedding(x_source)\n",
    "        x_packed = pack_padded_sequence(x_embedded, x_lengths.detach().cpu().numpy(), batch_first=True)\n",
    "\n",
    "        out, hidden  = self.rnn(x_packed)\n",
    "        hidden = hidden.permute(1, 0, 2)\n",
    "        hidden = hidden.contiguous().view(hidden.size(0), -1)\n",
    "        out, _ = pad_packed_sequence(out, batch_first=True)\n",
    "        return out, hidden"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "c36a47ae-e939-41d8-ab1d-23a46b2c0d00",
   "metadata": {},
   "outputs": [],
   "source": [
    "NERmodel = SeqToSeq(source_vocab_size=len(vectorizer.NARRATIVE_vocab), \n",
    "                 source_embedding_size=args.source_embedding_size, \n",
    "                 target_vocab_size=len(vectorizer.target_vocab),\n",
    "                 target_embedding_size=args.target_embedding_size, \n",
    "                 encoding_size=args.encoding_size,\n",
    "                 target_bos_index=vectorizer.target_vocab.begin_seq_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "9dc1b67f-61d4-4a09-a7a5-084b636353ef",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "180"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(vectorizer.target_vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "b368d93b-8c6e-4486-be71-e0978148465b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SeqToSeq(\n",
       "  (encoder): Encoder(\n",
       "    (source_embedding): Embedding(13225, 128, padding_idx=0)\n",
       "    (rnn): GRU(128, 128, batch_first=True, bidirectional=True)\n",
       "  )\n",
       "  (decoder): Decoder(\n",
       "    (target_embedding): Embedding(180, 128, padding_idx=0)\n",
       "    (rnn): GRUCell(384, 256)\n",
       "    (hidden_map): Linear(in_features=256, out_features=256, bias=True)\n",
       "    (classifier1): Linear(in_features=512, out_features=360, bias=True)\n",
       "    (classifier2): Linear(in_features=360, out_features=180, bias=True)\n",
       "    (relu): ReLU()\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "NERmodel"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a6ff83bd-9d0d-4937-a4c7-57b3ffde260c",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Set up"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "23fd0e40-69b3-424f-a763-dcd33582e14d",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_state = make_train_state(args)\n",
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "args.device = torch.device(\"cuda\" if args.cuda else \"cpu\")\n",
    "NERmodel = NERmodel.to(args.device)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e8abd5fa-3b52-41bb-98c8-ebc80346f4ba",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## Train\n",
    "\n",
    "Try to run if model dosent saved, then reset this kernel to free up GPU or CPU memory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "21b08f4d-557e-473f-8c60-2c0cc2ea4ee1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epochs: 000 | Train loss 3.3870 Train acc 75.156 Val loss 2.1036 Val acc 87.708\n",
      "Epochs: 001 | Train loss 2.0260 Train acc 82.505 Val loss 1.9685 Val acc 87.956\n",
      "Epochs: 002 | Train loss 1.8694 Train acc 87.647 Val loss 1.8300 Val acc 87.410\n",
      "Epochs: 003 | Train loss 1.7977 Train acc 87.693 Val loss 1.7637 Val acc 87.958\n",
      "Epochs: 004 | Train loss 1.7539 Train acc 87.791 Val loss 1.7213 Val acc 87.998\n",
      "Epochs: 005 | Train loss 1.7164 Train acc 87.894 Val loss 1.7388 Val acc 87.638\n",
      "Epochs: 006 | Train loss 1.6904 Train acc 88.078 Val loss 1.6670 Val acc 88.830\n",
      "Epochs: 007 | Train loss 1.6525 Train acc 88.603 Val loss 1.6221 Val acc 88.916\n",
      "Epochs: 008 | Train loss 1.6219 Train acc 88.590 Val loss 1.6116 Val acc 88.691\n",
      "Epochs: 009 | Train loss 1.5931 Train acc 88.498 Val loss 1.6176 Val acc 88.454\n",
      "Epochs: 010 | Train loss 1.5625 Train acc 88.486 Val loss 1.5743 Val acc 88.397\n",
      "Epochs: 011 | Train loss 1.5230 Train acc 88.318 Val loss 1.5613 Val acc 87.922\n",
      "Epochs: 012 | Train loss 1.4754 Train acc 88.171 Val loss 1.4751 Val acc 87.995\n",
      "Epochs: 013 | Train loss 1.4224 Train acc 87.483 Val loss 1.4332 Val acc 84.046\n",
      "Epochs: 014 | Train loss 1.3846 Train acc 86.764 Val loss 1.3865 Val acc 86.726\n",
      "Epochs: 015 | Train loss 1.3260 Train acc 87.552 Val loss 1.3315 Val acc 86.197\n",
      "Epochs: 016 | Train loss 1.2737 Train acc 86.544 Val loss 1.2655 Val acc 87.126\n",
      "Epochs: 017 | Train loss 1.2316 Train acc 86.364 Val loss 1.2410 Val acc 87.367\n",
      "Epochs: 018 | Train loss 1.1864 Train acc 85.880 Val loss 1.2532 Val acc 87.724\n",
      "Epochs: 019 | Train loss 1.1616 Train acc 86.428 Val loss 1.1382 Val acc 87.321\n",
      "Epochs: 020 | Train loss 1.1149 Train acc 86.055 Val loss 1.0950 Val acc 86.725\n",
      "Epochs: 021 | Train loss 1.0446 Train acc 86.190 Val loss 1.0776 Val acc 86.873\n",
      "Epochs: 022 | Train loss 1.0335 Train acc 84.310 Val loss 1.0338 Val acc 83.945\n",
      "Epochs: 023 | Train loss 1.0259 Train acc 85.625 Val loss 1.0275 Val acc 85.484\n",
      "Epochs: 024 | Train loss 0.9553 Train acc 85.136 Val loss 1.0110 Val acc 87.547\n",
      "Epochs: 025 | Train loss 0.8830 Train acc 85.834 Val loss 0.9131 Val acc 85.215\n",
      "Epochs: 026 | Train loss 0.8813 Train acc 83.830 Val loss 0.9376 Val acc 85.352\n",
      "Epochs: 027 | Train loss 0.8556 Train acc 84.587 Val loss 0.8652 Val acc 87.245\n",
      "Epochs: 028 | Train loss 0.7884 Train acc 85.448 Val loss 0.8464 Val acc 84.288\n",
      "Epochs: 029 | Train loss 0.7331 Train acc 85.047 Val loss 0.8078 Val acc 87.033\n",
      "Epochs: 031 | Train loss 0.7583 Train acc 83.667 Val loss 0.7506 Val acc 83.701\n",
      "Epochs: 032 | Train loss 0.6922 Train acc 84.883 Val loss 0.7372 Val acc 83.566\n",
      "Epochs: 033 | Train loss 0.6436 Train acc 85.463 Val loss 0.6840 Val acc 85.602\n",
      "Epochs: 035 | Train loss 0.6107 Train acc 85.479 Val loss 0.6537 Val acc 86.403\n",
      "Epochs: 037 | Train loss 0.6164 Train acc 84.791 Val loss 0.6761 Val acc 86.452\n",
      "Epochs: 038 | Train loss 0.5853 Train acc 85.424 Val loss 0.7308 Val acc 80.505\n",
      "Epochs: 040 | Train loss 0.5390 Train acc 85.935 Val loss 0.6674 Val acc 85.981\n",
      "Epochs: 041 | Train loss 0.5302 Train acc 85.724 Val loss 0.6786 Val acc 81.680\n",
      "Epochs: 042 | Train loss 0.4985 Train acc 86.267 Val loss 0.5674 Val acc 87.185\n",
      "Epochs: 045 | Train loss 0.5192 Train acc 85.157 Val loss 0.5417 Val acc 87.379\n",
      "Epochs: 046 | Train loss 0.4897 Train acc 85.527 Val loss 0.6692 Val acc 86.855\n",
      "Epochs: 048 | Train loss 0.4278 Train acc 87.590 Val loss 0.5603 Val acc 85.541\n",
      "Epochs: 049 | Train loss 0.4071 Train acc 87.940 Val loss 0.5599 Val acc 84.478\n",
      "Epochs: 051 | Train loss 0.4756 Train acc 85.615 Val loss 0.5625 Val acc 83.484\n",
      "Epochs: 052 | Train loss 0.4456 Train acc 86.493 Val loss 0.5104 Val acc 86.466\n",
      "Epochs: 053 | Train loss 0.3964 Train acc 88.372 Val loss 0.4855 Val acc 87.271\n",
      "Epochs: 054 | Train loss 0.3795 Train acc 88.223 Val loss 0.4822 Val acc 87.192\n",
      "Epochs: 057 | Train loss 0.6014 Train acc 82.821 Val loss 0.7595 Val acc 81.772\n",
      "Epochs: 058 | Train loss 0.5319 Train acc 85.015 Val loss 0.5500 Val acc 87.270\n",
      "Epochs: 059 | Train loss 0.4490 Train acc 87.481 Val loss 0.5065 Val acc 85.357\n",
      "Epochs: 060 | Train loss 0.4006 Train acc 88.377 Val loss 0.5082 Val acc 86.834\n",
      "Epochs: 061 | Train loss 0.3540 Train acc 89.445 Val loss 0.4727 Val acc 87.540\n",
      "Epochs: 062 | Train loss 0.3220 Train acc 90.027 Val loss 0.4517 Val acc 88.338\n",
      "Epochs: 065 | Train loss 0.4695 Train acc 85.001 Val loss 0.5223 Val acc 88.786\n",
      "Epochs: 066 | Train loss 0.4012 Train acc 87.926 Val loss 0.5478 Val acc 85.134\n",
      "Epochs: 067 | Train loss 0.3662 Train acc 88.979 Val loss 0.4828 Val acc 86.053\n",
      "Epochs: 068 | Train loss 0.3319 Train acc 89.680 Val loss 0.4455 Val acc 88.075\n",
      "Epochs: 069 | Train loss 0.3116 Train acc 90.290 Val loss 0.4812 Val acc 87.155\n",
      "Epochs: 072 | Train loss 0.4914 Train acc 84.948 Val loss 0.4805 Val acc 85.900\n",
      "Epochs: 073 | Train loss 0.4149 Train acc 87.063 Val loss 0.5017 Val acc 87.192\n",
      "Epochs: 074 | Train loss 0.3585 Train acc 89.039 Val loss 0.4902 Val acc 85.821\n",
      "Epochs: 075 | Train loss 0.3204 Train acc 90.043 Val loss 0.4671 Val acc 87.706\n",
      "Epochs: 076 | Train loss 0.2928 Train acc 90.655 Val loss 0.4041 Val acc 89.358\n",
      "Epochs: 077 | Train loss 0.2703 Train acc 91.591 Val loss 0.4169 Val acc 89.267\n",
      "Epochs: 080 | Train loss 0.4173 Train acc 86.932 Val loss 0.5251 Val acc 83.672\n",
      "Epochs: 081 | Train loss 0.3544 Train acc 89.447 Val loss 0.4539 Val acc 86.572\n",
      "Epochs: 082 | Train loss 0.3151 Train acc 89.554 Val loss 0.4584 Val acc 89.026\n",
      "Epochs: 083 | Train loss 0.2921 Train acc 90.256 Val loss 0.4256 Val acc 89.813\n",
      "Epochs: 084 | Train loss 0.2700 Train acc 91.520 Val loss 0.4500 Val acc 87.300\n",
      "Epochs: 088 | Train loss 0.3343 Train acc 89.223 Val loss 0.4119 Val acc 88.768\n",
      "Epochs: 089 | Train loss 0.2966 Train acc 90.272 Val loss 0.4136 Val acc 89.131\n",
      "Epochs: 090 | Train loss 0.2613 Train acc 91.392 Val loss 0.4224 Val acc 90.223\n",
      "Epochs: 091 | Train loss 0.2318 Train acc 92.830 Val loss 0.4328 Val acc 88.615\n",
      "Epochs: 094 | Train loss 0.4705 Train acc 87.069 Val loss 0.5323 Val acc 81.152\n",
      "Epochs: 095 | Train loss 0.3833 Train acc 87.536 Val loss 0.4819 Val acc 88.641\n",
      "Epochs: 096 | Train loss 0.3332 Train acc 89.059 Val loss 0.4782 Val acc 87.135\n",
      "Epochs: 097 | Train loss 0.2887 Train acc 91.150 Val loss 0.4141 Val acc 88.204\n",
      "Epochs: 098 | Train loss 0.2532 Train acc 91.935 Val loss 0.4330 Val acc 88.894\n",
      "Epochs: 099 | Train loss 0.2334 Train acc 92.429 Val loss 0.4072 Val acc 90.306\n",
      "Epochs: 100 | Train loss 0.2126 Train acc 93.336 Val loss 0.4878 Val acc 87.993\n",
      "Epochs: 102 | Train loss 0.3247 Train acc 89.660 Val loss 0.4516 Val acc 88.912\n",
      "Epochs: 103 | Train loss 0.2703 Train acc 90.421 Val loss 0.4544 Val acc 90.761\n",
      "Epochs: 108 | Train loss 0.2908 Train acc 90.261 Val loss 0.4717 Val acc 85.939\n",
      "Epochs: 109 | Train loss 0.2645 Train acc 90.964 Val loss 0.4408 Val acc 87.988\n",
      "Epochs: 110 | Train loss 0.2236 Train acc 92.544 Val loss 0.3911 Val acc 89.831\n",
      "Epochs: 111 | Train loss 0.2071 Train acc 93.093 Val loss 0.4041 Val acc 90.362\n",
      "Epochs: 112 | Train loss 0.1888 Train acc 93.711 Val loss 0.4204 Val acc 90.751\n",
      "Epochs: 113 | Train loss 0.1795 Train acc 94.329 Val loss 0.6348 Val acc 85.960\n",
      "Epochs: 115 | Train loss 0.7240 Train acc 81.611 Val loss 0.6953 Val acc 77.746\n",
      "Epochs: 116 | Train loss 0.5649 Train acc 85.200 Val loss 0.6111 Val acc 84.591\n",
      "Epochs: 117 | Train loss 0.4676 Train acc 86.626 Val loss 0.5085 Val acc 87.916\n",
      "Epochs: 118 | Train loss 0.3808 Train acc 89.542 Val loss 0.4367 Val acc 88.424\n",
      "Epochs: 119 | Train loss 0.3245 Train acc 90.029 Val loss 0.4349 Val acc 88.287\n",
      "Epochs: 120 | Train loss 0.2747 Train acc 91.420 Val loss 0.4218 Val acc 89.223\n",
      "Epochs: 121 | Train loss 0.2385 Train acc 92.610 Val loss 0.3943 Val acc 89.500\n",
      "Epochs: 122 | Train loss 0.2138 Train acc 93.247 Val loss 0.3806 Val acc 90.846\n",
      "Epochs: 123 | Train loss 0.1941 Train acc 93.650 Val loss 0.3927 Val acc 91.177\n",
      "Epochs: 126 | Train loss 0.2722 Train acc 90.284 Val loss 0.4133 Val acc 88.954\n",
      "Epochs: 127 | Train loss 0.2331 Train acc 92.820 Val loss 0.3670 Val acc 89.487\n",
      "Epochs: 128 | Train loss 0.2120 Train acc 92.773 Val loss 0.4029 Val acc 91.177\n",
      "Epochs: 129 | Train loss 0.1918 Train acc 93.402 Val loss 0.3997 Val acc 90.525\n",
      "Epochs: 130 | Train loss 0.1731 Train acc 94.679 Val loss 0.3937 Val acc 89.917\n",
      "Epochs: 133 | Train loss 0.3607 Train acc 88.637 Val loss 0.5435 Val acc 85.817\n",
      "Epochs: 134 | Train loss 0.3176 Train acc 89.662 Val loss 0.4587 Val acc 89.324\n",
      "Epochs: 135 | Train loss 0.2548 Train acc 91.448 Val loss 0.4100 Val acc 88.311\n",
      "Epochs: 136 | Train loss 0.2318 Train acc 92.563 Val loss 0.3845 Val acc 89.870\n",
      "Epochs: 137 | Train loss 0.2131 Train acc 92.675 Val loss 0.4034 Val acc 90.511\n",
      "Epochs: 138 | Train loss 0.1862 Train acc 93.995 Val loss 0.4370 Val acc 89.100\n",
      "Epochs: 139 | Train loss 0.1817 Train acc 93.932 Val loss 0.4133 Val acc 91.424\n",
      "Epochs: 140 | Train loss 0.1630 Train acc 94.628 Val loss 0.3894 Val acc 90.872\n",
      "Epochs: 143 | Train loss 0.4113 Train acc 87.064 Val loss 0.5183 Val acc 86.688\n",
      "Epochs: 144 | Train loss 0.3222 Train acc 89.827 Val loss 0.4844 Val acc 86.800\n",
      "Epochs: 145 | Train loss 0.2701 Train acc 90.750 Val loss 0.4340 Val acc 89.669\n",
      "Epochs: 146 | Train loss 0.2337 Train acc 92.615 Val loss 0.3961 Val acc 89.273\n",
      "Epochs: 147 | Train loss 0.2036 Train acc 93.220 Val loss 0.4230 Val acc 90.321\n",
      "Epochs: 148 | Train loss 0.1835 Train acc 94.138 Val loss 0.4093 Val acc 89.890\n",
      "Epochs: 149 | Train loss 0.1620 Train acc 94.813 Val loss 0.3839 Val acc 90.868\n",
      "Epochs: 150 | Train loss 0.1515 Train acc 95.006 Val loss 0.3771 Val acc 91.336\n",
      "Epochs: 151 | Train loss 0.1451 Train acc 95.268 Val loss 0.3958 Val acc 91.475\n",
      "Epochs: 152 | Train loss 0.1320 Train acc 95.793 Val loss 0.4207 Val acc 91.322\n",
      "Epochs: 154 | Train loss 0.4119 Train acc 87.499 Val loss 0.6959 Val acc 88.601\n",
      "Epochs: 155 | Train loss 0.3206 Train acc 89.516 Val loss 0.4748 Val acc 86.213\n",
      "Epochs: 156 | Train loss 0.2641 Train acc 92.153 Val loss 0.4239 Val acc 87.906\n",
      "Epochs: 157 | Train loss 0.2318 Train acc 91.487 Val loss 0.4030 Val acc 90.632\n",
      "Epochs: 158 | Train loss 0.1982 Train acc 93.913 Val loss 0.3963 Val acc 89.244\n",
      "Epochs: 159 | Train loss 0.1741 Train acc 94.079 Val loss 0.4182 Val acc 90.787\n",
      "Epochs: 160 | Train loss 0.1516 Train acc 95.074 Val loss 0.3691 Val acc 91.602\n",
      "Epochs: 161 | Train loss 0.1426 Train acc 95.399 Val loss 0.4082 Val acc 90.742\n",
      "Epochs: 165 | Train loss 0.3059 Train acc 91.137 Val loss 0.4712 Val acc 87.773\n",
      "Epochs: 166 | Train loss 0.2627 Train acc 91.077 Val loss 0.4585 Val acc 91.109\n",
      "Epochs: 167 | Train loss 0.2203 Train acc 92.408 Val loss 0.3855 Val acc 89.622\n",
      "Epochs: 168 | Train loss 0.1856 Train acc 94.183 Val loss 0.3671 Val acc 91.087\n",
      "Epochs: 169 | Train loss 0.1623 Train acc 94.354 Val loss 0.3886 Val acc 91.262\n",
      "Epochs: 170 | Train loss 0.1467 Train acc 95.575 Val loss 0.4046 Val acc 90.605\n",
      "Epochs: 171 | Train loss 0.1341 Train acc 95.319 Val loss 0.3931 Val acc 92.065\n",
      "Epochs: 174 | Train loss 0.5012 Train acc 86.610 Val loss 0.5370 Val acc 88.888\n",
      "Epochs: 175 | Train loss 0.3499 Train acc 88.168 Val loss 0.5347 Val acc 85.457\n",
      "Epochs: 176 | Train loss 0.2918 Train acc 90.824 Val loss 0.3882 Val acc 90.158\n",
      "Epochs: 177 | Train loss 0.2431 Train acc 91.948 Val loss 0.3935 Val acc 90.214\n",
      "Epochs: 178 | Train loss 0.1998 Train acc 93.763 Val loss 0.3850 Val acc 91.267\n",
      "Epochs: 179 | Train loss 0.1709 Train acc 94.461 Val loss 0.3886 Val acc 91.564\n",
      "Epochs: 180 | Train loss 0.1492 Train acc 95.174 Val loss 0.3779 Val acc 91.496\n",
      "Epochs: 181 | Train loss 0.1314 Train acc 95.928 Val loss 0.4061 Val acc 91.195\n",
      "Epochs: 182 | Train loss 0.1201 Train acc 96.254 Val loss 0.4332 Val acc 91.416\n",
      "Epochs: 183 | Train loss 0.1197 Train acc 96.106 Val loss 0.4050 Val acc 91.830\n",
      "Epochs: 186 | Train loss 0.2399 Train acc 91.802 Val loss 0.4049 Val acc 88.793\n",
      "Epochs: 187 | Train loss 0.1884 Train acc 93.931 Val loss 0.4273 Val acc 89.774\n",
      "Epochs: 188 | Train loss 0.1588 Train acc 94.377 Val loss 0.3782 Val acc 91.450\n",
      "Epochs: 189 | Train loss 0.1343 Train acc 95.788 Val loss 0.3785 Val acc 91.649\n",
      "Epochs: 190 | Train loss 0.1234 Train acc 95.996 Val loss 0.4230 Val acc 91.350\n",
      "Epochs: 191 | Train loss 0.1171 Train acc 96.089 Val loss 0.4149 Val acc 92.106\n",
      "Epochs: 194 | Train loss 0.3491 Train acc 89.199 Val loss 0.3829 Val acc 88.282\n",
      "Epochs: 195 | Train loss 0.2811 Train acc 90.803 Val loss 0.4400 Val acc 89.293\n",
      "Epochs: 196 | Train loss 0.2329 Train acc 92.233 Val loss 0.3851 Val acc 90.305\n",
      "Epochs: 197 | Train loss 0.1902 Train acc 94.029 Val loss 0.3610 Val acc 91.456\n",
      "Epochs: 198 | Train loss 0.1596 Train acc 94.925 Val loss 0.3835 Val acc 91.156\n",
      "Epochs: 199 | Train loss 0.1414 Train acc 95.476 Val loss 0.3855 Val acc 91.262\n"
     ]
    }
   ],
   "source": [
    "optimizer = optim.Adam(NERmodel.parameters(), lr=args.lr)\n",
    "mask_index = vectorizer.target_vocab.mask_index\n",
    "# dataset.class_weights = dataset.class_weights.to(args.device)\n",
    "\n",
    "for epoch_index in range(args.num_epochs):\n",
    "    dataset.set_split('train')\n",
    "    batch_generator = generate_batches(dataset, batch_size=args.batch_train, device=args.device)\n",
    "    running_loss = 0.0\n",
    "    running_acc = 0.0\n",
    "    NERmodel.train()\n",
    "\n",
    "    for batch_index, batch_dict in enumerate(batch_generator):  \n",
    "        # step 1. zero the gradients\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        # step 2. compute the output\n",
    "        y_pred = NERmodel(batch_dict['x_source'],  batch_dict['x_length'], batch_dict['x_target'])\n",
    "\n",
    "        # step 3. compute the loss\n",
    "        loss = sequence_loss(y_pred, batch_dict['y_target'], mask_index)\n",
    "\n",
    "        # step 4. use loss to produce gradients\n",
    "        loss.backward()\n",
    "\n",
    "        # step 5. use optimizer to take gradient step\n",
    "        optimizer.step()\n",
    "\n",
    "        # compute the running loss and running accuracy\n",
    "        running_loss += (loss.item() - running_loss) / (batch_index + 1)\n",
    "\n",
    "        acc_t = compute_accuracy(y_pred, batch_dict['y_target'], mask_index)\n",
    "        running_acc += (acc_t - running_acc) / (batch_index + 1)\n",
    "    \n",
    "    train_state['train_loss'].append(running_loss)\n",
    "    train_state['train_acc'].append(running_acc)\n",
    "    \n",
    "    dataset.set_split('val')\n",
    "    args.batch_valid = args.batch_valid if args.batch_valid else len(dataset)\n",
    "    batch_generator = generate_batches(dataset,  batch_size=args.batch_valid,  device=args.device)\n",
    "    running_loss = 0.\n",
    "    running_acc = 0.\n",
    "    NERmodel.eval()\n",
    "\n",
    "    for batch_index, batch_dict in enumerate(batch_generator):\n",
    "        # compute the output\n",
    "        y_pred = NERmodel(batch_dict['x_source'],  batch_dict['x_length'],  batch_dict['x_target'])\n",
    "\n",
    "        # step 1. compute the loss\n",
    "        loss = sequence_loss(y_pred, batch_dict['y_target'], mask_index)\n",
    "\n",
    "        # compute the running loss and accuracy\n",
    "        running_loss += (loss.item() - running_loss) / (batch_index + 1)\n",
    "\n",
    "        acc_t = compute_accuracy(y_pred, batch_dict['y_target'], mask_index)\n",
    "        running_acc += (acc_t - running_acc) / (batch_index + 1)\n",
    "\n",
    "    train_state['val_loss'].append(running_loss)\n",
    "    train_state['val_acc'].append(running_acc)\n",
    "    try:\n",
    "        if train_state['train_loss'][-1]< train_state['train_loss'][-2]:\n",
    "            pretty_print(epoch_index, train_state['train_loss'][-1], train_state['train_acc'][-1], train_state['val_loss'][-1], train_state['val_acc'][-1])\n",
    "    except:\n",
    "        pretty_print(epoch_index, train_state['train_loss'][-1], train_state['train_acc'][-1], train_state['val_loss'][-1], train_state['val_acc'][-1])\n",
    "    train_state = update_train_state(args=args, model=NERmodel, train_state=train_state)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3d2a9c2b-584b-4fdd-aa83-af33e973db67",
   "metadata": {},
   "source": [
    "## Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "d53050eb-2b00-4fa5-8acf-ca9e75b30b40",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test loss 0.4125 Test acc 90.968\n"
     ]
    }
   ],
   "source": [
    "args.device = torch.device(\"cpu\")\n",
    "# checkpoint = torch.load(args.model_state_file)\n",
    "# NERmodel.load_state_dict(checkpoint['model_state_dict'])\n",
    "NERmodel.load_state_dict(torch.load(train_state['model_filename']))\n",
    "NERmodel = NERmodel.to(args.device)\n",
    "\n",
    "dataset.set_split('test')\n",
    "args.batch_test = len(dataset)\n",
    "batch_generator = generate_batches(dataset,  batch_size=args.batch_test,  device=args.device)\n",
    "\n",
    "running_loss = 0.\n",
    "running_acc = 0.\n",
    "NERmodel.eval()\n",
    "mask_index = vectorizer.target_vocab.mask_index\n",
    "for batch_index, batch_dict in enumerate(batch_generator):\n",
    "    # compute the output\n",
    "    y_pred = NERmodel(batch_dict['x_source'],  batch_dict['x_length'],  batch_dict['x_target'])\n",
    "\n",
    "    # step 1. compute the loss\n",
    "    loss = sequence_loss(y_pred, batch_dict['y_target'], mask_index)\n",
    "\n",
    "    # compute the running loss and accuracy\n",
    "    running_loss += (loss.item() - running_loss) / (batch_index + 1)\n",
    "\n",
    "    acc_t = compute_accuracy(y_pred, batch_dict['y_target'], mask_index)\n",
    "    running_acc += (acc_t - running_acc) / (batch_index + 1)\n",
    "    y_true = batch_dict['y_target']\n",
    "\n",
    "train_state['test_loss'] = running_loss\n",
    "train_state['test_acc'] = running_acc\n",
    "\n",
    "print('Test loss', str(running_loss)[: 6], 'Test acc', str(running_acc)[: 6])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "c5d5e8dd-fded-42bb-8ecd-d1c60f4cbfb1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7278163374265884"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred, y_true = normalize_sizes(y_pred, y_true)\n",
    "y_true = y_true.cpu().detach().numpy()\n",
    "_, y_pred_indices = y_pred.max(dim=1)\n",
    "\n",
    "y_pred_indices = y_pred_indices.cpu().detach().numpy()\n",
    "df = pd.DataFrame({'y_true':y_true, 'y_pred':y_pred_indices})\n",
    "df = df[df['y_true'] != 0]\n",
    "df = df[df['y_true'] != 179] # Remvoe \"O\" tag\n",
    "accuracy_score(df['y_true'], df['y_pred'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "39cea9fe-6d9c-4329-ba2a-c0e802a8766e",
   "metadata": {},
   "source": [
    "Out of 179 classes (exclude \"O\" tag) the performance of the mode  is not good in terms of accuracy, But as it have 179 entiyies the performance is better than baseline (0.56%   1/179)\n",
    "\n",
    "During our test we found our Performance improved from 0.27 to 0.72 which around 45% of performance improvement, this demonstrate Seq2Seq with self attention will perfrom model performance significantly\n",
    "\n",
    "To view the confusion matrix comment out code below - long time to run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "f72fe30d-e32f-4cf0-a5a3-bfddd8e252c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plot is too big\n",
    "\n",
    "# from matplotlib.pyplot import figure\n",
    "# cmp = ConfusionMatrixDisplay.from_predictions(df['y_true'], df['y_pred'], normalize='pred')\n",
    "# fig, ax = plt.subplots(figsize=(80,80))\n",
    "# cmp.plot(ax=ax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f58de374-5bf1-4d52-916a-df2f9b46b867",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "69fe9698-2486-47c4-9cc5-7d9777a1ec28",
   "metadata": {
    "tags": []
   },
   "source": [
    "# View the entity\n",
    "By just look at the accuracy of prediciton will not make sense, as some labels in the sentinces may misclassified by human or some entites are similar, to check the actual performance we need to look at it result\n",
    "\n",
    "Uncomment code out if needed, "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6bb255b9-0bcc-434a-bb2a-4feca81d0610",
   "metadata": {},
   "source": [
    "Bu viewing the text we can see the the model is catching up some newly tagged entities that make send like the one below\n",
    "```\n",
    "right         O                                        B-body_part/arm/shoulder/right_shoulder\n",
    "shoulder      B-body_part/arm/shoulder                 I-body_part/arm/shoulder/right_shoulder\n",
    "\n",
    "\n",
    "sudden                          O                       B-body_part/back\n",
    "back                            B-body_part/back        B-body_part/back\n",
    "\n",
    "high                 O                      B-accident_cause/fall\n",
    "fall                 B-accident_cause/fall  B-accident_cause/fall\n",
    "```\n",
    "Which make sense \n",
    "\n",
    "\n",
    "But also sometime this mode is also catching up some information that suppose not be in the entities categories such as \n",
    "```\n",
    "his                             O                                         B-body_part/arm/hand/left_hand\n",
    "left                            B-body_part/arm/hand/left_hand            I-body_part/arm/hand/left_hand\n",
    "hand                            I-body_part/arm/hand/left_hand            O\n",
    "and                             O                                         B-body_part/neck\n",
    "his                             O                                         B-body_part/neck\n",
    "neck                            B-body_part/neck                          O\n",
    "when                            O                                         B-body_part/neck\n",
    "\n",
    "\n",
    "muscle                          B-injury/muscle         O\n",
    "fatigue                         O                       B-injury/muscle\n",
    "```\n",
    "We can see there's slightly shift to the actual prediction\n",
    "\n",
    "which it taged the words in font of hand as `B-body_part/arm/hand` but not the `hand` word\n",
    "\n",
    "Also by looking at the prediction result we can see it actually catching `pronoun` following by `Adj`, `Noun` etc. which is understandable. \n",
    "\n",
    "```\n",
    "his       O                                 B-body_part/leg/foot/left_foot\n",
    "left      B-body_part/leg/foot/left_foot    B-body_part/leg/foot/left_foot\n",
    "foot      I-body_part/leg/foot/left_foot    I-body_part/leg/foot/left_foot\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "08781f88-4748-4fc1-943a-b4af0360f941",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "# Print 50 samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "b1608dfb-4990-4729-a946-efee5ac5b6b1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------  ----------------------------------------  ----------------------------------------\n",
      "employee                        B-person/employee                         B-person/employee\n",
      "when                            O                                         O\n",
      "work                            O                                         O\n",
      "in                              O                                         O\n",
      "his                             O                                         O\n",
      "area                            O                                         O\n",
      "cone                            O                                         O\n",
      "crusher                         B-equipment/mechanical_equipment/crusher  B-equipment/mechanical_equipment/crusher\n",
      "he                              O                                         O\n",
      "be                              O                                         O\n",
      "wear                            O                                         O\n",
      "his                             O                                         O\n",
      "personal-protective-equipment-  O                                         O\n",
      "and                             O                                         O\n",
      "harness                         O                                         O\n",
      "and                             O                                         O\n",
      "he                              O                                         O\n",
      "slip                            O                                         O\n",
      "and                             O                                         O\n",
      "fell                            O                                         O\n",
      "ft-                             O                                         O\n",
      "from                            O                                         O\n",
      "the                             O                                         O\n",
      "ground                          O                                         O\n",
      "never                           O                                         O\n",
      "lose                            O                                         O\n",
      "consciousness                   O                                         O\n",
      "he                              O                                         O\n",
      "be                              O                                         O\n",
      "complain                        O                                         O\n",
      "about                           O                                         O\n",
      "his                             O                                         B-body_part/arm/hand/left_hand\n",
      "left                            B-body_part/arm/hand/left_hand            I-body_part/arm/hand/left_hand\n",
      "hand                            I-body_part/arm/hand/left_hand            O\n",
      "and                             O                                         B-body_part/neck\n",
      "his                             O                                         B-body_part/neck\n",
      "neck                            B-body_part/neck                          O\n",
      "when                            O                                         B-body_part/neck\n",
      "em                              O                                         O\n",
      "arrive                          O                                         O\n",
      "he                              O                                         O\n",
      "be                              O                                         O\n",
      "triage                          O                                         O\n",
      "he                              O                                         O\n",
      "refuse                          O                                         O\n",
      "transportation                  O                                         O\n",
      "next                            O                                         O\n",
      "day                             O                                         O\n",
      "go                              O                                         O\n",
      "to                              O                                         O\n",
      "see                             O                                         O\n",
      "doctor                          O                                         O\n",
      "could                           O                                         O\n",
      "not                             O                                         O\n",
      "stand                           O                                         O\n",
      "the                             O                                         O\n",
      "pain                            O                                         O\n",
      "<END>                           <END>                                     O\n",
      "------------------------------  ----------------------------------------  ----------------------------------------\n",
      "-------------------------------  -----------------  -----------------\n",
      "employee                         B-person/employee  B-person/employee\n",
      "be                               O                  O\n",
      "instal                           O                  O\n",
      "roof                             O                  O\n",
      "bolt                             O                  O\n",
      "in                               O                  O\n",
      "the                              O                  O\n",
      "number                           O                  O\n",
      "entry                            O                  O\n",
      "face                             O                  O\n",
      "on                               O                  O\n",
      "mobile-mining-unit-number-9003-  O                  O\n",
      "section                          O                  O\n",
      "when                             O                  O\n",
      "the                              O                  O\n",
      "left                             O                  O\n",
      "rib                              O                  O\n",
      "roll                             O                  O\n",
      "off                              O                  O\n",
      "hit                              O                  O\n",
      "the                              O                  O\n",
      "employee                         B-person/employee  B-person/employee\n",
      "from                             O                  O\n",
      "behind                           O                  O\n",
      "the                              O                  O\n",
      "rib                              O                  O\n",
      "injured                          O                  O\n",
      "the                              O                  O\n",
      "low                              O                  O\n",
      "part                             O                  O\n",
      "of                               O                  O\n",
      "the                              O                  O\n",
      "employee's                       O                  O\n",
      "leg                              O                  B-body_part/leg\n",
      "<END>                            <END>              B-body_part/leg\n",
      "-------------------------------  -----------------  -----------------\n",
      "------------  ---------------------------  ---------------------------\n",
      "while         O                            O\n",
      "backing       O                            O\n",
      "loader        O                            O\n",
      "up            O                            O\n",
      "from          O                            O\n",
      "get           O                            O\n",
      "bucket        O                            O\n",
      "of            O                            O\n",
      "coal          O                            O\n",
      "right         O                            O\n",
      "rear          O                            O\n",
      "wheel         O                            O\n",
      "drop          O                            O\n",
      "down          O                            O\n",
      "in            O                            O\n",
      "hole-bounce-  O                            B-person/operator\n",
      "operator      B-person/operator            O\n",
      "and           O                            O\n",
      "injured       O                            O\n",
      "his           O                            O\n",
      "low           B-body_part/back/lower_back  B-body_part/back/lower_back\n",
      "back          I-body_part/back/lower_back  I-body_part/back/lower_back\n",
      "from          O                            O\n",
      "shock         O                            O\n",
      "above         O                            O\n",
      "and           O                            O\n",
      "bounce        O                            O\n",
      "<END>         <END>                        <END>\n",
      "------------  ---------------------------  ---------------------------\n",
      "------------------------------  ----------------------  --------------------------------------\n",
      "miner                           O                       O\n",
      "suffer                          O                       O\n",
      "low                             O                       O\n",
      "left                            O                       B-body_part/arm/shoulder/left_shoulder\n",
      "back                            B-body_part/back        I-body_part/arm/shoulder/left_shoulder\n",
      "muscle                          B-injury/muscle         B-body_part/back\n",
      "strain                          B-injury/muscle/strain  O\n",
      "miner                           O                       O\n",
      "be                              O                       O\n",
      "on                              O                       O\n",
      "knee                            O                       O\n",
      "move                            O                       O\n",
      "cam                             O                       O\n",
      "to                              O                       O\n",
      "set                             O                       O\n",
      "screen                          O                       O\n",
      "deck                            O                       O\n",
      "follow                          O                       O\n",
      "normal                          O                       O\n",
      "procedure                       O                       O\n",
      "on                              O                       O\n",
      "rotex                           O                       O\n",
      "size                            O                       O\n",
      "machine                         O                       O\n",
      "and                             O                       O\n",
      "report                          O                       O\n",
      "sudden                          O                       B-body_part/back\n",
      "back                            B-body_part/back        B-body_part/back\n",
      "pain                            O                       O\n",
      "miner                           O                       O\n",
      "be                              O                       O\n",
      "experienced                     O                       O\n",
      "in                              O                       O\n",
      "this                            O                       O\n",
      "task                            O                       O\n",
      "miner                           O                       O\n",
      "be                              O                       O\n",
      "wear                            O                       O\n",
      "assign                          O                       O\n",
      "personal-protective-equipment-  O                       O\n",
      "possible                        O                       O\n",
      "factor                          O                       O\n",
      "include                         O                       O\n",
      "muscle                          B-injury/muscle         O\n",
      "fatigue                         O                       B-injury/muscle\n",
      "from                            O                       O\n",
      "screen                          O                       O\n",
      "change                          O                       O\n",
      "inadequate                      O                       O\n",
      "rest                            O                       O\n",
      "period                          O                       O\n",
      "or                              O                       O\n",
      "stress                          O                       O\n",
      "of                              O                       O\n",
      "come                            O                       O\n",
      "erect                           O                       O\n",
      "from                            O                       <END>\n",
      "kneel                           O                       <END>\n",
      "<END>                           <END>                   <END>\n",
      "------------------------------  ----------------------  --------------------------------------\n",
      "----------  ----------------------------------------  ----------------------------------------\n",
      "instal      O                                         O\n",
      "bit         O                                         O\n",
      "sleeve      O                                         O\n",
      "on          O                                         O\n",
      "continuous  B-vehicle/heavy_vehicle/continuous_miner  B-vehicle/heavy_vehicle/continuous_miner\n",
      "miner       I-vehicle/heavy_vehicle/continuous_miner  I-vehicle/heavy_vehicle/continuous_miner\n",
      "with        O                                         O\n",
      "two         O                                         O\n",
      "pound       O                                         O\n",
      "hammer      O                                         O\n",
      "when        O                                         O\n",
      "sliver      O                                         O\n",
      "of          O                                         O\n",
      "metal       O                                         O\n",
      "broke       O                                         O\n",
      "off         O                                         O\n",
      "of          O                                         O\n",
      "the         O                                         O\n",
      "sleeve      O                                         O\n",
      "go          O                                         O\n",
      "through     O                                         O\n",
      "mechanic    O                                         O\n",
      "glove       O                                         O\n",
      "and         O                                         O\n",
      "cut         O                                         O\n",
      "knuckle     O                                         O\n",
      "of          O                                         O\n",
      "left        O                                         O\n",
      "index       O                                         O\n",
      "finger      O                                         O\n",
      "stitch      O                                         O\n",
      "be          O                                         O\n",
      "require     O                                         O\n",
      "to          O                                         O\n",
      "close       O                                         O\n",
      "the         O                                         O\n",
      "wound       O                                         O\n",
      "he          O                                         O\n",
      "be          O                                         O\n",
      "release     O                                         O\n",
      "and         O                                         O\n",
      "return      O                                         O\n",
      "to          O                                         O\n",
      "work        O                                         O\n",
      "the         O                                         O\n",
      "next        O                                         O\n",
      "day         O                                         <END>\n",
      "<END>       <END>                                     <END>\n",
      "----------  ----------------------------------------  ----------------------------------------\n",
      "---------  --------------------  --------------------\n",
      "three      O                     O\n",
      "miner      O                     O\n",
      "be         O                     O\n",
      "cleaning   O                     O\n",
      "up         O                     O\n",
      "one        O                     O\n",
      "be         O                     O\n",
      "operating  O                     O\n",
      "wheel      O                     O\n",
      "loader     O                     O\n",
      "and        O                     O\n",
      "the        O                     O\n",
      "other      O                     O\n",
      "two        O                     O\n",
      "be         O                     O\n",
      "throw      O                     O\n",
      "scrap      O                     O\n",
      "steel      O                     O\n",
      "into       O                     O\n",
      "the        O                     O\n",
      "bucket     O                     O\n",
      "to         O                     O\n",
      "be         O                     O\n",
      "take       O                     O\n",
      "to         O                     O\n",
      "dumpster   O                     O\n",
      "the        O                     B-person/operator\n",
      "loader     O                     B-person/operator\n",
      "operator   B-person/operator     O\n",
      "thought    O                     O\n",
      "one        O                     O\n",
      "of         O                     O\n",
      "the        O                     O\n",
      "other      O                     O\n",
      "miner      O                     O\n",
      "signal     O                     O\n",
      "operator   B-person/operator     O\n",
      "to         O                     B-person/operator\n",
      "low        O                     B-person/operator\n",
      "the        O                     B-person/operator\n",
      "bucket     O                     B-person/operator\n",
      "operator   B-person/operator     O\n",
      "then       O                     O\n",
      "lower      O                     O\n",
      "the        O                     O\n",
      "bucket     O                     O\n",
      "onto       O                     O\n",
      "one        O                     O\n",
      "of         O                     O\n",
      "the        O                     O\n",
      "miner      O                     B-body_part/leg/foot\n",
      "foot       B-body_part/leg/foot  B-body_part/leg/foot\n",
      "<END>      <END>                 <END>\n",
      "---------  --------------------  --------------------\n",
      "------------  -------------------------------  -------------------------------\n",
      "fan           O                                O\n",
      "belt          O                                O\n",
      "be            O                                O\n",
      "instal        O                                O\n",
      "on            O                                O\n",
      "the           O                                O\n",
      "compressor    O                                O\n",
      "for           O                                O\n",
      "the           O                                O\n",
      "on            O                                O\n",
      "brookville    O                                O\n",
      "motor         O                                O\n",
      "the           O                                O\n",
      "fan           O                                O\n",
      "belt          O                                O\n",
      "have          O                                O\n",
      "twist         O                                O\n",
      "in            O                                O\n",
      "it            O                                O\n",
      "after         O                                O\n",
      "installation  O                                O\n",
      "the           O                                O\n",
      "victim        O                                O\n",
      "be            O                                O\n",
      "not           O                                O\n",
      "involve       O                                O\n",
      "in            O                                O\n",
      "the           O                                O\n",
      "installation  O                                O\n",
      "of            O                                O\n",
      "the           O                                O\n",
      "belt          O                                O\n",
      "but           O                                O\n",
      "want          O                                O\n",
      "to            O                                O\n",
      "fix           O                                O\n",
      "the           O                                O\n",
      "twist         O                                O\n",
      "at            O                                O\n",
      "this          O                                O\n",
      "time          O                                O\n",
      "the           O                                O\n",
      "motorman      O                                O\n",
      "set           O                                O\n",
      "the           O                                O\n",
      "breaker       O                                O\n",
      "on            O                                O\n",
      "the           O                                O\n",
      "motor         O                                O\n",
      "the           O                                O\n",
      "victim        O                                O\n",
      "reach         O                                O\n",
      "for           O                                O\n",
      "the           O                                O\n",
      "belt          O                                O\n",
      "a             O                                O\n",
      "the           O                                O\n",
      "breaker       O                                O\n",
      "be            O                                O\n",
      "set           O                                B-body_part/arm/hand\n",
      "compressor    O                                B-body_part/arm/hand\n",
      "start         O                                O\n",
      "hand          B-body_part/arm/hand             B-accident_cause/caught_between\n",
      "be            O                                B-accident_cause/caught_between\n",
      "caught        B-accident_cause/caught_between  I-accident_cause/caught_between\n",
      "between       I-accident_cause/caught_between  O\n",
      "belt          O                                O\n",
      "and           O                                <END>\n",
      "pull          O                                <END>\n",
      "<END>         <END>                            <END>\n",
      "------------  -------------------------------  -------------------------------\n",
      "-------------------  ---------------------  ---------------------\n",
      "north                O                      O\n",
      "to                   O                      O\n",
      "block                O                      O\n",
      "entry                O                      O\n",
      "mobile-mining-unit-  O                      O\n",
      "004.                 O                      O\n",
      "support              O                      O\n",
      "ft                   O                      O\n",
      "3/4                  O                      O\n",
      "inch                 O                      O\n",
      "resin                O                      O\n",
      "bolt                 O                      O\n",
      "pan                  O                      O\n",
      "brow                 O                      O\n",
      "strap                O                      O\n",
      "screen               O                      O\n",
      "17                   O                      O\n",
      "ft                   O                      O\n",
      "wide                 O                      O\n",
      "70                   O                      O\n",
      "ft                   O                      O\n",
      "long                 O                      O\n",
      "ft                   O                      O\n",
      "high                 O                      B-accident_cause/fall\n",
      "fall                 B-accident_cause/fall  B-accident_cause/fall\n",
      "dangered             O                      O\n",
      "off                  O                      O\n",
      "additional           O                      O\n",
      "roof                 O                      O\n",
      "support              O                      O\n",
      "instal               O                      O\n",
      "around               O                      O\n",
      "fall                 B-accident_cause/fall  B-accident_cause/fall\n",
      "and                  O                      O\n",
      "clean                O                      O\n",
      "up                   O                      O\n",
      "plan                 O                      O\n",
      "post                 O                      O\n",
      "<END>                <END>                  <END>\n",
      "-------------------  ---------------------  ---------------------\n",
      "--------  --------------------------------  --------------------------------\n",
      "employee  B-person/employee                 B-person/employee\n",
      "have      O                                 O\n",
      "just      O                                 O\n",
      "park      O                                 O\n",
      "the       O                                 O\n",
      "tractor   O                                 O\n",
      "by        O                                 O\n",
      "the       O                                 O\n",
      "ore       O                                 O\n",
      "muck      O                                 O\n",
      "bay       O                                 O\n",
      "he        O                                 O\n",
      "step      O                                 O\n",
      "off       O                                 O\n",
      "of        O                                 O\n",
      "the       O                                 O\n",
      "tractor   O                                 O\n",
      "and       O                                 O\n",
      "his       O                                 O\n",
      "left      B-body_part/leg/foot/left_foot    O\n",
      "foot      I-body_part/leg/foot/left_foot    B-body_part/leg/foot/left_foot\n",
      "come      O                                 I-body_part/leg/foot/left_foot\n",
      "down      O                                 O\n",
      "on        O                                 O\n",
      "small     O                                 O\n",
      "rock      O                                 O\n",
      "he        O                                 O\n",
      "twist     O                                 O\n",
      "his       O                                 O\n",
      "left      B-body_part/leg/ankle/left_ankle  B-body_part/leg/ankle/left_ankle\n",
      "ankle     I-body_part/leg/ankle/left_ankle  I-body_part/leg/ankle/left_ankle\n",
      "<END>     <END>                             <END>\n",
      "--------  --------------------------------  --------------------------------\n",
      "----------  --------------------  --------------------\n",
      "employee    B-person/employee     B-person/employee\n",
      "be          O                     O\n",
      "change      O                     O\n",
      "feed        O                     O\n",
      "chute       B-equipment/chute     O\n",
      "rubber      O                     O\n",
      "skirt       O                     O\n",
      "while       O                     O\n",
      "reposition  O                     O\n",
      "himself     O                     O\n",
      "employee    B-person/employee     B-person/employee\n",
      "grabbed     O                     O\n",
      "skirt       O                     O\n",
      "clamp       O                     O\n",
      "that        O                     O\n",
      "be          O                     O\n",
      "hang        O                     O\n",
      "knock       O                     O\n",
      "it          O                     O\n",
      "down        O                     O\n",
      "on          O                     O\n",
      "his         O                     B-body_part/arm/hand\n",
      "hand        B-body_part/arm/hand  O\n",
      "break       O                     O\n",
      "the         O                     O\n",
      "tip         O                     O\n",
      "of          O                     O\n",
      "his         O                     <END>\n",
      "finger      O                     <END>\n",
      "<END>       <END>                 <END>\n",
      "----------  --------------------  --------------------\n",
      "--------  ------------------------  ------------------------\n",
      "employee  B-person/employee         B-person/employee\n",
      "remove    O                         O\n",
      "rock      O                         O\n",
      "stuck     O                         O\n",
      "between   O                         O\n",
      "the       O                         O\n",
      "tire      O                         O\n",
      "of        O                         O\n",
      "water     O                         O\n",
      "truck     O                         O\n",
      "and       O                         O\n",
      "strain    O                         O\n",
      "his       O                         O\n",
      "shoulder  B-body_part/arm/shoulder  B-body_part/arm/shoulder\n",
      "torn      O                         O\n",
      "right     O                         O\n",
      "bicep     O                         O\n",
      "be        O                         O\n",
      "diagnose  O                         O\n",
      "<END>     <END>                     <END>\n",
      "--------  ------------------------  ------------------------\n",
      "--------  -----------------------------  -----------------------------\n",
      "employee  B-person/employee              B-person/employee\n",
      "have      O                              O\n",
      "change    O                              O\n",
      "light     O                              O\n",
      "on        O                              O\n",
      "cab       O                              O\n",
      "of        O                              O\n",
      "d10n      O                              O\n",
      "dozer     B-vehicle/heavy_vehicle/dozer  O\n",
      "a         O                              B-vehicle/heavy_vehicle/dozer\n",
      "he        O                              O\n",
      "change    O                              O\n",
      "body      B-body_part/body               B-body_part/body\n",
      "position  O                              O\n",
      "he        O                              O\n",
      "either    O                              O\n",
      "slip      O                              O\n",
      "or        O                              O\n",
      "tripped   O                              O\n",
      "and       O                              O\n",
      "fell      O                              O\n",
      "off       O                              O\n",
      "dozer     B-vehicle/heavy_vehicle/dozer  B-vehicle/heavy_vehicle/dozer\n",
      "to        O                              O\n",
      "the       O                              O\n",
      "ground    O                              <END>\n",
      "<END>     <END>                          <END>\n",
      "--------  -----------------------------  -----------------------------\n",
      "--------  ---------------------------  ---------------------------\n",
      "employee  B-person/employee            B-person/employee\n",
      "drive     B-activity/driving           O\n",
      "rock      O                            O\n",
      "truck     O                            O\n",
      "up        O                            O\n",
      "out       O                            O\n",
      "of        O                            O\n",
      "steep     O                            O\n",
      "pit       O                            O\n",
      "when      O                            O\n",
      "the       O                            O\n",
      "load      O                            O\n",
      "shift     O                            O\n",
      "to        O                            O\n",
      "the       O                            O\n",
      "rear      O                            O\n",
      "cause     O                            O\n",
      "the       O                            O\n",
      "truck     O                            O\n",
      "front     O                            O\n",
      "end       O                            O\n",
      "to        O                            O\n",
      "raise     O                            O\n",
      "up        O                            O\n",
      "4-5       O                            O\n",
      "foot      O                            O\n",
      "and       O                            O\n",
      "then      O                            O\n",
      "slam      O                            O\n",
      "down      O                            O\n",
      "jarring   O                            O\n",
      "the       O                            O\n",
      "truck     O                            O\n",
      "cause     O                            O\n",
      "pain      O                            B-body_part/back/lower_back\n",
      "in        O                            I-body_part/back/lower_back\n",
      "low       B-body_part/back/lower_back  <END>\n",
      "back      I-body_part/back/lower_back  <END>\n",
      "<END>     <END>                        <END>\n",
      "--------  ---------------------------  ---------------------------\n",
      "--------  -----  -----\n",
      "highwall  O      O\n",
      "failure   O      O\n",
      "measure   O      O\n",
      "40        O      O\n",
      "high      O      O\n",
      "thick     O      O\n",
      "30        O      O\n",
      "wide      O      <END>\n",
      "<END>     <END>  <END>\n",
      "--------  -----  -----\n",
      "--------  ------------------------  ------------------------\n",
      "employee  B-person/employee         B-person/employee\n",
      "be        O                         O\n",
      "injured   O                         O\n",
      "on        O                         O\n",
      "7/27/06   O                         O\n",
      "he        O                         O\n",
      "return    O                         O\n",
      "to        O                         O\n",
      "work      O                         O\n",
      "on        O                         O\n",
      "8/9/06    O                         O\n",
      "on        O                         O\n",
      "regular   O                         O\n",
      "duty      O                         O\n",
      "he        O                         O\n",
      "state     O                         O\n",
      "that      O                         O\n",
      "his       O                         O\n",
      "shoulder  B-body_part/arm/shoulder  B-body_part/arm/shoulder\n",
      "start     O                         O\n",
      "hurt      O                         O\n",
      "again     O                         O\n",
      "on        O                         O\n",
      "8/25/06   O                         O\n",
      "<END>     <END>                     <END>\n",
      "--------  ------------------------  ------------------------\n",
      "----------  ---------------------------------  ---------------------------------\n",
      "employee    B-person/employee                  B-person/employee\n",
      "be          O                                  O\n",
      "enlarge     O                                  O\n",
      "pilot       O                                  O\n",
      "hole        O                                  O\n",
      "in          O                                  O\n",
      "guard       O                                  O\n",
      "he          O                                  O\n",
      "be          O                                  O\n",
      "building    O                                  O\n",
      "with        O                                  O\n",
      "electric    O                                  O\n",
      "drill       O                                  O\n",
      "when        O                                  O\n",
      "bit         O                                  O\n",
      "caught      O                                  O\n",
      "and         O                                  O\n",
      "force       O                                  O\n",
      "drill       O                                  O\n",
      "from        O                                  O\n",
      "his         O                                  B-body_part/arm/hand\n",
      "hand        B-body_part/arm/hand               B-body_part/arm/hand\n",
      "the         O                                  O\n",
      "drill       O                                  O\n",
      "spun        O                                  O\n",
      "around      O                                  O\n",
      "and         O                                  O\n",
      "the         O                                  O\n",
      "handle      O                                  O\n",
      "struck      O                                  O\n",
      "his         O                                  B-body_part/arm/wrist/right_wrist\n",
      "right       B-body_part/arm/wrist/right_wrist  B-body_part/arm/wrist/right_wrist\n",
      "wrist       I-body_part/arm/wrist/right_wrist  I-body_part/arm/wrist/right_wrist\n",
      "he          O                                  O\n",
      "broke       O                                  O\n",
      "his         O                                  O\n",
      "4th         O                                  O\n",
      "metatarsal  O                                  O\n",
      "bone        O                                  O\n",
      "in          O                                  O\n",
      "his         O                                  O\n",
      "right       B-body_part/arm/wrist/right_wrist  B-body_part/arm/wrist/right_wrist\n",
      "wrist       I-body_part/arm/wrist/right_wrist  I-body_part/arm/wrist/right_wrist\n",
      "<END>       <END>                              <END>\n",
      "----------  ---------------------------------  ---------------------------------\n",
      "--------  -----------------  -----------------\n",
      "bolter    O                  B-person/operator\n",
      "operator  B-person/operator  B-person/operator\n",
      "be        O                  O\n",
      "bolting   O                  O\n",
      "roof      O                  O\n",
      "in        O                  O\n",
      "number    O                  O\n",
      "entry     O                  O\n",
      "on        O                  O\n",
      "1st       O                  O\n",
      "right     O                  O\n",
      "section   O                  O\n",
      "top       O                  O\n",
      "be        O                  O\n",
      "real      O                  O\n",
      "hard      O                  O\n",
      "and       O                  O\n",
      "bit       O                  O\n",
      "tore      O                  O\n",
      "up        O                  O\n",
      "he        O                  O\n",
      "be        O                  O\n",
      "knock     O                  O\n",
      "the       O                  O\n",
      "bit       O                  O\n",
      "off       O                  O\n",
      "the       O                  O\n",
      "drill     O                  O\n",
      "steel     O                  O\n",
      "to        O                  O\n",
      "change    O                  O\n",
      "it        O                  O\n",
      "when      O                  O\n",
      "he        O                  O\n",
      "struck    O                  O\n",
      "his       O                  O\n",
      "left      O                  O\n",
      "index     O                  O\n",
      "finger    O                  O\n",
      "between   O                  O\n",
      "bolter    O                  O\n",
      "and       O                  O\n",
      "drill     O                  O\n",
      "steel     O                  O\n",
      "bruising  O                  O\n",
      "finger    O                  O\n",
      "he        O                  O\n",
      "be        O                  O\n",
      "off       O                  O\n",
      "day       O                  <END>\n",
      "<END>     <END>              <END>\n",
      "--------  -----------------  -----------------\n",
      "----------  ----------------------------------------  ----------------------------------------\n",
      "on          O                                         O\n",
      "november    O                                         O\n",
      "21          O                                         O\n",
      "02          O                                         O\n",
      "the         O                                         O\n",
      "employee    B-person/employee                         B-person/employee\n",
      "be          O                                         O\n",
      "told        O                                         O\n",
      "he          O                                         O\n",
      "be          O                                         O\n",
      "go          O                                         O\n",
      "to          O                                         O\n",
      "be          O                                         O\n",
      "laid        O                                         O\n",
      "off         O                                         O\n",
      "on          O                                         O\n",
      "11/22/02    O                                         O\n",
      "he          O                                         O\n",
      "then        O                                         O\n",
      "state       O                                         O\n",
      "that        O                                         O\n",
      "he          O                                         O\n",
      "have        O                                         O\n",
      "injured     O                                         O\n",
      "his         O                                         B-body_part/arm/shoulder\n",
      "shoulder    B-body_part/arm/shoulder                  B-body_part/arm/shoulder\n",
      "shovel      O                                         O\n",
      "rock        O                                         O\n",
      "at          O                                         O\n",
      "the         O                                         O\n",
      "crusher     B-equipment/mechanical_equipment/crusher  O\n",
      "on          O                                         O\n",
      "10/31/02    O                                         B-equipment/mechanical_equipment/crusher\n",
      "and         O                                         O\n",
      "have        O                                         O\n",
      "not         O                                         O\n",
      "report      O                                         O\n",
      "it          O                                         O\n",
      "previously  O                                         O\n",
      "<END>       <END>                                     <END>\n",
      "----------  ----------------------------------------  ----------------------------------------\n",
      "------------  -----------------  -----------------\n",
      "employee      B-person/employee  B-person/employee\n",
      "be            O                  O\n",
      "cleaning      O                  O\n",
      "breakroom     O                  O\n",
      "which         O                  O\n",
      "involve       O                  O\n",
      "move          O                  O\n",
      "furniture     O                  O\n",
      "to            O                  O\n",
      "clean         O                  O\n",
      "behind        O                  O\n",
      "it            O                  O\n",
      "when          O                  O\n",
      "the           O                  O\n",
      "refrigerator  O                  O\n",
      "be            O                  O\n",
      "move          O                  O\n",
      "the           O                  O\n",
      "employee      B-person/employee  B-person/employee\n",
      "felt          O                  O\n",
      "pain          O                  O\n",
      "in            O                  O\n",
      "the           O                  O\n",
      "middle        O                  O\n",
      "right         O                  O\n",
      "side          O                  O\n",
      "of            O                  O\n",
      "her           O                  O\n",
      "back          B-body_part/back   B-body_part/back\n",
      "<END>         <END>              <END>\n",
      "------------  -----------------  -----------------\n",
      "-------------  -----------------  -----------------\n",
      "injured        O                  O\n",
      "be             O                  O\n",
      "use            O                  O\n",
      "wash           O                  O\n",
      "down           O                  O\n",
      "hose           O                  O\n",
      "to             O                  O\n",
      "clean          O                  O\n",
      "on             O                  O\n",
      "and            O                  O\n",
      "around         O                  O\n",
      "the            O                  O\n",
      "east           O                  O\n",
      "number-        O                  O\n",
      "belt           O                  O\n",
      "drive          O                  O\n",
      "after          O                  O\n",
      "employee's     O                  O\n",
      "work           O                  O\n",
      "be             O                  O\n",
      "complete       O                  B-person/employee\n",
      "employee       B-person/employee  O\n",
      "be             O                  O\n",
      "go             O                  O\n",
      "to             O                  O\n",
      "cut            O                  O\n",
      "the            O                  O\n",
      "wash           O                  O\n",
      "down           O                  O\n",
      "hose           O                  O\n",
      "off            O                  O\n",
      "the            O                  O\n",
      "injured        O                  O\n",
      "slipped/fell   O                  O\n",
      "while          O                  O\n",
      "cut            O                  O\n",
      "the            O                  O\n",
      "handle         O                  O\n",
      "off            O                  O\n",
      "on             O                  O\n",
      "the            O                  B-person/employee\n",
      "hose           O                  B-person/employee\n",
      "employee       B-person/employee  O\n",
      "then           O                  O\n",
      "inadvertently  O                  O\n",
      "pressurize     O                  O\n",
      "the            O                  O\n",
      "hose           O                  O\n",
      "the            O                  O\n",
      "hose           O                  O\n",
      "come           O                  O\n",
      "loose          O                  O\n",
      "from           O                  O\n",
      "employee's     O                  O\n",
      "hand           O                  O\n",
      "and            O                  O\n",
      "struck         O                  O\n",
      "employee       B-person/employee  B-person/employee\n",
      "in             O                  B-person/employee\n",
      "the            O                  O\n",
      "left           O                  O\n",
      "side           O                  O\n",
      "of             O                  O\n",
      "employee's     O                  O\n",
      "forehead       O                  O\n",
      "near           O                  O\n",
      "hairline       O                  O\n",
      "cause          O                  O\n",
      "cut            O                  <END>\n",
      "<END>          <END>              <END>\n",
      "-------------  -----------------  -----------------\n",
      "----------------  ----------------------------  -------------------\n",
      "employee          B-person/employee             B-person/employee\n",
      "attempt           O                             O\n",
      "to                O                             O\n",
      "reinstall         O                             O\n",
      "an                O                             O\n",
      "air-conditioner-  O                             O\n",
      "vent              O                             O\n",
      "louver            O                             B-person/employee\n",
      "employee          B-person/employee             O\n",
      "place             O                             O\n",
      "finger            B-body_part/arm/hand/fingers  O\n",
      "beside            O                             O\n",
      "and               O                             O\n",
      "over              O                             O\n",
      "the               O                             O\n",
      "edge              O                             O\n",
      "of                O                             O\n",
      "the               O                             O\n",
      "louver            O                             O\n",
      "try               O                             O\n",
      "to                O                             O\n",
      "hold              O                             O\n",
      "the               O                             O\n",
      "screw             O                             O\n",
      "so                O                             B-person/employee\n",
      "employee          B-person/employee             O\n",
      "could             O                             O\n",
      "screw             O                             O\n",
      "the               O                             B-body_part/back\n",
      "louver            O                             O\n",
      "back              B-body_part/back              O\n",
      "on                O                             O\n",
      "the               O                             O\n",
      "louver            O                             O\n",
      "abruptly          O                             O\n",
      "turn              O                             O\n",
      "result            O                             O\n",
      "in                O                             O\n",
      "right             O                             O\n",
      "middle            O                             O\n",
      "finger            O                             O\n",
      "and               O                             O\n",
      "right             O                             O\n",
      "ring              O                             O\n",
      "finger            O                             O\n",
      "to                O                             O\n",
      "make              O                             O\n",
      "contact           O                             O\n",
      "with              O                             O\n",
      "the               O                             B-equipment/cage\n",
      "squirrel          O                             B-equipment/cage\n",
      "cage              B-equipment/cage              B-person/employee\n",
      "fan               O                             B-person/employee\n",
      "employee          B-person/employee             B-injury/laceration\n",
      "suffer            O                             B-injury/laceration\n",
      "laceration        B-injury/laceration           O\n",
      "and               O                             O\n",
      "receive           O                             O\n",
      "stitch            O                             O\n",
      "to                O                             O\n",
      "the               O                             O\n",
      "right             O                             O\n",
      "3rd               O                             <END>\n",
      "finger            O                             <END>\n",
      "<END>             <END>                         <END>\n",
      "----------------  ----------------------------  -------------------\n",
      "---------  ---------------------------  ---------------------------\n",
      "employee   B-person/employee            B-person/employee\n",
      "be         O                            O\n",
      "drive      B-activity/driving           B-activity/driving\n",
      "cu106      O                            O\n",
      "boom       O                            O\n",
      "rig        O                            O\n",
      "around     O                            O\n",
      "rib        O                            O\n",
      "at         O                            O\n",
      "main       O                            O\n",
      "shop       O                            O\n",
      "he         O                            O\n",
      "hit        O                            O\n",
      "pothole    O                            O\n",
      "which      O                            O\n",
      "cause      O                            O\n",
      "the        O                            O\n",
      "air        O                            O\n",
      "ride       O                            O\n",
      "seat       O                            O\n",
      "to         O                            O\n",
      "hit        O                            O\n",
      "bottom     O                            O\n",
      "and        O                            O\n",
      "jar        O                            O\n",
      "his        O                            O\n",
      "low        B-body_part/back/lower_back  B-body_part/back/lower_back\n",
      "back       I-body_part/back/lower_back  I-body_part/back/lower_back\n",
      "he         O                            I-body_part/back/lower_back\n",
      "say        O                            O\n",
      "he         O                            O\n",
      "felt       O                            O\n",
      "no         O                            O\n",
      "immediate  O                            O\n",
      "pain       O                            O\n",
      "later      O                            O\n",
      "in         O                            O\n",
      "the        O                            O\n",
      "day        O                            O\n",
      "the        O                            O\n",
      "pain       O                            O\n",
      "increase   O                            O\n",
      "when       O                            O\n",
      "he         O                            O\n",
      "go         O                            O\n",
      "home       O                            O\n",
      "for        O                            O\n",
      "the        O                            O\n",
      "weekend    O                            O\n",
      "he         O                            O\n",
      "start      O                            O\n",
      "take       O                            O\n",
      "oxycontin  O                            O\n",
      "injury     B-injury                     O\n",
      "report     O                            O\n",
      "on         O                            O\n",
      "1/10/11    O                            O\n",
      "<END>      <END>                        O\n",
      "---------  ---------------------------  ---------------------------\n",
      "--------  --------------------------------------  --------------------------------------\n",
      "employee  B-person/employee                       B-person/employee\n",
      "be        O                                       O\n",
      "use       O                                       O\n",
      "prybar    O                                       O\n",
      "to        O                                       O\n",
      "move      O                                       O\n",
      "heavy     O                                       O\n",
      "object    O                                       O\n",
      "when      O                                       O\n",
      "the       O                                       O\n",
      "prybar    O                                       O\n",
      "slip      O                                       O\n",
      "and       O                                       O\n",
      "pinch     O                                       O\n",
      "his       O                                       O\n",
      "right     B-body_part/arm/hand/thumb/right_thumb  B-body_part/arm/hand/thumb/right_thumb\n",
      "thumb     I-body_part/arm/hand/thumb/right_thumb  I-body_part/arm/hand/thumb/right_thumb\n",
      "against   O                                       O\n",
      "concrete  O                                       O\n",
      "wall      O                                       O\n",
      "directly  O                                       O\n",
      "behind    O                                       O\n",
      "him       O                                       <END>\n",
      "<END>     <END>                                   <END>\n",
      "--------  --------------------------------------  --------------------------------------\n",
      "------------  ---------------------------------------  ---------------------------------------\n",
      "employee      B-person/employee                        B-person/employee\n",
      "board         O                                        O\n",
      "rubber        O                                        O\n",
      "tire          O                                        O\n",
      "dozer         B-vehicle/heavy_vehicle/dozer            B-vehicle/heavy_vehicle/dozer\n",
      "without       O                                        O\n",
      "incident      O                                        O\n",
      "at-start-     O                                        O\n",
      "of            O                                        O\n",
      "shift         O                                        O\n",
      "during        O                                        O\n",
      "shift         O                                        O\n",
      "experienced   O                                        O\n",
      "pain          O                                        O\n",
      "in-area-      O                                        O\n",
      "of            O                                        O\n",
      "right         B-body_part/arm/shoulder/right_shoulder  O\n",
      "shoulder      I-body_part/arm/shoulder/right_shoulder  B-body_part/arm/shoulder/right_shoulder\n",
      "blade         O                                        I-body_part/arm/shoulder/right_shoulder\n",
      "saw           O                                        O\n",
      "physician     O                                        O\n",
      "on            O                                        O\n",
      "2-5           O                                        O\n",
      "be            O                                        O\n",
      "diagnose      O                                        O\n",
      "with          O                                        O\n",
      "right         O                                        B-body_part/arm/shoulder/right_shoulder\n",
      "shoulder      B-body_part/arm/shoulder                 I-body_part/arm/shoulder/right_shoulder\n",
      "tendonitis    O                                        O\n",
      "be            O                                        O\n",
      "take          O                                        O\n",
      "off           O                                        O\n",
      "work          O                                        O\n",
      "on            O                                        O\n",
      "2-20-03.      O                                        O\n",
      "receive       O                                        O\n",
      "chart         O                                        O\n",
      "note          O                                        O\n",
      "on            O                                        O\n",
      "3-21-03.      O                                        O\n",
      "lose          O                                        O\n",
      "time          O                                        O\n",
      "restrict      O                                        O\n",
      "work          O                                        O\n",
      "occupational  O                                        O\n",
      "illness       O                                        <END>\n",
      "<END>         <END>                                    <END>\n",
      "------------  ---------------------------------------  ---------------------------------------\n",
      "--------  -----------------------------------------  -----------------------------------------\n",
      "employee  B-person/employee                          B-person/employee\n",
      "walk      B-activity/walking                         O\n",
      "along     O                                          O\n",
      "flat      O                                          O\n",
      "surface   O                                          O\n",
      "number    O                                          O\n",
      "101       O                                          O\n",
      "conveyor  B-equipment/mechanical_equipment/conveyor  B-equipment/mechanical_equipment/conveyor\n",
      "catwalk   O                                          O\n",
      "step      O                                          O\n",
      "on        O                                          O\n",
      "fist      O                                          O\n",
      "size      O                                          O\n",
      "rock      O                                          O\n",
      "twist     O                                          B-body_part/leg/knee/left_knee\n",
      "left      B-body_part/leg/knee/left_knee             I-body_part/leg/knee/left_knee\n",
      "knee      I-body_part/leg/knee/left_knee             O\n",
      "fell      O                                          O\n",
      "down      O                                          O\n",
      "come      O                                          O\n",
      "down      O                                          O\n",
      "on        O                                          B-body_part/leg/knee/left_knee\n",
      "left      B-body_part/leg/knee/left_knee             I-body_part/leg/knee/left_knee\n",
      "knee      I-body_part/leg/knee/left_knee             <END>\n",
      "<END>     <END>                                      <END>\n",
      "--------  -----------------------------------------  -----------------------------------------\n",
      "--------  ------------------------------  ------------------------------\n",
      "he        O                               O\n",
      "be        O                               O\n",
      "shovel    O                               O\n",
      "at        O                               O\n",
      "the       O                               O\n",
      "headgate  O                               O\n",
      "drive     O                               O\n",
      "area      O                               O\n",
      "on        O                               O\n",
      "the       O                               O\n",
      "longwall  O                               O\n",
      "face      O                               O\n",
      "he        O                               O\n",
      "stood     O                               O\n",
      "up        O                               O\n",
      "and       O                               O\n",
      "cut       O                               O\n",
      "his       O                               O\n",
      "right     B-body_part/head/ear/right_ear  O\n",
      "ear       I-body_part/head/ear/right_ear  B-body_part/head/ear/right_ear\n",
      "on        O                               I-body_part/head/ear/right_ear\n",
      "roof      O                               O\n",
      "strap     O                               O\n",
      "the       O                               O\n",
      "result    O                               O\n",
      "be        O                               O\n",
      "suture    O                               O\n",
      "<END>     <END>                           O\n",
      "--------  ------------------------------  ------------------------------\n",
      "-------------  ---------------------  ---------------------\n",
      "an             O                      O\n",
      "unplanned      O                      O\n",
      "roof           O                      O\n",
      "fall           B-accident_cause/fall  B-accident_cause/fall\n",
      "have           O                      O\n",
      "occur          O                      O\n",
      "above          O                      O\n",
      "permanent      O                      O\n",
      "anchorage      O                      O\n",
      "on             O                      O\n",
      "the            O                      O\n",
      "11             O                      O\n",
      "north          O                      O\n",
      "miner          O                      O\n",
      "section        O                      O\n",
      "this           O                      O\n",
      "fall           B-accident_cause/fall  B-accident_cause/fall\n",
      "exists         O                      B-accident_cause/fall\n",
      "in             O                      O\n",
      "the            O                      O\n",
      "number         O                      O\n",
      "belt           O                      O\n",
      "entry          O                      O\n",
      "one            O                      O\n",
      "cross-cut      O                      O\n",
      "outby          O                      O\n",
      "spad           O                      O\n",
      "number         O                      O\n",
      "566            O                      O\n",
      "and            O                      O\n",
      "measure        O                      O\n",
      "approximately  O                      O\n",
      "50'-           O                      O\n",
      "20             O                      O\n",
      "''             O                      O\n",
      "10             O                      O\n",
      "high           O                      O\n",
      "<END>          <END>                  O\n",
      "-------------  ---------------------  ---------------------\n",
      "------------  --------------------  --------------------\n",
      "maintenance   O                     O\n",
      "worker        B-person/worker       B-person/worker\n",
      "have          O                     O\n",
      "be            O                     O\n",
      "cut           O                     O\n",
      "weld          O                     O\n",
      "and           O                     O\n",
      "preform       O                     O\n",
      "regular       O                     O\n",
      "maintenance   O                     O\n",
      "duty          O                     O\n",
      "throughout    O                     O\n",
      "the           O                     O\n",
      "day           O                     O\n",
      "employee      B-person/employee     B-person/employee\n",
      "be            O                     O\n",
      "perspire      O                     O\n",
      "and           O                     O\n",
      "use           O                     O\n",
      "sleeve        O                     O\n",
      "to            O                     O\n",
      "wipe          O                     O\n",
      "perspiration  O                     O\n",
      "from          O                     O\n",
      "cheek         O                     O\n",
      "few           O                     O\n",
      "minute        O                     O\n",
      "later         O                     O\n",
      "employee      B-person/employee     B-person/employee\n",
      "felt          O                     O\n",
      "like          O                     B-body_part/head/eye\n",
      "eye           B-body_part/head/eye  B-body_part/head/eye\n",
      "have          O                     O\n",
      "something     O                     O\n",
      "in            O                     O\n",
      "it            O                     O\n",
      "employee      B-person/employee     B-person/employee\n",
      "wash          O                     O\n",
      "eye           B-body_part/head/eye  O\n",
      "out           O                     B-body_part/head/eye\n",
      "but           O                     O\n",
      "eye           B-body_part/head/eye  O\n",
      "be            O                     B-body_part/head/eye\n",
      "still         O                     O\n",
      "irritated     O                     O\n",
      "take          O                     O\n",
      "employee      B-person/employee     O\n",
      "to            O                     O\n",
      "doctor        O                     O\n",
      "eye           B-body_part/head/eye  O\n",
      "be            O                     O\n",
      "scratch       O                     O\n",
      "prescription  O                     <END>\n",
      "give          O                     <END>\n",
      "to            O                     <END>\n",
      "prevent       O                     <END>\n",
      "infection     O                     <END>\n",
      "<END>         <END>                 <END>\n",
      "------------  --------------------  --------------------\n",
      "--------  -------------------------------  -------------------------------\n",
      "employee  B-person/employee                B-person/employee\n",
      "be        O                                O\n",
      "building  O                                O\n",
      "stop      O                                O\n",
      "when      O                                O\n",
      "he        O                                O\n",
      "mash      O                                O\n",
      "finger    O                                O\n",
      "on        O                                O\n",
      "right     B-body_part/arm/hand/right_hand  B-body_part/arm/hand/right_hand\n",
      "hand      I-body_part/arm/hand/right_hand  I-body_part/arm/hand/right_hand\n",
      "<END>     <END>                            <END>\n",
      "--------  -------------------------------  -------------------------------\n",
      "---------  ------------------------  ------------------------\n",
      "employee   B-person/employee         B-person/employee\n",
      "be         O                         O\n",
      "attend     O                         O\n",
      "annual     O                         O\n",
      "safety     O                         O\n",
      "retrain    O                         O\n",
      "during     O                         O\n",
      "lunch      O                         B-injury/fracture/break\n",
      "break      B-injury/fracture/break   B-injury/fracture/break\n",
      "two        O                         O\n",
      "employee   O                         O\n",
      "be         O                         B-person/employee\n",
      "``         O                         O\n",
      "horse      O                         O\n",
      "play       O                         O\n",
      "''         O                         O\n",
      "employee   B-person/employee         B-person/employee\n",
      "attempt    O                         O\n",
      "to         O                         O\n",
      "pick       O                         O\n",
      "up         O                         O\n",
      "another    O                         O\n",
      "employee   B-person/employee         B-person/employee\n",
      "and        O                         O\n",
      "throw      O                         O\n",
      "him        O                         O\n",
      "over       O                         O\n",
      "his        O                         O\n",
      "shoulder   B-body_part/arm/shoulder  B-body_part/arm/shoulder\n",
      "both       O                         B-body_part/arm/shoulder\n",
      "employee   O                         B-person/employee\n",
      "fell       O                         O\n",
      "to         O                         O\n",
      "the        O                         O\n",
      "ground     O                         O\n",
      "cause      O                         B-person/employee\n",
      "employee   B-person/employee         B-person/employee\n",
      "to         O                         O\n",
      "dislocate  O                         O\n",
      "his        O                         O\n",
      "hip        O                         <END>\n",
      "<END>      <END>                     <END>\n",
      "---------  ------------------------  ------------------------\n",
      "--------------------------------------  ---------------------  ---------------------\n",
      "roof                                    O                      O\n",
      "fall                                    B-accident_cause/fall  B-accident_cause/fall\n",
      "be                                      O                      O\n",
      "discover                                O                      O\n",
      "in                                      O                      O\n",
      "the                                     O                      O\n",
      "primary                                 O                      O\n",
      "escape                                  O                      O\n",
      "way                                     O                      O\n",
      "cross-cuts                              O                      O\n",
      "inby                                    O                      O\n",
      "number                                  O                      O\n",
      "belt                                    O                      O\n",
      "drive                                   O                      O\n",
      "at                                      O                      O\n",
      "spad                                    O                      O\n",
      "number                                  O                      O\n",
      "1203.                                   O                      O\n",
      "the                                     O                      O\n",
      "fall                                    B-accident_cause/fall  B-accident_cause/fall\n",
      "be                                      O                      O\n",
      "approximately                           O                      O\n",
      "30                                      O                      O\n",
      "long                                    O                      O\n",
      "19                                      O                      O\n",
      "wide-and-                               O                      O\n",
      "have                                    O                      O\n",
      "fall                                    O                      O\n",
      "above                                   O                      O\n",
      "permanent                               O                      O\n",
      "roof                                    O                      O\n",
      "support                                 O                      B-accident_cause/fall\n",
      "the                                     O                      B-accident_cause/fall\n",
      "fall                                    B-accident_cause/fall  O\n",
      "be                                      O                      O\n",
      "discover                                O                      O\n",
      "and                                     O                      O\n",
      "report                                  O                      O\n",
      "to                                      O                      O\n",
      "mine-safety-and-health-administration-  O                      <END>\n",
      "<END>                                   <END>                  <END>\n",
      "--------------------------------------  ---------------------  ---------------------\n",
      "-------------  -----------------  -----------------\n",
      "the            O                  O\n",
      "number         O                  O\n",
      "rock           O                  O\n",
      "come           O                  O\n",
      "from           O                  O\n",
      "the            O                  O\n",
      "shot           O                  O\n",
      "in             O                  O\n",
      "big            O                  O\n",
      "piece          O                  O\n",
      "we             O                  O\n",
      "be             O                  O\n",
      "building       O                  O\n",
      "ramp           O                  O\n",
      "with           O                  O\n",
      "this           O                  O\n",
      "material       O                  O\n",
      "without        O                  O\n",
      "any            O                  O\n",
      "problem        O                  O\n",
      "it             O                  O\n",
      "start          O                  O\n",
      "to             O                  O\n",
      "rain           O                  O\n",
      "make           O                  O\n",
      "the            O                  O\n",
      "boulder        O                  O\n",
      "slippery       O                  B-person/employee\n",
      "employee       B-person/employee  B-person/employee\n",
      "climbed        O                  O\n",
      "his            O                  O\n",
      "front          O                  O\n",
      "tire           O                  O\n",
      "up             O                  O\n",
      "on             O                  O\n",
      "boulder        O                  O\n",
      "and            O                  O\n",
      "it             O                  O\n",
      "slip           O                  O\n",
      "off            O                  O\n",
      "the            O                  O\n",
      "other          O                  O\n",
      "side           O                  O\n",
      "bring          O                  O\n",
      "the            O                  O\n",
      "truck          O                  O\n",
      "down           O                  O\n",
      "fast           O                  O\n",
      "and            O                  O\n",
      "hard           O                  O\n",
      "jarring        O                  O\n",
      "and            O                  O\n",
      "rock           O                  O\n",
      "him            O                  O\n",
      "suddenly       O                  O\n",
      "this           O                  O\n",
      "motion         O                  O\n",
      "be             O                  O\n",
      "what           O                  O\n",
      "be             O                  O\n",
      "suspect        O                  O\n",
      "to             O                  O\n",
      "have           O                  O\n",
      "cause          O                  O\n",
      "the            O                  O\n",
      "cervical       O                  O\n",
      "radiculopathy  O                  O\n",
      "<END>          <END>              O\n",
      "-------------  -----------------  -----------------\n",
      "---------  ------------------------------  ------------------------------\n",
      "driver     B-person/driver                 B-person/driver\n",
      "and        O                               O\n",
      "passenger  O                               O\n",
      "where      O                               O\n",
      "drive      B-activity/driving              O\n",
      "down       O                               O\n",
      "haul       O                               O\n",
      "road       O                               O\n",
      "to         O                               O\n",
      "pete       O                               O\n",
      "bajo       O                               O\n",
      "pit        O                               O\n",
      "from       O                               O\n",
      "leeville   O                               O\n",
      "when       O                               O\n",
      "try        O                               O\n",
      "to         O                               O\n",
      "make       O                               O\n",
      "left       B-body_part/arm/hand/left_hand  B-body_part/arm/hand/left_hand\n",
      "hand       I-body_part/arm/hand/left_hand  I-body_part/arm/hand/left_hand\n",
      "turn       O                               O\n",
      "the        O                               O\n",
      "truck      O                               O\n",
      "roll       O                               O\n",
      "onto       O                               O\n",
      "the        O                               O\n",
      "passenger  O                               O\n",
      "side       O                               <END>\n",
      "<END>      <END>                           <END>\n",
      "---------  ------------------------------  ------------------------------\n",
      "--------------  ---------------------------  ---------------------------\n",
      "employee        B-person/employee            B-person/employee\n",
      "state           O                            O\n",
      "while           O                            O\n",
      "loosening       O                            O\n",
      "pack            O                            O\n",
      "gland           O                            O\n",
      "on              O                            O\n",
      "shuttle         O                            O\n",
      "car             B-vehicle/light_vehicle/car  B-vehicle/light_vehicle/car\n",
      "during          O                            O\n",
      "repair          O                            O\n",
      "struck          O                            B-body_part/arm/hand\n",
      "hand            B-body_part/arm/hand         B-body_part/arm/hand\n",
      "against-sharp-  O                            O\n",
      "metal           O                            O\n",
      "<END>           <END>                        <END>\n",
      "--------------  ---------------------------  ---------------------------\n",
      "----------  -------------------  -------------------\n",
      "employee    B-person/employee    B-person/employee\n",
      "left        O                    O\n",
      "the         O                    O\n",
      "section     O                    O\n",
      "and         O                    O\n",
      "go          O                    O\n",
      "outside     O                    O\n",
      "without     O                    O\n",
      "tell        O                    O\n",
      "his         O                    O\n",
      "supervisor  B-person/supervisor  B-person/supervisor\n",
      "supervisor  B-person/supervisor  O\n",
      "call        O                    O\n",
      "outside     O                    O\n",
      "and         O                    B-person/employee\n",
      "told        O                    B-person/employee\n",
      "employee    B-person/employee    O\n",
      "to          O                    O\n",
      "return      O                    O\n",
      "to          O                    O\n",
      "the         O                    O\n",
      "section     O                    O\n",
      "or          O                    O\n",
      "he          O                    O\n",
      "consider    O                    O\n",
      "him         O                    O\n",
      "quit        O                    O\n",
      "for         O                    B-activity/walking\n",
      "walk        B-activity/walking   O\n",
      "off         O                    O\n",
      "the         O                    B-person/employee\n",
      "job         O                    O\n",
      "employee    B-person/employee    O\n",
      "left        O                    O\n",
      "mine        O                    O\n",
      "site        O                    O\n",
      "and         O                    O\n",
      "go          O                    O\n",
      "home        O                    O\n",
      "after       O                    O\n",
      "he          O                    O\n",
      "get         O                    O\n",
      "home        O                    O\n",
      "employee    B-person/employee    O\n",
      "call        O                    B-body_part/back\n",
      "back        B-body_part/back     B-body_part/back\n",
      "to          O                    O\n",
      "the         O                    O\n",
      "mine        O                    O\n",
      "and         O                    O\n",
      "told        O                    O\n",
      "the         O                    O\n",
      "guard       O                    O\n",
      "that        O                    O\n",
      "he          O                    O\n",
      "hurt        O                    O\n",
      "his         O                    O\n",
      "back        B-body_part/back     O\n",
      "by          O                    O\n",
      "pull        O                    O\n",
      "piece       O                    O\n",
      "of          O                    O\n",
      "brattice    O                    O\n",
      "cloth       O                    <END>\n",
      "<END>       <END>                <END>\n",
      "----------  -------------------  -------------------\n",
      "--------------------------------------  ------------------  ----------------\n",
      "number                                  O                   O\n",
      "34                                      O                   O\n",
      "service                                 O                   O\n",
      "hoist                                   O                   O\n",
      "go                                      O                   O\n",
      "down                                    O                   O\n",
      "at                                      O                   O\n",
      "6:55                                    O                   O\n",
      "be                                      O                   O\n",
      "contact                                 O                   O\n",
      "electrician                             O                   O\n",
      "at                                      O                   O\n",
      "6:55                                    O                   O\n",
      "be                                      O                   O\n",
      "call                                    O                   O\n",
      "mine-safety-and-health-administration-  O                   O\n",
      "at                                      O                   O\n",
      "7:25                                    O                   O\n",
      "be                                      O                   O\n",
      "electrician                             O                   O\n",
      "found                                   O                   O\n",
      "bad                                     O                   O\n",
      "contact                                 O                   O\n",
      "hoist                                   O                   O\n",
      "be                                      O                   O\n",
      "back                                    B-body_part/back    B-body_part/back\n",
      "up                                      O                   O\n",
      "and                                     O                   O\n",
      "run                                     B-activity/running  O\n",
      "again                                   O                   O\n",
      "at                                      O                   O\n",
      "10:20                                   O                   O\n",
      "be                                      O                   O\n",
      "call                                    O                   O\n",
      "mine-safety-and-health-administration-  O                   O\n",
      "at                                      O                   O\n",
      "10:36                                   O                   O\n",
      "to                                      O                   O\n",
      "report                                  O                   O\n",
      "hoist                                   O                   O\n",
      "be                                      O                   O\n",
      "in                                      O                   O\n",
      "service                                 O                   <END>\n",
      "<END>                                   <END>               <END>\n",
      "--------------------------------------  ------------------  ----------------\n",
      "------------  -----------------  -----------------\n",
      "negligence    O                  O\n",
      "of            O                  O\n",
      "employee-he-  O                  O\n",
      "be            O                  O\n",
      "stand         O                  O\n",
      "where         O                  O\n",
      "he            O                  O\n",
      "should        O                  O\n",
      "not           O                  O\n",
      "have          O                  O\n",
      "be            O                  O\n",
      "he            O                  O\n",
      "be            O                  O\n",
      "in            O                  O\n",
      "blind         O                  O\n",
      "spot          O                  O\n",
      "of            O                  O\n",
      "operator      B-person/operator  B-person/operator\n",
      "<END>         <END>              <END>\n",
      "------------  -----------------  -----------------\n",
      "-------------  ---------------------  ---------------------\n",
      "roof           O                      O\n",
      "fall           B-accident_cause/fall  B-accident_cause/fall\n",
      "be             O                      O\n",
      "found          O                      O\n",
      "in             O                      O\n",
      "entry          O                      O\n",
      "cross-cut      O                      O\n",
      "70-71          O                      O\n",
      "west           O                      O\n",
      "intake         O                      O\n",
      "area           O                      O\n",
      "be             O                      O\n",
      "mine           O                      O\n",
      "in             O                      O\n",
      "august         O                      O\n",
      "of             O                      O\n",
      "2009           O                      O\n",
      "and            O                      O\n",
      "have           O                      O\n",
      "be             O                      O\n",
      "support        O                      O\n",
      "with           O                      O\n",
      "4-6            O                      O\n",
      "resin          O                      O\n",
      "pin            O                      O\n",
      "no             O                      O\n",
      "injury         B-injury               O\n",
      "or             O                      O\n",
      "property       O                      O\n",
      "loss           O                      O\n",
      "occur          O                      O\n",
      "there          O                      O\n",
      "be             O                      O\n",
      "no             O                      O\n",
      "plan           O                      O\n",
      "to             O                      O\n",
      "clean          O                      O\n",
      "up             O                      B-accident_cause/fall\n",
      "fall           B-accident_cause/fall  B-accident_cause/fall\n",
      "travelway      O                      O\n",
      "be             O                      O\n",
      "be             O                      O\n",
      "rehabilitate   O                      O\n",
      "in             O                      O\n",
      "entry          O                      O\n",
      "and            O                      O\n",
      "will           O                      O\n",
      "serve          O                      O\n",
      "a              O                      O\n",
      "the            O                      O\n",
      "new            O                      O\n",
      "primary        O                      O\n",
      "escapeway      O                      O\n",
      "fall           B-accident_cause/fall  O\n",
      "be             O                      O\n",
      "approximately  O                      O\n",
      "35             O                      O\n",
      "long           O                      O\n",
      "20             O                      O\n",
      "wide           O                      O\n",
      "high           O                      O\n",
      "<END>          <END>                  O\n",
      "-------------  ---------------------  ---------------------\n",
      "-----------  -------------------  -------------------\n",
      "employee     B-person/employee    B-person/employee\n",
      "be           O                    O\n",
      "drive        B-activity/driving   B-activity/driving\n",
      "up           O                    O\n",
      "the          O                    O\n",
      "ramp         O                    O\n",
      "and          O                    O\n",
      "begin        O                    O\n",
      "to           O                    O\n",
      "feel         O                    O\n",
      "lightheaded  O                    O\n",
      "and          O                    O\n",
      "dizzy        O                    O\n",
      "employee     B-person/employee    B-person/employee\n",
      "stop         O                    O\n",
      "truck        O                    O\n",
      "and          O                    O\n",
      "call         O                    O\n",
      "supervisor   B-person/supervisor  B-person/supervisor\n",
      "and          O                    O\n",
      "get          O                    O\n",
      "out          O                    O\n",
      "of           O                    O\n",
      "truck        O                    O\n",
      "until        O                    O\n",
      "help         O                    O\n",
      "could        O                    O\n",
      "arrive       O                    O\n",
      "<END>        <END>                <END>\n",
      "-----------  -------------------  -------------------\n",
      "--------  -----------------  -----------------\n",
      "the       O                  O\n",
      "foreman   O                  O\n",
      "employee  B-person/employee  B-person/employee\n",
      "be        O                  O\n",
      "carry     O                  O\n",
      "pipe      B-equipment/pipe   B-equipment/pipe\n",
      "from      O                  O\n",
      "the       O                  O\n",
      "return    O                  O\n",
      "to        O                  O\n",
      "number    O                  O\n",
      "belt      O                  O\n",
      "he        O                  O\n",
      "slip      O                  O\n",
      "on        O                  O\n",
      "the       O                  O\n",
      "ice       O                  O\n",
      "and       O                  O\n",
      "fell      O                  O\n",
      "<END>     <END>              <END>\n",
      "--------  -----------------  -----------------\n",
      "--------  ---------------------------  ---------------------------\n",
      "employee  B-person/employee            B-person/employee\n",
      "be        O                            O\n",
      "tend      O                            O\n",
      "miner     O                            O\n",
      "cable     O                            O\n",
      "in        O                            O\n",
      "the       O                            O\n",
      "number    O                            O\n",
      "entry     O                            O\n",
      "of        O                            O\n",
      "11th      O                            O\n",
      "west      O                            O\n",
      "when      O                            O\n",
      "piece     O                            O\n",
      "of        O                            O\n",
      "the       O                            O\n",
      "rib       O                            O\n",
      "be        O                            O\n",
      "dislodge  O                            O\n",
      "and       O                            O\n",
      "struck    O                            O\n",
      "the       O                            B-person/employee\n",
      "employee  B-person/employee            O\n",
      "on        O                            O\n",
      "the       O                            O\n",
      "right     O                            O\n",
      "side      O                            O\n",
      "of        O                            O\n",
      "his       O                            B-body_part/body\n",
      "body      B-body_part/body             B-body_part/body\n",
      "and       O                            O\n",
      "knock     O                            O\n",
      "him       O                            O\n",
      "into      O                            O\n",
      "the       O                            O\n",
      "shuttle   O                            B-vehicle/light_vehicle/car\n",
      "car       B-vehicle/light_vehicle/car  O\n",
      "with      O                            O\n",
      "the       O                            O\n",
      "left      O                            O\n",
      "side      O                            O\n",
      "of        O                            B-body_part/body\n",
      "his       O                            B-body_part/body\n",
      "body      B-body_part/body             <END>\n",
      "<END>     <END>                        <END>\n",
      "--------  ---------------------------  ---------------------------\n",
      "----------  -------------------  -------------------\n",
      "employee    B-person/employee    B-person/employee\n",
      "be          O                    O\n",
      "tighten     O                    O\n",
      "ratchet     O                    O\n",
      "strap       O                    O\n",
      "the         O                    O\n",
      "strap       O                    O\n",
      "slip        O                    O\n",
      "off         O                    O\n",
      "and         O                    O\n",
      "hit         O                    O\n",
      "employee    B-person/employee    B-person/employee\n",
      "in          O                    O\n",
      "the         O                    O\n",
      "face        O                    O\n",
      "cause       O                    O\n",
      "laceration  B-injury/laceration  B-injury/laceration\n",
      "to          O                    B-injury/laceration\n",
      "employee's  O                    O\n",
      "chin        O                    O\n",
      "which       O                    O\n",
      "require     O                    O\n",
      "several     O                    O\n",
      "stitch      O                    O\n",
      "<END>       <END>                <END>\n",
      "----------  -------------------  -------------------\n",
      "------------  -----------------  -----------------\n",
      "employee      B-person/employee  B-person/employee\n",
      "be            O                  O\n",
      "rid           O                  O\n",
      "in            O                  O\n",
      "deisel        O                  O\n",
      "tractor       O                  O\n",
      "a             O                  O\n",
      "the           O                  O\n",
      "tractor       O                  O\n",
      "tram          O                  O\n",
      "down          O                  O\n",
      "the           O                  O\n",
      "supply        O                  O\n",
      "road          O                  O\n",
      "it            O                  O\n",
      "hit           O                  O\n",
      "some          O                  O\n",
      "hole          O                  O\n",
      "in            O                  O\n",
      "the           O                  O\n",
      "roadway       O                  O\n",
      "the           O                  O\n",
      "employee's    O                  O\n",
      "right         O                  O\n",
      "hip           O                  O\n",
      "start         O                  O\n",
      "to            O                  O\n",
      "hurt          O                  O\n",
      "he            O                  O\n",
      "go            O                  O\n",
      "to            O                  O\n",
      "the           O                  O\n",
      "chiropractor  O                  O\n",
      "on            O                  O\n",
      "1/2/04        O                  O\n",
      "for           O                  O\n",
      "an            O                  O\n",
      "adjustment    O                  O\n",
      "<END>         <END>              O\n",
      "------------  -----------------  -----------------\n",
      "---------------  ------------------------  ------------------------\n",
      "employee         B-person/employee         B-person/employee\n",
      "state            O                         O\n",
      "strain           O                         B-injury/muscle/strain\n",
      "back             B-body_part/back          B-body_part/back\n",
      "while            O                         O\n",
      "work             O                         O\n",
      "on               O                         O\n",
      "pipeline         O                         O\n",
      "for              O                         O\n",
      "the              O                         O\n",
      "dredge           O                         O\n",
      "state            O                         O\n",
      "work             O                         O\n",
      "out              O                         O\n",
      "of               O                         O\n",
      "designate        O                         O\n",
      "work             O                         O\n",
      "boat             O                         O\n",
      "and              O                         O\n",
      "gust             O                         O\n",
      "of               O                         O\n",
      "wind             O                         O\n",
      "blew             O                         O\n",
      "and              O                         O\n",
      "start            O                         O\n",
      "to               O                         O\n",
      "separate         O                         O\n",
      "the              O                         O\n",
      "pipeline         O                         B-person/employee\n",
      "employee         B-person/employee         B-person/employee\n",
      "state            O                         B-body_part/arm/shoulder\n",
      "shoulder         B-body_part/arm/shoulder  B-body_part/arm/shoulder\n",
      "popped           O                         O\n",
      "and              O                         O\n",
      "he               O                         O\n",
      "stop             O                         O\n",
      "work             O                         O\n",
      "and              O                         O\n",
      "stretch          O                         O\n",
      "he               O                         B-body_part/back\n",
      "go               O                         O\n",
      "back             B-body_part/back          O\n",
      "to               O                         O\n",
      "work             O                         O\n",
      "for              O                         O\n",
      "the              O                         O\n",
      "rest             O                         O\n",
      "of               O                         O\n",
      "the              O                         O\n",
      "day              O                         O\n",
      "with             O                         O\n",
      "no               O                         O\n",
      "problem          O                         O\n",
      "on               O                         O\n",
      "monday           O                         O\n",
      "6/13/05          O                         O\n",
      "he               O                         O\n",
      "report           O                         O\n",
      "to               O                         O\n",
      "the              O                         B-body_part/back\n",
      "emergency-room-  O                         B-body_part/back\n",
      "with             O                         O\n",
      "back             B-body_part/back          O\n",
      "pain             O                         O\n",
      "and              O                         O\n",
      "say              O                         O\n",
      "it               O                         O\n",
      "be               O                         <END>\n",
      "work             O                         <END>\n",
      "related          O                         <END>\n",
      "<END>            <END>                     <END>\n",
      "---------------  ------------------------  ------------------------\n",
      "--------  ------------------------------  ------------------------------\n",
      "employee  B-person/employee               B-person/employee\n",
      "be        O                               O\n",
      "attempt   O                               O\n",
      "to        O                               O\n",
      "rerail    O                               O\n",
      "mantrip   O                               O\n",
      "when      O                               O\n",
      "the       O                               O\n",
      "jack      O                               O\n",
      "fell      O                               O\n",
      "over      O                               O\n",
      "and       O                               O\n",
      "struck    O                               O\n",
      "employee  B-person/employee               B-person/employee\n",
      "in        O                               O\n",
      "the       O                               O\n",
      "left      B-body_part/leg/knee/left_knee  B-body_part/leg/knee/left_knee\n",
      "knee      I-body_part/leg/knee/left_knee  I-body_part/leg/knee/left_knee\n",
      "<END>     <END>                           <END>\n",
      "--------  ------------------------------  ------------------------------\n",
      "-----------  ---------------------  ---------------------\n",
      "during       O                      O\n",
      "an           O                      O\n",
      "altercation  O                      O\n",
      "with         O                      O\n",
      "another      O                      O\n",
      "employee     B-person/employee      B-person/employee\n",
      "injured      O                      O\n",
      "employee     B-person/employee      O\n",
      "be           O                      B-person/employee\n",
      "struck       O                      O\n",
      "in           O                      O\n",
      "nose         B-body_part/head/nose  O\n",
      "both         O                      B-body_part/head/nose\n",
      "sent         O                      O\n",
      "home         O                      O\n",
      "until        O                      O\n",
      "incident     O                      O\n",
      "investigate  O                      O\n",
      "injured      O                      O\n",
      "employee     B-person/employee      B-person/employee\n",
      "do           O                      O\n",
      "not          O                      O\n",
      "seek         O                      O\n",
      "medical      O                      O\n",
      "attention    O                      O\n",
      "for          O                      O\n",
      "11/2         O                      O\n",
      "week         O                      O\n",
      "safety       O                      O\n",
      "dept         O                      O\n",
      "be           O                      O\n",
      "notify       O                      O\n",
      "employee     B-person/employee      B-person/employee\n",
      "have         O                      B-person/employee\n",
      "miss         O                      O\n",
      "day          O                      O\n",
      "to           O                      O\n",
      "see          O                      O\n",
      "doctor       O                      O\n",
      "and          O                      O\n",
      "day          O                      O\n",
      "for          O                      O\n",
      "x-ray        O                      O\n",
      "all          O                      O\n",
      "negative     O                      O\n",
      "<END>        <END>                  O\n",
      "-----------  ---------------------  ---------------------\n",
      "----------  -----------------------------  -----------------------------\n",
      "employee    B-person/employee              B-person/employee\n",
      "be          O                              O\n",
      "use         O                              O\n",
      "dilute      O                              O\n",
      "caustic     O                              O\n",
      "solution    O                              O\n",
      "to          O                              O\n",
      "hose-down-  O                              O\n",
      "mud         O                              O\n",
      "and         O                              O\n",
      "scale       O                              O\n",
      "on-side-    O                              O\n",
      "of          O                              O\n",
      "tank        O                              O\n",
      "some        O                              O\n",
      "of          O                              O\n",
      "the         O                              O\n",
      "caustic     O                              O\n",
      "solution    O                              O\n",
      "splash      O                              O\n",
      "back        B-body_part/back               O\n",
      "and         O                              B-body_part/back\n",
      "get         O                              O\n",
      "into        O                              O\n",
      "the         O                              O\n",
      "employee's  O                              O\n",
      "left        B-body_part/head/eye/left_eye  O\n",
      "eye         I-body_part/head/eye/left_eye  B-body_part/head/eye/left_eye\n",
      "employee    B-person/employee              I-body_part/head/eye/left_eye\n",
      "be          O                              B-person/employee\n",
      "see         O                              O\n",
      "by          O                              O\n",
      "doctor      O                              O\n",
      "who         O                              O\n",
      "prescribed  O                              O\n",
      "antibiotic  O                              O\n",
      "to          O                              O\n",
      "treat       O                              O\n",
      "small       O                              O\n",
      "corneal     O                              O\n",
      "abrasion    O                              O\n",
      "there       O                              O\n",
      "be          O                              O\n",
      "no          O                              O\n",
      "lose        O                              O\n",
      "time        O                              O\n",
      "and         O                              O\n",
      "only        O                              O\n",
      "restrict    O                              O\n",
      "duty        O                              O\n",
      "<END>       <END>                          <END>\n",
      "----------  -----------------------------  -----------------------------\n",
      "--------------------  ---------------------------------  ---------------------------------\n",
      "employee              B-person/employee                  B-person/employee\n",
      "be                    O                                  O\n",
      "replace               O                                  O\n",
      "the                   O                                  O\n",
      "rear                  O                                  O\n",
      "suspension            O                                  O\n",
      "cylinder              O                                  O\n",
      "on                    O                                  O\n",
      "cat                   O                                  O\n",
      "793                   O                                  O\n",
      "truck                 O                                  O\n",
      "a                     O                                  O\n",
      "he                    O                                  O\n",
      "be                    O                                  O\n",
      "use                   O                                  O\n",
      "sledge                O                                  O\n",
      "hammer                O                                  O\n",
      "to                    O                                  O\n",
      "drive                 O                                  O\n",
      "the                   O                                  O\n",
      "top                   O                                  O\n",
      "strut                 O                                  O\n",
      "pin                   O                                  O\n",
      "from                  O                                  O\n",
      "it                    O                                  O\n",
      "bore                  O                                  O\n",
      "he                    O                                  O\n",
      "felt                  O                                  O\n",
      "pain                  O                                  O\n",
      "in                    O                                  O\n",
      "his                   O                                  B-body_part/arm/wrist/right_wrist\n",
      "right                 B-body_part/arm/wrist/right_wrist  I-body_part/arm/wrist/right_wrist\n",
      "wrist                 I-body_part/arm/wrist/right_wrist  O\n",
      "he                    O                                  O\n",
      "do                    O                                  O\n",
      "not                   O                                  O\n",
      "report                O                                  O\n",
      "the                   O                                  O\n",
      "incident              O                                  O\n",
      "at                    O                                  O\n",
      "that                  O                                  O\n",
      "time                  O                                  O\n",
      "on                    O                                  O\n",
      "9/22/06               O                                  O\n",
      "a                     O                                  O\n",
      "he                    O                                  O\n",
      "be                    O                                  O\n",
      "loosening/tightening  O                                  O\n",
      "the                   O                                  O\n",
      "wear                  O                                  O\n",
      "plate                 O                                  O\n",
      "retain                O                                  O\n",
      "bolt                  O                                  O\n",
      "on                    O                                  O\n",
      "24m                   O                                  O\n",
      "motor                 O                                  O\n",
      "grader                B-vehicle/heavy_vehicle/grader     O\n",
      "use                   O                                  O\n",
      "3/4                   O                                  O\n",
      "''                    O                                  O\n",
      "impact                O                                  O\n",
      "wrench                O                                  O\n",
      "he                    O                                  O\n",
      "again                 O                                  O\n",
      "experienced           O                                  O\n",
      "pain                  O                                  O\n",
      "in                    O                                  O\n",
      "wrist                 B-body_part/arm/wrist              <END>\n",
      "<END>                 <END>                              <END>\n",
      "--------------------  ---------------------------------  ---------------------------------\n",
      "--------  ---------------------------  ---------------------------\n",
      "employee  B-person/employee            B-person/employee\n",
      "be        O                            O\n",
      "replace   O                            O\n",
      "bag       O                            O\n",
      "in        O                            O\n",
      "baghouse  O                            O\n",
      "bending   O                            O\n",
      "over      O                            O\n",
      "and       O                            O\n",
      "pull      O                            O\n",
      "bag       O                            O\n",
      "strain    O                            O\n",
      "low       B-body_part/back/lower_back  B-body_part/back/lower_back\n",
      "back      I-body_part/back/lower_back  I-body_part/back/lower_back\n",
      "<END>     <END>                        <END>\n",
      "--------  ---------------------------  ---------------------------\n",
      "--------  -----------------  -----------------\n",
      "employee  B-person/employee  B-person/employee\n",
      "state     O                  O\n",
      "employee  B-person/employee  B-person/employee\n",
      "be        O                  O\n",
      "exit      O                  O\n",
      "the       O                  O\n",
      "connex    O                  O\n",
      "box       O                  O\n",
      "storage   O                  O\n",
      "trailer   O                  O\n",
      "when      O                  O\n",
      "employee  B-person/employee  B-person/employee\n",
      "slip      O                  O\n",
      "on        O                  O\n",
      "snow/ice  O                  O\n",
      "cause     O                  O\n",
      "employee  B-person/employee  B-person/employee\n",
      "to        O                  O\n",
      "catch     O                  O\n",
      "self      O                  O\n",
      "in        O                  O\n",
      "split     O                  O\n",
      "<END>     <END>              <END>\n",
      "--------  -----------------  -----------------\n"
     ]
    }
   ],
   "source": [
    "args.device = torch.device(\"cpu\")\n",
    "# checkpoint = torch.load(args.model_state_file)\n",
    "# NERmodel.load_state_dict(checkpoint['model_state_dict'])\n",
    "NERmodel.load_state_dict(torch.load(train_state['model_filename']))\n",
    "NERmodel = NERmodel.to(args.device)\n",
    "\n",
    "dataset.set_split('test')\n",
    "batch_generator = generate_batches(dataset,  batch_size=1, shuffle=False, device=args.device)\n",
    "\n",
    "running_loss = 0.\n",
    "running_acc = 0.\n",
    "NERmodel.eval()\n",
    "mask_index = vectorizer.target_vocab.mask_index\n",
    "i = 0\n",
    "for batch_index, batch_dict in enumerate(batch_generator):\n",
    "    y_pred = NERmodel(batch_dict['x_source'],  batch_dict['x_length'],  batch_dict['x_target'])\n",
    "    origin = batch_dict['x_source'][0]\n",
    "    y_true = batch_dict['y_target']\n",
    "    pred, true = normalize_sizes(y_pred, y_true)\n",
    "    _, y_pred_indices = pred.max(dim=1)\n",
    "    \n",
    "    final = []\n",
    "    for j in range(len(true)):\n",
    "        if true[j].item() != 0:\n",
    "            tmp = []\n",
    "            tmp.append(vectorizer.NARRATIVE_vocab.lookup_index(origin[j+1].item()))\n",
    "            tmp.append(vectorizer.target_vocab.lookup_index(true[j].item()))\n",
    "            tmp.append(vectorizer.target_vocab.lookup_index(y_pred_indices[j].item()))\n",
    "            \n",
    "            final.append(tmp)\n",
    "\n",
    "    print(tabulate(final))\n",
    "\n",
    "    i += 1\n",
    "    if i == 50:\n",
    "        break"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cc11ba5f-063f-4be5-8e9d-0a37cda758b8",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Comment \n",
    "We an clearly see Sequence to Sequence with Self attention perfrom better than naive Sequence to Sequence model.\n",
    "\n",
    "Also we found out the loss function will affect the model performance significantly, to loss function we adopted from the lab with come modification, so it can make \"O\" tag less important, so ican can pay more attention to other tags.\n",
    "```Python\n",
    "def sequence_loss(y_pred, y_true, mask_index, weight=None):\n",
    "    y_pred, y_true = normalize_sizes(y_pred, y_true)\n",
    "    indexs = []\n",
    "    # Make \"O\" less important\n",
    "    for i in range(len(y_true)):\n",
    "        if y_true[i].item() != 179 or i%4 ==0:\n",
    "            indexs.append(i)\n",
    "    return F.cross_entropy(y_pred[indexs], y_true[indexs], ignore_index=mask_index, weight=weight)\n",
    "```\n",
    "\n",
    "For future improvement we can investegate more one better loss fucntion that suitable for this dataset.\n",
    "\n",
    "Also due in the data perpation stage we removed punctuation and any word length < 1, this may affected the model performance. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "319ad2dd-4022-44a5-a3e8-4f60ef02e876",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}