combine_multitask_gen_sent.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "streaming-crawford",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package wordnet to\n",
      "[nltk_data]     /home/ujjwalaananth/nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n",
      "[nltk_data] Downloading package omw-1.4 to\n",
      "[nltk_data]     /home/ujjwalaananth/nltk_data...\n",
      "[nltk_data]   Package omw-1.4 is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import jsonlines\n",
    "import json\n",
    "import os\n",
    "\n",
    "import re\n",
    "import nltk\n",
    "nltk.download('wordnet')\n",
    "nltk.download('omw-1.4')\n",
    "\n",
    "from nltk.corpus import wordnet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "silent-transaction",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "addpred=False\n",
    "addanyway=False\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "headed-peter",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "\n",
    "tofile=True\n",
    "\n",
    "include_neg = True\n",
    "\n",
    "if(include_neg):\n",
    "    #dirs = ['mlee_processed_sent','CASIE_sent_neg_processed','cysecED_neg_processed','wikievent_sent_neg_processed','RAMS3_neg_processed','maven_processed_sent']\n",
    "    #opdir = 'multitask5sent_neg'\n",
    "\n",
    "    #dirs = ['mlee_processed_sent','wikievent_sent_neg_processed','RAMS_doc','maven_processed_sent']\n",
    "    #dirs = ['mlee_processed_doc','wikievent_sent_neg_processed','RAMS_doc','maven_processed_sent']\n",
    "    dirs = ['RAMS3_neg_processed']\n",
    "    opdir = 'multitask4sent_bi'\n",
    "    #opdir = 'multitask4mix_bin2'\n",
    "    \n",
    "# sent\n",
    "else:\n",
    "    #dirs = ['CASIE_sent_processed','cysecED_processed','wikievent_sent_processed','RAMS3_processed',]\n",
    "    dirs = ['CASIE_sent_processed','cysecED_processed','wikievent_sent_processed','RAMS3_processed','maven_processed_sent']\n",
    "    #opdir = 'multitask4sent'\n",
    "    opdir = 'multitask5sent'\n",
    "\n",
    "# sent, with negative\n",
    "\n",
    "# GROUP CYBERSEC AND GEN DATASETS\n",
    "domains={\n",
    "    #'cybersec':dirs[1:3],\n",
    "    'gen':['wikievent_sent_neg_processed','RAMS_doc'],\n",
    "    'biomed2k':['mlee_processed_sent']\n",
    "}\n",
    "\n",
    "impcols = ['text', 'src_label', 'event_triggers', 'event_types']\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "charged-probability",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RAMS3_neg_processed\n",
      "                                                text           src_label  \\\n",
      "0  Transportation officials are urging carpool an...                 NaN   \n",
      "1                                    ( Paul Duggan )                 NaN   \n",
      "2  -- A Baltimore prosecutor accused a police det...  homicide->life.die   \n",
      "3  The heated exchange came in the chaotic sixth ...                 NaN   \n",
      "4                     ( Derek Hawkins and Lynh Bui )                 NaN   \n",
      "\n",
      "  event_triggers event_types  \n",
      "0           NONE        NONE  \n",
      "1           NONE        NONE  \n",
      "2       homicide    life.die  \n",
      "3           NONE        NONE  \n",
      "4           NONE        NONE  \n",
      "(35535, 4)  df\n",
      "(35535, 5) (4245, 5)\n",
      "(35535, 5)\n"
     ]
    }
   ],
   "source": [
    "st = False\n",
    "for d in dirs:\n",
    "    trfile = d+'/train.csv'\n",
    "    tefile = d+'/test.csv'\n",
    "    \n",
    "    df1 = pd.read_csv(trfile)\n",
    "    df2 = pd.read_csv(tefile)\n",
    "    \n",
    "    df1 = df1.loc[:,impcols]\n",
    "    df2 = df2.loc[:,impcols]\n",
    "    \n",
    "    if('genia' in d):\n",
    "        df1 = df1.iloc[:2000,:]\n",
    "    \n",
    "    print(d)\n",
    "    print(df1.head())\n",
    "    print(df1.shape, ' df')\n",
    "    \n",
    "    df1['dataset'] = [d]*len(df1)\n",
    "    df2['dataset'] = [d]*len(df2)\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    if(not st):\n",
    "        st=True\n",
    "        dfn1 = df1\n",
    "        dfn2 = df2\n",
    "    else:\n",
    "        dfn1 = pd.concat([df1,dfn1],axis=0,ignore_index=True)\n",
    "        dfn2 = pd.concat([df2,dfn2],axis=0,ignore_index=True)\n",
    "        dfn1 = dfn1.sample(frac=1).reset_index(drop=True)\n",
    "        #dfn2 = dfn2.sample(frac=1).reset_index(drop=True)\n",
    "    print(dfn1.shape,dfn2.shape)\n",
    "    \n",
    "    if(include_neg):\n",
    "        dfn1 = dfn1.fillna('NONE')\n",
    "        dfn2 = dfn2.fillna('NONE')\n",
    "    else:\n",
    "        dfn1 = dfn1.dropna(subset=['src_label'])\n",
    "        dfn2 = dfn2.dropna(subset=['src_label'])\n",
    "    \n",
    "    \n",
    "    print(dfn1.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "brown-fields",
   "metadata": {},
   "outputs": [],
   "source": [
    "def unique_sents(df):\n",
    "    for sent in df.text.unique():\n",
    "        ds = df[df.text==sent]\n",
    "        if(len(ds)==1):\n",
    "            continue\n",
    "        ids = ds.index\n",
    "        tgs = ds.src_label.unique()\n",
    "        if('NONE' in tgs and len(tgs)>1):\n",
    "            tgs = [t for t in tgs if 'NONE' not in t]\n",
    "        df.loc[ids[0],'src_label'] = ' | '.join(tgs)\n",
    "        #print(df.loc[ids[0],'src_label'])\n",
    "        \n",
    "    print(df.shape)\n",
    "    df = df.drop_duplicates(subset=['text'])\n",
    "    print(df.shape,' after dropping dups')\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "senior-celtic",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "german-cornwall",
   "metadata": {},
   "outputs": [],
   "source": [
    "## PROCESSING WIKIEVENT\n",
    "\n",
    "for df in [dfn1,dfn2]:\n",
    "    \n",
    "    for i,r in df.iterrows():\n",
    "        if('NONE' in r['src_label'] or 'wiki' not in r['dataset']):\n",
    "            continue\n",
    "            \n",
    "        et = [p.strip() for p in r['event_types'].split('|')]\n",
    "        et = ['.'.join(p.split('.')[:-1]) for p in et]\n",
    "        r['event_types'] = ' | '.join(et)\n",
    "        \n",
    "        et = [p.strip() for p in r['src_label'].split('|')]\n",
    "        et = ['.'.join(p.split('.')[:-1]) for p in et]\n",
    "        r['src_label'] = ' | '.join(et)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "therapeutic-convenience",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(35535, 5)\n",
      "(27751, 5)  after dropping dups\n",
      "(4245, 5)\n",
      "(3456, 5)  after dropping dups\n"
     ]
    }
   ],
   "source": [
    "dfn1 = unique_sents(dfn1)\n",
    "dfn2 = unique_sents(dfn2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "criminal-processing",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dataset  RAMS3_neg_processed\n",
      "27751\n",
      "27751\n",
      "3456\n",
      "3456\n"
     ]
    }
   ],
   "source": [
    "for dataset in dfn1.dataset.unique():\n",
    "    print('dataset ',dataset)\n",
    "    \n",
    "    print(dfn1[dfn1.dataset==dataset].shape[0])\n",
    "    print(dfn1[dfn1.dataset==dataset].text.nunique())\n",
    "    \n",
    "    print(dfn2[dfn2.dataset==dataset].shape[0])\n",
    "    print(dfn2[dfn2.dataset==dataset].text.nunique())\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "deadly-cinema",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                                text           src_label  \\\n",
      "0  Transportation officials are urging carpool an...                NONE   \n",
      "1                                    ( Paul Duggan )                NONE   \n",
      "2  -- A Baltimore prosecutor accused a police det...  homicide->life.die   \n",
      "3  The heated exchange came in the chaotic sixth ...                NONE   \n",
      "4                     ( Derek Hawkins and Lynh Bui )                NONE   \n",
      "\n",
      "  event_triggers event_types              dataset  \n",
      "0           NONE        NONE  RAMS3_neg_processed  \n",
      "1           NONE        NONE  RAMS3_neg_processed  \n",
      "2       homicide    life.die  RAMS3_neg_processed  \n",
      "3           NONE        NONE  RAMS3_neg_processed  \n",
      "4           NONE        NONE  RAMS3_neg_processed  \n",
      "=======\n",
      "                                                text  \\\n",
      "0                         We are ashamed of them . \"   \n",
      "1  However , Mutko stopped short of admitting the...   \n",
      "2  \" We are very sorry that athletes who tried to...   \n",
      "3  We are very sorry because Russia is committed ...   \n",
      "4  English former heptathlete and Athens 2004 bro...   \n",
      "\n",
      "                        src_label event_triggers            event_types  \\\n",
      "0                            NONE           NONE                   NONE   \n",
      "1                            NONE           NONE                   NONE   \n",
      "2  deceive->contact.prevarication        deceive  contact.prevarication   \n",
      "3                            NONE           NONE                   NONE   \n",
      "4                            NONE           NONE                   NONE   \n",
      "\n",
      "               dataset  \n",
      "0  RAMS3_neg_processed  \n",
      "1  RAMS3_neg_processed  \n",
      "2  RAMS3_neg_processed  \n",
      "3  RAMS3_neg_processed  \n",
      "4  RAMS3_neg_processed  \n"
     ]
    }
   ],
   "source": [
    "print(dfn1.head())\n",
    "print('=======')\n",
    "print(dfn2.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "developmental-lawyer",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 3456 entries, 0 to 4244\n",
      "Data columns (total 5 columns):\n",
      " #   Column          Non-Null Count  Dtype \n",
      "---  ------          --------------  ----- \n",
      " 0   text            3456 non-null   object\n",
      " 1   src_label       3456 non-null   object\n",
      " 2   event_triggers  3456 non-null   object\n",
      " 3   event_types     3456 non-null   object\n",
      " 4   dataset         3456 non-null   object\n",
      "dtypes: object(5)\n",
      "memory usage: 162.0+ KB\n"
     ]
    }
   ],
   "source": [
    "dfn2.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "whole-class",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2913"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dctp={}\n",
    "dctg={}\n",
    "predevex=open('results/'+'multitask2EV_prin_1024_20_2bs_2DAYS'+'/generated_predictions.txt').readlines()\n",
    "\n",
    "len(predevex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "infrared-samba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['contact', 'deceive', 'deceive->contact', 'conflict', 'slaughtered']"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predevex=[p[2:-2] for p in predevex]\n",
    "predevex[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "underlying-minister",
   "metadata": {},
   "outputs": [],
   "source": [
    "dfl=[]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "common-keeping",
   "metadata": {},
   "outputs": [],
   "source": [
    "def getdef(wordl):\n",
    "    \n",
    "    \n",
    "    defs=''\n",
    "    \n",
    "    for word in wordl.split(' | '):\n",
    "        if(len(word.split())>1):\n",
    "            word = max(word.split(), key=len)\n",
    "            \n",
    "        if(len(word.split('.'))>1):\n",
    "            word = word.split('.')[1] if len(wordnet.synsets(word.split('.')[1]))>0 else word.split('.')[0]\n",
    "        if(len(wordnet.synsets(word))>0):\n",
    "            defs+= wordnet.synsets(word)[0].definition()+' | '\n",
    "            if(len(wordnet.synsets(word))>1):\n",
    "                defs+= wordnet.synsets(word)[1].definition()+' | '\n",
    "        else:\n",
    "            defs+= ''\n",
    "\n",
    "    return defs\n",
    "\n",
    "def cleanup(word,dataset,tpe=False):\n",
    "    ## for casie store second half\n",
    "    ## do substitution\n",
    "            \n",
    "    return re.sub(\"-\", \" \",word)\n",
    "\n",
    "\n",
    "def clean(w):\n",
    "      \n",
    "    w = w.strip()\n",
    "    changed=False\n",
    "    if(w[-1]=='|'):\n",
    "        #print(w)\n",
    "        changed=True\n",
    "        w = w[:-1].strip()\n",
    "        #print(w)\n",
    "        \n",
    "    w = ' | '.join(list(set([x.strip() for x in w.strip().split('|')])))\n",
    "    return w,changed\n",
    "\n",
    "def clean_src_label(opdf,relcol='src_label'):\n",
    "    z=[]\n",
    "    for i,r in opdf.iterrows():\n",
    "        srcs = [p.strip() for p in r[relcol].split('|')]\n",
    "        srcs = [s for s in srcs if 'NONE' not in s]\n",
    "        if(len(srcs)==0):\n",
    "            srcs = ['NONE']\n",
    "        z.append(' | '.join(srcs))\n",
    "    opdf[relcol] = z\n",
    "    return opdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "several-medication",
   "metadata": {},
   "outputs": [],
   "source": [
    "#def process(df,opind)\n",
    "dfl=[]\n",
    "\n",
    "z=0\n",
    "\n",
    "def process(df,opind,remove,testfile=False,include_neg=False):\n",
    "    tasks=['EV-CLF','EV-ID','EV-EX']#,'ARG-ID','ARG-EX']\n",
    "    rel_cols=['event_types','event_triggers','src_label']#,'args','src_label2']\n",
    "    \n",
    "    qns = ['What types of events are occuring in this text?',\n",
    "          'Which words or phrases denote events occurring in this text?',\n",
    "        'What are the events occurring in this text?']\n",
    "    \n",
    "    instr=['This input text gives information about specific types of ongoing events. The output should be the types of events occurring in the text. If there are no events, print NONE',\n",
    "           'You are given a text as input. The text gives information about ongoing events. An event nugget is a word or phrase that most clearly expresses the event occurrence. Your task is to identify the words or phrases that are event nuggets for events in the text, where event type is not given. If there are no events, print NONE',\n",
    "           'The text given as input discusses ongoing events. An event nugget is a word or phrase that most clearly expresses the event occurrence. Generate output in the format [event nugget->event type] for all events in the text. If there are no events, print NONE'\n",
    "           #'An event argument is an entity that is directly or indirectly involved in the event. It acts on or is acted on by the nugget. From the text given as input, extract the relevant arguments for the event in this text, where event type and nugget are not given',\n",
    "           #'The following text talks about events and related entities. An event argument is an actor that is directly or indirectly involved in the event. It acts on or is acted on by the nugget. The role it plays depends on the event type. Your task is to generate as output, the relevant arguments for the event in this text and classify their roles. The output format is [argument]->[role]'\n",
    "          ]\n",
    "    \n",
    "    example_c = ['INPUT: Our best analysis is that email addresses and allrecipes.com passwords were intercepted during account registration or login by our members.</s>OUTPUT: attack.</s>EXPLANATION: In this event, a malicious entity is attempting to cause harm by attacking the allrecipes.com. This is indicated by the phrase \"were intercepted\" which shows active hostile actions.',\n",
    "                 'INPUT: W-2 records for all companies and all employees were compromised.</s>OUTPUT: were compromised</s>EXPLANATION: The event occurrence is described by the phrase \"were compromised\" in the given text.',\n",
    "                 'INPUT: It has also been suggested that the criminals who stole the data feared detection if they sold it online and have kept it to themselves to avoid capture.</s>OUTPUT: stole->attack</s>EXPLANATION: The event is an attack, as indicated by the word \"steal\" which functions as the event nugget.'\n",
    "                ]\n",
    "    \n",
    "    example_b = ['INPUT: Left ventricular weight, body weight, and their ratio were not significantly altered by alinidine treatment.</s>OUTPUT: planned_process | regulation</s>EXPLANATION: The input contains multiple events of planned_process and regulation type',\n",
    "               'INPUT: Left ventricular weight, body weight, and their ratio were not significantly altered by alinidine treatment.</s>OUTPUT: treatment | altered</s>EXPLANATION: the words \"treatment\" and \"altered\" are salient words describing important events.',\n",
    "               'INPUT: Left ventricular weight, body weight, and their ratio were not significantly altered by alinidine treatment.</s>OUTPUT: treatment->planned_process | altered->regulation</s>EXPLANATION:  The word \"treatment\" in the input denotes a planned process, while the \"altered\" indicates the sentence talks about regulation.'\n",
    "               #'INPUT: Last spring, hackers got into the system at the ministry, which was then headed by now-Prime Minister Paolo Gentiloni, and the attacks carried on for more than four months but did not  gain access to classified information, the paper said. OUTPUT: system | hackers | the ministry |  EXPLANATION: The event here, classified as attack and triggered by the phrase \"got into\", directly involves the following entities: system, which is affected, the ministry, which is also affected, hackers, who carry it out.',\n",
    "               #'INPUT: As Action Fraud warns, confidence tricksters are phoning up schools and colleges pretending to be from the Department of Education. OUTPUT: Department of Education->Trusted-Entity | colleges->Victim | schools->Victim | tricksters->Attacker |  EXPLANATION: In the cybersecurity event described, the event is done by the tricksters in the role of Attacker. It occurs through the Department of Education, which functions as a Trusted-Entity. The event adversely affects two entities: colleges and schools, which thus act as Victim entities.'\n",
    "              ]\n",
    "    example_r = ['INPUT:  The leaflets carried several messages to the citizens attempting to reassure them that the advancing army \" would not target civilians , \" but warned them to avoid the known locations of Isis militants . The military operation is the most complex carried out in Iraq since US forces withdrew from the country in 2011 . Last week , the UN said it was bracing itself for the world \\'s biggest and most complex humanitarian effort following the battle , which it expects will displace up to one million people and see civilians used as human shields.</s>OUTPUT: conflict.</s>EXPLANATION: The event triggered by \"battle\" refers to an event of the type \"conflict\" which refers to a serious disagreement between two or more entities',\n",
    "               'INPUT: The information minister alleged that oil smuggled into Turkey was bought by the Turkish president\\'s son , who owns an oil company . Mr al - Zoubi said in an interview , All of the oil was delivered to a company that belongs to the son of Recep [ Tayyip ] Erdogan . This is why Turkey became anxious when Russia began delivering airstrikes against the IS infrastructure and destroyed more than 500 trucks with oil already.</s>OUTPUT: smuggled</s>EXPLANATION: The event describes goods being moved. The exact nugget from the text that describes this event is \"smuggled\"',\n",
    "               'INPUT: The Organization for Security and Cooperation In Europe \\'s ( OSCE ) Office for Democratic Institutions and Human Rights and the OSCE High Commissioner on National Minorities issued a report in September saying that since Russia \\'s land grab , fundamental freedoms had \" deteriorated radically \" for many in Crimea , especially for pro - Ukrainian activists , journalists , and the Crimean Tatar community.</s>OUTPUT: land grab->transaction.exchangebuysell</s>EXPLANATION: In this text, the event being discussed is the \"land grab\", which functions as the event nugget. The type of event it describes is a transaction, in which ownership of entities is transferred.'\n",
    "               #'INPUT: Russian President Vladimir Putin implicated in fatal poisoning of former KGB operative. That cautious stance is likely to disappoint Litvinenko \\'s wife , who called Thursday for Britain to expel Russian intelligence officials and enact new sanctions in response to the killing of her husband . OUTPUT: Russian President Vladimir Putin | former KGB operative | poisoning  OUTPUT: In the described event, the Russian President Vladimir Putin does the event, and the event occurs through the entity poisoning. The poisoning acts on a former KGB operative, who is the victim of this poisoning attack carried out by Putin.',\n",
    "               #'INPUT: Whatever the technicalities , the big picture is this : unless the level of Russian airstrikes dramatically decreases , this ceasefire will not hold because the moderate armed opposition can not lay down their weapons and will not lay down their weapons while they are being annihilated from the air . OUTPUT:  | Russian->attacker | the moderate armed opposition->target | airstrikes->instrument | the air->place  EXPLANATION: The entities involved in this event are the Russian, who are the attacker. The airstrikes are the instrument through which the attack is carried out, on the target which is the moderate armed opposition. The place argument for the event is the air.']\n",
    "              ]\n",
    "    prompts=['This passage talks about an event. Extract the type of event being talked about.',\n",
    "             'An event nugget is a word or phrase that most clearly expresses the event occurrence. Which word or phrase is the event nugget in this text?',\n",
    "             'This passage talks about an event. An event nugget is a word or phrase that most clearly expresses the event occurrence. Which word or phrase is the event nugget in this text and what type of event does it refer to?'\n",
    "             #'An event argument is an agent or actor that is directly or indirectly involved in the event. It acts on or is acted on by the nugget. Which are the relevant arguments for the event in this passage?',\n",
    "             #'An event argument is an agent or actor that is directly or indirectly involved in the event. It acts on or is acted on by the nugget in a particular role. Which are the relevant arguments and what roles do they play for the event in this passage?'\n",
    "            ]\n",
    "    \n",
    "    #\"You are given a context as in input. This passage gives information about ongoing event. Your job is to generate a event type as an output, where event trigger is not given.\"\n",
    "\n",
    "    #\"Prompt: \" + prompt + ', ' + 'context: ' + hdsk + ', ' + 'output: ?'\n",
    "    \n",
    "    #\"Definition: \" + + \"example_1: [input: , output: , explanation: ]\" + \"Input: , output: ?\"  \n",
    "    \n",
    "    text=[]\n",
    "    texto=[]\n",
    "    textevex=[]\n",
    "    textevp=[]\n",
    "    subtask=[]\n",
    "    dataset=[]\n",
    "    src_label=[]\n",
    "    text_p=[]\n",
    "    text_in=[]\n",
    "    text_in_only=[]\n",
    "    \n",
    "    context=[]    \n",
    "    text_golddef=[]\n",
    "    instruction=[]\n",
    "    examples=[]\n",
    "    \n",
    "    print(len(df),' is len')\n",
    "\n",
    "    for ind,row in df.iterrows():\n",
    "        \n",
    "        #if(row['text'] in example_b[0]):\n",
    "        #    continue\n",
    "        \n",
    "        for i,t in enumerate(tasks):\n",
    "            \n",
    "            #if(t=='EV-EX'):\n",
    "            #    dctg[row['text']]=row['src_label']\n",
    "                #print(predevex[ind][2:-2])\n",
    "                    \n",
    "            if(t=='EV-CLF'): # first task\n",
    "                dctg[row['text']]={}\n",
    "                ## for casie store second half\n",
    "                dctg[row['text']]['etyg']=cleanup(row[rel_cols[i]],row['dataset'],tpe=True)\n",
    "                \n",
    "            if(t=='EV-ID'):\n",
    "                dctg[row['text']]['evg']=cleanup(row[rel_cols[i]],row['dataset'],tpe=False)\n",
    "                \n",
    "            subtask.append(t)\n",
    "            \n",
    "            '''\n",
    "            if(t in domains['gen']):\n",
    "                example_rel = example_r\n",
    "            else:\n",
    "                example_rel = example_b\n",
    "            '''\n",
    "            lab = row[rel_cols[i]]\n",
    "            \n",
    "            if((include_neg and len(lab)==0) or lab=='NONE->NONE'):\n",
    "                lab = 'NONE'\n",
    "                \n",
    "            lab,changed = clean(lab)\n",
    "            \n",
    "            src_label.append(lab)\n",
    "            dataset.append(row['dataset'])\n",
    "            texto.append(row['text'])\n",
    "            text.append(subtask[-1]+'</s>'+dataset[-1]+'</s>'+row['text'])\n",
    "            #text_p.append(' PROMPT: '+prompts[i]+' INPUT: '+row['text']+' OUTPUT: ?')\n",
    "            text_in.append(instr[i]+' EXAMPLE_1: ['+example_b[i]+'] EXAMPLE_2: ['+example_r[i]+'] INPUT: '+row['text']+' OUTPUT: ?' )\n",
    "            #text_in.append('context: '+row['text']+'</s>question: '+qns[i] )\n",
    "            #text_in.append(instr[i]+'</s>EXAMPLE_1: ['+example_r[i]+'</s>EXAMPLE_2: ['+example_b[i]+'</s>EXAMPLE_3: ['+example_c[i]+']</s>INPUT: '+row['text']+'</s>OUTPUT: ?' )\n",
    "            \n",
    "            context.append(instr[i]+' EXAMPLE_1: ['+example_b[i]+'] EXAMPLE_2: ['+example_r[i]+']')\n",
    "            instruction.append(instr[i])\n",
    "            examples.append('<example1>'+example_b[i]+'</example1><example2>'+example_r[i]+'</example2>')\n",
    "            #text_golddef.append\n",
    "            '''\n",
    "            if('ARG-' in t):\n",
    "                if(testfile):\n",
    "                    ### currently using gold ev-ex labels.\n",
    "                    textevex.append('EVENT: '+dctg[row['text']]+' TEXT: '+text_in[-1])\n",
    "                    textevp.append('EVENT: '+dctg[row['text']].split('->')[0]+' TEXT: '+text_in[-1])\n",
    "                    ### currently using predicted ev-ex labels \n",
    "                    #textevex.append(dctp[row['text']]+'<s>'+text[-1])\n",
    "                else:\n",
    "                    textevex.append('EVENT: '+dctg[row['text']]+' TEXT: '+text_in[-1])\n",
    "                    textevp.append('EVENT: '+dctg[row['text']].split('->')[0]+' TEXT: '+text_in[-1])\n",
    "            else:\n",
    "                textevex.append('')\n",
    "                textevp.append('')\n",
    "            '''    \n",
    "            \n",
    "            \n",
    "\n",
    "    \n",
    "    #opdf = pd.DataFrame(data={'prompt':text_in_only,'examples':examples,'text':text,'question':texto,'text_pr':text_p,'text_in':text_in,'src_label':src_label,'completion':src_label,'subtask':subtask,'dataset':dataset})\n",
    "    opdf = pd.DataFrame(data={'text':text,'question':texto,'text_in':text_in,'src_label':src_label,'subtask':subtask,'dataset':dataset})\n",
    "    \n",
    "    '''\n",
    "    opdf['text_evg']=['']*len(opdf)\n",
    "    opdf['text_evp']=['']*len(opdf)\n",
    "    opdf['text_evtyg']=['']*len(opdf)\n",
    "    opdf['text_evtyp']=['']*len(opdf)\n",
    "    \n",
    "    for ip,r in opdf.iterrows():\n",
    "        tg=dctg[r['texto']]['evg']\n",
    "        tyg=dctg[r['texto']]['etyg']\n",
    "        \n",
    "        r['text_evg'] = 'EVENT: '+tg+' DEFINITION: '+getdef(tg)+' TASK: '+r['text_in']\n",
    "        r['text_evp'] = 'EVENT: '+tg+' DEFINITION: '+getdef(tg)+' TASK: '+r['text_in']\n",
    "        r['text_evtyg'] = 'EVENT: '+tyg+' DEFINITION: '+getdef(tyg)+' TASK: '+r['text_in']\n",
    "        r['text_evtyp'] = 'EVENT: '+tyg+' DEFINITION: '+getdef(tyg)+' TASK: '+r['text_in']\n",
    "    \n",
    "    '''\n",
    "    if(not testfile):\n",
    "        opdf = opdf.sample(frac=1).reset_index(drop=True)\n",
    "    else:\n",
    "        print('orilen ',len(opdf))\n",
    "        opdf = opdf.drop(index=remove).reset_index(drop=True)\n",
    "        print('newlen ',len(opdf))\n",
    "        \n",
    "    opdf = clean_src_label(opdf,relcol='src_label')\n",
    "    dfl.append(opdf)\n",
    "    \n",
    "    if(tofile):\n",
    "        opdf.to_csv(opfiles_csv[opind],index=False)\n",
    "        opdf.to_json(opfiles_jsonl[opind],orient='records',lines=True,force_ascii=True)\n",
    "        print('filed')\n",
    "    print(opdf.head())\n",
    "    print('--------------------')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "confirmed-neighbor",
   "metadata": {},
   "outputs": [],
   "source": [
    "#remove=[4302,3029,11,120,2693]\n",
    "remove=[]\n",
    "### fix this!!!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "awful-likelihood",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "27751  is len\n",
      "filed\n",
      "                                                text  \\\n",
      "0  EV-CLF</s>RAMS3_neg_processed</s>It ’s no surp...   \n",
      "1  EV-EX</s>RAMS3_neg_processed</s>The three days...   \n",
      "2  EV-ID</s>RAMS3_neg_processed</s>Over the past ...   \n",
      "3  EV-ID</s>RAMS3_neg_processed</s>View of the ho...   \n",
      "4  EV-CLF</s>RAMS3_neg_processed</s>The moderator...   \n",
      "\n",
      "                                            question  \\\n",
      "0  It ’s no surprise that an organization like IS...   \n",
      "1  The three days of relentless bombing meant man...   \n",
      "2  Over the past two years , ISIS has publicly ex...   \n",
      "3  View of the hole from the “ storage space ” wh...   \n",
      "4  The moderator , Fox News ’ Chris Wallace , ask...   \n",
      "\n",
      "                                             text_in             src_label  \\\n",
      "0  This input text gives information about specif...                  NONE   \n",
      "1  The text given as input discusses ongoing even...  wounded->life.injure   \n",
      "2  You are given a text as input. The text gives ...              executed   \n",
      "3  You are given a text as input. The text gives ...                  NONE   \n",
      "4  This input text gives information about specif...                  NONE   \n",
      "\n",
      "  subtask              dataset  \n",
      "0  EV-CLF  RAMS3_neg_processed  \n",
      "1   EV-EX  RAMS3_neg_processed  \n",
      "2   EV-ID  RAMS3_neg_processed  \n",
      "3   EV-ID  RAMS3_neg_processed  \n",
      "4  EV-CLF  RAMS3_neg_processed  \n",
      "--------------------\n",
      "3456  is len\n",
      "orilen  10368\n",
      "newlen  10368\n",
      "filed\n",
      "                                                text  \\\n",
      "0  EV-CLF</s>RAMS3_neg_processed</s>We are ashame...   \n",
      "1  EV-ID</s>RAMS3_neg_processed</s>We are ashamed...   \n",
      "2  EV-EX</s>RAMS3_neg_processed</s>We are ashamed...   \n",
      "3  EV-CLF</s>RAMS3_neg_processed</s>However , Mut...   \n",
      "4  EV-ID</s>RAMS3_neg_processed</s>However , Mutk...   \n",
      "\n",
      "                                            question  \\\n",
      "0                         We are ashamed of them . \"   \n",
      "1                         We are ashamed of them . \"   \n",
      "2                         We are ashamed of them . \"   \n",
      "3  However , Mutko stopped short of admitting the...   \n",
      "4  However , Mutko stopped short of admitting the...   \n",
      "\n",
      "                                             text_in src_label subtask  \\\n",
      "0  This input text gives information about specif...      NONE  EV-CLF   \n",
      "1  You are given a text as input. The text gives ...      NONE   EV-ID   \n",
      "2  The text given as input discusses ongoing even...      NONE   EV-EX   \n",
      "3  This input text gives information about specif...      NONE  EV-CLF   \n",
      "4  You are given a text as input. The text gives ...      NONE   EV-ID   \n",
      "\n",
      "               dataset  \n",
      "0  RAMS3_neg_processed  \n",
      "1  RAMS3_neg_processed  \n",
      "2  RAMS3_neg_processed  \n",
      "3  RAMS3_neg_processed  \n",
      "4  RAMS3_neg_processed  \n",
      "--------------------\n"
     ]
    }
   ],
   "source": [
    "\n",
    "opfiles_csv = [opdir+'/train.csv',opdir+'/test.csv']\n",
    "opfiles_jsonl = [opdir+'/train.jsonl',opdir+'/test.jsonl']\n",
    "testfile=[False,True]\n",
    "for i,df in enumerate([dfn1,dfn2]):\n",
    "    process(df,i,remove,testfile[i],include_neg=include_neg)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "imposed-termination",
   "metadata": {},
   "outputs": [],
   "source": [
    "#dctp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "stylish-steal",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'gen': ['wikievent_sent_neg_processed', 'RAMS_doc'],\n",
       " 'biomed2k': ['mlee_processed_sent']}"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "domains"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "entertaining-worship",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>question</th>\n",
       "      <th>text_in</th>\n",
       "      <th>src_label</th>\n",
       "      <th>subtask</th>\n",
       "      <th>dataset</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>EV-CLF&lt;/s&gt;RAMS3_neg_processed&lt;/s&gt;It ’s no surp...</td>\n",
       "      <td>It ’s no surprise that an organization like IS...</td>\n",
       "      <td>This input text gives information about specif...</td>\n",
       "      <td>NONE</td>\n",
       "      <td>EV-CLF</td>\n",
       "      <td>RAMS3_neg_processed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>EV-EX&lt;/s&gt;RAMS3_neg_processed&lt;/s&gt;The three days...</td>\n",
       "      <td>The three days of relentless bombing meant man...</td>\n",
       "      <td>The text given as input discusses ongoing even...</td>\n",
       "      <td>wounded-&gt;life.injure</td>\n",
       "      <td>EV-EX</td>\n",
       "      <td>RAMS3_neg_processed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>EV-ID&lt;/s&gt;RAMS3_neg_processed&lt;/s&gt;Over the past ...</td>\n",
       "      <td>Over the past two years , ISIS has publicly ex...</td>\n",
       "      <td>You are given a text as input. The text gives ...</td>\n",
       "      <td>executed</td>\n",
       "      <td>EV-ID</td>\n",
       "      <td>RAMS3_neg_processed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>EV-ID&lt;/s&gt;RAMS3_neg_processed&lt;/s&gt;View of the ho...</td>\n",
       "      <td>View of the hole from the “ storage space ” wh...</td>\n",
       "      <td>You are given a text as input. The text gives ...</td>\n",
       "      <td>NONE</td>\n",
       "      <td>EV-ID</td>\n",
       "      <td>RAMS3_neg_processed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>EV-CLF&lt;/s&gt;RAMS3_neg_processed&lt;/s&gt;The moderator...</td>\n",
       "      <td>The moderator , Fox News ’ Chris Wallace , ask...</td>\n",
       "      <td>This input text gives information about specif...</td>\n",
       "      <td>NONE</td>\n",
       "      <td>EV-CLF</td>\n",
       "      <td>RAMS3_neg_processed</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text  \\\n",
       "0  EV-CLF</s>RAMS3_neg_processed</s>It ’s no surp...   \n",
       "1  EV-EX</s>RAMS3_neg_processed</s>The three days...   \n",
       "2  EV-ID</s>RAMS3_neg_processed</s>Over the past ...   \n",
       "3  EV-ID</s>RAMS3_neg_processed</s>View of the ho...   \n",
       "4  EV-CLF</s>RAMS3_neg_processed</s>The moderator...   \n",
       "\n",
       "                                            question  \\\n",
       "0  It ’s no surprise that an organization like IS...   \n",
       "1  The three days of relentless bombing meant man...   \n",
       "2  Over the past two years , ISIS has publicly ex...   \n",
       "3  View of the hole from the “ storage space ” wh...   \n",
       "4  The moderator , Fox News ’ Chris Wallace , ask...   \n",
       "\n",
       "                                             text_in             src_label  \\\n",
       "0  This input text gives information about specif...                  NONE   \n",
       "1  The text given as input discusses ongoing even...  wounded->life.injure   \n",
       "2  You are given a text as input. The text gives ...              executed   \n",
       "3  You are given a text as input. The text gives ...                  NONE   \n",
       "4  This input text gives information about specif...                  NONE   \n",
       "\n",
       "  subtask              dataset  \n",
       "0  EV-CLF  RAMS3_neg_processed  \n",
       "1   EV-EX  RAMS3_neg_processed  \n",
       "2   EV-ID  RAMS3_neg_processed  \n",
       "3   EV-ID  RAMS3_neg_processed  \n",
       "4  EV-CLF  RAMS3_neg_processed  "
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#a=pd.read_csv(opdir+'/train.csv')\n",
    "#b=pd.read_csv(opdir+'/test.csv')\n",
    "a=dfl[0]\n",
    "b=dfl[1]\n",
    "a.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "fundamental-component",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>question</th>\n",
       "      <th>text_in</th>\n",
       "      <th>src_label</th>\n",
       "      <th>subtask</th>\n",
       "      <th>dataset</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>EV-CLF&lt;/s&gt;RAMS3_neg_processed&lt;/s&gt;We are ashame...</td>\n",
       "      <td>We are ashamed of them . \"</td>\n",
       "      <td>This input text gives information about specif...</td>\n",
       "      <td>NONE</td>\n",
       "      <td>EV-CLF</td>\n",
       "      <td>RAMS3_neg_processed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>EV-ID&lt;/s&gt;RAMS3_neg_processed&lt;/s&gt;We are ashamed...</td>\n",
       "      <td>We are ashamed of them . \"</td>\n",
       "      <td>You are given a text as input. The text gives ...</td>\n",
       "      <td>NONE</td>\n",
       "      <td>EV-ID</td>\n",
       "      <td>RAMS3_neg_processed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>EV-EX&lt;/s&gt;RAMS3_neg_processed&lt;/s&gt;We are ashamed...</td>\n",
       "      <td>We are ashamed of them . \"</td>\n",
       "      <td>The text given as input discusses ongoing even...</td>\n",
       "      <td>NONE</td>\n",
       "      <td>EV-EX</td>\n",
       "      <td>RAMS3_neg_processed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>EV-CLF&lt;/s&gt;RAMS3_neg_processed&lt;/s&gt;However , Mut...</td>\n",
       "      <td>However , Mutko stopped short of admitting the...</td>\n",
       "      <td>This input text gives information about specif...</td>\n",
       "      <td>NONE</td>\n",
       "      <td>EV-CLF</td>\n",
       "      <td>RAMS3_neg_processed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>EV-ID&lt;/s&gt;RAMS3_neg_processed&lt;/s&gt;However , Mutk...</td>\n",
       "      <td>However , Mutko stopped short of admitting the...</td>\n",
       "      <td>You are given a text as input. The text gives ...</td>\n",
       "      <td>NONE</td>\n",
       "      <td>EV-ID</td>\n",
       "      <td>RAMS3_neg_processed</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text  \\\n",
       "0  EV-CLF</s>RAMS3_neg_processed</s>We are ashame...   \n",
       "1  EV-ID</s>RAMS3_neg_processed</s>We are ashamed...   \n",
       "2  EV-EX</s>RAMS3_neg_processed</s>We are ashamed...   \n",
       "3  EV-CLF</s>RAMS3_neg_processed</s>However , Mut...   \n",
       "4  EV-ID</s>RAMS3_neg_processed</s>However , Mutk...   \n",
       "\n",
       "                                            question  \\\n",
       "0                         We are ashamed of them . \"   \n",
       "1                         We are ashamed of them . \"   \n",
       "2                         We are ashamed of them . \"   \n",
       "3  However , Mutko stopped short of admitting the...   \n",
       "4  However , Mutko stopped short of admitting the...   \n",
       "\n",
       "                                             text_in src_label subtask  \\\n",
       "0  This input text gives information about specif...      NONE  EV-CLF   \n",
       "1  You are given a text as input. The text gives ...      NONE   EV-ID   \n",
       "2  The text given as input discusses ongoing even...      NONE   EV-EX   \n",
       "3  This input text gives information about specif...      NONE  EV-CLF   \n",
       "4  You are given a text as input. The text gives ...      NONE   EV-ID   \n",
       "\n",
       "               dataset  \n",
       "0  RAMS3_neg_processed  \n",
       "1  RAMS3_neg_processed  \n",
       "2  RAMS3_neg_processed  \n",
       "3  RAMS3_neg_processed  \n",
       "4  RAMS3_neg_processed  "
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "every-arlington",
   "metadata": {},
   "outputs": [],
   "source": [
    "#b['text_evg'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fifty-evidence",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "acting-newspaper",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "provincial-philip",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    83253.000000\n",
       "mean       217.153880\n",
       "std         17.766421\n",
       "min        188.000000\n",
       "25%        206.000000\n",
       "50%        215.000000\n",
       "75%        225.000000\n",
       "95%        246.000000\n",
       "max        578.000000\n",
       "dtype: float64"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "z=[]\n",
    "for i,row in a.iterrows():\n",
    "    z.append(len(row['text_in'].split()))\n",
    "    \n",
    "pd.Series(z).describe(percentiles=[.25,.5,.75,.95])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "sustainable-drunk",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-34-09c245ff25f2>:1: FutureWarning: The default dtype for empty Series will be 'object' instead of 'float64' in a future version. Specify a dtype explicitly to silence this warning.\n",
      "  pd.Series([p for p in z if p>4100]).describe()\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "count    0.0\n",
       "mean     NaN\n",
       "std      NaN\n",
       "min      NaN\n",
       "25%      NaN\n",
       "50%      NaN\n",
       "75%      NaN\n",
       "max      NaN\n",
       "dtype: float64"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.Series([p for p in z if p>4100]).describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "boxed-brook",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 83253 entries, 0 to 83252\n",
      "Data columns (total 6 columns):\n",
      " #   Column     Non-Null Count  Dtype \n",
      "---  ------     --------------  ----- \n",
      " 0   text       83253 non-null  object\n",
      " 1   question   83253 non-null  object\n",
      " 2   text_in    83253 non-null  object\n",
      " 3   src_label  83253 non-null  object\n",
      " 4   subtask    83253 non-null  object\n",
      " 5   dataset    83253 non-null  object\n",
      "dtypes: object(6)\n",
      "memory usage: 3.8+ MB\n"
     ]
    }
   ],
   "source": [
    "a.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "wireless-lucas",
   "metadata": {},
   "outputs": [],
   "source": [
    "#len(b[b.subtask.str.contains('ARG')])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "adult-activity",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\nop=['train','test']\\n\\nfor i,df in enumerate([a,b]):\\n    df_ev = df[df['subtask'].str.contains('EV-')]\\n    df_arg = df[df['subtask'].str.contains('ARG-')]\\n    dfl.append(df_ev)\\n    dfl.append(df_arg)\\n    \\n    if(addpred and i==1 and addanyway):\\n        for ix,r in df_ev.iterrows():\\n            #print(predevex[ix],r['subtask'],r['src_label'])\\n            if(r['subtask']=='EV-CLF'): ## first task\\n                if(r['texto'] not in dctp.keys()):\\n                    dctp[r['texto']]={}\\n                dctp[r['texto']]['etyp'] = cleanup(predevex[ix],r['dataset'],tpe=True)\\n            if(r['subtask']=='EV-ID'):\\n                if(r['texto'] not in dctp.keys()):\\n                    dctp[r['texto']]={}\\n                dctp[r['texto']]['evp'] = cleanup(predevex[ix],r['dataset'],tpe=False)\\n            if(r['subtask']=='EV-EX'):\\n                if(r['texto'] not in dctp.keys()):\\n                    dctp[r['texto']]={}\\n                if('evp' not in dctp[r['texto']].keys()):\\n                    dctp[r['texto']]['evp'] = cleanup(predevex[ix].split('->')[0],r['dataset'],tpe=False)\\n                if('evtyp' not in dctp[r['texto']].keys()):    \\n                    dctp[r['texto']]['evtyp'] = cleanup(predevex[ix].split('->')[-1],r['dataset'],tpe=True)\\n            \\n    \\n        for ix,r in df_ev.iterrows():\\n            \\n                \\n            tp = dctp[r['texto']]['evp']\\n            typ = dctp[r['texto']]['evtyp']\\n            r['text_evp'] = 'EVENT: '+tp+' DEFINITION: '+getdef(tp)+' TASK: '+r['text_in']\\n            r['text_evtyp'] = 'EVENT: '+typ+' DEFINITION: '+getdef(typ)+' TASK: '+r['text_in']\\n    \\n    \\n            \\n    \\n    if(tofile):\\n        df_ev.to_csv(opdir+'/'+op[i]+'ev.csv',index=False)\\n        df_ev.to_json(opdir+'/'+op[i]+'ev.jsonl',orient='records',lines=True,force_ascii=True)\\n        print(df_ev.head())\\n        print('-------------------------')\\n        df_arg.to_csv(opdir+'/'+op[i]+'arg.csv',index=False)\\n        df_arg.to_json(opdir+'/'+op[i]+'arg.jsonl',orient='records',lines=True,force_ascii=True)\\n        print(df_arg.head())\\n        print('filed')\\n\\n\""
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "op=['train','test']\n",
    "\n",
    "for i,df in enumerate([a,b]):\n",
    "    df_ev = df[df['subtask'].str.contains('EV-')]\n",
    "    df_arg = df[df['subtask'].str.contains('ARG-')]\n",
    "    dfl.append(df_ev)\n",
    "    dfl.append(df_arg)\n",
    "    \n",
    "    if(addpred and i==1 and addanyway):\n",
    "        for ix,r in df_ev.iterrows():\n",
    "            #print(predevex[ix],r['subtask'],r['src_label'])\n",
    "            if(r['subtask']=='EV-CLF'): ## first task\n",
    "                if(r['texto'] not in dctp.keys()):\n",
    "                    dctp[r['texto']]={}\n",
    "                dctp[r['texto']]['etyp'] = cleanup(predevex[ix],r['dataset'],tpe=True)\n",
    "            if(r['subtask']=='EV-ID'):\n",
    "                if(r['texto'] not in dctp.keys()):\n",
    "                    dctp[r['texto']]={}\n",
    "                dctp[r['texto']]['evp'] = cleanup(predevex[ix],r['dataset'],tpe=False)\n",
    "            if(r['subtask']=='EV-EX'):\n",
    "                if(r['texto'] not in dctp.keys()):\n",
    "                    dctp[r['texto']]={}\n",
    "                if('evp' not in dctp[r['texto']].keys()):\n",
    "                    dctp[r['texto']]['evp'] = cleanup(predevex[ix].split('->')[0],r['dataset'],tpe=False)\n",
    "                if('evtyp' not in dctp[r['texto']].keys()):    \n",
    "                    dctp[r['texto']]['evtyp'] = cleanup(predevex[ix].split('->')[-1],r['dataset'],tpe=True)\n",
    "            \n",
    "    \n",
    "        for ix,r in df_ev.iterrows():\n",
    "            \n",
    "                \n",
    "            tp = dctp[r['texto']]['evp']\n",
    "            typ = dctp[r['texto']]['evtyp']\n",
    "            r['text_evp'] = 'EVENT: '+tp+' DEFINITION: '+getdef(tp)+' TASK: '+r['text_in']\n",
    "            r['text_evtyp'] = 'EVENT: '+typ+' DEFINITION: '+getdef(typ)+' TASK: '+r['text_in']\n",
    "    \n",
    "    \n",
    "            \n",
    "    \n",
    "    if(tofile):\n",
    "        df_ev.to_csv(opdir+'/'+op[i]+'ev.csv',index=False)\n",
    "        df_ev.to_json(opdir+'/'+op[i]+'ev.jsonl',orient='records',lines=True,force_ascii=True)\n",
    "        print(df_ev.head())\n",
    "        print('-------------------------')\n",
    "        df_arg.to_csv(opdir+'/'+op[i]+'arg.csv',index=False)\n",
    "        df_arg.to_json(opdir+'/'+op[i]+'arg.jsonl',orient='records',lines=True,force_ascii=True)\n",
    "        print(df_arg.head())\n",
    "        print('filed')\n",
    "\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "seeing-proceeding",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'df_ev.head()'"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''df_ev.head()'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "educated-cooper",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "filed\n",
      "filed\n",
      "filed\n",
      "filed\n",
      "filed\n",
      "filed\n"
     ]
    }
   ],
   "source": [
    "op=['train','test']\n",
    "\n",
    "dss = ['EV-ID','EV-CLF','EV-EX']\n",
    "\n",
    "for dat in dss:\n",
    "    for i,df in enumerate([a,b]):\n",
    "        df_c = df[df['subtask'].str.contains(dat)]\n",
    "        \n",
    "        dfl.append(df_c)\n",
    "        \n",
    "        if(tofile):\n",
    "\n",
    "            df_c.to_csv(opdir+'/'+op[i]+dat.replace('-','').lower()+'.csv',index=False)\n",
    "            df_c.to_json(opdir+'/'+op[i]+dat.replace('-','').lower()+'.jsonl',orient='records',lines=True,force_ascii=True)\n",
    "\n",
    "            print('filed')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "sophisticated-moldova",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "83253\n",
      "filed\n",
      "10368\n",
      "filed\n",
      "0\n",
      "filed\n",
      "0\n",
      "filed\n",
      "0\n",
      "filed\n",
      "0\n",
      "filed\n",
      "0\n",
      "filed\n",
      "0\n",
      "filed\n"
     ]
    }
   ],
   "source": [
    "op=['train','test']\n",
    "\n",
    "dss = ['RAMS','wikievent','maven','ml']\n",
    "\n",
    "for dat in dss:\n",
    "    for i,df in enumerate([a,b]):\n",
    "        df_c = df[df['dataset'].str.contains(dat)]\n",
    "        \n",
    "        dfl.append(df_c)\n",
    "        \n",
    "        if(tofile):\n",
    "\n",
    "            df_c.to_csv(opdir+'/'+op[i]+dat.lower()+'.csv',index=False)\n",
    "            df_c.to_json(opdir+'/'+op[i]+dat.lower()+'.jsonl',orient='records',lines=True,force_ascii=True)\n",
    "            \n",
    "            print(len(df_c))\n",
    "            print('filed')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "spanish-neutral",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>question</th>\n",
       "      <th>text_in</th>\n",
       "      <th>src_label</th>\n",
       "      <th>subtask</th>\n",
       "      <th>dataset</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [text, question, text_in, src_label, subtask, dataset]\n",
       "Index: []"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_c.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "retired-collective",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "frequent-johnson",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "text         EV-CLF</s>RAMS3_neg_processed</s>We are ashame...\n",
       "question                            We are ashamed of them . \"\n",
       "text_in      This input text gives information about specif...\n",
       "src_label                                                 NONE\n",
       "subtask                                                 EV-CLF\n",
       "dataset                                    RAMS3_neg_processed\n",
       "Name: 0, dtype: object"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b.iloc[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "successful-brazilian",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "initial-computer",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "looking-sudan",
   "metadata": {},
   "outputs": [],
   "source": [
    "#dft=dfl[1]\n",
    "#dft.iloc[2693]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "dutch-boards",
   "metadata": {},
   "outputs": [],
   "source": [
    "dftrain=dfl[0]\n",
    "dftest=dfl[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "editorial-diary",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "filed\n",
      "filed\n",
      "filed\n",
      "filed\n",
      "filed\n",
      "filed\n",
      "filed\n",
      "filed\n",
      "filed\n",
      "filed\n",
      "filed\n",
      "filed\n",
      "filed\n",
      "filed\n",
      "filed\n",
      "filed\n",
      "filed\n",
      "filed\n",
      "filed\n",
      "filed\n",
      "filed\n",
      "filed\n",
      "filed\n",
      "filed\n"
     ]
    }
   ],
   "source": [
    "for dat in dss:\n",
    "\n",
    "    for t in ['EV-EX','EV-CLF','EV-ID']:#,'ARG-EX','ARG-ID']:\n",
    "        tk=t.lower().replace('-','')\n",
    "        pt = dftest[dftest['subtask']==t]\n",
    "        pt = pt[pt.dataset.str.contains(dat)]\n",
    "        if(tofile):\n",
    "            pt.to_csv(opdir+'/test'+dat.lower()+tk+'.csv',index=False)\n",
    "            pt.to_json(opdir+'/test'+dat.lower()+tk+'.jsonl',orient='records',lines=True,force_ascii=True)\n",
    "            print('filed')\n",
    "\n",
    "        pt = dftrain[dftrain['subtask']==t]\n",
    "        pt = pt[pt.dataset.str.contains(dat)]\n",
    "        if(tofile):\n",
    "            pt.to_csv(opdir+'/train'+dat.lower()+tk+'.csv',index=False)\n",
    "            pt.to_json(opdir+'/train'+dat.lower()+tk+'.jsonl',orient='records',lines=True,force_ascii=True)\n",
    "            print('filed')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "colonial-demand",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "typical-fifty",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"tasks=['EV-CLF','EV-ID','EV-EX','ARG-ID','ARG-EX']\\nfor d in ['CASIE','RAMS']\\n    for t in tasks:\""
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''tasks=['EV-CLF','EV-ID','EV-EX','ARG-ID','ARG-EX']\n",
    "for d in ['CASIE','RAMS']\n",
    "    for t in tasks:'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "subjective-priority",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "for dm in domains.keys():\n",
    "    pt = dftrain[dftrain.dataset.isin(domains[dm])]\n",
    "    pt.to_csv(opdir+'/train'+dm.lower()+'.csv',index=False)\n",
    "    \n",
    "    pt = dftest[dftest.dataset.isin(domains[dm])]\n",
    "    pt.to_csv(opdir+'/test'+dm.lower()+'.csv',index=False)\n",
    "    #dftest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "medical-browse",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'gen': ['wikievent_sent_neg_processed', 'RAMS_doc'],\n",
       " 'biomed2k': ['mlee_processed_sent']}"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "domains"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "greater-development",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "count    83253.000000\n",
      "mean       217.153880\n",
      "std         17.766421\n",
      "min        188.000000\n",
      "25%        206.000000\n",
      "50%        215.000000\n",
      "75%        225.000000\n",
      "max        578.000000\n",
      "Name: text_in, dtype: float64\n",
      "\n",
      "\n",
      "\n",
      "count    83253.000000\n",
      "mean        27.820547\n",
      "std         17.647194\n",
      "min          1.000000\n",
      "25%         16.000000\n",
      "50%         25.000000\n",
      "75%         36.000000\n",
      "max        386.000000\n",
      "Name: text, dtype: float64\n",
      "---------------------------\n",
      "count    10368.000000\n",
      "mean       216.593750\n",
      "std         16.991997\n",
      "min        188.000000\n",
      "25%        205.000000\n",
      "50%        214.000000\n",
      "75%        225.000000\n",
      "max        399.000000\n",
      "Name: text_in, dtype: float64\n",
      "\n",
      "\n",
      "\n",
      "count    10368.000000\n",
      "mean        27.260417\n",
      "std         16.867286\n",
      "min          1.000000\n",
      "25%         16.000000\n",
      "50%         25.000000\n",
      "75%         35.000000\n",
      "max        207.000000\n",
      "Name: text, dtype: float64\n",
      "---------------------------\n"
     ]
    }
   ],
   "source": [
    "for x in ['train.csv','test.csv']:\n",
    "    df = pd.read_csv(opdir+'/'+x)\n",
    "    print(df.text_in.str.split().str.len().describe())\n",
    "    print('\\n\\n')\n",
    "    #print(df.prompt.str.split().str.len().describe())\n",
    "    #print('\\n\\n')\n",
    "    print(df.text.str.split().str.len().describe())\n",
    "    print('---------------------------')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "satellite-banking",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Series([], Name: dataset, dtype: int64)\n",
      "\n",
      "\n",
      "\n",
      "RAMS3_neg_processed    6\n",
      "Name: dataset, dtype: int64\n",
      "-----------------------\n",
      "Series([], Name: dataset, dtype: int64)\n",
      "\n",
      "\n",
      "\n",
      "Series([], Name: dataset, dtype: int64)\n",
      "-----------------------\n"
     ]
    }
   ],
   "source": [
    "for x in ['train.csv','test.csv']:\n",
    "    df = pd.read_csv(opdir+'/'+x)\n",
    "    print(df[df.text.str.split().str.len()>512]['dataset'].value_counts())\n",
    "    print('\\n\\n')\n",
    "    #print(df[df.prompt.str.split().str.len()>512]['dataset'].value_counts())\n",
    "    #print('\\n\\n')\n",
    "    print(df[df.text_in.str.split().str.len()>512]['dataset'].value_counts())\n",
    "    print('-----------------------')\n",
    "#print(df[df.text_in.str.len()>1024]['src_label'].iloc[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "connected-privilege",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Series([], Name: dataset, dtype: int64)\n",
      "\n",
      "\n",
      "\n",
      "Series([], Name: dataset, dtype: int64)\n",
      "-----------------------\n",
      "Series([], Name: dataset, dtype: int64)\n",
      "\n",
      "\n",
      "\n",
      "Series([], Name: dataset, dtype: int64)\n",
      "-----------------------\n"
     ]
    }
   ],
   "source": [
    "for x in ['train.csv','test.csv']:\n",
    "    df = pd.read_csv(opdir+'/'+x)\n",
    "    print(df[df.text.str.split().str.len()>1024]['dataset'].value_counts())\n",
    "    print('\\n\\n')\n",
    "    #print(df[df.prompt.str.split().str.len()>1024]['dataset'].value_counts())\n",
    "    #print('\\n\\n')\n",
    "    print(df[df.text_in.str.split().str.len()>1024]['dataset'].value_counts())\n",
    "    print('-----------------------')\n",
    "#print(df[df.text_in.str.len()>1024]['src_label'].iloc[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "physical-judgment",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Series([], Name: dataset, dtype: int64)\n",
      "\n",
      "\n",
      "\n",
      "Series([], Name: dataset, dtype: int64)\n",
      "-------------------\n",
      "Series([], Name: dataset, dtype: int64)\n",
      "\n",
      "\n",
      "\n",
      "Series([], Name: dataset, dtype: int64)\n",
      "-------------------\n"
     ]
    }
   ],
   "source": [
    "for x in ['train.csv','test.csv']:\n",
    "    df = pd.read_csv(opdir+'/'+x)\n",
    "    print(df[df.text.str.split().str.len()>2048]['dataset'].value_counts())\n",
    "    print('\\n\\n')\n",
    "    #print(df[df.prompt.str.split().str.len()>2048]['dataset'].value_counts())\n",
    "    #print('\\n\\n')\n",
    "    print(df[df.text_in.str.split().str.len()>2048]['dataset'].value_counts())\n",
    "    print('-------------------')\n",
    "#print(df[df.text_in.str.len()>1024]['src_label'].iloc[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "boolean-greek",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RAMS3_neg_processed    27751.0\n",
      "Name: dataset, dtype: float64\n",
      "RAMS3_neg_processed    3456.0\n",
      "Name: dataset, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "for x in ['train.csv','test.csv']:\n",
    "    df = pd.read_csv(opdir+'/'+x)\n",
    "    print(df.dataset.value_counts()/3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "opposite-obligation",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "exceptional-thing",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"for x in ['train.csv','test.csv']:\\n    df = pd.read_csv(opdir+'/'+x)\\n    print(df[df.dataset.str.contains('genia') & df.subtask.str.contains('EX')]['src_label'].value_counts())\""
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''for x in ['train.csv','test.csv']:\n",
    "    df = pd.read_csv(opdir+'/'+x)\n",
    "    print(df[df.dataset.str.contains('genia') & df.subtask.str.contains('EX')]['src_label'].value_counts())'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "athletic-candy",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}