playground.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "Filename: data/pairs/junit4.csv\nDataset size: 180381\nClass sizes:\n-1    156728\n 0     23285\n 1       368\nName: label, dtype: int64\n"
    }
   ],
   "source": [
    "from helpers import dataset_parser, dataset_filter\n",
    "dataset_parser.parse('data/source/junit4.md', 'data/pairs/junit4.csv', skip_untagged=False)\n",
    "dataset_filter.print_statistics('data/pairs/junit4.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "Filename: data/pairs/gson.csv\nDataset size: 95266\nClass sizes:\n-1    92781\n 0     2303\n 1      182\nName: label, dtype: int64\nFilename: data/pairs/mockito.csv\nDataset size: 176121\nClass sizes:\n-1    167210\n 0      8499\n 1       412\nName: label, dtype: int64\nFilename: data/pairs/slf4j.csv\nDataset size: 11628\nClass sizes:\n-1    10353\n 0     1031\n 1      244\nName: label, dtype: int64\n"
    }
   ],
   "source": [
    "from helpers import dataset_parser, dataset_filter\n",
    "dataset_parser.parse('data/source/gson.md', 'data/pairs/gson.csv', skip_untagged=False)\n",
    "dataset_filter.print_statistics('data/pairs/gson.csv')\n",
    "dataset_parser.parse('data/source/mockito.md', 'data/pairs/mockito.csv', skip_untagged=False)\n",
    "dataset_filter.print_statistics('data/pairs/mockito.csv')\n",
    "dataset_parser.parse('data/source/slf4j.md', 'data/pairs/slf4j.csv', skip_untagged=False)\n",
    "dataset_filter.print_statistics('data/pairs/slf4j.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "Filename: data/filtered/gson.csv\nDataset size: 364\nClass sizes:\n1    182\n0    182\nName: label, dtype: int64\nFilename: data/filtered/junit4.csv\nDataset size: 736\nClass sizes:\n1    368\n0    368\nName: label, dtype: int64\nFilename: data/filtered/mockito.csv\nDataset size: 824\nClass sizes:\n1    412\n0    412\nName: label, dtype: int64\nFilename: data/filtered/slf4j.csv\nDataset size: 488\nClass sizes:\n1    244\n0    244\nName: label, dtype: int64\n"
    }
   ],
   "source": [
    "from helpers import dataset_filter\n",
    "for name in ['gson', 'junit4', 'mockito', 'slf4j']:\n",
    "    dataset_filter.randomized_filter('data/pairs/'+name+'.csv', 'data/filtered/'+name+'.csv')\n",
    "    dataset_filter.print_statistics('data/filtered/'+name+'.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "Filename: data/train.csv\nDataset size: 1676\nClass sizes:\n1    838\n0    838\nName: label, dtype: int64\n"
    }
   ],
   "source": [
    "from helpers import dataset_filter\n",
    "filenames = ['data/filtered/'+name+'.csv' for name in ['gson', 'mockito', 'slf4j']]\n",
    "dataset_filter.concatenate_datasets(filenames, 'data/train.csv')\n",
    "dataset_filter.print_statistics('data/train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": "Using TensorFlow backend.\nThreshold = 0.5833, train accuracy = 0.9690\nThreshold = 0.7762, train accuracy = 0.9284\nThreshold = 0.4439, train accuracy = 0.9141\nThreshold = 0.0600, train accuracy = 0.9588\nThreshold = 0.8687, train accuracy = 0.9905\nThreshold = -2.1924, train accuracy = 0.9529\n+-------------+------------+--------+--------+------+------+------+------+\n| Algorithm   |   Accuracy |     F1 |    ROC |   TP |   TN |   FP |   FN |\n+=============+============+========+========+======+======+======+======+\n| LCS         |     0.9497 | 0.9483 | 0.9952 |  339 |  360 |    8 |   29 |\n+-------------+------------+--------+--------+------+------+------+------+\n| COS         |     0.8899 | 0.8771 | 0.9938 |  289 |  366 |    2 |   79 |\n+-------------+------------+--------+--------+------+------+------+------+\n| LEV         |     0.9389 | 0.9386 | 0.9817 |  344 |  347 |   21 |   24 |\n+-------------+------------+--------+--------+------+------+------+------+\n| LSH         |     0.9443 | 0.9428 | 0.9815 |  338 |  357 |   11 |   30 |\n+-------------+------------+--------+--------+------+------+------+------+\n| Siam        |     0.9035 | 0.8973 | 0.9772 |  310 |  355 |   13 |   58 |\n+-------------+------------+--------+--------+------+------+------+------+\n| WMD         |     0.9470 | 0.9453 | 0.9923 |  337 |  360 |    8 |   31 |\n+-------------+------------+--------+--------+------+------+------+------+\n"
    }
   ],
   "source": [
    "from visualization import table\n",
    "table.similarity_metrics()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": "Using TensorFlow backend.\nThreshold = 0.5689, train accuracy = 0.9958\n"
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "{'accuracy': 0.9239130434782609,\n 'f1': 0.9270833333333334,\n 'roc': 0.9700717745746692,\n 'tp': 356,\n 'tn': 324,\n 'fp': 44,\n 'fn': 12}"
     },
     "metadata": {},
     "execution_count": 1
    }
   ],
   "source": [
    "from metrics import x_feature_metrics\n",
    "x_feature_metrics.siamx_evaluate('data/train.csv', 'data/test.csv', True, 'SiamX')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": "Using TensorFlow backend.\n+-------------+------------+--------------+-------------+---------------+--------+--------+\n| Algorithm   |   Accuracy |   + Accuracy |   Precision |   + Precision |     F1 |   + F1 |\n+=============+============+==============+=============+===============+========+========+\n| LCS         |     0.9497 |       0.9606 |      0.9769 |        0.9748 | 0.9483 | 0.9600 |\n+-------------+------------+--------------+-------------+---------------+--------+--------+\n| COS         |     0.8899 |       0.9565 |      0.9931 |        0.9719 | 0.8771 | 0.9558 |\n+-------------+------------+--------------+-------------+---------------+--------+--------+\n| LEV         |     0.9389 |       0.9443 |      0.9425 |        0.9225 | 0.9386 | 0.9457 |\n+-------------+------------+--------------+-------------+---------------+--------+--------+\n| LSH         |     0.9443 |       0.9402 |      0.9685 |        0.9709 | 0.9428 | 0.9382 |\n+-------------+------------+--------------+-------------+---------------+--------+--------+\n| WMD         |     0.9470 |       0.9579 |      0.9768 |        0.9542 | 0.9453 | 0.9581 |\n+-------------+------------+--------------+-------------+---------------+--------+--------+\n| Siam        |     0.9035 |       0.9239 |      0.9598 |        0.8900 | 0.8973 | 0.9271 |\n+-------------+------------+--------------+-------------+---------------+--------+--------+\n"
    }
   ],
   "source": [
    "from visualization import table\n",
    "table.feature_metrics()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "for name in ['gson', 'junit4', 'mockito', 'slf4j']:\n",
    "    df = pd.read_csv('data/pairs/'+name+'.csv', index_col=0)\n",
    "    df = df[df['label'] == -1]\n",
    "    df.to_csv('data/unlabeled/'+name+'.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": "Using TensorFlow backend.\n+-------------+------+\n| Algorithm   |   UP |\n+=============+======+\n| LCS         |  110 |\n+-------------+------+\n| COS         |  208 |\n+-------------+------+\n| LEV         |  345 |\n+-------------+------+\n| LSH         |  103 |\n+-------------+------+\n| WMD         |  369 |\n+-------------+------+\n| SiamX       | 4287 |\n+-------------+------+\n"
    }
   ],
   "source": [
    "from metrics.unlabeled import label_unlabeled\n",
    "\n",
    "label_unlabeled('data/unlabeled/junit4.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": "Using TensorFlow backend.\n+-------------+------+\n| Algorithm   |   UP |\n+=============+======+\n| LCS         |  110 |\n+-------------+------+\n| COS         |  208 |\n+-------------+------+\n| LEV         |  345 |\n+-------------+------+\n| LSH         |  103 |\n+-------------+------+\n| WMD         |  369 |\n+-------------+------+\n| SiamX       | 4287 |\n+-------------+------+\n"
    }
   ],
   "source": [
    "from metrics.unlabeled import label_unlabeled\n",
    "\n",
    "label_unlabeled('data/unlabeled/junit4.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "                                                comment1  \\\n0      This reader walks the elements of a JsonElemen...   \n1      This reader walks the elements of a JsonElemen...   \n2      This reader walks the elements of a JsonElemen...   \n3      This reader walks the elements of a JsonElemen...   \n4      This reader walks the elements of a JsonElemen...   \n...                                                  ...   \n95261  Returns the next available {@link JsonElement}...   \n95262  Returns the next available {@link JsonElement}...   \n95263  Returns true if a {@link JsonElement} is avail...   \n95264  Returns true if a {@link JsonElement} is avail...   \n95265  This optional {@link Iterator} method is not r...   \n\n                                                comment2           name1  \\\n0                            Creates an empty JsonArray.  JsonTreeReader   \n1      Constructs a new type literal. Derives represe...  JsonTreeReader   \n2            Unsafe. Constructs a type literal manually.  JsonTreeReader   \n3      Returns the type from super class's type param...  JsonTreeReader   \n4      Returns the raw (non-generic) type for this type.  JsonTreeReader   \n...                                                  ...             ...   \n95261  This optional {@link Iterator} method is not r...            next   \n95262  A streaming parser that allows reading of mult...            next   \n95263  This optional {@link Iterator} method is not r...         hasNext   \n95264  A streaming parser that allows reading of mult...         hasNext   \n95265  A streaming parser that allows reading of mult...          remove   \n\n                            name2  \\\n0                       JsonArray   \n1                       TypeToken   \n2                       TypeToken   \n3      getSuperclassTypeParameter   \n4                      getRawType   \n...                           ...   \n95261                      remove   \n95262            JsonStreamParser   \n95263                      remove   \n95264            JsonStreamParser   \n95265            JsonStreamParser   \n\n                                                   meta1  \\\n0      <!-- META {\"entityType\": \"Class\", \"entitySigna...   \n1      <!-- META {\"entityType\": \"Class\", \"entitySigna...   \n2      <!-- META {\"entityType\": \"Class\", \"entitySigna...   \n3      <!-- META {\"entityType\": \"Class\", \"entitySigna...   \n4      <!-- META {\"entityType\": \"Class\", \"entitySigna...   \n...                                                  ...   \n95261  <!-- META {\"entityType\": \"Method\", \"entitySign...   \n95262  <!-- META {\"entityType\": \"Method\", \"entitySign...   \n95263  <!-- META {\"entityType\": \"Method\", \"entitySign...   \n95264  <!-- META {\"entityType\": \"Method\", \"entitySign...   \n95265  <!-- META {\"entityType\": \"Method\", \"entitySign...   \n\n                                                   meta2  label  \n0      <!-- META {\"entityType\": \"Method\", \"entitySign...     -1  \n1      <!-- META {\"entityType\": \"Method\", \"entitySign...     -1  \n2      <!-- META {\"entityType\": \"Method\", \"entitySign...     -1  \n3      <!-- META {\"entityType\": \"Method\", \"entitySign...     -1  \n4      <!-- META {\"entityType\": \"Method\", \"entitySign...     -1  \n...                                                  ...    ...  \n95261  <!-- META {\"entityType\": \"Method\", \"entitySign...     -1  \n95262  <!-- META {\"entityType\": \"Class\", \"entitySigna...     -1  \n95263  <!-- META {\"entityType\": \"Method\", \"entitySign...     -1  \n95264  <!-- META {\"entityType\": \"Class\", \"entitySigna...     -1  \n95265  <!-- META {\"entityType\": \"Class\", \"entitySigna...     -1  \n\n[95266 rows x 7 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>comment1</th>\n      <th>comment2</th>\n      <th>name1</th>\n      <th>name2</th>\n      <th>meta1</th>\n      <th>meta2</th>\n      <th>label</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>This reader walks the elements of a JsonElemen...</td>\n      <td>Creates an empty JsonArray.</td>\n      <td>JsonTreeReader</td>\n      <td>JsonArray</td>\n      <td>&lt;!-- META {\"entityType\": \"Class\", \"entitySigna...</td>\n      <td>&lt;!-- META {\"entityType\": \"Method\", \"entitySign...</td>\n      <td>-1</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>This reader walks the elements of a JsonElemen...</td>\n      <td>Constructs a new type literal. Derives represe...</td>\n      <td>JsonTreeReader</td>\n      <td>TypeToken</td>\n      <td>&lt;!-- META {\"entityType\": \"Class\", \"entitySigna...</td>\n      <td>&lt;!-- META {\"entityType\": \"Method\", \"entitySign...</td>\n      <td>-1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>This reader walks the elements of a JsonElemen...</td>\n      <td>Unsafe. Constructs a type literal manually.</td>\n      <td>JsonTreeReader</td>\n      <td>TypeToken</td>\n      <td>&lt;!-- META {\"entityType\": \"Class\", \"entitySigna...</td>\n      <td>&lt;!-- META {\"entityType\": \"Method\", \"entitySign...</td>\n      <td>-1</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>This reader walks the elements of a JsonElemen...</td>\n      <td>Returns the type from super class's type param...</td>\n      <td>JsonTreeReader</td>\n      <td>getSuperclassTypeParameter</td>\n      <td>&lt;!-- META {\"entityType\": \"Class\", \"entitySigna...</td>\n      <td>&lt;!-- META {\"entityType\": \"Method\", \"entitySign...</td>\n      <td>-1</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>This reader walks the elements of a JsonElemen...</td>\n      <td>Returns the raw (non-generic) type for this type.</td>\n      <td>JsonTreeReader</td>\n      <td>getRawType</td>\n      <td>&lt;!-- META {\"entityType\": \"Class\", \"entitySigna...</td>\n      <td>&lt;!-- META {\"entityType\": \"Method\", \"entitySign...</td>\n      <td>-1</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>95261</th>\n      <td>Returns the next available {@link JsonElement}...</td>\n      <td>This optional {@link Iterator} method is not r...</td>\n      <td>next</td>\n      <td>remove</td>\n      <td>&lt;!-- META {\"entityType\": \"Method\", \"entitySign...</td>\n      <td>&lt;!-- META {\"entityType\": \"Method\", \"entitySign...</td>\n      <td>-1</td>\n    </tr>\n    <tr>\n      <th>95262</th>\n      <td>Returns the next available {@link JsonElement}...</td>\n      <td>A streaming parser that allows reading of mult...</td>\n      <td>next</td>\n      <td>JsonStreamParser</td>\n      <td>&lt;!-- META {\"entityType\": \"Method\", \"entitySign...</td>\n      <td>&lt;!-- META {\"entityType\": \"Class\", \"entitySigna...</td>\n      <td>-1</td>\n    </tr>\n    <tr>\n      <th>95263</th>\n      <td>Returns true if a {@link JsonElement} is avail...</td>\n      <td>This optional {@link Iterator} method is not r...</td>\n      <td>hasNext</td>\n      <td>remove</td>\n      <td>&lt;!-- META {\"entityType\": \"Method\", \"entitySign...</td>\n      <td>&lt;!-- META {\"entityType\": \"Method\", \"entitySign...</td>\n      <td>-1</td>\n    </tr>\n    <tr>\n      <th>95264</th>\n      <td>Returns true if a {@link JsonElement} is avail...</td>\n      <td>A streaming parser that allows reading of mult...</td>\n      <td>hasNext</td>\n      <td>JsonStreamParser</td>\n      <td>&lt;!-- META {\"entityType\": \"Method\", \"entitySign...</td>\n      <td>&lt;!-- META {\"entityType\": \"Class\", \"entitySigna...</td>\n      <td>-1</td>\n    </tr>\n    <tr>\n      <th>95265</th>\n      <td>This optional {@link Iterator} method is not r...</td>\n      <td>A streaming parser that allows reading of mult...</td>\n      <td>remove</td>\n      <td>JsonStreamParser</td>\n      <td>&lt;!-- META {\"entityType\": \"Method\", \"entitySign...</td>\n      <td>&lt;!-- META {\"entityType\": \"Class\", \"entitySigna...</td>\n      <td>-1</td>\n    </tr>\n  </tbody>\n</table>\n<p>95266 rows × 7 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 6
    }
   ],
   "source": [
    "import pandas as pd \n",
    "pd.read_csv('data/pairs/gson.csv', index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "===========  ===========  =============  ==================  ==================  ===========  ===============  ===============\nAlgorithm      Precision    + Precision    + Unl. Precision    + Full Precision    + Unl. TP    + Unl. Recall    + Full Recall\n===========  ===========  =============  ==================  ==================  ===========  ===============  ===============\nLCS               0.9769         0.9748              0.9000              0.9572           99           0.4267           0.7450\nCOS               0.9931         0.9719              0.7163              0.8777          149           0.6422           0.8250\nLEV               0.9425         0.9225              0.4058              0.6790          140           0.6034           0.8283\nLSH               0.9685         0.9709              0.8058              0.9329           83           0.3578           0.6950\nWMD               0.9768         0.9542              0.1843              0.5703           68           0.2931           0.7033\nSiam              0.9598         0.8900              0.0287              0.1022          123           0.5302           0.7983\n===========  ===========  =============  ==================  ==================  ===========  ===============  ===============\n"
    }
   ],
   "source": [
    "from tabulate import tabulate\n",
    "headers = ['Algorithm', 'Precision', '+ Precision', '+ Unl. Precision', '+ Full Precision', '+ Unl. TP', '+ Unl. Recall', '+ Full Recall']\n",
    "table = [['LCS',0.9769,0.9748,99/110,(348+99)/(110+348+9),99,99/232,(348+99)/(232+368)],\n",
    "['COS',0.9931,0.9719,149/208,(346+149)/(208+346+10),149,149/232,(346+149)/(232+368)],\n",
    "['LEV',0.9425,0.9225,140/345,(357+140)/(345+357+30),140,140/232,(357+140)/(232+368)],\n",
    "['LSH',0.9685,0.9709,83/103,(334+83)/(103+334+10),83,83/232,(334+83)/(232+368)],\n",
    "['WMD',0.9768,0.9542,68/369,(354+68)/(369+354+17),68,68/232,(354+68)/(232+368)],\n",
    "['Siam',0.9598,0.8900,123/4287,(356+123)/(4287+356+44),123,123/232,(356+123)/(232+368)]]\n",
    "print(tabulate(table, headers, tablefmt='rst', floatfmt='.4f'))\n",
    "# Всего 232 пары\n",
    "# Почему семантические алгоритмы так плохо отработали?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "===========  ========  ==========\nAlgorithm      Recall    + Recall\n===========  ========  ==========\nLCS            0.9212      0.9457\nCOS            0.7853      0.9402\nLEV            0.9348      0.9701\nLSH            0.9185      0.9076\ndoc2vec        0.9620      0.9973\nWMD            0.9158      0.9620\nWMDK (new)     0.8777      0.9457\nSiam           0.8424      0.9674\nSiam (new)     0.0000      0.9728\n===========  ========  ==========\n"
    }
   ],
   "source": [
    "from tabulate import tabulate\n",
    "headers = ['Algorithm', 'Recall', '+ Recall']\n",
    "table = \\\n",
    "[['LCS',339/368,348/368],\n",
    "['COS',289/368,346/368],\n",
    "['LEV',344/368,357/368],\n",
    "['LSH',338/368,334/368],\n",
    "['doc2vec',354/368,367/368],\n",
    "['WMD',337/368,354/368],\n",
    "['WMDK (new)',323/368,348/368],\n",
    "['Siam',310/368,356/368],\n",
    "['Siam (new)',0,358/368]]\n",
    "print(tabulate(table, headers, tablefmt='rst', floatfmt='.4f'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "[0.837863167760075,\n 0.8505154639175257,\n 0.7462462462462464,\n 0.7965616045845272,\n 0.6298507462686567,\n 0.1811991677700019]"
     },
     "metadata": {},
     "execution_count": 36
    }
   ],
   "source": [
    "full_prec = [(348+99)/(110+348+9),(346+149)/(208+346+10),(357+140)/(345+357+30),(334+83)/(103+334+10),(354+68)/(369+354+17),(356+123)/(4287+356+44)]\n",
    "full_rec = [(348+99)/(232+368),(346+149)/(232+368),(357+140)/(232+368),(334+83)/(232+368),(354+68)/(232+368),(356+123)/(232+368)]\n",
    "full_f1 = [2*p*r/(p+r) for p, r in zip(full_prec, full_rec)]\n",
    "full_f1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "===========  ==========  ============  ======  ======  ===========\nAlgorithm      Accuracy    + Accuracy      F1    + F1    + Full F1\n===========  ==========  ============  ======  ======  ===========\nLCS              0.9497        0.9606  0.9483  0.9600       0.8379\nCOS              0.8899        0.9565  0.8771  0.9558       0.8505\nLEV              0.9389        0.9443  0.9386  0.9457       0.7462\nLSH              0.9443        0.9402  0.9428  0.9382       0.7966\nWMD              0.9470        0.9579  0.9453  0.9581       0.6299\nSiam             0.9035        0.9239  0.8973  0.9271       0.1812\n===========  ==========  ============  ======  ======  ===========\n"
    }
   ],
   "source": [
    "from tabulate import tabulate\n",
    "headers = ['Algorithm', 'Accuracy', '+ Accuracy', 'F1', '+ F1', '+ Full F1']\n",
    "table = [['LCS',0.9497,0.9606,0.9483,0.9600],\n",
    "['COS',0.8899,0.9565,0.8771,0.9558],\n",
    "['LEV',0.9389,0.9443,0.9386,0.9457],\n",
    "['LSH',0.9443,0.9402,0.9428,0.9382],\n",
    "['WMD',0.9470,0.9579,0.9453,0.9581],\n",
    "['Siam',0.9035,0.9239,0.8973,0.9271]]\n",
    "for i in range(len(table)):\n",
    "    table[i].append(full_f1[i])\n",
    "print(tabulate(table, headers, tablefmt='rst', floatfmt='.4f'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "LCS {'accuracy': 0.9605978260869565, 'precision': 0.9747899159663865, 'f1': 0.96, 'tp': 348, 'tn': 359, 'fp': 9, 'fn': 20}\nCOS {'accuracy': 0.9565217391304348, 'precision': 0.9719101123595506, 'f1': 0.9558011049723757, 'tp': 346, 'tn': 358, 'fp': 10, 'fn': 22}\nLEV {'accuracy': 0.9442934782608695, 'precision': 0.9224806201550387, 'f1': 0.9456953642384106, 'tp': 357, 'tn': 338, 'fp': 30, 'fn': 11}\nLSH {'accuracy': 0.9402173913043478, 'precision': 0.9709302325581395, 'f1': 0.9382022471910113, 'tp': 334, 'tn': 358, 'fp': 10, 'fn': 34}\nWMD {'accuracy': 0.9578804347826086, 'precision': 0.954177897574124, 'f1': 0.9580514208389717, 'tp': 354, 'tn': 351, 'fp': 17, 'fn': 14}\nSiam {'accuracy': 0.9239130434782609, 'f1': 0.9270833333333334, 'precision': 0.89, 'roc': 0.9700717745746692, 'tp': 356, 'tn': 324, 'fp': 44, 'fn': 12}\n"
    }
   ],
   "source": [
    "from metrics import metrics, x_feature_metrics\n",
    "from helpers.similarity_generator import all_algorithms, get_algorithm_by_name\n",
    "\n",
    "for name, alg in all_algorithms():\n",
    "    if name == 'Siam':\n",
    "        x = x_feature_metrics.siamx_evaluate('data/train.csv', 'data/test.csv')\n",
    "    else:\n",
    "        x = x_feature_metrics.lr_evaluate('data/train.csv', 'data/test.csv', alg, name)\n",
    "    print(name, x)"
   ]
  }
 ],
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python_defaultSpec_1598644156568",
   "display_name": "Python 3.8.2 64-bit"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}