playground.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('data/labels.tsv',sep='\\t', header=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.set_index('sample')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "del df['Unnamed: 0']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'df' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-00cf07b74dcd>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdf\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'df' is not defined"
     ]
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('data/labels.tsv', sep=\"\\t\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-4-e36d6349d324>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'data/A.tsv'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0msep\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'\\t'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mheader\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindex_col\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/.conda/envs/omicsnew/lib/python3.6/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36mread_csv\u001b[0;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, dialect, error_bad_lines, warn_bad_lines, delim_whitespace, low_memory, memory_map, float_precision)\u001b[0m\n\u001b[1;32m    686\u001b[0m     )\n\u001b[1;32m    687\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 688\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    689\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    690\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/omicsnew/lib/python3.6/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_read\u001b[0;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[1;32m    458\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    459\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 460\u001b[0;31m         \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mparser\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnrows\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    461\u001b[0m     \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    462\u001b[0m         \u001b[0mparser\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/omicsnew/lib/python3.6/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36mread\u001b[0;34m(self, nrows)\u001b[0m\n\u001b[1;32m   1196\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnrows\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1197\u001b[0m         \u001b[0mnrows\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_validate_integer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"nrows\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnrows\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1198\u001b[0;31m         \u001b[0mret\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnrows\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1199\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1200\u001b[0m         \u001b[0;31m# May alter columns / col_dict\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/omicsnew/lib/python3.6/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36mread\u001b[0;34m(self, nrows)\u001b[0m\n\u001b[1;32m   2155\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnrows\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2156\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2157\u001b[0;31m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reader\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnrows\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2158\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mStopIteration\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2159\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_first_chunk\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader.read\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._read_low_memory\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._read_rows\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._convert_column_data\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._convert_tokens\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._convert_with_dtype\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/omicsnew/lib/python3.6/site-packages/pandas/core/dtypes/common.py\u001b[0m in \u001b[0;36mis_categorical_dtype\u001b[0;34m(arr_or_dtype)\u001b[0m\n\u001b[1;32m    528\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    529\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 530\u001b[0;31m \u001b[0;32mdef\u001b[0m \u001b[0mis_categorical_dtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marr_or_dtype\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mbool\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    531\u001b[0m     \"\"\"\n\u001b[1;32m    532\u001b[0m     \u001b[0mCheck\u001b[0m \u001b[0mwhether\u001b[0m \u001b[0man\u001b[0m \u001b[0marray\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mlike\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mdtype\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mof\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mCategorical\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "df = pd.read_csv('data/A.tsv',sep='\\t', header=0, index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TCGA-OR-A5JP-01A</th>\n",
       "      <th>TCGA-OR-A5JE-01A</th>\n",
       "      <th>TCGA-OR-A5JG-01A</th>\n",
       "      <th>TCGA-OR-A5L9-01A</th>\n",
       "      <th>TCGA-OR-A5JR-01A</th>\n",
       "      <th>TCGA-OR-A5KU-01A</th>\n",
       "      <th>TCGA-OR-A5LS-01A</th>\n",
       "      <th>TCGA-OR-A5J7-01A</th>\n",
       "      <th>TCGA-OR-A5JQ-01A</th>\n",
       "      <th>TCGA-OR-A5JS-01A</th>\n",
       "      <th>...</th>\n",
       "      <th>TCGA-VD-AA8S-01B</th>\n",
       "      <th>TCGA-VD-A8KI-01A</th>\n",
       "      <th>TCGA-WC-A87U-01A</th>\n",
       "      <th>TCGA-V4-A9EO-01A</th>\n",
       "      <th>TCGA-V4-A9EU-01A</th>\n",
       "      <th>TCGA-WC-A87T-01A</th>\n",
       "      <th>TCGA-V4-A9F5-01A</th>\n",
       "      <th>TCGA-V4-A9EA-01A</th>\n",
       "      <th>TCGA-RZ-AB0B-01A</th>\n",
       "      <th>TCGA-V4-A9F8-01A</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Ensembl_ID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ENSG00000242268.2</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>9.486642</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>10.056346</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000270112.3</th>\n",
       "      <td>10.689655</td>\n",
       "      <td>14.408626</td>\n",
       "      <td>14.022621</td>\n",
       "      <td>11.291444</td>\n",
       "      <td>10.221394</td>\n",
       "      <td>12.423503</td>\n",
       "      <td>12.830424</td>\n",
       "      <td>12.758888</td>\n",
       "      <td>11.547426</td>\n",
       "      <td>12.572065</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000167578.15</th>\n",
       "      <td>18.536987</td>\n",
       "      <td>18.684183</td>\n",
       "      <td>17.334107</td>\n",
       "      <td>19.713465</td>\n",
       "      <td>16.761630</td>\n",
       "      <td>17.762472</td>\n",
       "      <td>18.114361</td>\n",
       "      <td>19.068519</td>\n",
       "      <td>17.474470</td>\n",
       "      <td>17.860718</td>\n",
       "      <td>...</td>\n",
       "      <td>17.351977</td>\n",
       "      <td>17.851041</td>\n",
       "      <td>16.776477</td>\n",
       "      <td>16.145792</td>\n",
       "      <td>16.049705</td>\n",
       "      <td>16.271634</td>\n",
       "      <td>17.341920</td>\n",
       "      <td>17.227315</td>\n",
       "      <td>16.560319</td>\n",
       "      <td>17.083097</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000273842.1</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000078237.5</th>\n",
       "      <td>17.847476</td>\n",
       "      <td>18.227483</td>\n",
       "      <td>17.287893</td>\n",
       "      <td>16.722624</td>\n",
       "      <td>17.157762</td>\n",
       "      <td>17.001996</td>\n",
       "      <td>18.648729</td>\n",
       "      <td>18.076084</td>\n",
       "      <td>15.817248</td>\n",
       "      <td>17.761824</td>\n",
       "      <td>...</td>\n",
       "      <td>16.440650</td>\n",
       "      <td>17.138136</td>\n",
       "      <td>17.471388</td>\n",
       "      <td>17.326540</td>\n",
       "      <td>17.296212</td>\n",
       "      <td>17.713902</td>\n",
       "      <td>17.458762</td>\n",
       "      <td>14.621734</td>\n",
       "      <td>17.190721</td>\n",
       "      <td>15.859768</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000105063.17</th>\n",
       "      <td>19.730227</td>\n",
       "      <td>18.747937</td>\n",
       "      <td>19.723083</td>\n",
       "      <td>18.493479</td>\n",
       "      <td>18.643556</td>\n",
       "      <td>19.052817</td>\n",
       "      <td>19.677209</td>\n",
       "      <td>19.008640</td>\n",
       "      <td>18.342219</td>\n",
       "      <td>19.227682</td>\n",
       "      <td>...</td>\n",
       "      <td>19.553375</td>\n",
       "      <td>19.683503</td>\n",
       "      <td>18.670548</td>\n",
       "      <td>18.662499</td>\n",
       "      <td>18.990272</td>\n",
       "      <td>18.417189</td>\n",
       "      <td>19.600265</td>\n",
       "      <td>20.501112</td>\n",
       "      <td>19.076780</td>\n",
       "      <td>19.959899</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000231119.2</th>\n",
       "      <td>11.102080</td>\n",
       "      <td>13.367429</td>\n",
       "      <td>11.675085</td>\n",
       "      <td>13.376072</td>\n",
       "      <td>12.720505</td>\n",
       "      <td>12.742984</td>\n",
       "      <td>11.494331</td>\n",
       "      <td>8.957909</td>\n",
       "      <td>11.240291</td>\n",
       "      <td>11.994285</td>\n",
       "      <td>...</td>\n",
       "      <td>12.406394</td>\n",
       "      <td>12.497455</td>\n",
       "      <td>12.230291</td>\n",
       "      <td>12.523760</td>\n",
       "      <td>12.347437</td>\n",
       "      <td>11.890642</td>\n",
       "      <td>12.015016</td>\n",
       "      <td>13.464473</td>\n",
       "      <td>12.106187</td>\n",
       "      <td>12.401127</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000280861.1</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000123685.7</th>\n",
       "      <td>11.102346</td>\n",
       "      <td>13.675791</td>\n",
       "      <td>10.938679</td>\n",
       "      <td>13.528328</td>\n",
       "      <td>12.505076</td>\n",
       "      <td>12.251482</td>\n",
       "      <td>12.561450</td>\n",
       "      <td>12.071083</td>\n",
       "      <td>13.962517</td>\n",
       "      <td>11.632082</td>\n",
       "      <td>...</td>\n",
       "      <td>13.121154</td>\n",
       "      <td>13.299675</td>\n",
       "      <td>12.008214</td>\n",
       "      <td>11.076990</td>\n",
       "      <td>13.203189</td>\n",
       "      <td>11.820537</td>\n",
       "      <td>13.337001</td>\n",
       "      <td>12.742356</td>\n",
       "      <td>13.668115</td>\n",
       "      <td>14.025703</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000181518.3</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>9.005429</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>9.459400</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>60483 rows × 11057 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                    TCGA-OR-A5JP-01A  TCGA-OR-A5JE-01A  TCGA-OR-A5JG-01A  \\\n",
       "Ensembl_ID                                                                 \n",
       "ENSG00000242268.2           0.000000          0.000000          0.000000   \n",
       "ENSG00000270112.3          10.689655         14.408626         14.022621   \n",
       "ENSG00000167578.15         18.536987         18.684183         17.334107   \n",
       "ENSG00000273842.1           0.000000          0.000000          0.000000   \n",
       "ENSG00000078237.5          17.847476         18.227483         17.287893   \n",
       "...                              ...               ...               ...   \n",
       "ENSG00000105063.17         19.730227         18.747937         19.723083   \n",
       "ENSG00000231119.2          11.102080         13.367429         11.675085   \n",
       "ENSG00000280861.1           0.000000          0.000000          0.000000   \n",
       "ENSG00000123685.7          11.102346         13.675791         10.938679   \n",
       "ENSG00000181518.3           0.000000          9.005429          0.000000   \n",
       "\n",
       "                    TCGA-OR-A5L9-01A  TCGA-OR-A5JR-01A  TCGA-OR-A5KU-01A  \\\n",
       "Ensembl_ID                                                                 \n",
       "ENSG00000242268.2           0.000000          9.486642          0.000000   \n",
       "ENSG00000270112.3          11.291444         10.221394         12.423503   \n",
       "ENSG00000167578.15         19.713465         16.761630         17.762472   \n",
       "ENSG00000273842.1           0.000000          0.000000          0.000000   \n",
       "ENSG00000078237.5          16.722624         17.157762         17.001996   \n",
       "...                              ...               ...               ...   \n",
       "ENSG00000105063.17         18.493479         18.643556         19.052817   \n",
       "ENSG00000231119.2          13.376072         12.720505         12.742984   \n",
       "ENSG00000280861.1           0.000000          0.000000          0.000000   \n",
       "ENSG00000123685.7          13.528328         12.505076         12.251482   \n",
       "ENSG00000181518.3           0.000000          0.000000          0.000000   \n",
       "\n",
       "                    TCGA-OR-A5LS-01A  TCGA-OR-A5J7-01A  TCGA-OR-A5JQ-01A  \\\n",
       "Ensembl_ID                                                                 \n",
       "ENSG00000242268.2           0.000000          0.000000          0.000000   \n",
       "ENSG00000270112.3          12.830424         12.758888         11.547426   \n",
       "ENSG00000167578.15         18.114361         19.068519         17.474470   \n",
       "ENSG00000273842.1           0.000000          0.000000          0.000000   \n",
       "ENSG00000078237.5          18.648729         18.076084         15.817248   \n",
       "...                              ...               ...               ...   \n",
       "ENSG00000105063.17         19.677209         19.008640         18.342219   \n",
       "ENSG00000231119.2          11.494331          8.957909         11.240291   \n",
       "ENSG00000280861.1           0.000000          0.000000          0.000000   \n",
       "ENSG00000123685.7          12.561450         12.071083         13.962517   \n",
       "ENSG00000181518.3           0.000000          0.000000          0.000000   \n",
       "\n",
       "                    TCGA-OR-A5JS-01A  ...  TCGA-VD-AA8S-01B  TCGA-VD-A8KI-01A  \\\n",
       "Ensembl_ID                            ...                                       \n",
       "ENSG00000242268.2           0.000000  ...         10.056346          0.000000   \n",
       "ENSG00000270112.3          12.572065  ...          0.000000          0.000000   \n",
       "ENSG00000167578.15         17.860718  ...         17.351977         17.851041   \n",
       "ENSG00000273842.1           0.000000  ...          0.000000          0.000000   \n",
       "ENSG00000078237.5          17.761824  ...         16.440650         17.138136   \n",
       "...                              ...  ...               ...               ...   \n",
       "ENSG00000105063.17         19.227682  ...         19.553375         19.683503   \n",
       "ENSG00000231119.2          11.994285  ...         12.406394         12.497455   \n",
       "ENSG00000280861.1           0.000000  ...          0.000000          0.000000   \n",
       "ENSG00000123685.7          11.632082  ...         13.121154         13.299675   \n",
       "ENSG00000181518.3           0.000000  ...          0.000000          0.000000   \n",
       "\n",
       "                    TCGA-WC-A87U-01A  TCGA-V4-A9EO-01A  TCGA-V4-A9EU-01A  \\\n",
       "Ensembl_ID                                                                 \n",
       "ENSG00000242268.2           0.000000          0.000000          0.000000   \n",
       "ENSG00000270112.3           0.000000          0.000000          0.000000   \n",
       "ENSG00000167578.15         16.776477         16.145792         16.049705   \n",
       "ENSG00000273842.1           0.000000          0.000000          0.000000   \n",
       "ENSG00000078237.5          17.471388         17.326540         17.296212   \n",
       "...                              ...               ...               ...   \n",
       "ENSG00000105063.17         18.670548         18.662499         18.990272   \n",
       "ENSG00000231119.2          12.230291         12.523760         12.347437   \n",
       "ENSG00000280861.1           0.000000          0.000000          0.000000   \n",
       "ENSG00000123685.7          12.008214         11.076990         13.203189   \n",
       "ENSG00000181518.3           0.000000          0.000000          0.000000   \n",
       "\n",
       "                    TCGA-WC-A87T-01A  TCGA-V4-A9F5-01A  TCGA-V4-A9EA-01A  \\\n",
       "Ensembl_ID                                                                 \n",
       "ENSG00000242268.2           0.000000          0.000000          0.000000   \n",
       "ENSG00000270112.3           0.000000          0.000000          0.000000   \n",
       "ENSG00000167578.15         16.271634         17.341920         17.227315   \n",
       "ENSG00000273842.1           0.000000          0.000000          0.000000   \n",
       "ENSG00000078237.5          17.713902         17.458762         14.621734   \n",
       "...                              ...               ...               ...   \n",
       "ENSG00000105063.17         18.417189         19.600265         20.501112   \n",
       "ENSG00000231119.2          11.890642         12.015016         13.464473   \n",
       "ENSG00000280861.1           0.000000          0.000000          0.000000   \n",
       "ENSG00000123685.7          11.820537         13.337001         12.742356   \n",
       "ENSG00000181518.3           0.000000          9.459400          0.000000   \n",
       "\n",
       "                    TCGA-RZ-AB0B-01A  TCGA-V4-A9F8-01A  \n",
       "Ensembl_ID                                              \n",
       "ENSG00000242268.2           0.000000          0.000000  \n",
       "ENSG00000270112.3           0.000000          0.000000  \n",
       "ENSG00000167578.15         16.560319         17.083097  \n",
       "ENSG00000273842.1           0.000000          0.000000  \n",
       "ENSG00000078237.5          17.190721         15.859768  \n",
       "...                              ...               ...  \n",
       "ENSG00000105063.17         19.076780         19.959899  \n",
       "ENSG00000231119.2          12.106187         12.401127  \n",
       "ENSG00000280861.1           0.000000          0.000000  \n",
       "ENSG00000123685.7          13.668115         14.025703  \n",
       "ENSG00000181518.3           0.000000          0.000000  \n",
       "\n",
       "[60483 rows x 11057 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.set_index('Ensembl_ID')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_hdf('data/A.h5', key='df', mode='w')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.drop(columns=list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "list = ['TARGET-15-SJMPAL043513-04B', 'TARGET-15-SJMPAL040036-03B', 'TARGET-20-SJAML045735-09A', 'TARGET-20-SJAML045734-09A', 'TARGET-20-SJAML040268-09A', 'TARGET-15-SJMPAL043768-09B', 'TARGET-20-SJAML003320-09A', 'TARGET-20-SJAML003322-09A', 'TARGET-15-SJMPAL043507-04B', 'TARGET-15-SJMPAL044953-09A', 'TARGET-20-SJAML045739-09A', 'TARGET-20-SJAML045742-09A', 'TARGET-15-SJMPAL043767-09B', 'TARGET-15-SJMPAL012418-09A', 'TARGET-15-SJMPAL040039-09B', 'TARGET-20-SJAML045736-09A', 'TARGET-15-SJMPAL040036-03A', 'TARGET-20-SJAML045738-09A', 'TARGET-15-SJMPAL040039-09A', 'TARGET-20-SJAML007112-09A', 'TARGET-20-SJAML045737-09A', 'TARGET-20-SJAML007113-09A', 'TARGET-20-SJAML045741-09A', 'TARGET-15-SJMPAL043511-09B', 'TARGET-15-SJMPAL017973-09A', 'TARGET-20-SJAML016570-09A', 'TARGET-20-SJAML045740-09A', 'nan', 'TARGET-40-PAKUZU-01A', 'TARGET-40-0A4I4E-01A', 'TARGET-40-PAPFLB-01A']\n",
    "for s in list:\n",
    "    if(s in df.columns):\n",
    "        print(\"Da\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Ensembl_ID</th>\n",
       "      <th>TCGA-OR-A5JP-01A</th>\n",
       "      <th>TCGA-OR-A5JE-01A</th>\n",
       "      <th>TCGA-OR-A5JG-01A</th>\n",
       "      <th>TCGA-OR-A5L9-01A</th>\n",
       "      <th>TCGA-OR-A5JR-01A</th>\n",
       "      <th>TCGA-OR-A5KU-01A</th>\n",
       "      <th>TCGA-OR-A5LS-01A</th>\n",
       "      <th>TCGA-OR-A5J7-01A</th>\n",
       "      <th>TCGA-OR-A5JQ-01A</th>\n",
       "      <th>...</th>\n",
       "      <th>TARGET-50-PAJMKJ-01A</th>\n",
       "      <th>TARGET-50-CAAAAQ-11A</th>\n",
       "      <th>TARGET-50-PAKSCC-01A</th>\n",
       "      <th>TARGET-50-PAJNSL-11A</th>\n",
       "      <th>TARGET-50-PAJPAU-01A</th>\n",
       "      <th>TARGET-50-PAJNZU-01A</th>\n",
       "      <th>TARGET-50-PAJNNR-01A</th>\n",
       "      <th>TARGET-50-PAJNTJ-02A</th>\n",
       "      <th>TARGET-50-PAECJB-01A</th>\n",
       "      <th>TARGET-50-PALFRD-01A</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ENSG00000242268.2</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>9.486642</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>11.700035</td>\n",
       "      <td>10.041859</td>\n",
       "      <td>13.398458</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>10.617230</td>\n",
       "      <td>11.933609</td>\n",
       "      <td>14.140998</td>\n",
       "      <td>11.659218</td>\n",
       "      <td>10.662028</td>\n",
       "      <td>12.878131</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ENSG00000270112.3</td>\n",
       "      <td>10.689655</td>\n",
       "      <td>14.408626</td>\n",
       "      <td>14.022621</td>\n",
       "      <td>11.291444</td>\n",
       "      <td>10.221394</td>\n",
       "      <td>12.423503</td>\n",
       "      <td>12.830424</td>\n",
       "      <td>12.758888</td>\n",
       "      <td>11.547426</td>\n",
       "      <td>...</td>\n",
       "      <td>9.267574</td>\n",
       "      <td>12.513257</td>\n",
       "      <td>10.501003</td>\n",
       "      <td>10.452072</td>\n",
       "      <td>10.625798</td>\n",
       "      <td>8.310776</td>\n",
       "      <td>7.131909</td>\n",
       "      <td>7.678919</td>\n",
       "      <td>10.134942</td>\n",
       "      <td>11.116645</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ENSG00000167578.15</td>\n",
       "      <td>18.536987</td>\n",
       "      <td>18.684183</td>\n",
       "      <td>17.334107</td>\n",
       "      <td>19.713465</td>\n",
       "      <td>16.761630</td>\n",
       "      <td>17.762472</td>\n",
       "      <td>18.114361</td>\n",
       "      <td>19.068519</td>\n",
       "      <td>17.474470</td>\n",
       "      <td>...</td>\n",
       "      <td>15.541309</td>\n",
       "      <td>16.684341</td>\n",
       "      <td>15.905948</td>\n",
       "      <td>16.991286</td>\n",
       "      <td>15.066989</td>\n",
       "      <td>13.953978</td>\n",
       "      <td>15.969451</td>\n",
       "      <td>14.607776</td>\n",
       "      <td>14.387707</td>\n",
       "      <td>15.886538</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ENSG00000273842.1</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ENSG00000078237.5</td>\n",
       "      <td>17.847476</td>\n",
       "      <td>18.227483</td>\n",
       "      <td>17.287893</td>\n",
       "      <td>16.722624</td>\n",
       "      <td>17.157762</td>\n",
       "      <td>17.001996</td>\n",
       "      <td>18.648729</td>\n",
       "      <td>18.076084</td>\n",
       "      <td>15.817248</td>\n",
       "      <td>...</td>\n",
       "      <td>15.377730</td>\n",
       "      <td>16.438256</td>\n",
       "      <td>16.733394</td>\n",
       "      <td>16.149538</td>\n",
       "      <td>16.277784</td>\n",
       "      <td>15.673957</td>\n",
       "      <td>16.305087</td>\n",
       "      <td>15.916629</td>\n",
       "      <td>15.850915</td>\n",
       "      <td>16.188748</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60478</th>\n",
       "      <td>ENSG00000105063.17</td>\n",
       "      <td>19.730227</td>\n",
       "      <td>18.747937</td>\n",
       "      <td>19.723083</td>\n",
       "      <td>18.493479</td>\n",
       "      <td>18.643556</td>\n",
       "      <td>19.052817</td>\n",
       "      <td>19.677209</td>\n",
       "      <td>19.008640</td>\n",
       "      <td>18.342219</td>\n",
       "      <td>...</td>\n",
       "      <td>17.436096</td>\n",
       "      <td>17.673524</td>\n",
       "      <td>16.741095</td>\n",
       "      <td>18.039717</td>\n",
       "      <td>17.720028</td>\n",
       "      <td>17.640250</td>\n",
       "      <td>18.498134</td>\n",
       "      <td>17.969129</td>\n",
       "      <td>17.678311</td>\n",
       "      <td>17.436583</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60479</th>\n",
       "      <td>ENSG00000231119.2</td>\n",
       "      <td>11.102080</td>\n",
       "      <td>13.367429</td>\n",
       "      <td>11.675085</td>\n",
       "      <td>13.376072</td>\n",
       "      <td>12.720505</td>\n",
       "      <td>12.742984</td>\n",
       "      <td>11.494331</td>\n",
       "      <td>8.957909</td>\n",
       "      <td>11.240291</td>\n",
       "      <td>...</td>\n",
       "      <td>11.087928</td>\n",
       "      <td>14.485667</td>\n",
       "      <td>10.727946</td>\n",
       "      <td>14.209958</td>\n",
       "      <td>11.174525</td>\n",
       "      <td>11.584409</td>\n",
       "      <td>8.625349</td>\n",
       "      <td>11.708451</td>\n",
       "      <td>11.513964</td>\n",
       "      <td>11.229679</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60480</th>\n",
       "      <td>ENSG00000280861.1</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>11.193785</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60481</th>\n",
       "      <td>ENSG00000123685.7</td>\n",
       "      <td>11.102346</td>\n",
       "      <td>13.675791</td>\n",
       "      <td>10.938679</td>\n",
       "      <td>13.528328</td>\n",
       "      <td>12.505076</td>\n",
       "      <td>12.251482</td>\n",
       "      <td>12.561450</td>\n",
       "      <td>12.071083</td>\n",
       "      <td>13.962517</td>\n",
       "      <td>...</td>\n",
       "      <td>11.766018</td>\n",
       "      <td>13.129020</td>\n",
       "      <td>10.645799</td>\n",
       "      <td>13.245745</td>\n",
       "      <td>13.221623</td>\n",
       "      <td>11.999594</td>\n",
       "      <td>13.059531</td>\n",
       "      <td>15.225382</td>\n",
       "      <td>12.133453</td>\n",
       "      <td>12.436055</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60482</th>\n",
       "      <td>ENSG00000181518.3</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>9.005429</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>8.713134</td>\n",
       "      <td>9.029384</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.440917</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>60483 rows × 11738 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               Ensembl_ID  TCGA-OR-A5JP-01A  TCGA-OR-A5JE-01A  \\\n",
       "0       ENSG00000242268.2          0.000000          0.000000   \n",
       "1       ENSG00000270112.3         10.689655         14.408626   \n",
       "2      ENSG00000167578.15         18.536987         18.684183   \n",
       "3       ENSG00000273842.1          0.000000          0.000000   \n",
       "4       ENSG00000078237.5         17.847476         18.227483   \n",
       "...                   ...               ...               ...   \n",
       "60478  ENSG00000105063.17         19.730227         18.747937   \n",
       "60479   ENSG00000231119.2         11.102080         13.367429   \n",
       "60480   ENSG00000280861.1          0.000000          0.000000   \n",
       "60481   ENSG00000123685.7         11.102346         13.675791   \n",
       "60482   ENSG00000181518.3          0.000000          9.005429   \n",
       "\n",
       "       TCGA-OR-A5JG-01A  TCGA-OR-A5L9-01A  TCGA-OR-A5JR-01A  TCGA-OR-A5KU-01A  \\\n",
       "0              0.000000          0.000000          9.486642          0.000000   \n",
       "1             14.022621         11.291444         10.221394         12.423503   \n",
       "2             17.334107         19.713465         16.761630         17.762472   \n",
       "3              0.000000          0.000000          0.000000          0.000000   \n",
       "4             17.287893         16.722624         17.157762         17.001996   \n",
       "...                 ...               ...               ...               ...   \n",
       "60478         19.723083         18.493479         18.643556         19.052817   \n",
       "60479         11.675085         13.376072         12.720505         12.742984   \n",
       "60480          0.000000          0.000000          0.000000          0.000000   \n",
       "60481         10.938679         13.528328         12.505076         12.251482   \n",
       "60482          0.000000          0.000000          0.000000          0.000000   \n",
       "\n",
       "       TCGA-OR-A5LS-01A  TCGA-OR-A5J7-01A  TCGA-OR-A5JQ-01A  ...  \\\n",
       "0              0.000000          0.000000          0.000000  ...   \n",
       "1             12.830424         12.758888         11.547426  ...   \n",
       "2             18.114361         19.068519         17.474470  ...   \n",
       "3              0.000000          0.000000          0.000000  ...   \n",
       "4             18.648729         18.076084         15.817248  ...   \n",
       "...                 ...               ...               ...  ...   \n",
       "60478         19.677209         19.008640         18.342219  ...   \n",
       "60479         11.494331          8.957909         11.240291  ...   \n",
       "60480          0.000000          0.000000          0.000000  ...   \n",
       "60481         12.561450         12.071083         13.962517  ...   \n",
       "60482          0.000000          0.000000          0.000000  ...   \n",
       "\n",
       "       TARGET-50-PAJMKJ-01A  TARGET-50-CAAAAQ-11A  TARGET-50-PAKSCC-01A  \\\n",
       "0                 11.700035             10.041859             13.398458   \n",
       "1                  9.267574             12.513257             10.501003   \n",
       "2                 15.541309             16.684341             15.905948   \n",
       "3                  0.000000              0.000000              0.000000   \n",
       "4                 15.377730             16.438256             16.733394   \n",
       "...                     ...                   ...                   ...   \n",
       "60478             17.436096             17.673524             16.741095   \n",
       "60479             11.087928             14.485667             10.727946   \n",
       "60480              0.000000              0.000000              0.000000   \n",
       "60481             11.766018             13.129020             10.645799   \n",
       "60482              0.000000              0.000000              0.000000   \n",
       "\n",
       "       TARGET-50-PAJNSL-11A  TARGET-50-PAJPAU-01A  TARGET-50-PAJNZU-01A  \\\n",
       "0                  0.000000             10.617230             11.933609   \n",
       "1                 10.452072             10.625798              8.310776   \n",
       "2                 16.991286             15.066989             13.953978   \n",
       "3                  0.000000              0.000000              0.000000   \n",
       "4                 16.149538             16.277784             15.673957   \n",
       "...                     ...                   ...                   ...   \n",
       "60478             18.039717             17.720028             17.640250   \n",
       "60479             14.209958             11.174525             11.584409   \n",
       "60480              0.000000             11.193785              0.000000   \n",
       "60481             13.245745             13.221623             11.999594   \n",
       "60482              0.000000              8.713134              9.029384   \n",
       "\n",
       "       TARGET-50-PAJNNR-01A  TARGET-50-PAJNTJ-02A  TARGET-50-PAECJB-01A  \\\n",
       "0                 14.140998             11.659218             10.662028   \n",
       "1                  7.131909              7.678919             10.134942   \n",
       "2                 15.969451             14.607776             14.387707   \n",
       "3                  0.000000              0.000000              0.000000   \n",
       "4                 16.305087             15.916629             15.850915   \n",
       "...                     ...                   ...                   ...   \n",
       "60478             18.498134             17.969129             17.678311   \n",
       "60479              8.625349             11.708451             11.513964   \n",
       "60480              0.000000              0.000000              0.000000   \n",
       "60481             13.059531             15.225382             12.133453   \n",
       "60482              0.000000              0.000000              7.440917   \n",
       "\n",
       "       TARGET-50-PALFRD-01A  \n",
       "0                 12.878131  \n",
       "1                 11.116645  \n",
       "2                 15.886538  \n",
       "3                  0.000000  \n",
       "4                 16.188748  \n",
       "...                     ...  \n",
       "60478             17.436583  \n",
       "60479             11.229679  \n",
       "60480              0.000000  \n",
       "60481             12.436055  \n",
       "60482              0.000000  \n",
       "\n",
       "[60483 rows x 11738 columns]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.rename(columns={\"xena_sample\": \"Ensembl_ID\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('data/A_new.tsv', sep=\"\\t\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "sam = pd.read_csv('data/sample_A.tsv',sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TCGA-OR-A5JG-01A</th>\n",
       "      <th>TCGA-PK-A5HB-01A</th>\n",
       "      <th>TCGA-OR-A5JE-01A</th>\n",
       "      <th>TCGA-OR-A5KU-01A</th>\n",
       "      <th>TCGA-OR-A5L9-01A</th>\n",
       "      <th>TCGA-OR-A5JQ-01A</th>\n",
       "      <th>TCGA-OR-A5K4-01A</th>\n",
       "      <th>TCGA-OR-A5JL-01A</th>\n",
       "      <th>TCGA-OR-A5LS-01A</th>\n",
       "      <th>TCGA-OR-A5K2-01A</th>\n",
       "      <th>...</th>\n",
       "      <th>TCGA-V4-A9ES-01A</th>\n",
       "      <th>TCGA-VD-A8KI-01A</th>\n",
       "      <th>TCGA-VD-A8KL-01A</th>\n",
       "      <th>TCGA-V4-A9F7-01A</th>\n",
       "      <th>TCGA-V4-A9EO-01A</th>\n",
       "      <th>TCGA-WC-A87T-01A</th>\n",
       "      <th>TCGA-WC-AA9A-01A</th>\n",
       "      <th>TCGA-V4-A9EA-01A</th>\n",
       "      <th>TCGA-VD-A8KD-01A</th>\n",
       "      <th>TCGA-V4-A9EX-01A</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Composite Element REF</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>cg00000029</th>\n",
       "      <td>0.909143</td>\n",
       "      <td>0.860316</td>\n",
       "      <td>0.288861</td>\n",
       "      <td>0.089729</td>\n",
       "      <td>0.153293</td>\n",
       "      <td>0.602064</td>\n",
       "      <td>0.257048</td>\n",
       "      <td>0.127663</td>\n",
       "      <td>0.553619</td>\n",
       "      <td>0.300479</td>\n",
       "      <td>...</td>\n",
       "      <td>0.080110</td>\n",
       "      <td>0.111513</td>\n",
       "      <td>0.306704</td>\n",
       "      <td>0.086071</td>\n",
       "      <td>0.187295</td>\n",
       "      <td>0.658342</td>\n",
       "      <td>0.749078</td>\n",
       "      <td>0.069619</td>\n",
       "      <td>0.232657</td>\n",
       "      <td>0.666916</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cg00000108</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cg00000109</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cg00000165</th>\n",
       "      <td>0.870551</td>\n",
       "      <td>0.283920</td>\n",
       "      <td>0.831907</td>\n",
       "      <td>0.166760</td>\n",
       "      <td>0.099543</td>\n",
       "      <td>0.100943</td>\n",
       "      <td>0.078832</td>\n",
       "      <td>0.108161</td>\n",
       "      <td>0.133266</td>\n",
       "      <td>0.857255</td>\n",
       "      <td>...</td>\n",
       "      <td>0.870173</td>\n",
       "      <td>0.229565</td>\n",
       "      <td>0.176223</td>\n",
       "      <td>0.882137</td>\n",
       "      <td>0.096754</td>\n",
       "      <td>0.221580</td>\n",
       "      <td>0.141446</td>\n",
       "      <td>0.169049</td>\n",
       "      <td>0.129985</td>\n",
       "      <td>0.376554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cg00000236</th>\n",
       "      <td>0.917024</td>\n",
       "      <td>0.923508</td>\n",
       "      <td>0.913890</td>\n",
       "      <td>0.920367</td>\n",
       "      <td>0.897255</td>\n",
       "      <td>0.897362</td>\n",
       "      <td>0.844196</td>\n",
       "      <td>0.729874</td>\n",
       "      <td>0.894317</td>\n",
       "      <td>0.895123</td>\n",
       "      <td>...</td>\n",
       "      <td>0.905589</td>\n",
       "      <td>0.809304</td>\n",
       "      <td>0.908222</td>\n",
       "      <td>0.912263</td>\n",
       "      <td>0.895674</td>\n",
       "      <td>0.900149</td>\n",
       "      <td>0.899253</td>\n",
       "      <td>0.915486</td>\n",
       "      <td>0.918003</td>\n",
       "      <td>0.888085</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rs9363764</th>\n",
       "      <td>0.930699</td>\n",
       "      <td>0.573377</td>\n",
       "      <td>0.556809</td>\n",
       "      <td>0.129690</td>\n",
       "      <td>0.213745</td>\n",
       "      <td>0.249980</td>\n",
       "      <td>0.945352</td>\n",
       "      <td>0.942386</td>\n",
       "      <td>0.928414</td>\n",
       "      <td>0.890524</td>\n",
       "      <td>...</td>\n",
       "      <td>0.546015</td>\n",
       "      <td>0.897847</td>\n",
       "      <td>0.931831</td>\n",
       "      <td>0.540128</td>\n",
       "      <td>0.594019</td>\n",
       "      <td>0.939899</td>\n",
       "      <td>0.536003</td>\n",
       "      <td>0.949373</td>\n",
       "      <td>0.518650</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rs939290</th>\n",
       "      <td>0.027085</td>\n",
       "      <td>0.968857</td>\n",
       "      <td>0.419837</td>\n",
       "      <td>0.526697</td>\n",
       "      <td>0.026462</td>\n",
       "      <td>0.512725</td>\n",
       "      <td>0.965603</td>\n",
       "      <td>0.197435</td>\n",
       "      <td>0.964697</td>\n",
       "      <td>0.417380</td>\n",
       "      <td>...</td>\n",
       "      <td>0.053167</td>\n",
       "      <td>0.125388</td>\n",
       "      <td>0.949923</td>\n",
       "      <td>0.025734</td>\n",
       "      <td>0.049256</td>\n",
       "      <td>0.962960</td>\n",
       "      <td>0.948573</td>\n",
       "      <td>0.966269</td>\n",
       "      <td>0.217953</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rs951295</th>\n",
       "      <td>0.868674</td>\n",
       "      <td>0.527440</td>\n",
       "      <td>0.592132</td>\n",
       "      <td>0.590947</td>\n",
       "      <td>0.039633</td>\n",
       "      <td>0.354050</td>\n",
       "      <td>0.031389</td>\n",
       "      <td>0.516984</td>\n",
       "      <td>0.970434</td>\n",
       "      <td>0.461114</td>\n",
       "      <td>...</td>\n",
       "      <td>0.033507</td>\n",
       "      <td>0.043929</td>\n",
       "      <td>0.606104</td>\n",
       "      <td>0.523771</td>\n",
       "      <td>0.042678</td>\n",
       "      <td>0.527032</td>\n",
       "      <td>0.514941</td>\n",
       "      <td>0.967873</td>\n",
       "      <td>0.031468</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rs966367</th>\n",
       "      <td>0.939304</td>\n",
       "      <td>0.475860</td>\n",
       "      <td>0.037113</td>\n",
       "      <td>0.040915</td>\n",
       "      <td>0.036176</td>\n",
       "      <td>0.152047</td>\n",
       "      <td>0.048417</td>\n",
       "      <td>0.041763</td>\n",
       "      <td>0.097237</td>\n",
       "      <td>0.616806</td>\n",
       "      <td>...</td>\n",
       "      <td>0.479625</td>\n",
       "      <td>0.458178</td>\n",
       "      <td>0.036470</td>\n",
       "      <td>0.038780</td>\n",
       "      <td>0.042968</td>\n",
       "      <td>0.480868</td>\n",
       "      <td>0.045003</td>\n",
       "      <td>0.038112</td>\n",
       "      <td>0.940854</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rs9839873</th>\n",
       "      <td>0.230432</td>\n",
       "      <td>0.934253</td>\n",
       "      <td>0.541317</td>\n",
       "      <td>0.927163</td>\n",
       "      <td>0.228178</td>\n",
       "      <td>0.942741</td>\n",
       "      <td>0.108393</td>\n",
       "      <td>0.902634</td>\n",
       "      <td>0.899957</td>\n",
       "      <td>0.903889</td>\n",
       "      <td>...</td>\n",
       "      <td>0.863696</td>\n",
       "      <td>0.843952</td>\n",
       "      <td>0.878990</td>\n",
       "      <td>0.697567</td>\n",
       "      <td>0.868092</td>\n",
       "      <td>0.708051</td>\n",
       "      <td>0.154651</td>\n",
       "      <td>0.925066</td>\n",
       "      <td>0.846994</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>485577 rows × 9735 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                       TCGA-OR-A5JG-01A  TCGA-PK-A5HB-01A  TCGA-OR-A5JE-01A  \\\n",
       "Composite Element REF                                                         \n",
       "cg00000029                     0.909143          0.860316          0.288861   \n",
       "cg00000108                          NaN               NaN               NaN   \n",
       "cg00000109                          NaN               NaN               NaN   \n",
       "cg00000165                     0.870551          0.283920          0.831907   \n",
       "cg00000236                     0.917024          0.923508          0.913890   \n",
       "...                                 ...               ...               ...   \n",
       "rs9363764                      0.930699          0.573377          0.556809   \n",
       "rs939290                       0.027085          0.968857          0.419837   \n",
       "rs951295                       0.868674          0.527440          0.592132   \n",
       "rs966367                       0.939304          0.475860          0.037113   \n",
       "rs9839873                      0.230432          0.934253          0.541317   \n",
       "\n",
       "                       TCGA-OR-A5KU-01A  TCGA-OR-A5L9-01A  TCGA-OR-A5JQ-01A  \\\n",
       "Composite Element REF                                                         \n",
       "cg00000029                     0.089729          0.153293          0.602064   \n",
       "cg00000108                          NaN               NaN               NaN   \n",
       "cg00000109                          NaN               NaN               NaN   \n",
       "cg00000165                     0.166760          0.099543          0.100943   \n",
       "cg00000236                     0.920367          0.897255          0.897362   \n",
       "...                                 ...               ...               ...   \n",
       "rs9363764                      0.129690          0.213745          0.249980   \n",
       "rs939290                       0.526697          0.026462          0.512725   \n",
       "rs951295                       0.590947          0.039633          0.354050   \n",
       "rs966367                       0.040915          0.036176          0.152047   \n",
       "rs9839873                      0.927163          0.228178          0.942741   \n",
       "\n",
       "                       TCGA-OR-A5K4-01A  TCGA-OR-A5JL-01A  TCGA-OR-A5LS-01A  \\\n",
       "Composite Element REF                                                         \n",
       "cg00000029                     0.257048          0.127663          0.553619   \n",
       "cg00000108                          NaN               NaN               NaN   \n",
       "cg00000109                          NaN               NaN               NaN   \n",
       "cg00000165                     0.078832          0.108161          0.133266   \n",
       "cg00000236                     0.844196          0.729874          0.894317   \n",
       "...                                 ...               ...               ...   \n",
       "rs9363764                      0.945352          0.942386          0.928414   \n",
       "rs939290                       0.965603          0.197435          0.964697   \n",
       "rs951295                       0.031389          0.516984          0.970434   \n",
       "rs966367                       0.048417          0.041763          0.097237   \n",
       "rs9839873                      0.108393          0.902634          0.899957   \n",
       "\n",
       "                       TCGA-OR-A5K2-01A  ...  TCGA-V4-A9ES-01A  \\\n",
       "Composite Element REF                    ...                     \n",
       "cg00000029                     0.300479  ...          0.080110   \n",
       "cg00000108                          NaN  ...               NaN   \n",
       "cg00000109                          NaN  ...               NaN   \n",
       "cg00000165                     0.857255  ...          0.870173   \n",
       "cg00000236                     0.895123  ...          0.905589   \n",
       "...                                 ...  ...               ...   \n",
       "rs9363764                      0.890524  ...          0.546015   \n",
       "rs939290                       0.417380  ...          0.053167   \n",
       "rs951295                       0.461114  ...          0.033507   \n",
       "rs966367                       0.616806  ...          0.479625   \n",
       "rs9839873                      0.903889  ...          0.863696   \n",
       "\n",
       "                       TCGA-VD-A8KI-01A  TCGA-VD-A8KL-01A  TCGA-V4-A9F7-01A  \\\n",
       "Composite Element REF                                                         \n",
       "cg00000029                     0.111513          0.306704          0.086071   \n",
       "cg00000108                          NaN               NaN               NaN   \n",
       "cg00000109                          NaN               NaN               NaN   \n",
       "cg00000165                     0.229565          0.176223          0.882137   \n",
       "cg00000236                     0.809304          0.908222          0.912263   \n",
       "...                                 ...               ...               ...   \n",
       "rs9363764                      0.897847          0.931831          0.540128   \n",
       "rs939290                       0.125388          0.949923          0.025734   \n",
       "rs951295                       0.043929          0.606104          0.523771   \n",
       "rs966367                       0.458178          0.036470          0.038780   \n",
       "rs9839873                      0.843952          0.878990          0.697567   \n",
       "\n",
       "                       TCGA-V4-A9EO-01A  TCGA-WC-A87T-01A  TCGA-WC-AA9A-01A  \\\n",
       "Composite Element REF                                                         \n",
       "cg00000029                     0.187295          0.658342          0.749078   \n",
       "cg00000108                          NaN               NaN               NaN   \n",
       "cg00000109                          NaN               NaN               NaN   \n",
       "cg00000165                     0.096754          0.221580          0.141446   \n",
       "cg00000236                     0.895674          0.900149          0.899253   \n",
       "...                                 ...               ...               ...   \n",
       "rs9363764                      0.594019          0.939899          0.536003   \n",
       "rs939290                       0.049256          0.962960          0.948573   \n",
       "rs951295                       0.042678          0.527032          0.514941   \n",
       "rs966367                       0.042968          0.480868          0.045003   \n",
       "rs9839873                      0.868092          0.708051          0.154651   \n",
       "\n",
       "                       TCGA-V4-A9EA-01A  TCGA-VD-A8KD-01A  TCGA-V4-A9EX-01A  \n",
       "Composite Element REF                                                        \n",
       "cg00000029                     0.069619          0.232657          0.666916  \n",
       "cg00000108                          NaN               NaN               NaN  \n",
       "cg00000109                          NaN               NaN               NaN  \n",
       "cg00000165                     0.169049          0.129985          0.376554  \n",
       "cg00000236                     0.915486          0.918003          0.888085  \n",
       "...                                 ...               ...               ...  \n",
       "rs9363764                      0.949373          0.518650               NaN  \n",
       "rs939290                       0.966269          0.217953               NaN  \n",
       "rs951295                       0.967873          0.031468               NaN  \n",
       "rs966367                       0.038112          0.940854               NaN  \n",
       "rs9839873                      0.925066          0.846994               NaN  \n",
       "\n",
       "[485577 rows x 9735 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('data/B.tsv',sep='\\t', header=0, index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.set_index(\"Composite Element REF\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.index.isnull().any()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1 = df.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(\"B_samples.npy\",df.columns.to_numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(\"B_features.npy\",df.index.to_numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['TCGA-OR-A5JP-01A', 'TCGA-OR-A5JG-01A', 'TCGA-PK-A5HB-01A', ...,\n",
       "       'TCGA-V4-A9EA-01A', 'TCGA-VD-A8KD-01A', 'TCGA-V4-A9EX-01A'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = np.load(\"data/B_samples.npy\", allow_pickle=True)\n",
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['cg00000029', 'cg00000108', 'cg00000109', ..., 'rs951295',\n",
       "       'rs966367', 'rs9839873'], dtype=object)"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b = np.load(\"data/B_features.npy\", allow_pickle=True)\n",
    "b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "v = np.load(\"data/B.npy\", allow_pickle=True)\n",
    "v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "ename": "Error",
     "evalue": "Session cannot generate requests",
     "output_type": "error",
     "traceback": [
      "Error: Session cannot generate requests",
      "at S.executeCodeCell (/home/sayed.hashim/.vscode-server/extensions/ms-toolsai.jupyter-2021.10.1001414422/out/client/extension.js:66:301742)",
      "at S.execute (/home/sayed.hashim/.vscode-server/extensions/ms-toolsai.jupyter-2021.10.1001414422/out/client/extension.js:66:300732)",
      "at S.start (/home/sayed.hashim/.vscode-server/extensions/ms-toolsai.jupyter-2021.10.1001414422/out/client/extension.js:66:296408)",
      "at async t.CellExecutionQueue.executeQueuedCells (/home/sayed.hashim/.vscode-server/extensions/ms-toolsai.jupyter-2021.10.1001414422/out/client/extension.js:66:312326)",
      "at async t.CellExecutionQueue.start (/home/sayed.hashim/.vscode-server/extensions/ms-toolsai.jupyter-2021.10.1001414422/out/client/extension.js:66:311862)"
     ]
    }
   ],
   "source": [
    "Bdf = pd.DataFrame(data=v, index=b, columns=a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "ename": "Error",
     "evalue": "Session cannot generate requests",
     "output_type": "error",
     "traceback": [
      "Error: Session cannot generate requests",
      "at S.executeCodeCell (/home/sayed.hashim/.vscode-server/extensions/ms-toolsai.jupyter-2021.10.1001414422/out/client/extension.js:66:301742)",
      "at S.execute (/home/sayed.hashim/.vscode-server/extensions/ms-toolsai.jupyter-2021.10.1001414422/out/client/extension.js:66:300732)",
      "at S.start (/home/sayed.hashim/.vscode-server/extensions/ms-toolsai.jupyter-2021.10.1001414422/out/client/extension.js:66:296408)",
      "at async t.CellExecutionQueue.executeQueuedCells (/home/sayed.hashim/.vscode-server/extensions/ms-toolsai.jupyter-2021.10.1001414422/out/client/extension.js:66:312326)",
      "at async t.CellExecutionQueue.start (/home/sayed.hashim/.vscode-server/extensions/ms-toolsai.jupyter-2021.10.1001414422/out/client/extension.js:66:311862)"
     ]
    }
   ],
   "source": [
    "1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1.to_csv('data/B_withoutna.tsv', sep=\"\\t\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1.to_hdf('data/B_withoutna.h5', key='df', mode='w')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.set_index('Composite Element REF')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Composite Element REF</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>cg00000029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>cg00000108</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>cg00000109</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>cg00000165</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>cg00000236</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485572</th>\n",
       "      <td>rs9363764</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485573</th>\n",
       "      <td>rs939290</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485574</th>\n",
       "      <td>rs951295</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485575</th>\n",
       "      <td>rs966367</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485576</th>\n",
       "      <td>rs9839873</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>485577 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Composite Element REF\n",
       "0                 cg00000029\n",
       "1                 cg00000108\n",
       "2                 cg00000109\n",
       "3                 cg00000165\n",
       "4                 cg00000236\n",
       "...                      ...\n",
       "485572             rs9363764\n",
       "485573              rs939290\n",
       "485574              rs951295\n",
       "485575              rs966367\n",
       "485576             rs9839873\n",
       "\n",
       "[485577 rows x 1 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(df.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.DataFrame(df.index).to_csv('B_withna_rownames.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_hdf('data/B.h5', key='df', mode='w')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_hdf('data/B.h5', header=0, index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('data/B.tsv',sep='\\t', header=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.set_index('Composite Element REF')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('data/B_new.tsv', sep=\"\\t\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_hdf('data/B.h5', key='df', mode='w')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TCGA-OR-A5JP-01A</th>\n",
       "      <th>TCGA-OR-A5JG-01A</th>\n",
       "      <th>TCGA-PK-A5HB-01A</th>\n",
       "      <th>TCGA-OR-A5JE-01A</th>\n",
       "      <th>TCGA-OR-A5KU-01A</th>\n",
       "      <th>TCGA-OR-A5L9-01A</th>\n",
       "      <th>TCGA-OR-A5JQ-01A</th>\n",
       "      <th>TCGA-OR-A5K4-01A</th>\n",
       "      <th>TCGA-OR-A5JL-01A</th>\n",
       "      <th>TCGA-OR-A5LS-01A</th>\n",
       "      <th>...</th>\n",
       "      <th>TCGA-V4-A9ES-01A</th>\n",
       "      <th>TCGA-VD-A8KI-01A</th>\n",
       "      <th>TCGA-VD-A8KL-01A</th>\n",
       "      <th>TCGA-V4-A9F7-01A</th>\n",
       "      <th>TCGA-V4-A9EO-01A</th>\n",
       "      <th>TCGA-WC-A87T-01A</th>\n",
       "      <th>TCGA-WC-AA9A-01A</th>\n",
       "      <th>TCGA-V4-A9EA-01A</th>\n",
       "      <th>TCGA-VD-A8KD-01A</th>\n",
       "      <th>TCGA-V4-A9EX-01A</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Composite Element REF</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>cg00000029</th>\n",
       "      <td>0.098070</td>\n",
       "      <td>0.909143</td>\n",
       "      <td>0.860316</td>\n",
       "      <td>0.288861</td>\n",
       "      <td>0.089729</td>\n",
       "      <td>0.153293</td>\n",
       "      <td>0.602064</td>\n",
       "      <td>0.257048</td>\n",
       "      <td>0.127663</td>\n",
       "      <td>0.553619</td>\n",
       "      <td>...</td>\n",
       "      <td>0.080110</td>\n",
       "      <td>0.111513</td>\n",
       "      <td>0.306704</td>\n",
       "      <td>0.086071</td>\n",
       "      <td>0.187295</td>\n",
       "      <td>0.658342</td>\n",
       "      <td>0.749078</td>\n",
       "      <td>0.069619</td>\n",
       "      <td>0.232657</td>\n",
       "      <td>0.666916</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cg00000108</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cg00000109</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cg00000165</th>\n",
       "      <td>0.883701</td>\n",
       "      <td>0.870551</td>\n",
       "      <td>0.283920</td>\n",
       "      <td>0.831907</td>\n",
       "      <td>0.166760</td>\n",
       "      <td>0.099543</td>\n",
       "      <td>0.100943</td>\n",
       "      <td>0.078832</td>\n",
       "      <td>0.108161</td>\n",
       "      <td>0.133266</td>\n",
       "      <td>...</td>\n",
       "      <td>0.870173</td>\n",
       "      <td>0.229565</td>\n",
       "      <td>0.176223</td>\n",
       "      <td>0.882137</td>\n",
       "      <td>0.096754</td>\n",
       "      <td>0.221580</td>\n",
       "      <td>0.141446</td>\n",
       "      <td>0.169049</td>\n",
       "      <td>0.129985</td>\n",
       "      <td>0.376554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cg00000236</th>\n",
       "      <td>0.920635</td>\n",
       "      <td>0.917024</td>\n",
       "      <td>0.923508</td>\n",
       "      <td>0.913890</td>\n",
       "      <td>0.920367</td>\n",
       "      <td>0.897255</td>\n",
       "      <td>0.897362</td>\n",
       "      <td>0.844196</td>\n",
       "      <td>0.729874</td>\n",
       "      <td>0.894317</td>\n",
       "      <td>...</td>\n",
       "      <td>0.905589</td>\n",
       "      <td>0.809304</td>\n",
       "      <td>0.908222</td>\n",
       "      <td>0.912263</td>\n",
       "      <td>0.895674</td>\n",
       "      <td>0.900149</td>\n",
       "      <td>0.899253</td>\n",
       "      <td>0.915486</td>\n",
       "      <td>0.918003</td>\n",
       "      <td>0.888085</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rs9363764</th>\n",
       "      <td>0.935990</td>\n",
       "      <td>0.930699</td>\n",
       "      <td>0.573377</td>\n",
       "      <td>0.556809</td>\n",
       "      <td>0.129690</td>\n",
       "      <td>0.213745</td>\n",
       "      <td>0.249980</td>\n",
       "      <td>0.945352</td>\n",
       "      <td>0.942386</td>\n",
       "      <td>0.928414</td>\n",
       "      <td>...</td>\n",
       "      <td>0.546015</td>\n",
       "      <td>0.897847</td>\n",
       "      <td>0.931831</td>\n",
       "      <td>0.540128</td>\n",
       "      <td>0.594019</td>\n",
       "      <td>0.939899</td>\n",
       "      <td>0.536003</td>\n",
       "      <td>0.949373</td>\n",
       "      <td>0.518650</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rs939290</th>\n",
       "      <td>0.157168</td>\n",
       "      <td>0.027085</td>\n",
       "      <td>0.968857</td>\n",
       "      <td>0.419837</td>\n",
       "      <td>0.526697</td>\n",
       "      <td>0.026462</td>\n",
       "      <td>0.512725</td>\n",
       "      <td>0.965603</td>\n",
       "      <td>0.197435</td>\n",
       "      <td>0.964697</td>\n",
       "      <td>...</td>\n",
       "      <td>0.053167</td>\n",
       "      <td>0.125388</td>\n",
       "      <td>0.949923</td>\n",
       "      <td>0.025734</td>\n",
       "      <td>0.049256</td>\n",
       "      <td>0.962960</td>\n",
       "      <td>0.948573</td>\n",
       "      <td>0.966269</td>\n",
       "      <td>0.217953</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rs951295</th>\n",
       "      <td>0.038993</td>\n",
       "      <td>0.868674</td>\n",
       "      <td>0.527440</td>\n",
       "      <td>0.592132</td>\n",
       "      <td>0.590947</td>\n",
       "      <td>0.039633</td>\n",
       "      <td>0.354050</td>\n",
       "      <td>0.031389</td>\n",
       "      <td>0.516984</td>\n",
       "      <td>0.970434</td>\n",
       "      <td>...</td>\n",
       "      <td>0.033507</td>\n",
       "      <td>0.043929</td>\n",
       "      <td>0.606104</td>\n",
       "      <td>0.523771</td>\n",
       "      <td>0.042678</td>\n",
       "      <td>0.527032</td>\n",
       "      <td>0.514941</td>\n",
       "      <td>0.967873</td>\n",
       "      <td>0.031468</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rs966367</th>\n",
       "      <td>0.035533</td>\n",
       "      <td>0.939304</td>\n",
       "      <td>0.475860</td>\n",
       "      <td>0.037113</td>\n",
       "      <td>0.040915</td>\n",
       "      <td>0.036176</td>\n",
       "      <td>0.152047</td>\n",
       "      <td>0.048417</td>\n",
       "      <td>0.041763</td>\n",
       "      <td>0.097237</td>\n",
       "      <td>...</td>\n",
       "      <td>0.479625</td>\n",
       "      <td>0.458178</td>\n",
       "      <td>0.036470</td>\n",
       "      <td>0.038780</td>\n",
       "      <td>0.042968</td>\n",
       "      <td>0.480868</td>\n",
       "      <td>0.045003</td>\n",
       "      <td>0.038112</td>\n",
       "      <td>0.940854</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rs9839873</th>\n",
       "      <td>0.913430</td>\n",
       "      <td>0.230432</td>\n",
       "      <td>0.934253</td>\n",
       "      <td>0.541317</td>\n",
       "      <td>0.927163</td>\n",
       "      <td>0.228178</td>\n",
       "      <td>0.942741</td>\n",
       "      <td>0.108393</td>\n",
       "      <td>0.902634</td>\n",
       "      <td>0.899957</td>\n",
       "      <td>...</td>\n",
       "      <td>0.863696</td>\n",
       "      <td>0.843952</td>\n",
       "      <td>0.878990</td>\n",
       "      <td>0.697567</td>\n",
       "      <td>0.868092</td>\n",
       "      <td>0.708051</td>\n",
       "      <td>0.154651</td>\n",
       "      <td>0.925066</td>\n",
       "      <td>0.846994</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>485577 rows × 9736 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                       TCGA-OR-A5JP-01A  TCGA-OR-A5JG-01A  TCGA-PK-A5HB-01A  \\\n",
       "Composite Element REF                                                         \n",
       "cg00000029                     0.098070          0.909143          0.860316   \n",
       "cg00000108                          NaN               NaN               NaN   \n",
       "cg00000109                          NaN               NaN               NaN   \n",
       "cg00000165                     0.883701          0.870551          0.283920   \n",
       "cg00000236                     0.920635          0.917024          0.923508   \n",
       "...                                 ...               ...               ...   \n",
       "rs9363764                      0.935990          0.930699          0.573377   \n",
       "rs939290                       0.157168          0.027085          0.968857   \n",
       "rs951295                       0.038993          0.868674          0.527440   \n",
       "rs966367                       0.035533          0.939304          0.475860   \n",
       "rs9839873                      0.913430          0.230432          0.934253   \n",
       "\n",
       "                       TCGA-OR-A5JE-01A  TCGA-OR-A5KU-01A  TCGA-OR-A5L9-01A  \\\n",
       "Composite Element REF                                                         \n",
       "cg00000029                     0.288861          0.089729          0.153293   \n",
       "cg00000108                          NaN               NaN               NaN   \n",
       "cg00000109                          NaN               NaN               NaN   \n",
       "cg00000165                     0.831907          0.166760          0.099543   \n",
       "cg00000236                     0.913890          0.920367          0.897255   \n",
       "...                                 ...               ...               ...   \n",
       "rs9363764                      0.556809          0.129690          0.213745   \n",
       "rs939290                       0.419837          0.526697          0.026462   \n",
       "rs951295                       0.592132          0.590947          0.039633   \n",
       "rs966367                       0.037113          0.040915          0.036176   \n",
       "rs9839873                      0.541317          0.927163          0.228178   \n",
       "\n",
       "                       TCGA-OR-A5JQ-01A  TCGA-OR-A5K4-01A  TCGA-OR-A5JL-01A  \\\n",
       "Composite Element REF                                                         \n",
       "cg00000029                     0.602064          0.257048          0.127663   \n",
       "cg00000108                          NaN               NaN               NaN   \n",
       "cg00000109                          NaN               NaN               NaN   \n",
       "cg00000165                     0.100943          0.078832          0.108161   \n",
       "cg00000236                     0.897362          0.844196          0.729874   \n",
       "...                                 ...               ...               ...   \n",
       "rs9363764                      0.249980          0.945352          0.942386   \n",
       "rs939290                       0.512725          0.965603          0.197435   \n",
       "rs951295                       0.354050          0.031389          0.516984   \n",
       "rs966367                       0.152047          0.048417          0.041763   \n",
       "rs9839873                      0.942741          0.108393          0.902634   \n",
       "\n",
       "                       TCGA-OR-A5LS-01A  ...  TCGA-V4-A9ES-01A  \\\n",
       "Composite Element REF                    ...                     \n",
       "cg00000029                     0.553619  ...          0.080110   \n",
       "cg00000108                          NaN  ...               NaN   \n",
       "cg00000109                          NaN  ...               NaN   \n",
       "cg00000165                     0.133266  ...          0.870173   \n",
       "cg00000236                     0.894317  ...          0.905589   \n",
       "...                                 ...  ...               ...   \n",
       "rs9363764                      0.928414  ...          0.546015   \n",
       "rs939290                       0.964697  ...          0.053167   \n",
       "rs951295                       0.970434  ...          0.033507   \n",
       "rs966367                       0.097237  ...          0.479625   \n",
       "rs9839873                      0.899957  ...          0.863696   \n",
       "\n",
       "                       TCGA-VD-A8KI-01A  TCGA-VD-A8KL-01A  TCGA-V4-A9F7-01A  \\\n",
       "Composite Element REF                                                         \n",
       "cg00000029                     0.111513          0.306704          0.086071   \n",
       "cg00000108                          NaN               NaN               NaN   \n",
       "cg00000109                          NaN               NaN               NaN   \n",
       "cg00000165                     0.229565          0.176223          0.882137   \n",
       "cg00000236                     0.809304          0.908222          0.912263   \n",
       "...                                 ...               ...               ...   \n",
       "rs9363764                      0.897847          0.931831          0.540128   \n",
       "rs939290                       0.125388          0.949923          0.025734   \n",
       "rs951295                       0.043929          0.606104          0.523771   \n",
       "rs966367                       0.458178          0.036470          0.038780   \n",
       "rs9839873                      0.843952          0.878990          0.697567   \n",
       "\n",
       "                       TCGA-V4-A9EO-01A  TCGA-WC-A87T-01A  TCGA-WC-AA9A-01A  \\\n",
       "Composite Element REF                                                         \n",
       "cg00000029                     0.187295          0.658342          0.749078   \n",
       "cg00000108                          NaN               NaN               NaN   \n",
       "cg00000109                          NaN               NaN               NaN   \n",
       "cg00000165                     0.096754          0.221580          0.141446   \n",
       "cg00000236                     0.895674          0.900149          0.899253   \n",
       "...                                 ...               ...               ...   \n",
       "rs9363764                      0.594019          0.939899          0.536003   \n",
       "rs939290                       0.049256          0.962960          0.948573   \n",
       "rs951295                       0.042678          0.527032          0.514941   \n",
       "rs966367                       0.042968          0.480868          0.045003   \n",
       "rs9839873                      0.868092          0.708051          0.154651   \n",
       "\n",
       "                       TCGA-V4-A9EA-01A  TCGA-VD-A8KD-01A  TCGA-V4-A9EX-01A  \n",
       "Composite Element REF                                                        \n",
       "cg00000029                     0.069619          0.232657          0.666916  \n",
       "cg00000108                          NaN               NaN               NaN  \n",
       "cg00000109                          NaN               NaN               NaN  \n",
       "cg00000165                     0.169049          0.129985          0.376554  \n",
       "cg00000236                     0.915486          0.918003          0.888085  \n",
       "...                                 ...               ...               ...  \n",
       "rs9363764                      0.949373          0.518650               NaN  \n",
       "rs939290                       0.966269          0.217953               NaN  \n",
       "rs951295                       0.967873          0.031468               NaN  \n",
       "rs966367                       0.038112          0.940854               NaN  \n",
       "rs9839873                      0.925066          0.846994               NaN  \n",
       "\n",
       "[485577 rows x 9736 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "d = pd.DataFrame(\n",
    "\t[[21, 72, 67],\n",
    "\t[23, 78, 69],\n",
    "\t[32, 74, 56],\n",
    "\t[52, 54, 76]],\n",
    "\tcolumns=['a', 'b', 'c'])\n",
    "\n",
    "#convert dataframe to numpy array\n",
    "arr = d.to_numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[21, 72, 67],\n",
       "       [23, 78, 69],\n",
       "       [32, 74, 56],\n",
       "       [52, 54, 76]])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "arr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>a</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>72</td>\n",
       "      <td>67</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>78</td>\n",
       "      <td>69</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>74</td>\n",
       "      <td>56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>54</td>\n",
       "      <td>76</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     b   c\n",
       "a         \n",
       "21  72  67\n",
       "23  78  69\n",
       "32  74  56\n",
       "52  54  76"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# dr = dr.set_index('a')\n",
    "dr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>72</td>\n",
       "      <td>67</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>78</td>\n",
       "      <td>69</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>74</td>\n",
       "      <td>56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>54</td>\n",
       "      <td>76</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    b   c\n",
       "0  72  67\n",
       "1  78  69\n",
       "2  74  56\n",
       "3  54  76"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dr.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "dr = d.rename_axis(None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>72</td>\n",
       "      <td>67</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>78</td>\n",
       "      <td>69</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>74</td>\n",
       "      <td>56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>54</td>\n",
       "      <td>76</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     b   c\n",
       "21  72  67\n",
       "23  78  69\n",
       "32  74  56\n",
       "52  54  76"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TCGA-OR-A5JP-01A</th>\n",
       "      <th>TCGA-OR-A5JG-01A</th>\n",
       "      <th>TCGA-PK-A5HB-01A</th>\n",
       "      <th>TCGA-OR-A5JE-01A</th>\n",
       "      <th>TCGA-OR-A5KU-01A</th>\n",
       "      <th>TCGA-OR-A5L9-01A</th>\n",
       "      <th>TCGA-OR-A5JQ-01A</th>\n",
       "      <th>TCGA-OR-A5K4-01A</th>\n",
       "      <th>TCGA-OR-A5JL-01A</th>\n",
       "      <th>TCGA-OR-A5LS-01A</th>\n",
       "      <th>...</th>\n",
       "      <th>TCGA-V4-A9ES-01A</th>\n",
       "      <th>TCGA-VD-A8KI-01A</th>\n",
       "      <th>TCGA-VD-A8KL-01A</th>\n",
       "      <th>TCGA-V4-A9F7-01A</th>\n",
       "      <th>TCGA-V4-A9EO-01A</th>\n",
       "      <th>TCGA-WC-A87T-01A</th>\n",
       "      <th>TCGA-WC-AA9A-01A</th>\n",
       "      <th>TCGA-V4-A9EA-01A</th>\n",
       "      <th>TCGA-VD-A8KD-01A</th>\n",
       "      <th>TCGA-V4-A9EX-01A</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Composite Element REF</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>cg00000029</th>\n",
       "      <td>0.098070</td>\n",
       "      <td>0.909143</td>\n",
       "      <td>0.860316</td>\n",
       "      <td>0.288861</td>\n",
       "      <td>0.089729</td>\n",
       "      <td>0.153293</td>\n",
       "      <td>0.602064</td>\n",
       "      <td>0.257048</td>\n",
       "      <td>0.127663</td>\n",
       "      <td>0.553619</td>\n",
       "      <td>...</td>\n",
       "      <td>0.080110</td>\n",
       "      <td>0.111513</td>\n",
       "      <td>0.306704</td>\n",
       "      <td>0.086071</td>\n",
       "      <td>0.187295</td>\n",
       "      <td>0.658342</td>\n",
       "      <td>0.749078</td>\n",
       "      <td>0.069619</td>\n",
       "      <td>0.232657</td>\n",
       "      <td>0.666916</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cg00000108</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cg00000109</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cg00000165</th>\n",
       "      <td>0.883701</td>\n",
       "      <td>0.870551</td>\n",
       "      <td>0.283920</td>\n",
       "      <td>0.831907</td>\n",
       "      <td>0.166760</td>\n",
       "      <td>0.099543</td>\n",
       "      <td>0.100943</td>\n",
       "      <td>0.078832</td>\n",
       "      <td>0.108161</td>\n",
       "      <td>0.133266</td>\n",
       "      <td>...</td>\n",
       "      <td>0.870173</td>\n",
       "      <td>0.229565</td>\n",
       "      <td>0.176223</td>\n",
       "      <td>0.882137</td>\n",
       "      <td>0.096754</td>\n",
       "      <td>0.221580</td>\n",
       "      <td>0.141446</td>\n",
       "      <td>0.169049</td>\n",
       "      <td>0.129985</td>\n",
       "      <td>0.376554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cg00000236</th>\n",
       "      <td>0.920635</td>\n",
       "      <td>0.917024</td>\n",
       "      <td>0.923508</td>\n",
       "      <td>0.913890</td>\n",
       "      <td>0.920367</td>\n",
       "      <td>0.897255</td>\n",
       "      <td>0.897362</td>\n",
       "      <td>0.844196</td>\n",
       "      <td>0.729874</td>\n",
       "      <td>0.894317</td>\n",
       "      <td>...</td>\n",
       "      <td>0.905589</td>\n",
       "      <td>0.809304</td>\n",
       "      <td>0.908222</td>\n",
       "      <td>0.912263</td>\n",
       "      <td>0.895674</td>\n",
       "      <td>0.900149</td>\n",
       "      <td>0.899253</td>\n",
       "      <td>0.915486</td>\n",
       "      <td>0.918003</td>\n",
       "      <td>0.888085</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rs9363764</th>\n",
       "      <td>0.935990</td>\n",
       "      <td>0.930699</td>\n",
       "      <td>0.573377</td>\n",
       "      <td>0.556809</td>\n",
       "      <td>0.129690</td>\n",
       "      <td>0.213745</td>\n",
       "      <td>0.249980</td>\n",
       "      <td>0.945352</td>\n",
       "      <td>0.942386</td>\n",
       "      <td>0.928414</td>\n",
       "      <td>...</td>\n",
       "      <td>0.546015</td>\n",
       "      <td>0.897847</td>\n",
       "      <td>0.931831</td>\n",
       "      <td>0.540128</td>\n",
       "      <td>0.594019</td>\n",
       "      <td>0.939899</td>\n",
       "      <td>0.536003</td>\n",
       "      <td>0.949373</td>\n",
       "      <td>0.518650</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rs939290</th>\n",
       "      <td>0.157168</td>\n",
       "      <td>0.027085</td>\n",
       "      <td>0.968857</td>\n",
       "      <td>0.419837</td>\n",
       "      <td>0.526697</td>\n",
       "      <td>0.026462</td>\n",
       "      <td>0.512725</td>\n",
       "      <td>0.965603</td>\n",
       "      <td>0.197435</td>\n",
       "      <td>0.964697</td>\n",
       "      <td>...</td>\n",
       "      <td>0.053167</td>\n",
       "      <td>0.125388</td>\n",
       "      <td>0.949923</td>\n",
       "      <td>0.025734</td>\n",
       "      <td>0.049256</td>\n",
       "      <td>0.962960</td>\n",
       "      <td>0.948573</td>\n",
       "      <td>0.966269</td>\n",
       "      <td>0.217953</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rs951295</th>\n",
       "      <td>0.038993</td>\n",
       "      <td>0.868674</td>\n",
       "      <td>0.527440</td>\n",
       "      <td>0.592132</td>\n",
       "      <td>0.590947</td>\n",
       "      <td>0.039633</td>\n",
       "      <td>0.354050</td>\n",
       "      <td>0.031389</td>\n",
       "      <td>0.516984</td>\n",
       "      <td>0.970434</td>\n",
       "      <td>...</td>\n",
       "      <td>0.033507</td>\n",
       "      <td>0.043929</td>\n",
       "      <td>0.606104</td>\n",
       "      <td>0.523771</td>\n",
       "      <td>0.042678</td>\n",
       "      <td>0.527032</td>\n",
       "      <td>0.514941</td>\n",
       "      <td>0.967873</td>\n",
       "      <td>0.031468</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rs966367</th>\n",
       "      <td>0.035533</td>\n",
       "      <td>0.939304</td>\n",
       "      <td>0.475860</td>\n",
       "      <td>0.037113</td>\n",
       "      <td>0.040915</td>\n",
       "      <td>0.036176</td>\n",
       "      <td>0.152047</td>\n",
       "      <td>0.048417</td>\n",
       "      <td>0.041763</td>\n",
       "      <td>0.097237</td>\n",
       "      <td>...</td>\n",
       "      <td>0.479625</td>\n",
       "      <td>0.458178</td>\n",
       "      <td>0.036470</td>\n",
       "      <td>0.038780</td>\n",
       "      <td>0.042968</td>\n",
       "      <td>0.480868</td>\n",
       "      <td>0.045003</td>\n",
       "      <td>0.038112</td>\n",
       "      <td>0.940854</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rs9839873</th>\n",
       "      <td>0.913430</td>\n",
       "      <td>0.230432</td>\n",
       "      <td>0.934253</td>\n",
       "      <td>0.541317</td>\n",
       "      <td>0.927163</td>\n",
       "      <td>0.228178</td>\n",
       "      <td>0.942741</td>\n",
       "      <td>0.108393</td>\n",
       "      <td>0.902634</td>\n",
       "      <td>0.899957</td>\n",
       "      <td>...</td>\n",
       "      <td>0.863696</td>\n",
       "      <td>0.843952</td>\n",
       "      <td>0.878990</td>\n",
       "      <td>0.697567</td>\n",
       "      <td>0.868092</td>\n",
       "      <td>0.708051</td>\n",
       "      <td>0.154651</td>\n",
       "      <td>0.925066</td>\n",
       "      <td>0.846994</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>485577 rows × 9736 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                       TCGA-OR-A5JP-01A  TCGA-OR-A5JG-01A  TCGA-PK-A5HB-01A  \\\n",
       "Composite Element REF                                                         \n",
       "cg00000029                     0.098070          0.909143          0.860316   \n",
       "cg00000108                          NaN               NaN               NaN   \n",
       "cg00000109                          NaN               NaN               NaN   \n",
       "cg00000165                     0.883701          0.870551          0.283920   \n",
       "cg00000236                     0.920635          0.917024          0.923508   \n",
       "...                                 ...               ...               ...   \n",
       "rs9363764                      0.935990          0.930699          0.573377   \n",
       "rs939290                       0.157168          0.027085          0.968857   \n",
       "rs951295                       0.038993          0.868674          0.527440   \n",
       "rs966367                       0.035533          0.939304          0.475860   \n",
       "rs9839873                      0.913430          0.230432          0.934253   \n",
       "\n",
       "                       TCGA-OR-A5JE-01A  TCGA-OR-A5KU-01A  TCGA-OR-A5L9-01A  \\\n",
       "Composite Element REF                                                         \n",
       "cg00000029                     0.288861          0.089729          0.153293   \n",
       "cg00000108                          NaN               NaN               NaN   \n",
       "cg00000109                          NaN               NaN               NaN   \n",
       "cg00000165                     0.831907          0.166760          0.099543   \n",
       "cg00000236                     0.913890          0.920367          0.897255   \n",
       "...                                 ...               ...               ...   \n",
       "rs9363764                      0.556809          0.129690          0.213745   \n",
       "rs939290                       0.419837          0.526697          0.026462   \n",
       "rs951295                       0.592132          0.590947          0.039633   \n",
       "rs966367                       0.037113          0.040915          0.036176   \n",
       "rs9839873                      0.541317          0.927163          0.228178   \n",
       "\n",
       "                       TCGA-OR-A5JQ-01A  TCGA-OR-A5K4-01A  TCGA-OR-A5JL-01A  \\\n",
       "Composite Element REF                                                         \n",
       "cg00000029                     0.602064          0.257048          0.127663   \n",
       "cg00000108                          NaN               NaN               NaN   \n",
       "cg00000109                          NaN               NaN               NaN   \n",
       "cg00000165                     0.100943          0.078832          0.108161   \n",
       "cg00000236                     0.897362          0.844196          0.729874   \n",
       "...                                 ...               ...               ...   \n",
       "rs9363764                      0.249980          0.945352          0.942386   \n",
       "rs939290                       0.512725          0.965603          0.197435   \n",
       "rs951295                       0.354050          0.031389          0.516984   \n",
       "rs966367                       0.152047          0.048417          0.041763   \n",
       "rs9839873                      0.942741          0.108393          0.902634   \n",
       "\n",
       "                       TCGA-OR-A5LS-01A  ...  TCGA-V4-A9ES-01A  \\\n",
       "Composite Element REF                    ...                     \n",
       "cg00000029                     0.553619  ...          0.080110   \n",
       "cg00000108                          NaN  ...               NaN   \n",
       "cg00000109                          NaN  ...               NaN   \n",
       "cg00000165                     0.133266  ...          0.870173   \n",
       "cg00000236                     0.894317  ...          0.905589   \n",
       "...                                 ...  ...               ...   \n",
       "rs9363764                      0.928414  ...          0.546015   \n",
       "rs939290                       0.964697  ...          0.053167   \n",
       "rs951295                       0.970434  ...          0.033507   \n",
       "rs966367                       0.097237  ...          0.479625   \n",
       "rs9839873                      0.899957  ...          0.863696   \n",
       "\n",
       "                       TCGA-VD-A8KI-01A  TCGA-VD-A8KL-01A  TCGA-V4-A9F7-01A  \\\n",
       "Composite Element REF                                                         \n",
       "cg00000029                     0.111513          0.306704          0.086071   \n",
       "cg00000108                          NaN               NaN               NaN   \n",
       "cg00000109                          NaN               NaN               NaN   \n",
       "cg00000165                     0.229565          0.176223          0.882137   \n",
       "cg00000236                     0.809304          0.908222          0.912263   \n",
       "...                                 ...               ...               ...   \n",
       "rs9363764                      0.897847          0.931831          0.540128   \n",
       "rs939290                       0.125388          0.949923          0.025734   \n",
       "rs951295                       0.043929          0.606104          0.523771   \n",
       "rs966367                       0.458178          0.036470          0.038780   \n",
       "rs9839873                      0.843952          0.878990          0.697567   \n",
       "\n",
       "                       TCGA-V4-A9EO-01A  TCGA-WC-A87T-01A  TCGA-WC-AA9A-01A  \\\n",
       "Composite Element REF                                                         \n",
       "cg00000029                     0.187295          0.658342          0.749078   \n",
       "cg00000108                          NaN               NaN               NaN   \n",
       "cg00000109                          NaN               NaN               NaN   \n",
       "cg00000165                     0.096754          0.221580          0.141446   \n",
       "cg00000236                     0.895674          0.900149          0.899253   \n",
       "...                                 ...               ...               ...   \n",
       "rs9363764                      0.594019          0.939899          0.536003   \n",
       "rs939290                       0.049256          0.962960          0.948573   \n",
       "rs951295                       0.042678          0.527032          0.514941   \n",
       "rs966367                       0.042968          0.480868          0.045003   \n",
       "rs9839873                      0.868092          0.708051          0.154651   \n",
       "\n",
       "                       TCGA-V4-A9EA-01A  TCGA-VD-A8KD-01A  TCGA-V4-A9EX-01A  \n",
       "Composite Element REF                                                        \n",
       "cg00000029                     0.069619          0.232657          0.666916  \n",
       "cg00000108                          NaN               NaN               NaN  \n",
       "cg00000109                          NaN               NaN               NaN  \n",
       "cg00000165                     0.169049          0.129985          0.376554  \n",
       "cg00000236                     0.915486          0.918003          0.888085  \n",
       "...                                 ...               ...               ...  \n",
       "rs9363764                      0.949373          0.518650               NaN  \n",
       "rs939290                       0.966269          0.217953               NaN  \n",
       "rs951295                       0.967873          0.031468               NaN  \n",
       "rs966367                       0.038112          0.940854               NaN  \n",
       "rs9839873                      0.925066          0.846994               NaN  \n",
       "\n",
       "[485577 rows x 9736 columns]"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2 = df.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TCGA-OR-A5JP-01A</th>\n",
       "      <th>TCGA-OR-A5JG-01A</th>\n",
       "      <th>TCGA-PK-A5HB-01A</th>\n",
       "      <th>TCGA-OR-A5JE-01A</th>\n",
       "      <th>TCGA-OR-A5KU-01A</th>\n",
       "      <th>TCGA-OR-A5L9-01A</th>\n",
       "      <th>TCGA-OR-A5JQ-01A</th>\n",
       "      <th>TCGA-OR-A5K4-01A</th>\n",
       "      <th>TCGA-OR-A5JL-01A</th>\n",
       "      <th>TCGA-OR-A5LS-01A</th>\n",
       "      <th>...</th>\n",
       "      <th>TCGA-V4-A9ES-01A</th>\n",
       "      <th>TCGA-VD-A8KI-01A</th>\n",
       "      <th>TCGA-VD-A8KL-01A</th>\n",
       "      <th>TCGA-V4-A9F7-01A</th>\n",
       "      <th>TCGA-V4-A9EO-01A</th>\n",
       "      <th>TCGA-WC-A87T-01A</th>\n",
       "      <th>TCGA-WC-AA9A-01A</th>\n",
       "      <th>TCGA-V4-A9EA-01A</th>\n",
       "      <th>TCGA-VD-A8KD-01A</th>\n",
       "      <th>TCGA-V4-A9EX-01A</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.098070</td>\n",
       "      <td>0.909143</td>\n",
       "      <td>0.860316</td>\n",
       "      <td>0.288861</td>\n",
       "      <td>0.089729</td>\n",
       "      <td>0.153293</td>\n",
       "      <td>0.602064</td>\n",
       "      <td>0.257048</td>\n",
       "      <td>0.127663</td>\n",
       "      <td>0.553619</td>\n",
       "      <td>...</td>\n",
       "      <td>0.080110</td>\n",
       "      <td>0.111513</td>\n",
       "      <td>0.306704</td>\n",
       "      <td>0.086071</td>\n",
       "      <td>0.187295</td>\n",
       "      <td>0.658342</td>\n",
       "      <td>0.749078</td>\n",
       "      <td>0.069619</td>\n",
       "      <td>0.232657</td>\n",
       "      <td>0.666916</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.883701</td>\n",
       "      <td>0.870551</td>\n",
       "      <td>0.283920</td>\n",
       "      <td>0.831907</td>\n",
       "      <td>0.166760</td>\n",
       "      <td>0.099543</td>\n",
       "      <td>0.100943</td>\n",
       "      <td>0.078832</td>\n",
       "      <td>0.108161</td>\n",
       "      <td>0.133266</td>\n",
       "      <td>...</td>\n",
       "      <td>0.870173</td>\n",
       "      <td>0.229565</td>\n",
       "      <td>0.176223</td>\n",
       "      <td>0.882137</td>\n",
       "      <td>0.096754</td>\n",
       "      <td>0.221580</td>\n",
       "      <td>0.141446</td>\n",
       "      <td>0.169049</td>\n",
       "      <td>0.129985</td>\n",
       "      <td>0.376554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.920635</td>\n",
       "      <td>0.917024</td>\n",
       "      <td>0.923508</td>\n",
       "      <td>0.913890</td>\n",
       "      <td>0.920367</td>\n",
       "      <td>0.897255</td>\n",
       "      <td>0.897362</td>\n",
       "      <td>0.844196</td>\n",
       "      <td>0.729874</td>\n",
       "      <td>0.894317</td>\n",
       "      <td>...</td>\n",
       "      <td>0.905589</td>\n",
       "      <td>0.809304</td>\n",
       "      <td>0.908222</td>\n",
       "      <td>0.912263</td>\n",
       "      <td>0.895674</td>\n",
       "      <td>0.900149</td>\n",
       "      <td>0.899253</td>\n",
       "      <td>0.915486</td>\n",
       "      <td>0.918003</td>\n",
       "      <td>0.888085</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485572</th>\n",
       "      <td>0.935990</td>\n",
       "      <td>0.930699</td>\n",
       "      <td>0.573377</td>\n",
       "      <td>0.556809</td>\n",
       "      <td>0.129690</td>\n",
       "      <td>0.213745</td>\n",
       "      <td>0.249980</td>\n",
       "      <td>0.945352</td>\n",
       "      <td>0.942386</td>\n",
       "      <td>0.928414</td>\n",
       "      <td>...</td>\n",
       "      <td>0.546015</td>\n",
       "      <td>0.897847</td>\n",
       "      <td>0.931831</td>\n",
       "      <td>0.540128</td>\n",
       "      <td>0.594019</td>\n",
       "      <td>0.939899</td>\n",
       "      <td>0.536003</td>\n",
       "      <td>0.949373</td>\n",
       "      <td>0.518650</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485573</th>\n",
       "      <td>0.157168</td>\n",
       "      <td>0.027085</td>\n",
       "      <td>0.968857</td>\n",
       "      <td>0.419837</td>\n",
       "      <td>0.526697</td>\n",
       "      <td>0.026462</td>\n",
       "      <td>0.512725</td>\n",
       "      <td>0.965603</td>\n",
       "      <td>0.197435</td>\n",
       "      <td>0.964697</td>\n",
       "      <td>...</td>\n",
       "      <td>0.053167</td>\n",
       "      <td>0.125388</td>\n",
       "      <td>0.949923</td>\n",
       "      <td>0.025734</td>\n",
       "      <td>0.049256</td>\n",
       "      <td>0.962960</td>\n",
       "      <td>0.948573</td>\n",
       "      <td>0.966269</td>\n",
       "      <td>0.217953</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485574</th>\n",
       "      <td>0.038993</td>\n",
       "      <td>0.868674</td>\n",
       "      <td>0.527440</td>\n",
       "      <td>0.592132</td>\n",
       "      <td>0.590947</td>\n",
       "      <td>0.039633</td>\n",
       "      <td>0.354050</td>\n",
       "      <td>0.031389</td>\n",
       "      <td>0.516984</td>\n",
       "      <td>0.970434</td>\n",
       "      <td>...</td>\n",
       "      <td>0.033507</td>\n",
       "      <td>0.043929</td>\n",
       "      <td>0.606104</td>\n",
       "      <td>0.523771</td>\n",
       "      <td>0.042678</td>\n",
       "      <td>0.527032</td>\n",
       "      <td>0.514941</td>\n",
       "      <td>0.967873</td>\n",
       "      <td>0.031468</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485575</th>\n",
       "      <td>0.035533</td>\n",
       "      <td>0.939304</td>\n",
       "      <td>0.475860</td>\n",
       "      <td>0.037113</td>\n",
       "      <td>0.040915</td>\n",
       "      <td>0.036176</td>\n",
       "      <td>0.152047</td>\n",
       "      <td>0.048417</td>\n",
       "      <td>0.041763</td>\n",
       "      <td>0.097237</td>\n",
       "      <td>...</td>\n",
       "      <td>0.479625</td>\n",
       "      <td>0.458178</td>\n",
       "      <td>0.036470</td>\n",
       "      <td>0.038780</td>\n",
       "      <td>0.042968</td>\n",
       "      <td>0.480868</td>\n",
       "      <td>0.045003</td>\n",
       "      <td>0.038112</td>\n",
       "      <td>0.940854</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485576</th>\n",
       "      <td>0.913430</td>\n",
       "      <td>0.230432</td>\n",
       "      <td>0.934253</td>\n",
       "      <td>0.541317</td>\n",
       "      <td>0.927163</td>\n",
       "      <td>0.228178</td>\n",
       "      <td>0.942741</td>\n",
       "      <td>0.108393</td>\n",
       "      <td>0.902634</td>\n",
       "      <td>0.899957</td>\n",
       "      <td>...</td>\n",
       "      <td>0.863696</td>\n",
       "      <td>0.843952</td>\n",
       "      <td>0.878990</td>\n",
       "      <td>0.697567</td>\n",
       "      <td>0.868092</td>\n",
       "      <td>0.708051</td>\n",
       "      <td>0.154651</td>\n",
       "      <td>0.925066</td>\n",
       "      <td>0.846994</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>485577 rows × 9736 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        TCGA-OR-A5JP-01A  TCGA-OR-A5JG-01A  TCGA-PK-A5HB-01A  \\\n",
       "0               0.098070          0.909143          0.860316   \n",
       "1                    NaN               NaN               NaN   \n",
       "2                    NaN               NaN               NaN   \n",
       "3               0.883701          0.870551          0.283920   \n",
       "4               0.920635          0.917024          0.923508   \n",
       "...                  ...               ...               ...   \n",
       "485572          0.935990          0.930699          0.573377   \n",
       "485573          0.157168          0.027085          0.968857   \n",
       "485574          0.038993          0.868674          0.527440   \n",
       "485575          0.035533          0.939304          0.475860   \n",
       "485576          0.913430          0.230432          0.934253   \n",
       "\n",
       "        TCGA-OR-A5JE-01A  TCGA-OR-A5KU-01A  TCGA-OR-A5L9-01A  \\\n",
       "0               0.288861          0.089729          0.153293   \n",
       "1                    NaN               NaN               NaN   \n",
       "2                    NaN               NaN               NaN   \n",
       "3               0.831907          0.166760          0.099543   \n",
       "4               0.913890          0.920367          0.897255   \n",
       "...                  ...               ...               ...   \n",
       "485572          0.556809          0.129690          0.213745   \n",
       "485573          0.419837          0.526697          0.026462   \n",
       "485574          0.592132          0.590947          0.039633   \n",
       "485575          0.037113          0.040915          0.036176   \n",
       "485576          0.541317          0.927163          0.228178   \n",
       "\n",
       "        TCGA-OR-A5JQ-01A  TCGA-OR-A5K4-01A  TCGA-OR-A5JL-01A  \\\n",
       "0               0.602064          0.257048          0.127663   \n",
       "1                    NaN               NaN               NaN   \n",
       "2                    NaN               NaN               NaN   \n",
       "3               0.100943          0.078832          0.108161   \n",
       "4               0.897362          0.844196          0.729874   \n",
       "...                  ...               ...               ...   \n",
       "485572          0.249980          0.945352          0.942386   \n",
       "485573          0.512725          0.965603          0.197435   \n",
       "485574          0.354050          0.031389          0.516984   \n",
       "485575          0.152047          0.048417          0.041763   \n",
       "485576          0.942741          0.108393          0.902634   \n",
       "\n",
       "        TCGA-OR-A5LS-01A  ...  TCGA-V4-A9ES-01A  TCGA-VD-A8KI-01A  \\\n",
       "0               0.553619  ...          0.080110          0.111513   \n",
       "1                    NaN  ...               NaN               NaN   \n",
       "2                    NaN  ...               NaN               NaN   \n",
       "3               0.133266  ...          0.870173          0.229565   \n",
       "4               0.894317  ...          0.905589          0.809304   \n",
       "...                  ...  ...               ...               ...   \n",
       "485572          0.928414  ...          0.546015          0.897847   \n",
       "485573          0.964697  ...          0.053167          0.125388   \n",
       "485574          0.970434  ...          0.033507          0.043929   \n",
       "485575          0.097237  ...          0.479625          0.458178   \n",
       "485576          0.899957  ...          0.863696          0.843952   \n",
       "\n",
       "        TCGA-VD-A8KL-01A  TCGA-V4-A9F7-01A  TCGA-V4-A9EO-01A  \\\n",
       "0               0.306704          0.086071          0.187295   \n",
       "1                    NaN               NaN               NaN   \n",
       "2                    NaN               NaN               NaN   \n",
       "3               0.176223          0.882137          0.096754   \n",
       "4               0.908222          0.912263          0.895674   \n",
       "...                  ...               ...               ...   \n",
       "485572          0.931831          0.540128          0.594019   \n",
       "485573          0.949923          0.025734          0.049256   \n",
       "485574          0.606104          0.523771          0.042678   \n",
       "485575          0.036470          0.038780          0.042968   \n",
       "485576          0.878990          0.697567          0.868092   \n",
       "\n",
       "        TCGA-WC-A87T-01A  TCGA-WC-AA9A-01A  TCGA-V4-A9EA-01A  \\\n",
       "0               0.658342          0.749078          0.069619   \n",
       "1                    NaN               NaN               NaN   \n",
       "2                    NaN               NaN               NaN   \n",
       "3               0.221580          0.141446          0.169049   \n",
       "4               0.900149          0.899253          0.915486   \n",
       "...                  ...               ...               ...   \n",
       "485572          0.939899          0.536003          0.949373   \n",
       "485573          0.962960          0.948573          0.966269   \n",
       "485574          0.527032          0.514941          0.967873   \n",
       "485575          0.480868          0.045003          0.038112   \n",
       "485576          0.708051          0.154651          0.925066   \n",
       "\n",
       "        TCGA-VD-A8KD-01A  TCGA-V4-A9EX-01A  \n",
       "0               0.232657          0.666916  \n",
       "1                    NaN               NaN  \n",
       "2                    NaN               NaN  \n",
       "3               0.129985          0.376554  \n",
       "4               0.918003          0.888085  \n",
       "...                  ...               ...  \n",
       "485572          0.518650               NaN  \n",
       "485573          0.217953               NaN  \n",
       "485574          0.031468               NaN  \n",
       "485575          0.940854               NaN  \n",
       "485576          0.846994               NaN  \n",
       "\n",
       "[485577 rows x 9736 columns]"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr = df2.to_numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(485577, 9736)"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "arr.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4727560255"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.count_nonzero(arr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9999963158722694"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "4727560255 / (485577 * 9736)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save('data/B_withna_onlyvalues.npy', arr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = 'B'\n",
    "Bvalues = np.load('data/'+dataset+'.npy', allow_pickle=True)\n",
    "Bfeatures = np.load('data/'+dataset+ '_features.npy', allow_pickle=True)\n",
    "Bsamples = np.load('data/'+dataset+ '_samples.npy', allow_pickle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = 'A'\n",
    "Avalues = np.load('data/'+dataset+'.npy', allow_pickle=True)\n",
    "Afeatures = np.load('data/'+dataset+ '_features.npy', allow_pickle=True)\n",
    "Asamples = np.load('data/'+dataset+ '_samples.npy', allow_pickle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "AandB = np.intersect1d(Asamples, Bsamples, return_indices=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(9081,)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "AandB[1].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "As = Asamples[AandB[1]]\n",
    "Av = Avalues[:,AandB[1]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60483, 9081)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Av.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "Bs = Bsamples[AandB[2]]\n",
    "Bv = Bvalues[:,AandB[2]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save('data/AB_inter/A_samples.npy', As)\n",
    "np.save('data/AB_inter/B_samples.npy', Bs)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save('data/AB_inter/A.npy', Av)\n",
    "np.save('data/AB_inter/B.npy', Bv)"
   ]
  },
  {
<<<<<<< HEAD
=======
<<<<<<< HEAD
>>>>>>> ec1f19016b7f7bcbd9e7373b8dd12cacbd84dc02
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Process test summary"
   ]
  },
  {
   "cell_type": "code",
<<<<<<< HEAD
   "execution_count": 28,
=======
   "execution_count": 1,
>>>>>>> ec1f19016b7f7bcbd9e7373b8dd12cacbd84dc02
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "import itertools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load txt file that contains 5 columns: accuracy, precision, recall, f1, roc_auc\n",
    "encoder, decoder, downstream = [1,4,8], [1,4], [1,2]\n",
    "# create new df with columns: reduction, accuracy, precision, recall, f1, roc_auc\n",
    "max_acc_df = pd.DataFrame(columns=['reduction', 'epoch', 'accuracy', 'precision', 'recall', 'f1', 'roc_auc', 'mean_metric'])\n",
    "max_mean_df = pd.DataFrame(columns=['reduction', 'epoch', 'accuracy', 'precision', 'recall', 'f1', 'roc_auc', 'mean_metric'])\n",
    "for ed, dd, dsd in list(itertools.product(encoder, decoder, downstream)):\n",
    "    path = os.path.join('checkpoints/subset/reduction/sub4', str(ed) + '_' + str(dd) +'_' + str(dsd), 'test_summary.txt')\n",
    "    # create df from path, where column names are epoch, accuracy, precision, recall, f1, roc_auc\n",
    "    # and ignore empty rows, and ignore last value of each line\n",
    "    df = pd.read_csv(path, sep='\\t', header=None, skiprows=1).dropna(axis=1, how='all')\n",
    "    # rename columns as accuracy, precision, recall, f1, roc_auc\n",
    "    df.columns = ['epoch', 'accuracy', 'precision', 'recall', 'f1', 'roc_auc']\n",
    "    # round values of df to 3 decimal places\n",
    "    df = df.round(3)\n",
    "    # select values of each column where accuracy is maximum\n",
    "    row_max = df.loc[df['accuracy'].idxmax()]\n",
    "    # select values of each column where mean of last 5 columns is maximum\n",
    "    row_mean = df.loc[df[['accuracy', 'precision', 'recall']].mean(axis=1).idxmax()]\n",
    "    # calculate mean of last 5 columns of row_max\n",
    "    row_mean_last5 = row_mean[['accuracy', 'precision', 'recall']].mean(axis=0).round(3)\n",
    "    # do the same thing above to row_max\n",
    "    row_max_last5 = row_max[['accuracy', 'precision', 'recall']].mean(axis=0).round(3)\n",
    "    # append reduction + row_max as a new row to max_acc_df\n",
    "    max_acc_df = max_acc_df.append(pd.DataFrame([[str(ed) + '_' + str(dd) +'_' + str(dsd)] + list(row_max.values) + [row_max_last5]], columns=max_acc_df.columns))\n",
    "    max_mean_df = max_mean_df.append(pd.DataFrame([[str(ed) + '_' + str(dd) +'_' + str(dsd)] + list(row_mean.values) + [row_mean_last5]], columns=max_mean_df.columns))\n",
    "max_acc_df = max_acc_df.set_index('reduction')\n",
    "max_mean_df = max_mean_df.set_index('reduction')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>epoch</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1</th>\n",
       "      <th>roc_auc</th>\n",
       "      <th>mean_metric</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>reduction</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1_1_1</th>\n",
       "      <td>130.0</td>\n",
       "      <td>0.890</td>\n",
       "      <td>0.881</td>\n",
       "      <td>0.908</td>\n",
       "      <td>0.859</td>\n",
       "      <td>0.997</td>\n",
       "      <td>0.893</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1_1_2</th>\n",
       "      <td>130.0</td>\n",
       "      <td>0.922</td>\n",
       "      <td>0.925</td>\n",
       "      <td>0.894</td>\n",
       "      <td>0.879</td>\n",
       "      <td>0.998</td>\n",
       "      <td>0.914</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1_4_1</th>\n",
       "      <td>51.0</td>\n",
       "      <td>0.931</td>\n",
       "      <td>0.902</td>\n",
       "      <td>0.941</td>\n",
       "      <td>0.905</td>\n",
       "      <td>0.999</td>\n",
       "      <td>0.925</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1_4_2</th>\n",
       "      <td>102.0</td>\n",
       "      <td>0.910</td>\n",
       "      <td>0.892</td>\n",
       "      <td>0.920</td>\n",
       "      <td>0.888</td>\n",
       "      <td>0.998</td>\n",
       "      <td>0.907</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4_1_1</th>\n",
       "      <td>33.0</td>\n",
       "      <td>0.962</td>\n",
       "      <td>0.930</td>\n",
       "      <td>0.909</td>\n",
       "      <td>0.909</td>\n",
       "      <td>0.999</td>\n",
       "      <td>0.934</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4_1_2</th>\n",
       "      <td>127.0</td>\n",
       "      <td>0.949</td>\n",
       "      <td>0.933</td>\n",
       "      <td>0.911</td>\n",
       "      <td>0.916</td>\n",
       "      <td>0.994</td>\n",
       "      <td>0.931</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4_4_1</th>\n",
       "      <td>51.0</td>\n",
       "      <td>0.954</td>\n",
       "      <td>0.937</td>\n",
       "      <td>0.939</td>\n",
       "      <td>0.929</td>\n",
       "      <td>0.999</td>\n",
       "      <td>0.943</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4_4_2</th>\n",
       "      <td>54.0</td>\n",
       "      <td>0.943</td>\n",
       "      <td>0.893</td>\n",
       "      <td>0.910</td>\n",
       "      <td>0.898</td>\n",
       "      <td>0.997</td>\n",
       "      <td>0.915</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8_1_1</th>\n",
       "      <td>127.0</td>\n",
       "      <td>0.945</td>\n",
       "      <td>0.927</td>\n",
       "      <td>0.935</td>\n",
       "      <td>0.928</td>\n",
       "      <td>0.999</td>\n",
       "      <td>0.936</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8_1_2</th>\n",
       "      <td>128.0</td>\n",
       "      <td>0.954</td>\n",
       "      <td>0.945</td>\n",
       "      <td>0.943</td>\n",
       "      <td>0.940</td>\n",
       "      <td>0.999</td>\n",
       "      <td>0.947</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8_4_1</th>\n",
       "      <td>126.0</td>\n",
       "      <td>0.944</td>\n",
       "      <td>0.947</td>\n",
       "      <td>0.907</td>\n",
       "      <td>0.907</td>\n",
       "      <td>0.999</td>\n",
       "      <td>0.933</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8_4_2</th>\n",
       "      <td>94.0</td>\n",
       "      <td>0.945</td>\n",
       "      <td>0.942</td>\n",
       "      <td>0.935</td>\n",
       "      <td>0.936</td>\n",
       "      <td>0.999</td>\n",
       "      <td>0.941</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           epoch  accuracy  precision  recall     f1  roc_auc  mean_metric\n",
       "reduction                                                                 \n",
       "1_1_1      130.0     0.890      0.881   0.908  0.859    0.997        0.893\n",
       "1_1_2      130.0     0.922      0.925   0.894  0.879    0.998        0.914\n",
       "1_4_1       51.0     0.931      0.902   0.941  0.905    0.999        0.925\n",
       "1_4_2      102.0     0.910      0.892   0.920  0.888    0.998        0.907\n",
       "4_1_1       33.0     0.962      0.930   0.909  0.909    0.999        0.934\n",
       "4_1_2      127.0     0.949      0.933   0.911  0.916    0.994        0.931\n",
       "4_4_1       51.0     0.954      0.937   0.939  0.929    0.999        0.943\n",
       "4_4_2       54.0     0.943      0.893   0.910  0.898    0.997        0.915\n",
       "8_1_1      127.0     0.945      0.927   0.935  0.928    0.999        0.936\n",
       "8_1_2      128.0     0.954      0.945   0.943  0.940    0.999        0.947\n",
       "8_4_1      126.0     0.944      0.947   0.907  0.907    0.999        0.933\n",
       "8_4_2       94.0     0.945      0.942   0.935  0.936    0.999        0.941"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_mean_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>epoch</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1</th>\n",
       "      <th>roc_auc</th>\n",
       "      <th>mean_metric</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>reduction</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1_1_1</th>\n",
       "      <td>32.0</td>\n",
       "      <td>0.914</td>\n",
       "      <td>0.892</td>\n",
       "      <td>0.864</td>\n",
       "      <td>0.850</td>\n",
       "      <td>0.999</td>\n",
       "      <td>0.890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1_1_2</th>\n",
       "      <td>130.0</td>\n",
       "      <td>0.922</td>\n",
       "      <td>0.925</td>\n",
       "      <td>0.894</td>\n",
       "      <td>0.879</td>\n",
       "      <td>0.998</td>\n",
       "      <td>0.914</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1_4_1</th>\n",
       "      <td>33.0</td>\n",
       "      <td>0.941</td>\n",
       "      <td>0.890</td>\n",
       "      <td>0.904</td>\n",
       "      <td>0.892</td>\n",
       "      <td>0.998</td>\n",
       "      <td>0.912</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1_4_2</th>\n",
       "      <td>102.0</td>\n",
       "      <td>0.910</td>\n",
       "      <td>0.892</td>\n",
       "      <td>0.920</td>\n",
       "      <td>0.888</td>\n",
       "      <td>0.998</td>\n",
       "      <td>0.907</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4_1_1</th>\n",
       "      <td>33.0</td>\n",
       "      <td>0.962</td>\n",
       "      <td>0.930</td>\n",
       "      <td>0.909</td>\n",
       "      <td>0.909</td>\n",
       "      <td>0.999</td>\n",
       "      <td>0.934</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4_1_2</th>\n",
       "      <td>127.0</td>\n",
       "      <td>0.949</td>\n",
       "      <td>0.933</td>\n",
       "      <td>0.911</td>\n",
       "      <td>0.916</td>\n",
       "      <td>0.994</td>\n",
       "      <td>0.931</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4_4_1</th>\n",
       "      <td>51.0</td>\n",
       "      <td>0.954</td>\n",
       "      <td>0.937</td>\n",
       "      <td>0.939</td>\n",
       "      <td>0.929</td>\n",
       "      <td>0.999</td>\n",
       "      <td>0.943</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4_4_2</th>\n",
       "      <td>54.0</td>\n",
       "      <td>0.943</td>\n",
       "      <td>0.893</td>\n",
       "      <td>0.910</td>\n",
       "      <td>0.898</td>\n",
       "      <td>0.997</td>\n",
       "      <td>0.915</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8_1_1</th>\n",
       "      <td>38.0</td>\n",
       "      <td>0.955</td>\n",
       "      <td>0.919</td>\n",
       "      <td>0.912</td>\n",
       "      <td>0.914</td>\n",
       "      <td>0.999</td>\n",
       "      <td>0.929</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8_1_2</th>\n",
       "      <td>128.0</td>\n",
       "      <td>0.954</td>\n",
       "      <td>0.945</td>\n",
       "      <td>0.943</td>\n",
       "      <td>0.940</td>\n",
       "      <td>0.999</td>\n",
       "      <td>0.947</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8_4_1</th>\n",
       "      <td>43.0</td>\n",
       "      <td>0.949</td>\n",
       "      <td>0.914</td>\n",
       "      <td>0.892</td>\n",
       "      <td>0.895</td>\n",
       "      <td>0.999</td>\n",
       "      <td>0.918</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8_4_2</th>\n",
       "      <td>46.0</td>\n",
       "      <td>0.954</td>\n",
       "      <td>0.881</td>\n",
       "      <td>0.874</td>\n",
       "      <td>0.874</td>\n",
       "      <td>0.998</td>\n",
       "      <td>0.903</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           epoch  accuracy  precision  recall     f1  roc_auc  mean_metric\n",
       "reduction                                                                 \n",
       "1_1_1       32.0     0.914      0.892   0.864  0.850    0.999        0.890\n",
       "1_1_2      130.0     0.922      0.925   0.894  0.879    0.998        0.914\n",
       "1_4_1       33.0     0.941      0.890   0.904  0.892    0.998        0.912\n",
       "1_4_2      102.0     0.910      0.892   0.920  0.888    0.998        0.907\n",
       "4_1_1       33.0     0.962      0.930   0.909  0.909    0.999        0.934\n",
       "4_1_2      127.0     0.949      0.933   0.911  0.916    0.994        0.931\n",
       "4_4_1       51.0     0.954      0.937   0.939  0.929    0.999        0.943\n",
       "4_4_2       54.0     0.943      0.893   0.910  0.898    0.997        0.915\n",
       "8_1_1       38.0     0.955      0.919   0.912  0.914    0.999        0.929\n",
       "8_1_2      128.0     0.954      0.945   0.943  0.940    0.999        0.947\n",
       "8_4_1       43.0     0.949      0.914   0.892  0.895    0.999        0.918\n",
       "8_4_2       46.0     0.954      0.881   0.874  0.874    0.998        0.903"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_acc_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_mean_df.to_csv('checkpoints/subset/reduction/sub4/max_mean_df.csv')"
   ]
  },
  {
   "cell_type": "code",
<<<<<<< HEAD
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
=======
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: 'checkpoints/subset/agg_method/8_4_2_sub4/max/test_summary.txt'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-4-a368c10f72e7>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      8\u001b[0m     \u001b[0;31m# create df from path, where column names are epoch, accuracy, precision, recall, f1, roc_auc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m     \u001b[0;31m# and ignore empty rows, and ignore last value of each line\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m     \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msep\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'\\t'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mheader\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mskiprows\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdropna\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhow\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'all'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m     \u001b[0;31m# rename columns as accuracy, precision, recall, f1, roc_auc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m     \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolumns\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m'epoch'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'accuracy'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'precision'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'recall'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'f1'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'roc_auc'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/omicsnew/lib/python3.6/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36mread_csv\u001b[0;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, dialect, error_bad_lines, warn_bad_lines, delim_whitespace, low_memory, memory_map, float_precision)\u001b[0m\n\u001b[1;32m    686\u001b[0m     )\n\u001b[1;32m    687\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 688\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    689\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    690\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/omicsnew/lib/python3.6/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_read\u001b[0;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[1;32m    452\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    453\u001b[0m     \u001b[0;31m# Create the parser.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 454\u001b[0;31m     \u001b[0mparser\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTextFileReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfp_or_buf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    455\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    456\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mchunksize\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0miterator\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/omicsnew/lib/python3.6/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, f, engine, **kwds)\u001b[0m\n\u001b[1;32m    946\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"has_index_names\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"has_index_names\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    947\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 948\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mengine\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    949\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    950\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/omicsnew/lib/python3.6/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_make_engine\u001b[0;34m(self, engine)\u001b[0m\n\u001b[1;32m   1178\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mengine\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"c\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1179\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"c\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1180\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mCParserWrapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1181\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1182\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"python\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/omicsnew/lib/python3.6/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, src, **kwds)\u001b[0m\n\u001b[1;32m   2008\u001b[0m         \u001b[0mkwds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"usecols\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0musecols\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2009\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2010\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reader\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mparsers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTextReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2011\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munnamed_cols\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reader\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munnamed_cols\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2012\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader.__cinit__\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._setup_parser_source\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'checkpoints/subset/agg_method/8_4_2_sub4/max/test_summary.txt'"
     ]
    }
   ],
>>>>>>> ec1f19016b7f7bcbd9e7373b8dd12cacbd84dc02
   "source": [
    "# load txt file that contains 5 columns: accuracy, precision, recall, f1, roc_auc\n",
    "agg_methods = ['max', 'mean', 'min', 'sum', 'random', 'subset_identity']\n",
    "# create new df with columns: reduction, accuracy, precision, recall, f1, roc_auc\n",
    "max_acc_df = pd.DataFrame(columns=['reduction', 'epoch', 'accuracy', 'precision', 'recall', 'f1', 'roc_auc', 'mean_metric'])\n",
    "max_mean_df = pd.DataFrame(columns=['reduction', 'epoch', 'accuracy', 'precision', 'recall', 'f1', 'roc_auc', 'mean_metric'])\n",
    "for agg_method in agg_methods:\n",
    "    path = os.path.join('checkpoints/subset/agg_method/8_4_2_sub4', agg_method, 'test_summary.txt')\n",
    "    # create df from path, where column names are epoch, accuracy, precision, recall, f1, roc_auc\n",
    "    # and ignore empty rows, and ignore last value of each line\n",
    "    df = pd.read_csv(path, sep='\\t', header=None, skiprows=1).dropna(axis=1, how='all')\n",
    "    # rename columns as accuracy, precision, recall, f1, roc_auc\n",
    "    df.columns = ['epoch', 'accuracy', 'precision', 'recall', 'f1', 'roc_auc']\n",
    "    # round values of df to 3 decimal places\n",
    "    df = df.round(3)\n",
    "    # select values of each column where accuracy is maximum\n",
    "    row_max = df.loc[df['accuracy'].idxmax()]\n",
    "    # select values of each column where mean of last 5 columns is maximum\n",
    "    row_mean = df.loc[df[['accuracy', 'precision', 'recall']].mean(axis=1).idxmax()]\n",
    "    # calculate mean of last 5 columns of row_max\n",
    "    row_mean_last5 = row_mean[['accuracy', 'precision', 'recall']].mean(axis=0).round(3)\n",
    "    # do the same thing above to row_max\n",
    "    row_max_last5 = row_max[['accuracy', 'precision', 'recall']].mean(axis=0).round(3)\n",
    "    # append reduction + row_max as a new row to max_acc_df\n",
    "    max_acc_df = max_acc_df.append(pd.DataFrame([[agg_method] + list(row_max.values) + [row_max_last5]], columns=max_acc_df.columns))\n",
    "    max_mean_df = max_mean_df.append(pd.DataFrame([[agg_method] + list(row_mean.values) + [row_mean_last5]], columns=max_mean_df.columns))\n",
    "max_acc_df = max_acc_df.set_index('reduction')\n",
    "max_mean_df = max_mean_df.set_index('reduction')\n"
   ]
  },
  {
   "cell_type": "code",
<<<<<<< HEAD
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_mean_df.to_csv('checkpoints/subset/agg_method/8_4_2_sub4/max_mean_df.csv')\n"
=======
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create new df with columns: reduction, accuracy, precision, recall, f1, roc_auc\n",
    "max_acc_df = pd.DataFrame(columns=['reduction', 'epoch', 'accuracy', 'precision', 'recall', 'f1', 'roc_auc', 'mean_metric'])\n",
    "max_mean_df = pd.DataFrame(columns=['reduction', 'epoch', 'accuracy', 'precision', 'recall', 'f1', 'roc_auc', 'mean_metric'])\n",
    "for subset_num in range(2, 13):\n",
    "    path = os.path.join('checkpoints/subset/subset_num/8_4_2', str(subset_num), 'test_summary.txt')\n",
    "    # create df from path, where column names are epoch, accuracy, precision, recall, f1, roc_auc\n",
    "    # and ignore empty rows, and ignore last value of each line\n",
    "    df = pd.read_csv(path, sep='\\t', header=None, skiprows=1).dropna(axis=1, how='all')\n",
    "    # rename columns as accuracy, precision, recall, f1, roc_auc\n",
    "    df.columns = ['epoch', 'accuracy', 'precision', 'recall', 'f1', 'roc_auc']\n",
    "    # round values of df to 3 decimal places\n",
    "    df = df.round(3)\n",
    "    # select values of each column where accuracy is maximum\n",
    "    row_max = df.loc[df['accuracy'].idxmax()]\n",
    "    # select values of each column where mean of last 5 columns is maximum\n",
    "    row_mean = df.loc[df[['accuracy', 'precision', 'recall']].mean(axis=1).idxmax()]\n",
    "    # calculate mean of last 5 columns of row_max\n",
    "    row_mean_last5 = row_mean[['accuracy', 'precision', 'recall']].mean(axis=0).round(3)\n",
    "    # do the same thing above to row_max\n",
    "    row_max_last5 = row_max[['accuracy', 'precision', 'recall']].mean(axis=0).round(3)\n",
    "    # append reduction + row_max as a new row to max_acc_df\n",
    "    max_acc_df = max_acc_df.append(pd.DataFrame([[subset_num] + list(row_max.values) + [row_max_last5]], columns=max_acc_df.columns))\n",
    "    max_mean_df = max_mean_df.append(pd.DataFrame([[subset_num] + list(row_mean.values) + [row_mean_last5]], columns=max_mean_df.columns))\n",
    "max_acc_df = max_acc_df.set_index('reduction')\n",
    "max_mean_df = max_mean_df.set_index('reduction')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_mean_df.to_csv('checkpoints/subset/subset_num/8_4_2/max_mean_df.csv')"
>>>>>>> ec1f19016b7f7bcbd9e7373b8dd12cacbd84dc02
   ]
  },
  {
   "cell_type": "code",
<<<<<<< HEAD
   "execution_count": 62,
=======
   "execution_count": 7,
>>>>>>> ec1f19016b7f7bcbd9e7373b8dd12cacbd84dc02
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>epoch</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1</th>\n",
       "      <th>roc_auc</th>\n",
       "      <th>mean_metric</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>reduction</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
<<<<<<< HEAD
       "      <th>max</th>\n",
       "      <td>50.0</td>\n",
       "      <td>0.905</td>\n",
       "      <td>0.845</td>\n",
       "      <td>0.824</td>\n",
       "      <td>0.812</td>\n",
       "      <td>0.997</td>\n",
       "      <td>0.858</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>118.0</td>\n",
       "      <td>0.943</td>\n",
       "      <td>0.927</td>\n",
       "      <td>0.935</td>\n",
       "      <td>0.923</td>\n",
       "      <td>0.999</td>\n",
       "      <td>0.935</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>119.0</td>\n",
       "      <td>0.935</td>\n",
       "      <td>0.907</td>\n",
       "      <td>0.932</td>\n",
       "      <td>0.913</td>\n",
       "      <td>0.997</td>\n",
       "      <td>0.925</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sum</th>\n",
       "      <td>107.0</td>\n",
       "      <td>0.951</td>\n",
       "      <td>0.923</td>\n",
       "      <td>0.941</td>\n",
       "      <td>0.927</td>\n",
       "      <td>0.999</td>\n",
       "      <td>0.938</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>random</th>\n",
       "      <td>125.0</td>\n",
       "      <td>0.897</td>\n",
       "      <td>0.869</td>\n",
       "      <td>0.814</td>\n",
       "      <td>0.825</td>\n",
       "      <td>0.996</td>\n",
       "      <td>0.860</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>subset_identity</th>\n",
       "      <td>46.0</td>\n",
       "      <td>0.954</td>\n",
       "      <td>0.881</td>\n",
       "      <td>0.874</td>\n",
       "      <td>0.874</td>\n",
       "      <td>0.998</td>\n",
       "      <td>0.903</td>\n",
=======
       "      <th>2</th>\n",
       "      <td>130.0</td>\n",
       "      <td>0.959</td>\n",
       "      <td>0.960</td>\n",
       "      <td>0.940</td>\n",
       "      <td>0.944</td>\n",
       "      <td>0.999</td>\n",
       "      <td>0.953</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>128.0</td>\n",
       "      <td>0.958</td>\n",
       "      <td>0.944</td>\n",
       "      <td>0.939</td>\n",
       "      <td>0.929</td>\n",
       "      <td>0.999</td>\n",
       "      <td>0.947</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9.0</td>\n",
       "      <td>0.963</td>\n",
       "      <td>0.950</td>\n",
       "      <td>0.958</td>\n",
       "      <td>0.952</td>\n",
       "      <td>0.999</td>\n",
       "      <td>0.957</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>57.0</td>\n",
       "      <td>0.959</td>\n",
       "      <td>0.913</td>\n",
       "      <td>0.909</td>\n",
       "      <td>0.903</td>\n",
       "      <td>0.999</td>\n",
       "      <td>0.927</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>61.0</td>\n",
       "      <td>0.961</td>\n",
       "      <td>0.912</td>\n",
       "      <td>0.930</td>\n",
       "      <td>0.920</td>\n",
       "      <td>0.999</td>\n",
       "      <td>0.934</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>48.0</td>\n",
       "      <td>0.952</td>\n",
       "      <td>0.869</td>\n",
       "      <td>0.880</td>\n",
       "      <td>0.874</td>\n",
       "      <td>0.999</td>\n",
       "      <td>0.900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>56.0</td>\n",
       "      <td>0.954</td>\n",
       "      <td>0.909</td>\n",
       "      <td>0.925</td>\n",
       "      <td>0.915</td>\n",
       "      <td>0.999</td>\n",
       "      <td>0.929</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>67.0</td>\n",
       "      <td>0.953</td>\n",
       "      <td>0.950</td>\n",
       "      <td>0.917</td>\n",
       "      <td>0.925</td>\n",
       "      <td>0.999</td>\n",
       "      <td>0.940</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>127.0</td>\n",
       "      <td>0.950</td>\n",
       "      <td>0.941</td>\n",
       "      <td>0.944</td>\n",
       "      <td>0.940</td>\n",
       "      <td>0.999</td>\n",
       "      <td>0.945</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>61.0</td>\n",
       "      <td>0.949</td>\n",
       "      <td>0.905</td>\n",
       "      <td>0.904</td>\n",
       "      <td>0.900</td>\n",
       "      <td>0.999</td>\n",
       "      <td>0.919</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>129.0</td>\n",
       "      <td>0.941</td>\n",
       "      <td>0.936</td>\n",
       "      <td>0.925</td>\n",
       "      <td>0.924</td>\n",
       "      <td>0.999</td>\n",
       "      <td>0.934</td>\n",
>>>>>>> ec1f19016b7f7bcbd9e7373b8dd12cacbd84dc02
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
<<<<<<< HEAD
       "                 epoch  accuracy  precision  recall     f1  roc_auc  \\\n",
       "reduction                                                             \n",
       "max               50.0     0.905      0.845   0.824  0.812    0.997   \n",
       "mean             118.0     0.943      0.927   0.935  0.923    0.999   \n",
       "min              119.0     0.935      0.907   0.932  0.913    0.997   \n",
       "sum              107.0     0.951      0.923   0.941  0.927    0.999   \n",
       "random           125.0     0.897      0.869   0.814  0.825    0.996   \n",
       "subset_identity   46.0     0.954      0.881   0.874  0.874    0.998   \n",
       "\n",
       "                 mean_metric  \n",
       "reduction                     \n",
       "max                    0.858  \n",
       "mean                   0.935  \n",
       "min                    0.925  \n",
       "sum                    0.938  \n",
       "random                 0.860  \n",
       "subset_identity        0.903  "
      ]
     },
     "execution_count": 62,
=======
       "           epoch  accuracy  precision  recall     f1  roc_auc  mean_metric\n",
       "reduction                                                                 \n",
       "2          130.0     0.959      0.960   0.940  0.944    0.999        0.953\n",
       "3          128.0     0.958      0.944   0.939  0.929    0.999        0.947\n",
       "4            9.0     0.963      0.950   0.958  0.952    0.999        0.957\n",
       "5           57.0     0.959      0.913   0.909  0.903    0.999        0.927\n",
       "6           61.0     0.961      0.912   0.930  0.920    0.999        0.934\n",
       "7           48.0     0.952      0.869   0.880  0.874    0.999        0.900\n",
       "8           56.0     0.954      0.909   0.925  0.915    0.999        0.929\n",
       "9           67.0     0.953      0.950   0.917  0.925    0.999        0.940\n",
       "10         127.0     0.950      0.941   0.944  0.940    0.999        0.945\n",
       "11          61.0     0.949      0.905   0.904  0.900    0.999        0.919\n",
       "12         129.0     0.941      0.936   0.925  0.924    0.999        0.934"
      ]
     },
     "execution_count": 7,
>>>>>>> ec1f19016b7f7bcbd9e7373b8dd12cacbd84dc02
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_acc_df"
   ]
  },
  {
<<<<<<< HEAD
=======
=======
>>>>>>> 58dac1a6b18b0968c9de4211201aa3473dc16eee
>>>>>>> ec1f19016b7f7bcbd9e7373b8dd12cacbd84dc02
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "f63e317dfb82ab324e4a81c90944a9621d0fbd6743c743ca3dc0836459aaf79d"
  },
  "kernelspec": {
   "display_name": "Python 3.6.12 64-bit ('omicsnew': conda)",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}