train_random_forest.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from   joblib import dump\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import random\n",
    "from   sklearn.ensemble import RandomForestClassifier\n",
    "from   sklearn.metrics import accuracy_score, roc_auc_score\n",
    "from   src.feature_importance import extract_feature_importance, plot_feature_importance, rank_feature_importance\n",
    "from   src.feature_selection import remove_correlated_features\n",
    "from   src.model_evaluation import adjusted_prediction, eval_sensitivity_specificity"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set feature files (contains patient info and pyradiomics features)\n",
    "feature_file_training_set = 'extracted_radiomics_features/extracted_example_features_train.csv' # RVC dataset\n",
    "label_file_training_set   = 'data/example_labels_train.csv'\n",
    "\n",
    "# Number of thresholds\n",
    "n_thresholds = 401\n",
    "\n",
    "# Meta info header columns\n",
    "n_header_cols = 22\n",
    "\n",
    "# Feature selection\n",
    "corr_threshold    = 0.8\n",
    "\n",
    "# Fix seed\n",
    "seed = 42\n",
    "np.random.seed(seed)\n",
    "random.seed(seed)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load pyradiomics features and labels\n",
    "df_features_training = pd.read_csv(feature_file_training_set)\n",
    "labels_training      = np.loadtxt(label_file_training_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>diagnostics_Versions_PyRadiomics</th>\n",
       "      <th>diagnostics_Versions_Numpy</th>\n",
       "      <th>diagnostics_Versions_SimpleITK</th>\n",
       "      <th>diagnostics_Versions_PyWavelet</th>\n",
       "      <th>diagnostics_Versions_Python</th>\n",
       "      <th>diagnostics_Configuration_Settings</th>\n",
       "      <th>diagnostics_Configuration_EnabledImageTypes</th>\n",
       "      <th>diagnostics_Image-original_Hash</th>\n",
       "      <th>diagnostics_Image-original_Dimensionality</th>\n",
       "      <th>diagnostics_Image-original_Spacing</th>\n",
       "      <th>...</th>\n",
       "      <th>original_glszm_SmallAreaHighGrayLevelEmphasis</th>\n",
       "      <th>original_glszm_SmallAreaLowGrayLevelEmphasis</th>\n",
       "      <th>original_glszm_ZoneEntropy</th>\n",
       "      <th>original_glszm_ZonePercentage</th>\n",
       "      <th>original_glszm_ZoneVariance</th>\n",
       "      <th>original_ngtdm_Busyness</th>\n",
       "      <th>original_ngtdm_Coarseness</th>\n",
       "      <th>original_ngtdm_Complexity</th>\n",
       "      <th>original_ngtdm_Contrast</th>\n",
       "      <th>original_ngtdm_Strength</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2.2.0</td>\n",
       "      <td>1.17.4</td>\n",
       "      <td>1.2.4</td>\n",
       "      <td>1.1.1</td>\n",
       "      <td>3.7.5</td>\n",
       "      <td>{'minimumROIDimensions': 2, 'minimumROISize': ...</td>\n",
       "      <td>{'Original': {}}</td>\n",
       "      <td>4d22955ee79b24bc744a8bad21195feda217d228</td>\n",
       "      <td>3D</td>\n",
       "      <td>(1.0, 1.0, 1.0)</td>\n",
       "      <td>...</td>\n",
       "      <td>8438.540262</td>\n",
       "      <td>0.010331</td>\n",
       "      <td>7.366261</td>\n",
       "      <td>0.949246</td>\n",
       "      <td>0.059280</td>\n",
       "      <td>0.019727</td>\n",
       "      <td>0.004660</td>\n",
       "      <td>440566.527274</td>\n",
       "      <td>6.871414</td>\n",
       "      <td>47.854486</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2.2.0</td>\n",
       "      <td>1.17.4</td>\n",
       "      <td>1.2.4</td>\n",
       "      <td>1.1.1</td>\n",
       "      <td>3.7.5</td>\n",
       "      <td>{'minimumROIDimensions': 2, 'minimumROISize': ...</td>\n",
       "      <td>{'Original': {}}</td>\n",
       "      <td>146aa3517683f2ad362e10fc3c2b7ffc2cc19f93</td>\n",
       "      <td>3D</td>\n",
       "      <td>(1.0, 1.0, 1.0)</td>\n",
       "      <td>...</td>\n",
       "      <td>8164.669110</td>\n",
       "      <td>0.009941</td>\n",
       "      <td>7.371531</td>\n",
       "      <td>0.935528</td>\n",
       "      <td>0.072963</td>\n",
       "      <td>0.019765</td>\n",
       "      <td>0.004604</td>\n",
       "      <td>437926.914601</td>\n",
       "      <td>6.528345</td>\n",
       "      <td>47.359422</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2.2.0</td>\n",
       "      <td>1.17.4</td>\n",
       "      <td>1.2.4</td>\n",
       "      <td>1.1.1</td>\n",
       "      <td>3.7.5</td>\n",
       "      <td>{'minimumROIDimensions': 2, 'minimumROISize': ...</td>\n",
       "      <td>{'Original': {}}</td>\n",
       "      <td>7969a8186618f072ad39980d13192cf134005c81</td>\n",
       "      <td>3D</td>\n",
       "      <td>(1.0, 1.0, 1.0)</td>\n",
       "      <td>...</td>\n",
       "      <td>9070.722630</td>\n",
       "      <td>0.007485</td>\n",
       "      <td>7.396664</td>\n",
       "      <td>0.934156</td>\n",
       "      <td>0.071390</td>\n",
       "      <td>0.018642</td>\n",
       "      <td>0.004304</td>\n",
       "      <td>467492.600158</td>\n",
       "      <td>7.125012</td>\n",
       "      <td>48.434773</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2.2.0</td>\n",
       "      <td>1.17.4</td>\n",
       "      <td>1.2.4</td>\n",
       "      <td>1.1.1</td>\n",
       "      <td>3.7.5</td>\n",
       "      <td>{'minimumROIDimensions': 2, 'minimumROISize': ...</td>\n",
       "      <td>{'Original': {}}</td>\n",
       "      <td>40a2e9b08312ca4524646d3d355787830f6da9be</td>\n",
       "      <td>3D</td>\n",
       "      <td>(1.0, 1.0, 1.0)</td>\n",
       "      <td>...</td>\n",
       "      <td>8511.088221</td>\n",
       "      <td>0.009914</td>\n",
       "      <td>7.395036</td>\n",
       "      <td>0.935528</td>\n",
       "      <td>0.072963</td>\n",
       "      <td>0.019946</td>\n",
       "      <td>0.004454</td>\n",
       "      <td>445117.530692</td>\n",
       "      <td>7.227345</td>\n",
       "      <td>47.024958</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2.2.0</td>\n",
       "      <td>1.17.4</td>\n",
       "      <td>1.2.4</td>\n",
       "      <td>1.1.1</td>\n",
       "      <td>3.7.5</td>\n",
       "      <td>{'minimumROIDimensions': 2, 'minimumROISize': ...</td>\n",
       "      <td>{'Original': {}}</td>\n",
       "      <td>aea3c45fcd3fa2c3a84b801e3ecf8f8504ed5ae6</td>\n",
       "      <td>3D</td>\n",
       "      <td>(1.0, 1.0, 1.0)</td>\n",
       "      <td>...</td>\n",
       "      <td>8514.572907</td>\n",
       "      <td>0.012318</td>\n",
       "      <td>7.350737</td>\n",
       "      <td>0.946502</td>\n",
       "      <td>0.067820</td>\n",
       "      <td>0.019127</td>\n",
       "      <td>0.004621</td>\n",
       "      <td>442918.835492</td>\n",
       "      <td>6.525662</td>\n",
       "      <td>48.885326</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2.2.0</td>\n",
       "      <td>1.17.4</td>\n",
       "      <td>1.2.4</td>\n",
       "      <td>1.1.1</td>\n",
       "      <td>3.7.5</td>\n",
       "      <td>{'minimumROIDimensions': 2, 'minimumROISize': ...</td>\n",
       "      <td>{'Original': {}}</td>\n",
       "      <td>7aaefab3873b1d9c1a61c977428f492a0c0d738e</td>\n",
       "      <td>3D</td>\n",
       "      <td>(1.0, 1.0, 1.0)</td>\n",
       "      <td>...</td>\n",
       "      <td>9116.926060</td>\n",
       "      <td>0.005309</td>\n",
       "      <td>7.385063</td>\n",
       "      <td>0.945130</td>\n",
       "      <td>0.060490</td>\n",
       "      <td>0.018740</td>\n",
       "      <td>0.004590</td>\n",
       "      <td>450771.399001</td>\n",
       "      <td>6.973380</td>\n",
       "      <td>48.189647</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2.2.0</td>\n",
       "      <td>1.17.4</td>\n",
       "      <td>1.2.4</td>\n",
       "      <td>1.1.1</td>\n",
       "      <td>3.7.5</td>\n",
       "      <td>{'minimumROIDimensions': 2, 'minimumROISize': ...</td>\n",
       "      <td>{'Original': {}}</td>\n",
       "      <td>06e1308cf71565cdcbd1c1cb8d70ef48e91904fd</td>\n",
       "      <td>3D</td>\n",
       "      <td>(1.0, 1.0, 1.0)</td>\n",
       "      <td>...</td>\n",
       "      <td>8497.367799</td>\n",
       "      <td>0.009018</td>\n",
       "      <td>7.439163</td>\n",
       "      <td>0.928669</td>\n",
       "      <td>0.085681</td>\n",
       "      <td>0.018802</td>\n",
       "      <td>0.004634</td>\n",
       "      <td>447927.445612</td>\n",
       "      <td>6.894316</td>\n",
       "      <td>49.523209</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>2.2.0</td>\n",
       "      <td>1.17.4</td>\n",
       "      <td>1.2.4</td>\n",
       "      <td>1.1.1</td>\n",
       "      <td>3.7.5</td>\n",
       "      <td>{'minimumROIDimensions': 2, 'minimumROISize': ...</td>\n",
       "      <td>{'Original': {}}</td>\n",
       "      <td>0fa3d174dd4a3ef39e91d51f59acafded8b6a15d</td>\n",
       "      <td>3D</td>\n",
       "      <td>(1.0, 1.0, 1.0)</td>\n",
       "      <td>...</td>\n",
       "      <td>8570.180525</td>\n",
       "      <td>0.009329</td>\n",
       "      <td>7.356454</td>\n",
       "      <td>0.936900</td>\n",
       "      <td>0.089168</td>\n",
       "      <td>0.018733</td>\n",
       "      <td>0.004744</td>\n",
       "      <td>421801.433828</td>\n",
       "      <td>6.260574</td>\n",
       "      <td>48.935080</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>2.2.0</td>\n",
       "      <td>1.17.4</td>\n",
       "      <td>1.2.4</td>\n",
       "      <td>1.1.1</td>\n",
       "      <td>3.7.5</td>\n",
       "      <td>{'minimumROIDimensions': 2, 'minimumROISize': ...</td>\n",
       "      <td>{'Original': {}}</td>\n",
       "      <td>082e4889ac3384ad9f51d02fa839b3127175ab5a</td>\n",
       "      <td>3D</td>\n",
       "      <td>(1.0, 1.0, 1.0)</td>\n",
       "      <td>...</td>\n",
       "      <td>8326.420083</td>\n",
       "      <td>0.006253</td>\n",
       "      <td>7.403529</td>\n",
       "      <td>0.939643</td>\n",
       "      <td>0.074706</td>\n",
       "      <td>0.020737</td>\n",
       "      <td>0.004495</td>\n",
       "      <td>444830.333652</td>\n",
       "      <td>7.393673</td>\n",
       "      <td>46.569711</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>2.2.0</td>\n",
       "      <td>1.17.4</td>\n",
       "      <td>1.2.4</td>\n",
       "      <td>1.1.1</td>\n",
       "      <td>3.7.5</td>\n",
       "      <td>{'minimumROIDimensions': 2, 'minimumROISize': ...</td>\n",
       "      <td>{'Original': {}}</td>\n",
       "      <td>0c86569ebfb1e671b9afaf672eeb9149acbb8b15</td>\n",
       "      <td>3D</td>\n",
       "      <td>(1.0, 1.0, 1.0)</td>\n",
       "      <td>...</td>\n",
       "      <td>8992.644274</td>\n",
       "      <td>0.012530</td>\n",
       "      <td>7.403597</td>\n",
       "      <td>0.941015</td>\n",
       "      <td>0.067500</td>\n",
       "      <td>0.019668</td>\n",
       "      <td>0.004228</td>\n",
       "      <td>479119.579990</td>\n",
       "      <td>7.758806</td>\n",
       "      <td>46.702033</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>2.2.0</td>\n",
       "      <td>1.17.4</td>\n",
       "      <td>1.2.4</td>\n",
       "      <td>1.1.1</td>\n",
       "      <td>3.7.5</td>\n",
       "      <td>{'minimumROIDimensions': 2, 'minimumROISize': ...</td>\n",
       "      <td>{'Original': {}}</td>\n",
       "      <td>78f4f2ded94b643a841f6d88d8c080451a407134</td>\n",
       "      <td>3D</td>\n",
       "      <td>(1.0, 1.0, 1.0)</td>\n",
       "      <td>...</td>\n",
       "      <td>8590.721977</td>\n",
       "      <td>0.008072</td>\n",
       "      <td>7.388018</td>\n",
       "      <td>0.934156</td>\n",
       "      <td>0.083138</td>\n",
       "      <td>0.021263</td>\n",
       "      <td>0.004344</td>\n",
       "      <td>464765.819210</td>\n",
       "      <td>7.632952</td>\n",
       "      <td>46.820508</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>2.2.0</td>\n",
       "      <td>1.17.4</td>\n",
       "      <td>1.2.4</td>\n",
       "      <td>1.1.1</td>\n",
       "      <td>3.7.5</td>\n",
       "      <td>{'minimumROIDimensions': 2, 'minimumROISize': ...</td>\n",
       "      <td>{'Original': {}}</td>\n",
       "      <td>b6d5de89dd93ead056dea1043cc0823d3cb3c5e3</td>\n",
       "      <td>3D</td>\n",
       "      <td>(1.0, 1.0, 1.0)</td>\n",
       "      <td>...</td>\n",
       "      <td>8416.874662</td>\n",
       "      <td>0.012580</td>\n",
       "      <td>7.399024</td>\n",
       "      <td>0.931413</td>\n",
       "      <td>0.091779</td>\n",
       "      <td>0.020217</td>\n",
       "      <td>0.004452</td>\n",
       "      <td>454684.899163</td>\n",
       "      <td>7.237960</td>\n",
       "      <td>47.410395</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>2.2.0</td>\n",
       "      <td>1.17.4</td>\n",
       "      <td>1.2.4</td>\n",
       "      <td>1.1.1</td>\n",
       "      <td>3.7.5</td>\n",
       "      <td>{'minimumROIDimensions': 2, 'minimumROISize': ...</td>\n",
       "      <td>{'Original': {}}</td>\n",
       "      <td>20d69fa0fc4d6c2fbfbe73538792fa6822e0965f</td>\n",
       "      <td>3D</td>\n",
       "      <td>(1.0, 1.0, 1.0)</td>\n",
       "      <td>...</td>\n",
       "      <td>8756.871685</td>\n",
       "      <td>0.007697</td>\n",
       "      <td>7.403029</td>\n",
       "      <td>0.936900</td>\n",
       "      <td>0.068670</td>\n",
       "      <td>0.019120</td>\n",
       "      <td>0.004452</td>\n",
       "      <td>452685.748160</td>\n",
       "      <td>7.018727</td>\n",
       "      <td>47.361418</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>2.2.0</td>\n",
       "      <td>1.17.4</td>\n",
       "      <td>1.2.4</td>\n",
       "      <td>1.1.1</td>\n",
       "      <td>3.7.5</td>\n",
       "      <td>{'minimumROIDimensions': 2, 'minimumROISize': ...</td>\n",
       "      <td>{'Original': {}}</td>\n",
       "      <td>b5fc41a3af6f73d77bf643d0f049098dca7890e4</td>\n",
       "      <td>3D</td>\n",
       "      <td>(1.0, 1.0, 1.0)</td>\n",
       "      <td>...</td>\n",
       "      <td>8781.859144</td>\n",
       "      <td>0.013584</td>\n",
       "      <td>7.357048</td>\n",
       "      <td>0.949246</td>\n",
       "      <td>0.059280</td>\n",
       "      <td>0.020053</td>\n",
       "      <td>0.004504</td>\n",
       "      <td>452517.076064</td>\n",
       "      <td>7.248649</td>\n",
       "      <td>48.803494</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>2.2.0</td>\n",
       "      <td>1.17.4</td>\n",
       "      <td>1.2.4</td>\n",
       "      <td>1.1.1</td>\n",
       "      <td>3.7.5</td>\n",
       "      <td>{'minimumROIDimensions': 2, 'minimumROISize': ...</td>\n",
       "      <td>{'Original': {}}</td>\n",
       "      <td>3a9e7b07fc539024904bb954b749d16372fe2b01</td>\n",
       "      <td>3D</td>\n",
       "      <td>(1.0, 1.0, 1.0)</td>\n",
       "      <td>...</td>\n",
       "      <td>8968.083658</td>\n",
       "      <td>0.011163</td>\n",
       "      <td>7.362323</td>\n",
       "      <td>0.938272</td>\n",
       "      <td>0.070233</td>\n",
       "      <td>0.019075</td>\n",
       "      <td>0.004346</td>\n",
       "      <td>471077.586864</td>\n",
       "      <td>7.149584</td>\n",
       "      <td>47.065635</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>2.2.0</td>\n",
       "      <td>1.17.4</td>\n",
       "      <td>1.2.4</td>\n",
       "      <td>1.1.1</td>\n",
       "      <td>3.7.5</td>\n",
       "      <td>{'minimumROIDimensions': 2, 'minimumROISize': ...</td>\n",
       "      <td>{'Original': {}}</td>\n",
       "      <td>d9d13326e417aa496f35c5afe250aa09ecb6ddc7</td>\n",
       "      <td>3D</td>\n",
       "      <td>(1.0, 1.0, 1.0)</td>\n",
       "      <td>...</td>\n",
       "      <td>9165.076898</td>\n",
       "      <td>0.010707</td>\n",
       "      <td>7.315335</td>\n",
       "      <td>0.953361</td>\n",
       "      <td>0.049405</td>\n",
       "      <td>0.018812</td>\n",
       "      <td>0.004412</td>\n",
       "      <td>446707.889200</td>\n",
       "      <td>7.128715</td>\n",
       "      <td>47.882917</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>2.2.0</td>\n",
       "      <td>1.17.4</td>\n",
       "      <td>1.2.4</td>\n",
       "      <td>1.1.1</td>\n",
       "      <td>3.7.5</td>\n",
       "      <td>{'minimumROIDimensions': 2, 'minimumROISize': ...</td>\n",
       "      <td>{'Original': {}}</td>\n",
       "      <td>9cea6f0d6e7c1585e84f89a8d5b21636d22457b4</td>\n",
       "      <td>3D</td>\n",
       "      <td>(1.0, 1.0, 1.0)</td>\n",
       "      <td>...</td>\n",
       "      <td>8515.010447</td>\n",
       "      <td>0.007957</td>\n",
       "      <td>7.358684</td>\n",
       "      <td>0.951989</td>\n",
       "      <td>0.053653</td>\n",
       "      <td>0.020990</td>\n",
       "      <td>0.004239</td>\n",
       "      <td>477161.074824</td>\n",
       "      <td>7.734372</td>\n",
       "      <td>46.579298</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>2.2.0</td>\n",
       "      <td>1.17.4</td>\n",
       "      <td>1.2.4</td>\n",
       "      <td>1.1.1</td>\n",
       "      <td>3.7.5</td>\n",
       "      <td>{'minimumROIDimensions': 2, 'minimumROISize': ...</td>\n",
       "      <td>{'Original': {}}</td>\n",
       "      <td>4a1e24a8db71110d9ae479470b0366914298a925</td>\n",
       "      <td>3D</td>\n",
       "      <td>(1.0, 1.0, 1.0)</td>\n",
       "      <td>...</td>\n",
       "      <td>8879.383455</td>\n",
       "      <td>0.009356</td>\n",
       "      <td>7.394948</td>\n",
       "      <td>0.939643</td>\n",
       "      <td>0.068867</td>\n",
       "      <td>0.018968</td>\n",
       "      <td>0.004564</td>\n",
       "      <td>446737.864964</td>\n",
       "      <td>6.768768</td>\n",
       "      <td>47.688605</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>2.2.0</td>\n",
       "      <td>1.17.4</td>\n",
       "      <td>1.2.4</td>\n",
       "      <td>1.1.1</td>\n",
       "      <td>3.7.5</td>\n",
       "      <td>{'minimumROIDimensions': 2, 'minimumROISize': ...</td>\n",
       "      <td>{'Original': {}}</td>\n",
       "      <td>0b005dcaddbf8a2bb120946aff8a5954a4335fc1</td>\n",
       "      <td>3D</td>\n",
       "      <td>(1.0, 1.0, 1.0)</td>\n",
       "      <td>...</td>\n",
       "      <td>8918.629549</td>\n",
       "      <td>0.008475</td>\n",
       "      <td>7.382586</td>\n",
       "      <td>0.942387</td>\n",
       "      <td>0.069043</td>\n",
       "      <td>0.020658</td>\n",
       "      <td>0.004257</td>\n",
       "      <td>469586.519333</td>\n",
       "      <td>7.541916</td>\n",
       "      <td>47.552034</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>2.2.0</td>\n",
       "      <td>1.17.4</td>\n",
       "      <td>1.2.4</td>\n",
       "      <td>1.1.1</td>\n",
       "      <td>3.7.5</td>\n",
       "      <td>{'minimumROIDimensions': 2, 'minimumROISize': ...</td>\n",
       "      <td>{'Original': {}}</td>\n",
       "      <td>8e80bd4f6a82f8820b197e3897b3181aed10e187</td>\n",
       "      <td>3D</td>\n",
       "      <td>(1.0, 1.0, 1.0)</td>\n",
       "      <td>...</td>\n",
       "      <td>8435.158713</td>\n",
       "      <td>0.007946</td>\n",
       "      <td>7.352103</td>\n",
       "      <td>0.943759</td>\n",
       "      <td>0.070577</td>\n",
       "      <td>0.019603</td>\n",
       "      <td>0.004533</td>\n",
       "      <td>433530.623896</td>\n",
       "      <td>6.975051</td>\n",
       "      <td>46.754217</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>2.2.0</td>\n",
       "      <td>1.17.4</td>\n",
       "      <td>1.2.4</td>\n",
       "      <td>1.1.1</td>\n",
       "      <td>3.7.5</td>\n",
       "      <td>{'minimumROIDimensions': 2, 'minimumROISize': ...</td>\n",
       "      <td>{'Original': {}}</td>\n",
       "      <td>dae2fe29c0cb0df15241945165bda7ed2585b5b1</td>\n",
       "      <td>3D</td>\n",
       "      <td>(1.0, 1.0, 1.0)</td>\n",
       "      <td>...</td>\n",
       "      <td>8779.234173</td>\n",
       "      <td>0.010754</td>\n",
       "      <td>7.366860</td>\n",
       "      <td>0.953361</td>\n",
       "      <td>0.046528</td>\n",
       "      <td>0.018833</td>\n",
       "      <td>0.004485</td>\n",
       "      <td>446142.272073</td>\n",
       "      <td>6.901914</td>\n",
       "      <td>49.539665</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>2.2.0</td>\n",
       "      <td>1.17.4</td>\n",
       "      <td>1.2.4</td>\n",
       "      <td>1.1.1</td>\n",
       "      <td>3.7.5</td>\n",
       "      <td>{'minimumROIDimensions': 2, 'minimumROISize': ...</td>\n",
       "      <td>{'Original': {}}</td>\n",
       "      <td>d716144058b8648fd99ca437fd80c65f395a41ca</td>\n",
       "      <td>3D</td>\n",
       "      <td>(1.0, 1.0, 1.0)</td>\n",
       "      <td>...</td>\n",
       "      <td>8521.435240</td>\n",
       "      <td>0.009918</td>\n",
       "      <td>7.425866</td>\n",
       "      <td>0.931413</td>\n",
       "      <td>0.074106</td>\n",
       "      <td>0.021018</td>\n",
       "      <td>0.004411</td>\n",
       "      <td>459803.278846</td>\n",
       "      <td>7.498261</td>\n",
       "      <td>47.648157</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>2.2.0</td>\n",
       "      <td>1.17.4</td>\n",
       "      <td>1.2.4</td>\n",
       "      <td>1.1.1</td>\n",
       "      <td>3.7.5</td>\n",
       "      <td>{'minimumROIDimensions': 2, 'minimumROISize': ...</td>\n",
       "      <td>{'Original': {}}</td>\n",
       "      <td>0ad9d28180008645be26f505a683797346b72b4d</td>\n",
       "      <td>3D</td>\n",
       "      <td>(1.0, 1.0, 1.0)</td>\n",
       "      <td>...</td>\n",
       "      <td>9204.345981</td>\n",
       "      <td>0.010076</td>\n",
       "      <td>7.338394</td>\n",
       "      <td>0.951989</td>\n",
       "      <td>0.050771</td>\n",
       "      <td>0.018895</td>\n",
       "      <td>0.004091</td>\n",
       "      <td>473866.335820</td>\n",
       "      <td>7.328811</td>\n",
       "      <td>47.820821</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>2.2.0</td>\n",
       "      <td>1.17.4</td>\n",
       "      <td>1.2.4</td>\n",
       "      <td>1.1.1</td>\n",
       "      <td>3.7.5</td>\n",
       "      <td>{'minimumROIDimensions': 2, 'minimumROISize': ...</td>\n",
       "      <td>{'Original': {}}</td>\n",
       "      <td>ba8ba081d7ee446e1217dfd5dc5904f645b626a0</td>\n",
       "      <td>3D</td>\n",
       "      <td>(1.0, 1.0, 1.0)</td>\n",
       "      <td>...</td>\n",
       "      <td>8835.039740</td>\n",
       "      <td>0.004428</td>\n",
       "      <td>7.346131</td>\n",
       "      <td>0.949246</td>\n",
       "      <td>0.053500</td>\n",
       "      <td>0.018660</td>\n",
       "      <td>0.004506</td>\n",
       "      <td>445632.115361</td>\n",
       "      <td>6.784510</td>\n",
       "      <td>47.928331</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>2.2.0</td>\n",
       "      <td>1.17.4</td>\n",
       "      <td>1.2.4</td>\n",
       "      <td>1.1.1</td>\n",
       "      <td>3.7.5</td>\n",
       "      <td>{'minimumROIDimensions': 2, 'minimumROISize': ...</td>\n",
       "      <td>{'Original': {}}</td>\n",
       "      <td>d7f9f65603d2600ad36e78ac2ce8cb0e678bfe3f</td>\n",
       "      <td>3D</td>\n",
       "      <td>(1.0, 1.0, 1.0)</td>\n",
       "      <td>...</td>\n",
       "      <td>8834.859448</td>\n",
       "      <td>0.003943</td>\n",
       "      <td>7.358777</td>\n",
       "      <td>0.939643</td>\n",
       "      <td>0.071786</td>\n",
       "      <td>0.020704</td>\n",
       "      <td>0.004155</td>\n",
       "      <td>457782.358934</td>\n",
       "      <td>7.678084</td>\n",
       "      <td>47.131456</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>25 rows × 129 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   diagnostics_Versions_PyRadiomics diagnostics_Versions_Numpy  \\\n",
       "0                             2.2.0                     1.17.4   \n",
       "1                             2.2.0                     1.17.4   \n",
       "2                             2.2.0                     1.17.4   \n",
       "3                             2.2.0                     1.17.4   \n",
       "4                             2.2.0                     1.17.4   \n",
       "5                             2.2.0                     1.17.4   \n",
       "6                             2.2.0                     1.17.4   \n",
       "7                             2.2.0                     1.17.4   \n",
       "8                             2.2.0                     1.17.4   \n",
       "9                             2.2.0                     1.17.4   \n",
       "10                            2.2.0                     1.17.4   \n",
       "11                            2.2.0                     1.17.4   \n",
       "12                            2.2.0                     1.17.4   \n",
       "13                            2.2.0                     1.17.4   \n",
       "14                            2.2.0                     1.17.4   \n",
       "15                            2.2.0                     1.17.4   \n",
       "16                            2.2.0                     1.17.4   \n",
       "17                            2.2.0                     1.17.4   \n",
       "18                            2.2.0                     1.17.4   \n",
       "19                            2.2.0                     1.17.4   \n",
       "20                            2.2.0                     1.17.4   \n",
       "21                            2.2.0                     1.17.4   \n",
       "22                            2.2.0                     1.17.4   \n",
       "23                            2.2.0                     1.17.4   \n",
       "24                            2.2.0                     1.17.4   \n",
       "\n",
       "   diagnostics_Versions_SimpleITK diagnostics_Versions_PyWavelet  \\\n",
       "0                           1.2.4                          1.1.1   \n",
       "1                           1.2.4                          1.1.1   \n",
       "2                           1.2.4                          1.1.1   \n",
       "3                           1.2.4                          1.1.1   \n",
       "4                           1.2.4                          1.1.1   \n",
       "5                           1.2.4                          1.1.1   \n",
       "6                           1.2.4                          1.1.1   \n",
       "7                           1.2.4                          1.1.1   \n",
       "8                           1.2.4                          1.1.1   \n",
       "9                           1.2.4                          1.1.1   \n",
       "10                          1.2.4                          1.1.1   \n",
       "11                          1.2.4                          1.1.1   \n",
       "12                          1.2.4                          1.1.1   \n",
       "13                          1.2.4                          1.1.1   \n",
       "14                          1.2.4                          1.1.1   \n",
       "15                          1.2.4                          1.1.1   \n",
       "16                          1.2.4                          1.1.1   \n",
       "17                          1.2.4                          1.1.1   \n",
       "18                          1.2.4                          1.1.1   \n",
       "19                          1.2.4                          1.1.1   \n",
       "20                          1.2.4                          1.1.1   \n",
       "21                          1.2.4                          1.1.1   \n",
       "22                          1.2.4                          1.1.1   \n",
       "23                          1.2.4                          1.1.1   \n",
       "24                          1.2.4                          1.1.1   \n",
       "\n",
       "   diagnostics_Versions_Python  \\\n",
       "0                        3.7.5   \n",
       "1                        3.7.5   \n",
       "2                        3.7.5   \n",
       "3                        3.7.5   \n",
       "4                        3.7.5   \n",
       "5                        3.7.5   \n",
       "6                        3.7.5   \n",
       "7                        3.7.5   \n",
       "8                        3.7.5   \n",
       "9                        3.7.5   \n",
       "10                       3.7.5   \n",
       "11                       3.7.5   \n",
       "12                       3.7.5   \n",
       "13                       3.7.5   \n",
       "14                       3.7.5   \n",
       "15                       3.7.5   \n",
       "16                       3.7.5   \n",
       "17                       3.7.5   \n",
       "18                       3.7.5   \n",
       "19                       3.7.5   \n",
       "20                       3.7.5   \n",
       "21                       3.7.5   \n",
       "22                       3.7.5   \n",
       "23                       3.7.5   \n",
       "24                       3.7.5   \n",
       "\n",
       "                   diagnostics_Configuration_Settings  \\\n",
       "0   {'minimumROIDimensions': 2, 'minimumROISize': ...   \n",
       "1   {'minimumROIDimensions': 2, 'minimumROISize': ...   \n",
       "2   {'minimumROIDimensions': 2, 'minimumROISize': ...   \n",
       "3   {'minimumROIDimensions': 2, 'minimumROISize': ...   \n",
       "4   {'minimumROIDimensions': 2, 'minimumROISize': ...   \n",
       "5   {'minimumROIDimensions': 2, 'minimumROISize': ...   \n",
       "6   {'minimumROIDimensions': 2, 'minimumROISize': ...   \n",
       "7   {'minimumROIDimensions': 2, 'minimumROISize': ...   \n",
       "8   {'minimumROIDimensions': 2, 'minimumROISize': ...   \n",
       "9   {'minimumROIDimensions': 2, 'minimumROISize': ...   \n",
       "10  {'minimumROIDimensions': 2, 'minimumROISize': ...   \n",
       "11  {'minimumROIDimensions': 2, 'minimumROISize': ...   \n",
       "12  {'minimumROIDimensions': 2, 'minimumROISize': ...   \n",
       "13  {'minimumROIDimensions': 2, 'minimumROISize': ...   \n",
       "14  {'minimumROIDimensions': 2, 'minimumROISize': ...   \n",
       "15  {'minimumROIDimensions': 2, 'minimumROISize': ...   \n",
       "16  {'minimumROIDimensions': 2, 'minimumROISize': ...   \n",
       "17  {'minimumROIDimensions': 2, 'minimumROISize': ...   \n",
       "18  {'minimumROIDimensions': 2, 'minimumROISize': ...   \n",
       "19  {'minimumROIDimensions': 2, 'minimumROISize': ...   \n",
       "20  {'minimumROIDimensions': 2, 'minimumROISize': ...   \n",
       "21  {'minimumROIDimensions': 2, 'minimumROISize': ...   \n",
       "22  {'minimumROIDimensions': 2, 'minimumROISize': ...   \n",
       "23  {'minimumROIDimensions': 2, 'minimumROISize': ...   \n",
       "24  {'minimumROIDimensions': 2, 'minimumROISize': ...   \n",
       "\n",
       "   diagnostics_Configuration_EnabledImageTypes  \\\n",
       "0                             {'Original': {}}   \n",
       "1                             {'Original': {}}   \n",
       "2                             {'Original': {}}   \n",
       "3                             {'Original': {}}   \n",
       "4                             {'Original': {}}   \n",
       "5                             {'Original': {}}   \n",
       "6                             {'Original': {}}   \n",
       "7                             {'Original': {}}   \n",
       "8                             {'Original': {}}   \n",
       "9                             {'Original': {}}   \n",
       "10                            {'Original': {}}   \n",
       "11                            {'Original': {}}   \n",
       "12                            {'Original': {}}   \n",
       "13                            {'Original': {}}   \n",
       "14                            {'Original': {}}   \n",
       "15                            {'Original': {}}   \n",
       "16                            {'Original': {}}   \n",
       "17                            {'Original': {}}   \n",
       "18                            {'Original': {}}   \n",
       "19                            {'Original': {}}   \n",
       "20                            {'Original': {}}   \n",
       "21                            {'Original': {}}   \n",
       "22                            {'Original': {}}   \n",
       "23                            {'Original': {}}   \n",
       "24                            {'Original': {}}   \n",
       "\n",
       "             diagnostics_Image-original_Hash  \\\n",
       "0   4d22955ee79b24bc744a8bad21195feda217d228   \n",
       "1   146aa3517683f2ad362e10fc3c2b7ffc2cc19f93   \n",
       "2   7969a8186618f072ad39980d13192cf134005c81   \n",
       "3   40a2e9b08312ca4524646d3d355787830f6da9be   \n",
       "4   aea3c45fcd3fa2c3a84b801e3ecf8f8504ed5ae6   \n",
       "5   7aaefab3873b1d9c1a61c977428f492a0c0d738e   \n",
       "6   06e1308cf71565cdcbd1c1cb8d70ef48e91904fd   \n",
       "7   0fa3d174dd4a3ef39e91d51f59acafded8b6a15d   \n",
       "8   082e4889ac3384ad9f51d02fa839b3127175ab5a   \n",
       "9   0c86569ebfb1e671b9afaf672eeb9149acbb8b15   \n",
       "10  78f4f2ded94b643a841f6d88d8c080451a407134   \n",
       "11  b6d5de89dd93ead056dea1043cc0823d3cb3c5e3   \n",
       "12  20d69fa0fc4d6c2fbfbe73538792fa6822e0965f   \n",
       "13  b5fc41a3af6f73d77bf643d0f049098dca7890e4   \n",
       "14  3a9e7b07fc539024904bb954b749d16372fe2b01   \n",
       "15  d9d13326e417aa496f35c5afe250aa09ecb6ddc7   \n",
       "16  9cea6f0d6e7c1585e84f89a8d5b21636d22457b4   \n",
       "17  4a1e24a8db71110d9ae479470b0366914298a925   \n",
       "18  0b005dcaddbf8a2bb120946aff8a5954a4335fc1   \n",
       "19  8e80bd4f6a82f8820b197e3897b3181aed10e187   \n",
       "20  dae2fe29c0cb0df15241945165bda7ed2585b5b1   \n",
       "21  d716144058b8648fd99ca437fd80c65f395a41ca   \n",
       "22  0ad9d28180008645be26f505a683797346b72b4d   \n",
       "23  ba8ba081d7ee446e1217dfd5dc5904f645b626a0   \n",
       "24  d7f9f65603d2600ad36e78ac2ce8cb0e678bfe3f   \n",
       "\n",
       "   diagnostics_Image-original_Dimensionality  \\\n",
       "0                                         3D   \n",
       "1                                         3D   \n",
       "2                                         3D   \n",
       "3                                         3D   \n",
       "4                                         3D   \n",
       "5                                         3D   \n",
       "6                                         3D   \n",
       "7                                         3D   \n",
       "8                                         3D   \n",
       "9                                         3D   \n",
       "10                                        3D   \n",
       "11                                        3D   \n",
       "12                                        3D   \n",
       "13                                        3D   \n",
       "14                                        3D   \n",
       "15                                        3D   \n",
       "16                                        3D   \n",
       "17                                        3D   \n",
       "18                                        3D   \n",
       "19                                        3D   \n",
       "20                                        3D   \n",
       "21                                        3D   \n",
       "22                                        3D   \n",
       "23                                        3D   \n",
       "24                                        3D   \n",
       "\n",
       "   diagnostics_Image-original_Spacing  ...  \\\n",
       "0                     (1.0, 1.0, 1.0)  ...   \n",
       "1                     (1.0, 1.0, 1.0)  ...   \n",
       "2                     (1.0, 1.0, 1.0)  ...   \n",
       "3                     (1.0, 1.0, 1.0)  ...   \n",
       "4                     (1.0, 1.0, 1.0)  ...   \n",
       "5                     (1.0, 1.0, 1.0)  ...   \n",
       "6                     (1.0, 1.0, 1.0)  ...   \n",
       "7                     (1.0, 1.0, 1.0)  ...   \n",
       "8                     (1.0, 1.0, 1.0)  ...   \n",
       "9                     (1.0, 1.0, 1.0)  ...   \n",
       "10                    (1.0, 1.0, 1.0)  ...   \n",
       "11                    (1.0, 1.0, 1.0)  ...   \n",
       "12                    (1.0, 1.0, 1.0)  ...   \n",
       "13                    (1.0, 1.0, 1.0)  ...   \n",
       "14                    (1.0, 1.0, 1.0)  ...   \n",
       "15                    (1.0, 1.0, 1.0)  ...   \n",
       "16                    (1.0, 1.0, 1.0)  ...   \n",
       "17                    (1.0, 1.0, 1.0)  ...   \n",
       "18                    (1.0, 1.0, 1.0)  ...   \n",
       "19                    (1.0, 1.0, 1.0)  ...   \n",
       "20                    (1.0, 1.0, 1.0)  ...   \n",
       "21                    (1.0, 1.0, 1.0)  ...   \n",
       "22                    (1.0, 1.0, 1.0)  ...   \n",
       "23                    (1.0, 1.0, 1.0)  ...   \n",
       "24                    (1.0, 1.0, 1.0)  ...   \n",
       "\n",
       "   original_glszm_SmallAreaHighGrayLevelEmphasis  \\\n",
       "0                                    8438.540262   \n",
       "1                                    8164.669110   \n",
       "2                                    9070.722630   \n",
       "3                                    8511.088221   \n",
       "4                                    8514.572907   \n",
       "5                                    9116.926060   \n",
       "6                                    8497.367799   \n",
       "7                                    8570.180525   \n",
       "8                                    8326.420083   \n",
       "9                                    8992.644274   \n",
       "10                                   8590.721977   \n",
       "11                                   8416.874662   \n",
       "12                                   8756.871685   \n",
       "13                                   8781.859144   \n",
       "14                                   8968.083658   \n",
       "15                                   9165.076898   \n",
       "16                                   8515.010447   \n",
       "17                                   8879.383455   \n",
       "18                                   8918.629549   \n",
       "19                                   8435.158713   \n",
       "20                                   8779.234173   \n",
       "21                                   8521.435240   \n",
       "22                                   9204.345981   \n",
       "23                                   8835.039740   \n",
       "24                                   8834.859448   \n",
       "\n",
       "    original_glszm_SmallAreaLowGrayLevelEmphasis  original_glszm_ZoneEntropy  \\\n",
       "0                                       0.010331                    7.366261   \n",
       "1                                       0.009941                    7.371531   \n",
       "2                                       0.007485                    7.396664   \n",
       "3                                       0.009914                    7.395036   \n",
       "4                                       0.012318                    7.350737   \n",
       "5                                       0.005309                    7.385063   \n",
       "6                                       0.009018                    7.439163   \n",
       "7                                       0.009329                    7.356454   \n",
       "8                                       0.006253                    7.403529   \n",
       "9                                       0.012530                    7.403597   \n",
       "10                                      0.008072                    7.388018   \n",
       "11                                      0.012580                    7.399024   \n",
       "12                                      0.007697                    7.403029   \n",
       "13                                      0.013584                    7.357048   \n",
       "14                                      0.011163                    7.362323   \n",
       "15                                      0.010707                    7.315335   \n",
       "16                                      0.007957                    7.358684   \n",
       "17                                      0.009356                    7.394948   \n",
       "18                                      0.008475                    7.382586   \n",
       "19                                      0.007946                    7.352103   \n",
       "20                                      0.010754                    7.366860   \n",
       "21                                      0.009918                    7.425866   \n",
       "22                                      0.010076                    7.338394   \n",
       "23                                      0.004428                    7.346131   \n",
       "24                                      0.003943                    7.358777   \n",
       "\n",
       "    original_glszm_ZonePercentage original_glszm_ZoneVariance  \\\n",
       "0                        0.949246                    0.059280   \n",
       "1                        0.935528                    0.072963   \n",
       "2                        0.934156                    0.071390   \n",
       "3                        0.935528                    0.072963   \n",
       "4                        0.946502                    0.067820   \n",
       "5                        0.945130                    0.060490   \n",
       "6                        0.928669                    0.085681   \n",
       "7                        0.936900                    0.089168   \n",
       "8                        0.939643                    0.074706   \n",
       "9                        0.941015                    0.067500   \n",
       "10                       0.934156                    0.083138   \n",
       "11                       0.931413                    0.091779   \n",
       "12                       0.936900                    0.068670   \n",
       "13                       0.949246                    0.059280   \n",
       "14                       0.938272                    0.070233   \n",
       "15                       0.953361                    0.049405   \n",
       "16                       0.951989                    0.053653   \n",
       "17                       0.939643                    0.068867   \n",
       "18                       0.942387                    0.069043   \n",
       "19                       0.943759                    0.070577   \n",
       "20                       0.953361                    0.046528   \n",
       "21                       0.931413                    0.074106   \n",
       "22                       0.951989                    0.050771   \n",
       "23                       0.949246                    0.053500   \n",
       "24                       0.939643                    0.071786   \n",
       "\n",
       "   original_ngtdm_Busyness original_ngtdm_Coarseness  \\\n",
       "0                 0.019727                  0.004660   \n",
       "1                 0.019765                  0.004604   \n",
       "2                 0.018642                  0.004304   \n",
       "3                 0.019946                  0.004454   \n",
       "4                 0.019127                  0.004621   \n",
       "5                 0.018740                  0.004590   \n",
       "6                 0.018802                  0.004634   \n",
       "7                 0.018733                  0.004744   \n",
       "8                 0.020737                  0.004495   \n",
       "9                 0.019668                  0.004228   \n",
       "10                0.021263                  0.004344   \n",
       "11                0.020217                  0.004452   \n",
       "12                0.019120                  0.004452   \n",
       "13                0.020053                  0.004504   \n",
       "14                0.019075                  0.004346   \n",
       "15                0.018812                  0.004412   \n",
       "16                0.020990                  0.004239   \n",
       "17                0.018968                  0.004564   \n",
       "18                0.020658                  0.004257   \n",
       "19                0.019603                  0.004533   \n",
       "20                0.018833                  0.004485   \n",
       "21                0.021018                  0.004411   \n",
       "22                0.018895                  0.004091   \n",
       "23                0.018660                  0.004506   \n",
       "24                0.020704                  0.004155   \n",
       "\n",
       "   original_ngtdm_Complexity  original_ngtdm_Contrast  original_ngtdm_Strength  \n",
       "0              440566.527274                 6.871414                47.854486  \n",
       "1              437926.914601                 6.528345                47.359422  \n",
       "2              467492.600158                 7.125012                48.434773  \n",
       "3              445117.530692                 7.227345                47.024958  \n",
       "4              442918.835492                 6.525662                48.885326  \n",
       "5              450771.399001                 6.973380                48.189647  \n",
       "6              447927.445612                 6.894316                49.523209  \n",
       "7              421801.433828                 6.260574                48.935080  \n",
       "8              444830.333652                 7.393673                46.569711  \n",
       "9              479119.579990                 7.758806                46.702033  \n",
       "10             464765.819210                 7.632952                46.820508  \n",
       "11             454684.899163                 7.237960                47.410395  \n",
       "12             452685.748160                 7.018727                47.361418  \n",
       "13             452517.076064                 7.248649                48.803494  \n",
       "14             471077.586864                 7.149584                47.065635  \n",
       "15             446707.889200                 7.128715                47.882917  \n",
       "16             477161.074824                 7.734372                46.579298  \n",
       "17             446737.864964                 6.768768                47.688605  \n",
       "18             469586.519333                 7.541916                47.552034  \n",
       "19             433530.623896                 6.975051                46.754217  \n",
       "20             446142.272073                 6.901914                49.539665  \n",
       "21             459803.278846                 7.498261                47.648157  \n",
       "22             473866.335820                 7.328811                47.820821  \n",
       "23             445632.115361                 6.784510                47.928331  \n",
       "24             457782.358934                 7.678084                47.131456  \n",
       "\n",
       "[25 rows x 129 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Inspect feature file\n",
    "df_features_training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>original_shape_Elongation</th>\n",
       "      <th>original_shape_Flatness</th>\n",
       "      <th>original_shape_LeastAxisLength</th>\n",
       "      <th>original_shape_MajorAxisLength</th>\n",
       "      <th>original_shape_Maximum2DDiameterColumn</th>\n",
       "      <th>original_shape_Maximum2DDiameterRow</th>\n",
       "      <th>original_shape_Maximum2DDiameterSlice</th>\n",
       "      <th>original_shape_Maximum3DDiameter</th>\n",
       "      <th>original_shape_MeshVolume</th>\n",
       "      <th>original_shape_MinorAxisLength</th>\n",
       "      <th>...</th>\n",
       "      <th>original_glszm_SmallAreaHighGrayLevelEmphasis</th>\n",
       "      <th>original_glszm_SmallAreaLowGrayLevelEmphasis</th>\n",
       "      <th>original_glszm_ZoneEntropy</th>\n",
       "      <th>original_glszm_ZonePercentage</th>\n",
       "      <th>original_glszm_ZoneVariance</th>\n",
       "      <th>original_ngtdm_Busyness</th>\n",
       "      <th>original_ngtdm_Coarseness</th>\n",
       "      <th>original_ngtdm_Complexity</th>\n",
       "      <th>original_ngtdm_Contrast</th>\n",
       "      <th>original_ngtdm_Strength</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>14.456832</td>\n",
       "      <td>716.166667</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>...</td>\n",
       "      <td>8438.540262</td>\n",
       "      <td>0.010331</td>\n",
       "      <td>7.366261</td>\n",
       "      <td>0.949246</td>\n",
       "      <td>0.059280</td>\n",
       "      <td>0.019727</td>\n",
       "      <td>0.004660</td>\n",
       "      <td>440566.527274</td>\n",
       "      <td>6.871414</td>\n",
       "      <td>47.854486</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>14.456832</td>\n",
       "      <td>716.166667</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>...</td>\n",
       "      <td>8164.669110</td>\n",
       "      <td>0.009941</td>\n",
       "      <td>7.371531</td>\n",
       "      <td>0.935528</td>\n",
       "      <td>0.072963</td>\n",
       "      <td>0.019765</td>\n",
       "      <td>0.004604</td>\n",
       "      <td>437926.914601</td>\n",
       "      <td>6.528345</td>\n",
       "      <td>47.359422</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>14.456832</td>\n",
       "      <td>716.166667</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>...</td>\n",
       "      <td>9070.722630</td>\n",
       "      <td>0.007485</td>\n",
       "      <td>7.396664</td>\n",
       "      <td>0.934156</td>\n",
       "      <td>0.071390</td>\n",
       "      <td>0.018642</td>\n",
       "      <td>0.004304</td>\n",
       "      <td>467492.600158</td>\n",
       "      <td>7.125012</td>\n",
       "      <td>48.434773</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>14.456832</td>\n",
       "      <td>716.166667</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>...</td>\n",
       "      <td>8511.088221</td>\n",
       "      <td>0.009914</td>\n",
       "      <td>7.395036</td>\n",
       "      <td>0.935528</td>\n",
       "      <td>0.072963</td>\n",
       "      <td>0.019946</td>\n",
       "      <td>0.004454</td>\n",
       "      <td>445117.530692</td>\n",
       "      <td>7.227345</td>\n",
       "      <td>47.024958</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>14.456832</td>\n",
       "      <td>716.166667</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>...</td>\n",
       "      <td>8514.572907</td>\n",
       "      <td>0.012318</td>\n",
       "      <td>7.350737</td>\n",
       "      <td>0.946502</td>\n",
       "      <td>0.067820</td>\n",
       "      <td>0.019127</td>\n",
       "      <td>0.004621</td>\n",
       "      <td>442918.835492</td>\n",
       "      <td>6.525662</td>\n",
       "      <td>48.885326</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>14.456832</td>\n",
       "      <td>716.166667</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>...</td>\n",
       "      <td>9116.926060</td>\n",
       "      <td>0.005309</td>\n",
       "      <td>7.385063</td>\n",
       "      <td>0.945130</td>\n",
       "      <td>0.060490</td>\n",
       "      <td>0.018740</td>\n",
       "      <td>0.004590</td>\n",
       "      <td>450771.399001</td>\n",
       "      <td>6.973380</td>\n",
       "      <td>48.189647</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>14.456832</td>\n",
       "      <td>716.166667</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>...</td>\n",
       "      <td>8497.367799</td>\n",
       "      <td>0.009018</td>\n",
       "      <td>7.439163</td>\n",
       "      <td>0.928669</td>\n",
       "      <td>0.085681</td>\n",
       "      <td>0.018802</td>\n",
       "      <td>0.004634</td>\n",
       "      <td>447927.445612</td>\n",
       "      <td>6.894316</td>\n",
       "      <td>49.523209</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>14.456832</td>\n",
       "      <td>716.166667</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>...</td>\n",
       "      <td>8570.180525</td>\n",
       "      <td>0.009329</td>\n",
       "      <td>7.356454</td>\n",
       "      <td>0.936900</td>\n",
       "      <td>0.089168</td>\n",
       "      <td>0.018733</td>\n",
       "      <td>0.004744</td>\n",
       "      <td>421801.433828</td>\n",
       "      <td>6.260574</td>\n",
       "      <td>48.935080</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>14.456832</td>\n",
       "      <td>716.166667</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>...</td>\n",
       "      <td>8326.420083</td>\n",
       "      <td>0.006253</td>\n",
       "      <td>7.403529</td>\n",
       "      <td>0.939643</td>\n",
       "      <td>0.074706</td>\n",
       "      <td>0.020737</td>\n",
       "      <td>0.004495</td>\n",
       "      <td>444830.333652</td>\n",
       "      <td>7.393673</td>\n",
       "      <td>46.569711</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>14.456832</td>\n",
       "      <td>716.166667</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>...</td>\n",
       "      <td>8992.644274</td>\n",
       "      <td>0.012530</td>\n",
       "      <td>7.403597</td>\n",
       "      <td>0.941015</td>\n",
       "      <td>0.067500</td>\n",
       "      <td>0.019668</td>\n",
       "      <td>0.004228</td>\n",
       "      <td>479119.579990</td>\n",
       "      <td>7.758806</td>\n",
       "      <td>46.702033</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>14.456832</td>\n",
       "      <td>716.166667</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>...</td>\n",
       "      <td>8590.721977</td>\n",
       "      <td>0.008072</td>\n",
       "      <td>7.388018</td>\n",
       "      <td>0.934156</td>\n",
       "      <td>0.083138</td>\n",
       "      <td>0.021263</td>\n",
       "      <td>0.004344</td>\n",
       "      <td>464765.819210</td>\n",
       "      <td>7.632952</td>\n",
       "      <td>46.820508</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>14.456832</td>\n",
       "      <td>716.166667</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>...</td>\n",
       "      <td>8416.874662</td>\n",
       "      <td>0.012580</td>\n",
       "      <td>7.399024</td>\n",
       "      <td>0.931413</td>\n",
       "      <td>0.091779</td>\n",
       "      <td>0.020217</td>\n",
       "      <td>0.004452</td>\n",
       "      <td>454684.899163</td>\n",
       "      <td>7.237960</td>\n",
       "      <td>47.410395</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>14.456832</td>\n",
       "      <td>716.166667</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>...</td>\n",
       "      <td>8756.871685</td>\n",
       "      <td>0.007697</td>\n",
       "      <td>7.403029</td>\n",
       "      <td>0.936900</td>\n",
       "      <td>0.068670</td>\n",
       "      <td>0.019120</td>\n",
       "      <td>0.004452</td>\n",
       "      <td>452685.748160</td>\n",
       "      <td>7.018727</td>\n",
       "      <td>47.361418</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>14.456832</td>\n",
       "      <td>716.166667</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>...</td>\n",
       "      <td>8781.859144</td>\n",
       "      <td>0.013584</td>\n",
       "      <td>7.357048</td>\n",
       "      <td>0.949246</td>\n",
       "      <td>0.059280</td>\n",
       "      <td>0.020053</td>\n",
       "      <td>0.004504</td>\n",
       "      <td>452517.076064</td>\n",
       "      <td>7.248649</td>\n",
       "      <td>48.803494</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>14.456832</td>\n",
       "      <td>716.166667</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>...</td>\n",
       "      <td>8968.083658</td>\n",
       "      <td>0.011163</td>\n",
       "      <td>7.362323</td>\n",
       "      <td>0.938272</td>\n",
       "      <td>0.070233</td>\n",
       "      <td>0.019075</td>\n",
       "      <td>0.004346</td>\n",
       "      <td>471077.586864</td>\n",
       "      <td>7.149584</td>\n",
       "      <td>47.065635</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>14.456832</td>\n",
       "      <td>716.166667</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>...</td>\n",
       "      <td>9165.076898</td>\n",
       "      <td>0.010707</td>\n",
       "      <td>7.315335</td>\n",
       "      <td>0.953361</td>\n",
       "      <td>0.049405</td>\n",
       "      <td>0.018812</td>\n",
       "      <td>0.004412</td>\n",
       "      <td>446707.889200</td>\n",
       "      <td>7.128715</td>\n",
       "      <td>47.882917</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>14.456832</td>\n",
       "      <td>716.166667</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>...</td>\n",
       "      <td>8515.010447</td>\n",
       "      <td>0.007957</td>\n",
       "      <td>7.358684</td>\n",
       "      <td>0.951989</td>\n",
       "      <td>0.053653</td>\n",
       "      <td>0.020990</td>\n",
       "      <td>0.004239</td>\n",
       "      <td>477161.074824</td>\n",
       "      <td>7.734372</td>\n",
       "      <td>46.579298</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>14.456832</td>\n",
       "      <td>716.166667</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>...</td>\n",
       "      <td>8879.383455</td>\n",
       "      <td>0.009356</td>\n",
       "      <td>7.394948</td>\n",
       "      <td>0.939643</td>\n",
       "      <td>0.068867</td>\n",
       "      <td>0.018968</td>\n",
       "      <td>0.004564</td>\n",
       "      <td>446737.864964</td>\n",
       "      <td>6.768768</td>\n",
       "      <td>47.688605</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>14.456832</td>\n",
       "      <td>716.166667</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>...</td>\n",
       "      <td>8918.629549</td>\n",
       "      <td>0.008475</td>\n",
       "      <td>7.382586</td>\n",
       "      <td>0.942387</td>\n",
       "      <td>0.069043</td>\n",
       "      <td>0.020658</td>\n",
       "      <td>0.004257</td>\n",
       "      <td>469586.519333</td>\n",
       "      <td>7.541916</td>\n",
       "      <td>47.552034</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>14.456832</td>\n",
       "      <td>716.166667</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>...</td>\n",
       "      <td>8435.158713</td>\n",
       "      <td>0.007946</td>\n",
       "      <td>7.352103</td>\n",
       "      <td>0.943759</td>\n",
       "      <td>0.070577</td>\n",
       "      <td>0.019603</td>\n",
       "      <td>0.004533</td>\n",
       "      <td>433530.623896</td>\n",
       "      <td>6.975051</td>\n",
       "      <td>46.754217</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>14.456832</td>\n",
       "      <td>716.166667</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>...</td>\n",
       "      <td>8779.234173</td>\n",
       "      <td>0.010754</td>\n",
       "      <td>7.366860</td>\n",
       "      <td>0.953361</td>\n",
       "      <td>0.046528</td>\n",
       "      <td>0.018833</td>\n",
       "      <td>0.004485</td>\n",
       "      <td>446142.272073</td>\n",
       "      <td>6.901914</td>\n",
       "      <td>49.539665</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>14.456832</td>\n",
       "      <td>716.166667</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>...</td>\n",
       "      <td>8521.435240</td>\n",
       "      <td>0.009918</td>\n",
       "      <td>7.425866</td>\n",
       "      <td>0.931413</td>\n",
       "      <td>0.074106</td>\n",
       "      <td>0.021018</td>\n",
       "      <td>0.004411</td>\n",
       "      <td>459803.278846</td>\n",
       "      <td>7.498261</td>\n",
       "      <td>47.648157</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>14.456832</td>\n",
       "      <td>716.166667</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>...</td>\n",
       "      <td>9204.345981</td>\n",
       "      <td>0.010076</td>\n",
       "      <td>7.338394</td>\n",
       "      <td>0.951989</td>\n",
       "      <td>0.050771</td>\n",
       "      <td>0.018895</td>\n",
       "      <td>0.004091</td>\n",
       "      <td>473866.335820</td>\n",
       "      <td>7.328811</td>\n",
       "      <td>47.820821</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>14.456832</td>\n",
       "      <td>716.166667</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>...</td>\n",
       "      <td>8835.039740</td>\n",
       "      <td>0.004428</td>\n",
       "      <td>7.346131</td>\n",
       "      <td>0.949246</td>\n",
       "      <td>0.053500</td>\n",
       "      <td>0.018660</td>\n",
       "      <td>0.004506</td>\n",
       "      <td>445632.115361</td>\n",
       "      <td>6.784510</td>\n",
       "      <td>47.928331</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>12.041595</td>\n",
       "      <td>14.456832</td>\n",
       "      <td>716.166667</td>\n",
       "      <td>10.327956</td>\n",
       "      <td>...</td>\n",
       "      <td>8834.859448</td>\n",
       "      <td>0.003943</td>\n",
       "      <td>7.358777</td>\n",
       "      <td>0.939643</td>\n",
       "      <td>0.071786</td>\n",
       "      <td>0.020704</td>\n",
       "      <td>0.004155</td>\n",
       "      <td>457782.358934</td>\n",
       "      <td>7.678084</td>\n",
       "      <td>47.131456</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>25 rows × 107 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    original_shape_Elongation  original_shape_Flatness  \\\n",
       "0                         1.0                      1.0   \n",
       "1                         1.0                      1.0   \n",
       "2                         1.0                      1.0   \n",
       "3                         1.0                      1.0   \n",
       "4                         1.0                      1.0   \n",
       "5                         1.0                      1.0   \n",
       "6                         1.0                      1.0   \n",
       "7                         1.0                      1.0   \n",
       "8                         1.0                      1.0   \n",
       "9                         1.0                      1.0   \n",
       "10                        1.0                      1.0   \n",
       "11                        1.0                      1.0   \n",
       "12                        1.0                      1.0   \n",
       "13                        1.0                      1.0   \n",
       "14                        1.0                      1.0   \n",
       "15                        1.0                      1.0   \n",
       "16                        1.0                      1.0   \n",
       "17                        1.0                      1.0   \n",
       "18                        1.0                      1.0   \n",
       "19                        1.0                      1.0   \n",
       "20                        1.0                      1.0   \n",
       "21                        1.0                      1.0   \n",
       "22                        1.0                      1.0   \n",
       "23                        1.0                      1.0   \n",
       "24                        1.0                      1.0   \n",
       "\n",
       "    original_shape_LeastAxisLength  original_shape_MajorAxisLength  \\\n",
       "0                        10.327956                       10.327956   \n",
       "1                        10.327956                       10.327956   \n",
       "2                        10.327956                       10.327956   \n",
       "3                        10.327956                       10.327956   \n",
       "4                        10.327956                       10.327956   \n",
       "5                        10.327956                       10.327956   \n",
       "6                        10.327956                       10.327956   \n",
       "7                        10.327956                       10.327956   \n",
       "8                        10.327956                       10.327956   \n",
       "9                        10.327956                       10.327956   \n",
       "10                       10.327956                       10.327956   \n",
       "11                       10.327956                       10.327956   \n",
       "12                       10.327956                       10.327956   \n",
       "13                       10.327956                       10.327956   \n",
       "14                       10.327956                       10.327956   \n",
       "15                       10.327956                       10.327956   \n",
       "16                       10.327956                       10.327956   \n",
       "17                       10.327956                       10.327956   \n",
       "18                       10.327956                       10.327956   \n",
       "19                       10.327956                       10.327956   \n",
       "20                       10.327956                       10.327956   \n",
       "21                       10.327956                       10.327956   \n",
       "22                       10.327956                       10.327956   \n",
       "23                       10.327956                       10.327956   \n",
       "24                       10.327956                       10.327956   \n",
       "\n",
       "    original_shape_Maximum2DDiameterColumn  \\\n",
       "0                                12.041595   \n",
       "1                                12.041595   \n",
       "2                                12.041595   \n",
       "3                                12.041595   \n",
       "4                                12.041595   \n",
       "5                                12.041595   \n",
       "6                                12.041595   \n",
       "7                                12.041595   \n",
       "8                                12.041595   \n",
       "9                                12.041595   \n",
       "10                               12.041595   \n",
       "11                               12.041595   \n",
       "12                               12.041595   \n",
       "13                               12.041595   \n",
       "14                               12.041595   \n",
       "15                               12.041595   \n",
       "16                               12.041595   \n",
       "17                               12.041595   \n",
       "18                               12.041595   \n",
       "19                               12.041595   \n",
       "20                               12.041595   \n",
       "21                               12.041595   \n",
       "22                               12.041595   \n",
       "23                               12.041595   \n",
       "24                               12.041595   \n",
       "\n",
       "    original_shape_Maximum2DDiameterRow  \\\n",
       "0                             12.041595   \n",
       "1                             12.041595   \n",
       "2                             12.041595   \n",
       "3                             12.041595   \n",
       "4                             12.041595   \n",
       "5                             12.041595   \n",
       "6                             12.041595   \n",
       "7                             12.041595   \n",
       "8                             12.041595   \n",
       "9                             12.041595   \n",
       "10                            12.041595   \n",
       "11                            12.041595   \n",
       "12                            12.041595   \n",
       "13                            12.041595   \n",
       "14                            12.041595   \n",
       "15                            12.041595   \n",
       "16                            12.041595   \n",
       "17                            12.041595   \n",
       "18                            12.041595   \n",
       "19                            12.041595   \n",
       "20                            12.041595   \n",
       "21                            12.041595   \n",
       "22                            12.041595   \n",
       "23                            12.041595   \n",
       "24                            12.041595   \n",
       "\n",
       "    original_shape_Maximum2DDiameterSlice  original_shape_Maximum3DDiameter  \\\n",
       "0                               12.041595                         14.456832   \n",
       "1                               12.041595                         14.456832   \n",
       "2                               12.041595                         14.456832   \n",
       "3                               12.041595                         14.456832   \n",
       "4                               12.041595                         14.456832   \n",
       "5                               12.041595                         14.456832   \n",
       "6                               12.041595                         14.456832   \n",
       "7                               12.041595                         14.456832   \n",
       "8                               12.041595                         14.456832   \n",
       "9                               12.041595                         14.456832   \n",
       "10                              12.041595                         14.456832   \n",
       "11                              12.041595                         14.456832   \n",
       "12                              12.041595                         14.456832   \n",
       "13                              12.041595                         14.456832   \n",
       "14                              12.041595                         14.456832   \n",
       "15                              12.041595                         14.456832   \n",
       "16                              12.041595                         14.456832   \n",
       "17                              12.041595                         14.456832   \n",
       "18                              12.041595                         14.456832   \n",
       "19                              12.041595                         14.456832   \n",
       "20                              12.041595                         14.456832   \n",
       "21                              12.041595                         14.456832   \n",
       "22                              12.041595                         14.456832   \n",
       "23                              12.041595                         14.456832   \n",
       "24                              12.041595                         14.456832   \n",
       "\n",
       "    original_shape_MeshVolume  original_shape_MinorAxisLength  ...  \\\n",
       "0                  716.166667                       10.327956  ...   \n",
       "1                  716.166667                       10.327956  ...   \n",
       "2                  716.166667                       10.327956  ...   \n",
       "3                  716.166667                       10.327956  ...   \n",
       "4                  716.166667                       10.327956  ...   \n",
       "5                  716.166667                       10.327956  ...   \n",
       "6                  716.166667                       10.327956  ...   \n",
       "7                  716.166667                       10.327956  ...   \n",
       "8                  716.166667                       10.327956  ...   \n",
       "9                  716.166667                       10.327956  ...   \n",
       "10                 716.166667                       10.327956  ...   \n",
       "11                 716.166667                       10.327956  ...   \n",
       "12                 716.166667                       10.327956  ...   \n",
       "13                 716.166667                       10.327956  ...   \n",
       "14                 716.166667                       10.327956  ...   \n",
       "15                 716.166667                       10.327956  ...   \n",
       "16                 716.166667                       10.327956  ...   \n",
       "17                 716.166667                       10.327956  ...   \n",
       "18                 716.166667                       10.327956  ...   \n",
       "19                 716.166667                       10.327956  ...   \n",
       "20                 716.166667                       10.327956  ...   \n",
       "21                 716.166667                       10.327956  ...   \n",
       "22                 716.166667                       10.327956  ...   \n",
       "23                 716.166667                       10.327956  ...   \n",
       "24                 716.166667                       10.327956  ...   \n",
       "\n",
       "    original_glszm_SmallAreaHighGrayLevelEmphasis  \\\n",
       "0                                     8438.540262   \n",
       "1                                     8164.669110   \n",
       "2                                     9070.722630   \n",
       "3                                     8511.088221   \n",
       "4                                     8514.572907   \n",
       "5                                     9116.926060   \n",
       "6                                     8497.367799   \n",
       "7                                     8570.180525   \n",
       "8                                     8326.420083   \n",
       "9                                     8992.644274   \n",
       "10                                    8590.721977   \n",
       "11                                    8416.874662   \n",
       "12                                    8756.871685   \n",
       "13                                    8781.859144   \n",
       "14                                    8968.083658   \n",
       "15                                    9165.076898   \n",
       "16                                    8515.010447   \n",
       "17                                    8879.383455   \n",
       "18                                    8918.629549   \n",
       "19                                    8435.158713   \n",
       "20                                    8779.234173   \n",
       "21                                    8521.435240   \n",
       "22                                    9204.345981   \n",
       "23                                    8835.039740   \n",
       "24                                    8834.859448   \n",
       "\n",
       "    original_glszm_SmallAreaLowGrayLevelEmphasis  original_glszm_ZoneEntropy  \\\n",
       "0                                       0.010331                    7.366261   \n",
       "1                                       0.009941                    7.371531   \n",
       "2                                       0.007485                    7.396664   \n",
       "3                                       0.009914                    7.395036   \n",
       "4                                       0.012318                    7.350737   \n",
       "5                                       0.005309                    7.385063   \n",
       "6                                       0.009018                    7.439163   \n",
       "7                                       0.009329                    7.356454   \n",
       "8                                       0.006253                    7.403529   \n",
       "9                                       0.012530                    7.403597   \n",
       "10                                      0.008072                    7.388018   \n",
       "11                                      0.012580                    7.399024   \n",
       "12                                      0.007697                    7.403029   \n",
       "13                                      0.013584                    7.357048   \n",
       "14                                      0.011163                    7.362323   \n",
       "15                                      0.010707                    7.315335   \n",
       "16                                      0.007957                    7.358684   \n",
       "17                                      0.009356                    7.394948   \n",
       "18                                      0.008475                    7.382586   \n",
       "19                                      0.007946                    7.352103   \n",
       "20                                      0.010754                    7.366860   \n",
       "21                                      0.009918                    7.425866   \n",
       "22                                      0.010076                    7.338394   \n",
       "23                                      0.004428                    7.346131   \n",
       "24                                      0.003943                    7.358777   \n",
       "\n",
       "    original_glszm_ZonePercentage  original_glszm_ZoneVariance  \\\n",
       "0                        0.949246                     0.059280   \n",
       "1                        0.935528                     0.072963   \n",
       "2                        0.934156                     0.071390   \n",
       "3                        0.935528                     0.072963   \n",
       "4                        0.946502                     0.067820   \n",
       "5                        0.945130                     0.060490   \n",
       "6                        0.928669                     0.085681   \n",
       "7                        0.936900                     0.089168   \n",
       "8                        0.939643                     0.074706   \n",
       "9                        0.941015                     0.067500   \n",
       "10                       0.934156                     0.083138   \n",
       "11                       0.931413                     0.091779   \n",
       "12                       0.936900                     0.068670   \n",
       "13                       0.949246                     0.059280   \n",
       "14                       0.938272                     0.070233   \n",
       "15                       0.953361                     0.049405   \n",
       "16                       0.951989                     0.053653   \n",
       "17                       0.939643                     0.068867   \n",
       "18                       0.942387                     0.069043   \n",
       "19                       0.943759                     0.070577   \n",
       "20                       0.953361                     0.046528   \n",
       "21                       0.931413                     0.074106   \n",
       "22                       0.951989                     0.050771   \n",
       "23                       0.949246                     0.053500   \n",
       "24                       0.939643                     0.071786   \n",
       "\n",
       "    original_ngtdm_Busyness  original_ngtdm_Coarseness  \\\n",
       "0                  0.019727                   0.004660   \n",
       "1                  0.019765                   0.004604   \n",
       "2                  0.018642                   0.004304   \n",
       "3                  0.019946                   0.004454   \n",
       "4                  0.019127                   0.004621   \n",
       "5                  0.018740                   0.004590   \n",
       "6                  0.018802                   0.004634   \n",
       "7                  0.018733                   0.004744   \n",
       "8                  0.020737                   0.004495   \n",
       "9                  0.019668                   0.004228   \n",
       "10                 0.021263                   0.004344   \n",
       "11                 0.020217                   0.004452   \n",
       "12                 0.019120                   0.004452   \n",
       "13                 0.020053                   0.004504   \n",
       "14                 0.019075                   0.004346   \n",
       "15                 0.018812                   0.004412   \n",
       "16                 0.020990                   0.004239   \n",
       "17                 0.018968                   0.004564   \n",
       "18                 0.020658                   0.004257   \n",
       "19                 0.019603                   0.004533   \n",
       "20                 0.018833                   0.004485   \n",
       "21                 0.021018                   0.004411   \n",
       "22                 0.018895                   0.004091   \n",
       "23                 0.018660                   0.004506   \n",
       "24                 0.020704                   0.004155   \n",
       "\n",
       "    original_ngtdm_Complexity  original_ngtdm_Contrast  \\\n",
       "0               440566.527274                 6.871414   \n",
       "1               437926.914601                 6.528345   \n",
       "2               467492.600158                 7.125012   \n",
       "3               445117.530692                 7.227345   \n",
       "4               442918.835492                 6.525662   \n",
       "5               450771.399001                 6.973380   \n",
       "6               447927.445612                 6.894316   \n",
       "7               421801.433828                 6.260574   \n",
       "8               444830.333652                 7.393673   \n",
       "9               479119.579990                 7.758806   \n",
       "10              464765.819210                 7.632952   \n",
       "11              454684.899163                 7.237960   \n",
       "12              452685.748160                 7.018727   \n",
       "13              452517.076064                 7.248649   \n",
       "14              471077.586864                 7.149584   \n",
       "15              446707.889200                 7.128715   \n",
       "16              477161.074824                 7.734372   \n",
       "17              446737.864964                 6.768768   \n",
       "18              469586.519333                 7.541916   \n",
       "19              433530.623896                 6.975051   \n",
       "20              446142.272073                 6.901914   \n",
       "21              459803.278846                 7.498261   \n",
       "22              473866.335820                 7.328811   \n",
       "23              445632.115361                 6.784510   \n",
       "24              457782.358934                 7.678084   \n",
       "\n",
       "    original_ngtdm_Strength  \n",
       "0                 47.854486  \n",
       "1                 47.359422  \n",
       "2                 48.434773  \n",
       "3                 47.024958  \n",
       "4                 48.885326  \n",
       "5                 48.189647  \n",
       "6                 49.523209  \n",
       "7                 48.935080  \n",
       "8                 46.569711  \n",
       "9                 46.702033  \n",
       "10                46.820508  \n",
       "11                47.410395  \n",
       "12                47.361418  \n",
       "13                48.803494  \n",
       "14                47.065635  \n",
       "15                47.882917  \n",
       "16                46.579298  \n",
       "17                47.688605  \n",
       "18                47.552034  \n",
       "19                46.754217  \n",
       "20                49.539665  \n",
       "21                47.648157  \n",
       "22                47.820821  \n",
       "23                47.928331  \n",
       "24                47.131456  \n",
       "\n",
       "[25 rows x 107 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Drop meta info header columns\n",
    "df_features_training.drop(columns = df_features_training.columns[:n_header_cols], inplace=True)\n",
    "df_features_training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0., 1., 0., 1., 0., 1., 0., 1., 0., 1., 0., 0., 0., 0., 0., 1., 0.,\n",
       "       0., 1., 1., 0., 1., 0., 1., 1.])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Inspect labels\n",
    "labels_training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature selection\n",
    "\n",
    "Of the features previously extracted, features highly correlated with another feature are identified and excluded from the analysis. This feature selection based on pair-wise feature correlation is applied to improve the machine learning training process and to enable optimized feature interpretability. For this purpose, we calculate a Pearson correlation matrix for all features extracted from the training set and exclude one feature of any feature pair with a Person correlation coefficient above 'corr_threshold'."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Remove correlated features...\n",
      "Correlation threshold: 0.8\n",
      "Drop 73 / 107  features ( 68.2 %).\n",
      "Number of remaining features: 34\n"
     ]
    }
   ],
   "source": [
    "# Identify and remove strongly correlated features in the training dataset\n",
    "df_features_training_uncorr, dropped_features = remove_correlated_features(df_features_training, corr_threshold=corr_threshold)\n",
    "\n",
    "# Save names of features which have been dropped and will not be used for training or evaluating the random forest model\n",
    "dropped_features = np.asarray(dropped_features)\n",
    "np.savetxt('feature_selection/dropped_features.csv', dropped_features, newline=',', fmt='%s')\n",
    "\n",
    "# Save names of features which have been selected and will be used for training and evaluating the random forest model\n",
    "feature_names = df_features_training_uncorr.columns.to_numpy()\n",
    "np.savetxt('feature_selection/selected_features.csv', feature_names, newline=',', fmt='%s')\n",
    "print(\"Number of remaining features:\", len(feature_names))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Random forest training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['trained_models/trained_example_random_forest_model.joblib']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Prepare the training data, i.e. select the feature columns and the label column, respectively, and ignore the rest.\n",
    "X_train = df_features_training_uncorr.to_numpy()\n",
    "y_train = labels_training\n",
    "\n",
    "# Set up the random forest model\n",
    "rf = RandomForestClassifier(n_estimators=1000, oob_score=True, random_state=0)\n",
    "\n",
    "# Train the model\n",
    "rf.fit(X_train, y_train)\n",
    "\n",
    "# Save model\n",
    "dump(rf, 'trained_models/trained_example_random_forest_model.joblib')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Internal validation (using out-of-bag predictions)\n",
    "\n",
    "We use the out-of-bag (OOB) predictions to get a point estimate of the performance that can be expected from the model for unseen data. In addition, we evaluate the out-of-bag predictions for a wide range of classification thresholds in order to pick and report three different models (A, B and C) later on (see below)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set up a wide range of classification thresholds\n",
    "thresholds = np.linspace(0.0,1.0,n_thresholds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Internal evaluation\n",
    "oob_bin_acc     = []\n",
    "oob_sensitivity = []\n",
    "oob_specificity = []\n",
    "\n",
    "oob_roc_auc = roc_auc_score(y_train, rf.oob_decision_function_[:, 1])\n",
    "\n",
    "# Apply different thresholds for classification\n",
    "for t in thresholds:\n",
    "\n",
    "    # Adjust out-of-bag prediction for the current threshold\n",
    "    y_pred_adj = adjusted_prediction(rf.oob_decision_function_, threshold=t, positive_label=1)\n",
    "\n",
    "    # Out-of-bag prediction scores for the current threshold\n",
    "    bin_acc_thr = accuracy_score(y_train, y_pred_adj)\n",
    "    roc_auc_thr = roc_auc_score(y_train, rf.oob_decision_function_[:, 1])\n",
    "    sensitivity_thr, specificity_thr = eval_sensitivity_specificity(y_train, y_pred_adj)\n",
    "\n",
    "    oob_bin_acc.append(bin_acc_thr)\n",
    "    oob_sensitivity.append(sensitivity_thr)\n",
    "    oob_specificity.append(specificity_thr)\n",
    "\n",
    "oob_bin_acc     = np.asarray(oob_bin_acc)\n",
    "oob_sensitivity = np.asarray(oob_sensitivity)\n",
    "oob_specificity = np.asarray(oob_specificity)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Search classification thresholds\n",
    "\n",
    "Sensitivity and specificity depend on the classification threshold, a parameter which is used to turn predicted class probabilities, i.e. the output of the random forest model (here: rf.oob_decision_function_) for a given input sample, into class predictions (benign versus premalignant).\n",
    "\n",
    "Here, we search for three different thresholds:\n",
    "1. A default threshold value of 0.5 (Model A)\n",
    "2. A threshold value which maximizes Youden's index (J=sensitivity+specificity+1) (Model B)\n",
    "3. A threshold that results in the highest possible specificity while achieving a sensitivity of at least 0.85 (Model C)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Maximum Youden's index J = 0.00 found for threshold T = 0.0000\n"
     ]
    }
   ],
   "source": [
    "# Find default threshold of 0.5\n",
    "ind_thr_50 = int(len(thresholds)/2.0)\n",
    "\n",
    "# Find threshold which maximizes Youden's index J\n",
    "J         = oob_sensitivity + oob_specificity - 1\n",
    "J_max     = np.amax(J)\n",
    "ind_J_max = np.argmax(J)\n",
    "print(\"Maximum Youden's index J = {:0.2f} found for threshold T = {:0.4f}\".format(J_max, thresholds[ind_J_max]))\n",
    "\n",
    "# Find threshold which yields sensitivity of 0.85\n",
    "ind_85 = 0\n",
    "while(oob_sensitivity[ind_85] >= 0.85):\n",
    "    ind_85 += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Internal out-of-bag (OOB) validation:\n",
      "\n",
      "oob_roc_auc: 0.23\n",
      "\n",
      "OOB scores for threshold of 0.5 (Model A):\n",
      "\n",
      "bin_acc:     0.36\n",
      "sensitivity: 0.00\n",
      "specifity:   0.64\n",
      "\n",
      "OOB scores for a maximized Youden's Index (Model B, threshold = 0.000):\n",
      "\n",
      "bin_acc:     0.44\n",
      "sensitivity: 1.00\n",
      "specifity:   0.00\n",
      "\n",
      "OOB scores for a sensitivity of 0.85 (Model C, threshold = 0.233):\n",
      "\n",
      "bin_acc:     0.36\n",
      "sensitivity: 0.82\n",
      "specifity:   0.00\n"
     ]
    }
   ],
   "source": [
    "print(\"Internal out-of-bag (OOB) validation:\\n\")\n",
    "print('oob_roc_auc: {:.2f}'.format(oob_roc_auc))\n",
    "\n",
    "print(\"\\nOOB scores for threshold of {:0.1f} (Model A):\\n\".format(thresholds[ind_thr_50]))  \n",
    "print('bin_acc:     {:0.2f}'.format(oob_bin_acc[ind_thr_50]))\n",
    "print('sensitivity: {:0.2f}'.format(oob_sensitivity[ind_thr_50]))\n",
    "print('specifity:   {:0.2f}'.format(oob_specificity[ind_thr_50]))\n",
    "\n",
    "print(\"\\nOOB scores for a maximized Youden's Index (Model B, threshold = {:0.3f}):\\n\".format(thresholds[ind_J_max]))  \n",
    "print('bin_acc:     {:0.2f}'.format(oob_bin_acc[ind_J_max]))\n",
    "print('sensitivity: {:0.2f}'.format(oob_sensitivity[ind_J_max]))\n",
    "print('specifity:   {:0.2f}'.format(oob_specificity[ind_J_max]))\n",
    "\n",
    "print(\"\\nOOB scores for a sensitivity of 0.85 (Model C, threshold = {:0.3f}):\\n\".format(thresholds[ind_85]))  \n",
    "print('bin_acc:     {:0.2f}'.format(oob_bin_acc[ind_85]))\n",
    "print('sensitivity: {:0.2f}'.format(oob_sensitivity[ind_85]))\n",
    "print('specifity:   {:0.2f}'.format(oob_specificity[ind_85]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature importance\n",
    "\n",
    "The Scikit-learn random forest implementation provides an internal estimate of feature importance, i.e. how much the class prediction (benign versus premalignant) of a trained model depends on a specific feature relative to all other features. Here, we evaluate the relative importance of the features used for training the random forest."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 1. feature  15: original_firstorder_90Percentile (0.096223)\n",
      " 2. feature  17: original_firstorder_Entropy (0.065844)\n",
      " 3. feature  14: original_firstorder_10Percentile (0.064274)\n",
      " 4. feature  24: original_glcm_InverseVariance (0.060633)\n",
      " 5. feature  23: original_glcm_Id (0.055962)\n",
      " 6. feature  22: original_glcm_Correlation (0.055184)\n",
      " 7. feature  19: original_firstorder_Maximum (0.054201)\n",
      " 8. feature  28: original_gldm_DependenceEntropy (0.051140)\n",
      " 9. feature  20: original_firstorder_Minimum (0.049560)\n",
      "10. feature  26: original_glcm_MCC (0.048734)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsgAAALICAYAAABiqwZ2AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAyrklEQVR4nO3deZhlV1kv4F93hkZQDEFUwJiBwAdJI2gLBiMCkSFM4fp4kSkhBoyIBEUug0qA0CGgogRlnoQIKFwcCKAYhgwIlwgmDHaDKwSCYRZIQAjQSei+f+xTsCiqUud0nzrVVfW+z1PPqdp7r7O/1dV16lfrrL32hl27dgUAABhsXOkCAABgbyIgAwBAR0AGAICOgAwAAB0BGQAAOvuudAF7m4suumhTkjsk+XyS76xwOQAALI99ktw0yQe2bNmyo98hIP+gOyT515UuAgCAmbhzkvf0G2YekKvqoCRnJrlHkg1J3pnkca21y8do+6wkP59kS5IDk5zUWnv1IseenOT/JDk0yaeSnNlae8kYJX4+SW51q1tl//33H+NwAABWm6uvvjqXXHJJMsp+vZkG5Kq6fpJzk+xIcmKSXUmemeS8qvqZ1tpVSzzFY5N8KMlbkzz8Os5zcpKXJnl2hgD+K0leVFUbWmsvXuIc30mS/fffP5s2bVqyTwAArGo/MKV21iPIJyc5LEm11i5Nkqr6SJKPJ3lUkucu0f5HW2s7q+rwLBKQq2rfJGckeU1r7SmjzedV1c2SnF5Vr2itXTOFvgAAsAbNehWL45JcOBeOk6S1dlmS9yZ5wFKNW2s7xzjHnZLcJMlr521/TZIbJ/mlsasFAGDdmXVAPjLJtgW2b09yxBTPkQXOs330OK3zAACwBs06IB+Y5MoFtl+R5EZTPEcWOM8V8/YDAMAPsMzbIrZtW2igGwCAtW7WAfnKLDxSvNjI8u6eI6Pz9Mt2zI0cX5ExbN682SoWAABr1I4dOxYdEJ31FIvt+d4c4d4RST46xXNkgfPMzT2e1nkAAFiDZh2Q35zkqKo6bG5DVR2S5OjRvml4X5IvJ3nYvO3HZxg9fu+UzgMAwBo06ykWL09ySpKzq+rUDDcKOT3JpzPc2CNJUlUHJ/lEkq2tta3d9rtkWMLtJ0ebfr6qvpEkrbW/Gz1eU1VPzXBjkM9muFHIMUkekeSxrbWrl7eLAACsZjMdQR7dKe+YJJdkWJf4dUkuS3JMa+0b3aEbkuyzQH3PSPLGJM8fff2Y0ddvnHeelyR5dJJfT3JOkockOaW19sJp9gcAgLVnw65du1a6hr3KRRdddEiSy1ykBwCwdnUX6R26ZcuWT/X7Zj0HGQAA9moCMgAAdARkAADoCMgAANARkAEAoCMgAwBAR0AGAICOgAwAAB0BGQAAOgIyAAB0BGQAAOgIyAAA0BGQAQCgIyADAEBHQAYAgI6ADAAAHQEZAAA6AvKEdu7cudIlTM1a6gsAwLTsu9IFrDYbN27Mtm3bVrqMqdi8efNKlwAAsNcxggwAAB0BGQAAOgIyAAB0BGQAAOgIyAAA0BGQAQCgIyADAEBHQAYAgI6ADAAAHQEZAAA6AjIAAHQEZAAA6AjIAADQEZABAKAjIAMAQEdABgCAjoAMAAAdARkAADoCMgAAdARkAADoCMgAANARkAEAoCMgAwBAR0AGAICOgAwAAB0BGQAAOgIyAAB0BGQAAOgIyAAA0BGQAQCgIyADAEBHQAYAgI6ADAAAHQEZAAA6AjIAAHQEZAAA6AjIAADQEZABAKAjIAMAQEdABgCAjoAMAAAdARkAADoCMgAAdARkAADoCMgAANARkAEAoCMgAwBAR0AGAICOgAwAAB0BGQAAOgIyAAB0BGQAAOgIyAAA0BGQAQCgIyADAEBHQAYAgI6ADAAAHQEZAAA6AjIAAHQEZAAA6AjIAADQEZABAKAjIAMAQEdABgCAjoAMAAAdARkAADoCMgAAdARkAADoCMgAANARkAEAoCMgAwBAR0AGAICOgAwAAB0BGQAAOgIyAAB0BGQAAOgIyAAA0BGQAQCgIyADAEBHQAYAgI6ADAAAHQEZAAA6AjIAAHQEZAAA6AjIAADQEZABAKAjIAMAQEdABgCAjoAMAAAdARkAADoCMgAAdARkAADoCMgAANARkAEAoCMgAwBAR0AGAICOgAwAAB0BGQAAOvvO+oRVdVCSM5PcI8mGJO9M8rjW2uVjtL1ektOTHJ/kgCQfSvLk1tq75x134yRPS3L/JDdN8oUk/5TkGa21L02rLwAArD0zHUGuqusnOTfJrZOcmOSEJLdMcl5V3WCMp3hlkpMzhN/7Jfl8knOq6vbdOTYkeXOShyZ5TpJ7jx4fnOQto/0AALCgWY8gn5zksCTVWrs0SarqI0k+nuRRSZ67WMOqul2G0PuI1tqrRtsuSLI9ydYkx40OvWWSX0zyqNbay0bbzq+qnUlenORWSdqU+wUAwBox6znIxyW5cC4cJ0lr7bIk703ygDHaXpPkDV3ba5O8Psm9qmrTaPP+o8f/mdf+q6NH864BAFjUrEeQj0xy9gLbtyd54BhtL2utfXOBtvsnOXz0+fYk707y1Kq6NMl/Jjkiw7SMt7XWPrb75QMAsNbNejT1wCRXLrD9iiQ32oO2c/vTWtuV5D4ZplF8IMnXk/xbkk8m+bXJSwYAYD2Z+SoWM/LyJEcl+e0kH0tymyTPSPJ3VXX/1trOpZ5g27ZtC27fsmXLFMtceRdddNFKlwAAsFeZdUC+MguPFC82Ojy/7cGLtE1GI8lVdd8kD0ly99bau0b73l1Vn0zy9gxLvy00zeP7bN68OZs2bVrqsFVvrQV+AIBx7NixY9EB0VlPsdieYS7xfEck+egYbQ8dLRU3v+3VSeYu/Lvt6PED8457/+jxNuOVCgDAejTrgPzmJEdV1WFzG6rqkCRHj/Zdl7ck2S/dxXxVtW+SByV5e2ttx2jzF0aPd5zX/hdGj5/drcoBAFgXZj3F4uVJTklydlWdmmRXhjvjfTrJS+cOqqqDk3wiydbW2tYkaa19sKrekOR5VbVfksuSPDrJoUke1p3jH5KckeSvq+r0DKtY3DrJ00fn+cdl7SEAAKvaTEeQW2tXJTkmySVJXpPkdRmC7jGttW90h25Iss8C9Z2U5FVJnpnh1tEHJTm2tXZxd47/yXCB3tuSPKl7fEuSO807DwAAfJ+Zr2LRWrs8Syy31lr7VIaQPH/7t5I8fvRxXe0/neSRu18lAADrlbvKAQBAR0AGAICOgAwAAB0BGQAAOgIyAAB0BGQAAOgIyAAA0BGQAQCgIyADAEBHQAYAgI6ADAAAHQEZAAA6AjIAAHQEZAAA6AjIAADQEZABAKAjIAMAQEdABgCAjoAMAAAdARkAADoCMgAAdARkAADoCMgAANARkAEAoCMgAwBAR0AGAICOgAwAAB0BGQAAOgIyAAB0BGQAAOgIyAAA0BGQAQCgIyADAEBHQAYAgI6ADAAAHQEZAAA6AjIAAHQEZAAA6AjIAADQEZABAKAjIAMAQEdABgCAjoAMAAAdARkAADoCMgAAdARkAADoCMgAANARkAEAoCMgAwBAR0AGAICOgAwAAB0BGQAAOgIyAAB0BGQAAOgIyAAA0BGQAQCgIyADAEBHQAYAgI6ADAAAHQEZAAA6AjIAAHQEZAAA6AjIAADQEZABAKAjIAMAQEdABgCAjoAMAAAdARkAADoCMgAAdARkAADoCMgAANARkAEAoCMgAwBAR0AGAICOgAwAAB0BGQAAOgIyAAB0BGQAAOgIyAAA0BGQAQCgIyADAEBHQAYAgI6ADAAAHQEZAAA6AjJj27lz50qXMDVrqS8AwHTtu9IFsHps3Lgx27ZtW+kypmLz5s0rXQIAsJcyggwAAB0BGQAAOgIyAAB0BGQAAOgIyAAA0BGQAQCgIyADAEBHQAYAgI6ADAAAHQEZxrSWbk+9lvoCANPmVtMwJrfaBoD1wQgyAAB0BGQAAOgIyAAA0BGQAQCgIyADAEBHQAYAgI6ADAAAHQEZAAA6AjIAAHQEZAAA6AjIAADQEZABAKAjIAMAQEdABgCAjoAMAAAdARkAADoCMgAAdARkAADoCMgAANARkAEAoLPvrE9YVQclOTPJPZJsSPLOJI9rrV0+RtvrJTk9yfFJDkjyoSRPbq29e4Fjbz469j5JbpTkc0le31r7w6l0BACANWmmI8hVdf0k5ya5dZITk5yQ5JZJzquqG4zxFK9McnKSpyW5X5LPJzmnqm4/7zyHJHl/klsl+d0k90xyWpJrp9ANWJd27ty50iVMxVrpBwDLZ9YjyCcnOSxJtdYuTZKq+kiSjyd5VJLnLtawqm6X5KFJHtFae9Vo2wVJtifZmuS47vCXJPlskru11q4Zbbtgul2B9WXjxo3Ztm3bSpexxzZv3rzSJQCwl5v1HOTjklw4F46TpLV2WZL3JnnAGG2vSfKGru21SV6f5F5VtSlJquoWSe6V5PldOAYAgLHMegT5yCRnL7B9e5IHjtH2stbaNxdou3+Sw0efHz3a/q2qekeSX07yzSRvSfL7rbWv7GbtAACsA7MOyAcmuXKB7VdkuJBud9vO7U+Sm40e/yrJa5I8O0N4fnaSI6rqjq21JSchLvZW8pYtW5ZquqpcdNFFYx+7nvue6P9a6v+kfQdgfZn5KhYzMDdt5PzW2mNGn59bVV/LaDpGkrct9SSbN2/Opk2blqnEvcdaCj2TWs99T9Z3/9dz3wEY7NixY9EB0VnPQb4yC48ULzY6PG7b5HsjyXNTKN4x77i3jx5/donzAACwjs06IG/PMJd4viOSfHSMtoeOloqb3/bqJJd2x10XazwBALCoWQfkNyc5qqoOm9swWrP46NG+6/KWJPulu5ivqvZN8qAkb2+t7RhtvjDJFzJMpegdO3r8wO4WDwDA2jfrOcgvT3JKkrOr6tQkuzLc7e7TSV46d1BVHZzkE0m2tta2Jklr7YNV9YYkz6uq/ZJcluTRSQ5N8rC5tq21a6vqD5K8uqpekuQfMlykd0aS8zPcqAQAABY00xHk1tpVSY5JckmGFSZelyHoHtNa+0Z36IYk+yxQ30lJXpXkmUn+KclBSY5trV087zxnJXl4kl/KMPL81CSvTXL/1tquKXcLAIA1ZOarWLTWLk/ya0sc86kMIXn+9m8lefzoY6nzvCZDCAcAgLHNeg4yAADs1QRkAADoCMgAANARkAEAoCMgAwBAR0AGAICOgAwAAB0BGQAAOgIyAAB0BGQAAOgIyAAA0BGQAQCgs++kDarqZ5M8NckvJzkgyR1baxdX1bOSvLu19i/TLREAAGZnohHkqvqlJO9LcuskfzOv/c4kvz290gAAYPYmnWLxx0nOSXJkksfP23dxkp+bRlEAALBSJg3IP5fkxa21XUl2zdv35SQ3mUpVAACwQiYNyN9Ocv1F9t00ydf2rBwAAFhZkwbk9yR5XFXt022bG0l+ZJJzp1IVAACskElXsXhqkvcm+XCSv8sQjk+squcm2ZLkDtMtDwAAZmuiEeTW2oczLO/2xSRPSbIhySmj3XdprbXplgew8nbu3LnSJUzNWuoLwHKZeB3k1trFSX6lqq6X5MAkX22tfXPqlQHsJTZu3Jht27atdBlTsXnz5pUuAWCvN1FArqr9kuzfWruqtfbtJJ/r9t0gydWttWumXCMAAMzMpCPIr0iyX5KHLrDvpUmuTvKIPS0KAABWyqSrWNwtydmL7Htzkl/Zs3IAAGBlTRqQfzzJfy+y70tJfmLPygFgb7OWLuxbS30Bls+kUyz+O8ltk5y3wL7bJvnKHlcEwF7FRYrAejPpCPJbkzy1qn6m31hVt82w7NtbplUYAACshElHkJ+W5B5JLqqqDyT5TJKbJ7ljksuSnDrd8gAAYLYmvVHIlzPcLe/ZGW4ScvvR4xlJ7jDaDwAAq9bu3CjkqxlGkp829WoAAGCFTToHGQDWjbW06sVa6gsst4lHkKvqxCQPSfLTSa43b/eu1totplEYAKw0K3jA+jTpraafmuQZSbYl+VCSHctQEwAArJhJR5AfmeQvWmu/vxzFAADASpt0DvKNY61jAADWsEkD8gVJbrcchQAAwN5g0ikWj0vyD1X1lST/nOSK+Qe01lwmCwDAqjVpQL5k9PiqRfbv2o3nBACAvcakYXZrhhAMAABr0kQBubV22jLVAQAAewV30gMAgM7u3Elv/yT3TlJZ+E56p0+jMAAAWAmT3knvZknek+SQDHORN4x29fOSBWQAAFatSadYPCfJl5L8dIZw/AtJDktyRpJLR58DAMCqNekUizsneUKSz42+3tla+1SSp1XVPkn+MskDplceAADM1u7cavpzo5uBXJXkRt2+c5PcdUp1AQDAipg0IH8myY+NPv9Eknt2++6Y5NvTKAoAAFbKpFMszktylyRvSvLSJC+sqtsnuSbJvUbbAABg1Zo0IJ+a5MAkaa29uKr2TfKgJNdP8qcZ7rQHAACr1qR30vtyki93Xz8/yfOnXRQAAKyUieYgV9W5VXXrRfbdqqrOnU5ZAMBK27lz50qXMDVrqS8sv0mnWNw1yQ0X2fcjGeYnAwBrwMaNG7Nt27aVLmMqNm/evNIlsIpMuopF8v13zevdIsk39qAWAABYcUuOIFfVSUlOGn25K8nLqurr8w77oSSbk7xruuUBAMBsjTOCvDPJd0YfG+Z9PffxlSQvTvLI5SkTAABmY8kR5NbaWUnOSpKqOi/Jo1tr/7nchQEAwEoYew5yVe2f5EeT/PTylQMAACtr7IDcWrs6yaFJrl2+cgAAYGVNuorFO5LcczkKAQCAvcGk6yA/P8lrR7eYflOSz2fesm+ttU9OpzQAAJi9SQPyBaPHxyf5/UWO2Wf3ywEA2Dvs3LkzGzfuzi0j9j5rqS+zMGlAPmnpQwAAVj93Ely/JgrIoyXfAABgzZp0BDlJUlUbkhyR5MAkVyT5aGttsVtQAwDAqjHxZJSq+s0MF+d9JMn5o8fPVZW76AEAsOpNFJCr6mFJXpbkP5I8Isl9Ro//keRlVfWQqVcIAAAzNOkUiycleV1r7YR528+qqtckeXKSv51KZQAArIi1tOrF7vRl0oBcGULyQl6bYW1kAABWsfW+gsekfxp8PclPLbLvp0b7AQBg1Zo0IL8tybOq6s79xqq6U5JnjvYDAMCqtTtzkI9Kcn5VfTbDahY/mWH0+NIsPv0CAABWhYlGkFtrX0hy+yS/l+R9Sf4nyYVJHpvkZ1trX5x2gQAAMEsT3yiktfbNJC8YfQAAwJqyu3fSOzzJHZPcPMlnkry/tfaJaRYGAAArYaKAXFXXS/KiJCck2afb9Z2qOivJY1prO6ZYHwAAzNSkq1j8WZKHJXl6ksOT/Mjo8bQMofk50ywOAABmbdIpFg9O8ozW2rO6bZ9MckZVJcnvJ/ndKdUGAAAzN+kI8qYk719k378l2X/PygEAgJU1aUB+Z5J7LrLvnknO3bNyAABgZU06xeK5SV5TVTdI8sYkX0zyE0l+Pcl9khxfVYfNHdxa++S0CgUAgFmYNCBfMHp8dJLf7rZvmLd/zj4BAIBVZNKAfNKyVAEAAHuJiQJya+2s5SoEAAD2BpNepAcAAGvaxLearqpjkzwwyUFJrjdv967W2l2mURgAAKyEiUaQq+pJSf45yf2S3CDJd+Z97Jx2gQAAMEuTjiCfkuSlSU5prX1nGeoBAIAVNekc5BsmeaNwDADAWjVpQD4nyVHLUQgAAOwNdmeKxT9W1a4kb09y5fwD3D0PAIDVbNKAvCvJ15OckeSZixzj7nkAAKxakwbkVyf5xSRnJvnPJFdPuyAAAFhJkwbkuyV5TGvt1ctQCwAArLhJL9L7UpIvLkchAACwN5g0IP9lkt+pKreoBgBgTZp0isWNkmxO8tGqekd+cBWLXa21p0+lMgAAWAGTBuSndJ/faoH9u5IIyAAArFoTBeTWmqkVAACsaQIvAAB0lhxBrqrvJLlTa+39VbUzwzSKxexqrU06bQMAAPYa44TZrUk+031+XQEZAABWtSUDcmvtGd3npy1rNQAAsMLMQQYAgI6ADAAAHQEZAAA6AjIAAHQEZAAA6AjIAADQEZABAKAjIAMAQEdABgCAjoAMAAAdARkAADoCMgAAdARkAADoCMgAANARkAEAoLPvrE9YVQclOTPJPZJsSPLOJI9rrV0+RtvrJTk9yfFJDkjyoSRPbq29+zraPDjJ3yb5bGvtp/a0fgAA1raZjiBX1fWTnJvk1klOTHJCklsmOa+qbjDGU7wyyclJnpbkfkk+n+Scqrr9Iuc7IMnzknxhD0sHAGCdmPUI8slJDktSrbVLk6SqPpLk40keleS5izWsqtsleWiSR7TWXjXadkGS7Um2JjlugWZ/muTDGYL03afXDQAA1qpZz0E+LsmFc+E4SVprlyV5b5IHjNH2miRv6Npem+T1Se5VVZv6g6vq6AxTMR4zndIBAFgPZh2Qj0yybYHt25McMUbby1pr31yg7f5JDp/bUFX7JXlZkuf0YRwAAJYy6ykWBya5coHtVyS50R60nds/58lJNiV59qQFztm2baEcn2zZsmV3n3KvdNFFF4197Hrue6L/a6n/67nvif573Ruf/q/f/q/nvicrsIrFcquqw5M8Jcmvtta+vbvPs3nz5mzatGnpA1e5tfYDMIn13Pdkffd/Pfc90f/13P/13PdE/9dz/xfq+44dOxYdEJ11QL4yC48ULzY6PL/twYu0Tb43kvyXGVbKuHC0ikUyTMHYMPp6R2vtWxPUDADAOjLrOcjbM8wlnu+IJB8do+2ho6Xi5re9Osml3df3yRCo5z4ekuRmo893e9oFAABr36wD8puTHFVVh81tqKpDkhw92ndd3pJkvyQP7Nrum+RBSd7eWtsx2vzgJHeb93FOki+PPn/BNDoCAMDaNOspFi9PckqSs6vq1CS7MtwZ79NJXjp3UFUdnOQTSba21rYmSWvtg1X1hiTPG61ScVmSRyc5NMnD5tq21i6cf9Kq+o0MUyvOX55uAQCwVsx0BLm1dlWSY5JckuQ1SV6XIege01r7RnfohiT7LFDfSUleleSZSf4pyUFJjm2tXbzMpQMAsE7MfBWL1trlSX5tiWM+lSEkz9/+rSSPH31Mcs7fmOR4AADWr1nPQQYAgL2agAwAAB0BGQAAOgIyAAB0BGQAAOgIyAAA0BGQAQCgIyADAEBHQAYAgI6ADAAAHQEZAAA6AjIAAHQEZAAA6AjIAADQEZABAKAjIAMAQEdABgCAjoAMAAAdARkAADoCMgAAdARkAADoCMgAANARkAEAoCMgAwBAR0AGAICOgAwAAB0BGQAAOgIyAAB0BGQAAOgIyAAA0BGQAQCgIyADAEBHQAYAgI6ADAAAHQEZAAA6AjIAAHQEZAAA6AjIAADQEZABAKAjIAMAQEdABgCAjoAMAAAdARkAADoCMgAAdARkAADoCMgAANARkAEAoCMgAwBAR0AGAICOgAwAAB0BGQAAOgIyAAB0BGQAAOgIyAAA0BGQAQCgIyADAEBHQAYAgI6ADAAAHQEZAAA6AjIAAHQEZAAA6AjIAADQEZABAKAjIAMAQEdABgCAjoAMAAAdARkAADoCMgAAdARkAADoCMgAANARkAEAoCMgAwBAR0AGAICOgAwAAB0BGQAAOgIyAAB0BGQAAOgIyAAA0BGQAQCgIyADAEBHQAYAgI6ADAAAHQEZAAA6AjIAAHQEZAAA6AjIAADQEZABAKAjIAMAQEdABgCAjoAMAAAdARkAADoCMgAAdARkAADoCMgAANARkAEAoCMgAwBAR0AGAICOgAwAAB0BGQAAOgIyAAB0BGQAAOgIyAAA0BGQAQCgIyADAEBHQAYAgI6ADAAAHQEZAAA6AjIAAHQEZAAA6AjIAADQEZABAKAjIAMAQEdABgCAjoAMAAAdARkAADoCMgAAdARkAADo7DvrE1bVQUnOTHKPJBuSvDPJ41prl4/R9npJTk9yfJIDknwoyZNba+/ujrlVksckuVuSw5J8PckHkjy1tfbhafYFAIC1Z6YjyFV1/STnJrl1khOTnJDklknOq6objPEUr0xycpKnJblfks8nOaeqbt8dc88M4fisJPdP8jtJbpLkwqraMp2eAACwVs16BPnkDKO61Vq7NEmq6iNJPp7kUUmeu1jDqrpdkocmeURr7VWjbRck2Z5ka5LjRoe+PskLW2u7urbnJvlUkt9L8vDpdgkAgLVk1nOQj0ty4Vw4TpLW2mVJ3pvkAWO0vSbJG7q212YIxPeqqk2jbV/uw/Fo29eSXJLk5tPoBAAAa9esA/KRSbYtsH17kiPGaHtZa+2bC7TdP8nhizWsqgOTbE7ysfFLBQBgPZr1FIsDk1y5wPYrktxoD9rO7V/M8zNcEPi8Jc7xXdu2LZTjky1b1tY05osuumjsY9dz3xP9X0v9X899T/Tf69749H/99n899z1ZgVUsZq2q/jDD3OVH9lM7lrJ58+Zs2rRp+QrbS6y1H4BJrOe+J+u7/+u574n+r+f+r+e+J/q/nvu/UN937Nix6IDorKdYXJmFR4oXGx0et23yvZHk76qq307yrCSnttb+aoI6AQBYp2YdkLdnmEs83xFJPjpG20NHS8XNb3t1ku8bHa6qE5K8KMmft9bO2L1yAQBYb2YdkN+c5KiqOmxuQ1UdkuTo0b7r8pYk+yV5YNd23yQPSvL21tqObvuvJnlVkle01p4wteoBAFjzZj0H+eVJTklydlWdmmRXhjvjfTrJS+cOqqqDk3wiydbW2tYkaa19sKrekOR5VbVfksuSPDrJoUke1rX95SR/m+TDSV5dVUd159/RWvvgMvYPAIBVbqYBubV2VVUdk+FW06/JsLLEuzLcavob3aEbkuyTHxzhPinJGUmemeFW0x9Ocmxr7eLumGOSbErycxnWV+79V5JDptEXAADWppmvYtFauzzJry1xzKcyhOT527+V5PGjj8XanpbktD2pEQCA9WvWc5ABAGCvJiADAEBHQAYAgI6ADAAAHQEZAAA6AjIAAHQEZAAA6AjIAADQEZABAKAjIAMAQEdABgCAjoAMAAAdARkAADoCMgAAdARkAADoCMgAANARkAEAoCMgAwBAR0AGAICOgAwAAB0BGQAAOgIyAAB0BGQAAOgIyAAA0BGQAQCgIyADAEBHQAYAgI6ADAAAHQEZAAA6AjIAAHQEZAAA6AjIAADQEZABAKAjIAMAQEdABgCAjoAMAAAdARkAADoCMgAAdARkAADoCMgAANARkAEAoCMgAwBAR0AGAICOgAwAAB0BGQAAOgIyAAB0BGQAAOgIyAAA0BGQAQCgIyADAEBHQAYAgI6ADAAAHQEZAAA6AjIAAHQEZAAA6AjIAADQEZABAKAjIAMAQEdABgCAjoAMAAAdARkAADoCMgAAdARkAADoCMgAANARkAEAoCMgAwBAR0AGAICOgAwAAB0BGQAAOgIyAAB0BGQAAOgIyAAA0BGQAQCgIyADAEBHQAYAgI6ADAAAHQEZAAA6AjIAAHQEZAAA6AjIAADQEZABAKAjIAMAQEdABgCAjoAMAAAdARkAADoCMgAAdARkAADoCMgAANARkAEAoCMgAwBAR0AGAICOgAwAAB0BGQAAOgIyAAB0BGQAAOgIyAAA0BGQAQCgIyADAEBHQAYAgI6ADAAAHQEZAAA6AjIAAHQEZAAA6AjIAADQEZABAKAjIAMAQEdABgCAjoAMAAAdARkAADoCMgAAdARkAADoCMgAANARkAEAoCMgAwBAR0AGAICOgAwAAB0BGQAAOvvO+oRVdVCSM5PcI8mGJO9M8rjW2uVjtL1ektOTHJ/kgCQfSvLk1tq75x23McmTkzwqyU8maUm2ttb+fmodAQBgTZrpCHJVXT/JuUluneTEJCckuWWS86rqBmM8xSuTnJzkaUnul+TzSc6pqtvPO+70JKcleUGSeye5MMkbq+o+e94LAADWslmPIJ+c5LAk1Vq7NEmq6iNJPp5htPe5izWsqtsleWiSR7TWXjXadkGS7Um2JjlutO3HkzwhyR+31v5s1Py8qjo8yR8n+edl6BcAAGvErOcgH5fkwrlwnCSttcuSvDfJA8Zoe02SN3Rtr03y+iT3qqpNo833SrJ/ktfOa//aJLetqkP3qAcAAKxpsx5BPjLJ2Qts357kgWO0vay19s0F2u6f5PDR50cm2ZHk0gWOS5Ijklx2HefZJ0muvvrqRQ/YtWvXEqWuDjt27Ji4zXrue6L/a6H/67nvif573Zuc/q/f/q/1vndZb5/5+2YdkA9McuUC269IcqM9aDu3f+7xq621+d/V+cct5qZJcskllyxx2Oq3bdu2lS5hxaznvifru//rue+J/q/n/q/nvif6v577P0bfb5rkE/2Gma9isQp8IMmdM1wA+J0VrgUAgOWxT4Zw/IH5O2YdkK/MwiPFi40Oz2978CJtk++NEF+Z5ICq2jBvFHn+cQvasmXLjiTvWaIWAABWv08stHHWF+nNzRGe74gkHx2j7aGjpeLmt70635tzvD3JpiS3WOC4jHEeAADWsVkH5DcnOaqqDpvbUFWHJDl6tO+6vCXJfuku5quqfZM8KMnbW2tzM7D/JcNqFw+b1/74JNtGq2YAAMCCNszyCsXRzUA+nORbSU5NsivDTT1+JMnPtNa+MTru4AxD3ltba1u79q/PsIzbEzOsRPHoDDcM+cXW2sXdcX+c5HFJ/ijJxRlC9KOSHNdae+vy9hIAgNVspnOQW2tXVdUxGW41/ZoMt5p+V4ZbTX+jO3RDhonT80e4T0pyRpJnZrjV9IeTHNuH45GnJPlGkt/L9241/evCMQAAS5npCDIAAOztLPO2Aqrqp5I8OcnPJ7ldkh9Kcmhr7VPzjlvsr5efba19aDlrXC7j9L2qTkvy9EWeYkdr7XrLXOayGfd7P6/NHyR5dpL3ttZ+aRZ1Loeq+t9JHpKh7z+e5PIk/5DkWa21ry/S5iUZpke9rrV2/KxqXQ7j9L+qtmR4l+y2SW6c5KsZpomd3lp73wqUPRVj9v1XMrxLeKckN0vyuSRvT/L01tp/r0Td0zLBa/6hSZ6T5O4Zrrl5f5Inttb+faYFT9G4P/dVdWSGKZdHJfnRJJ9K8qokzxvdNXdVmqD/N8rwvf9fGf5/vC/J77fW/mPWNU/TJK/7VXVUktMy/B/YL8knk5zRWnv9LGueM+uL9BgcnuTXMyxJ969LHPvqDL8w+o/VfBeTcfr+ivxgn++e5NosfTHn3m6S731GF7SemmRVB4SRJ2RYW/yPkhyb5MUZriN4R1X9wGtRVR2d4eLa/5llkctonP4fkGFFnv+T4XqLx462XVBVd5xxvdM0Tt9/O8MfBc8cHfPsJMclubCqfnjmFU/Xkj/3VXXjDEuMbs7wR+GDR7vOq6rbzKLIZbLk976qbpbk/CSHZbh+6P5J3pTkTzP8wbiajdP/DRkWIjg2w8/8r2UIiOeN/rhazcZ63a+q+yZ5d5IvJHlokgckeXmSFRsQM4K8Mt7dWvuJJKmq30xyz+s49rOttQtnU9ZMLNn31tpnknym31ZVJ2T4/3rWLIpcRpN875PhxeR1SSqr/+f1/q21L3VfX1BVV2T4nt41yblzO6pqvyQvzfDL8VGzLHIZLdn/1tq7MlyX8V1V9S9JvpzkhAwjiqvRON/731ngmEuSXJAhXP7VrIpdBuP83D86yU8k+eXW2idGx56bYRTtGRn+DVajcb7390vyY0mObq3NDQCdW1W3SPLwDKPvq9U4/T8uw2pex7TWzkuSqnpfhsUInpTkd2da8XQt2f+q+pEM7xa8qLX2uO7Yd86sygUYQV4BrbWdK13DStmDvp+Y5ItJzpliOTM3Sf+r6qFJfi7JHy5fRbMz70Vyztzdi24+b/sTM1yo+2fLWtQMTdj/3lVJdmR4B2VVGqfve/Dvs9cb8+f+qCQfnwvHo3ZXZRhxvt9oWdNVZ8zv6/6jx/nvFn01qzynjNn/45J8bi4cj9p9LcOo8gOWt8LlNWb/H5jkJkn+fCZFjWlV/sCtM4+uqidmeIviwgzz8ZZ8a34tqaqDktwtq3wu2iRG89HOTPKk1toVVbXSJS2Xu4wePza3oaoOzzCt5L6ttWvWcN+TBfqfJKO3HudugfoHo80vn2Fds7Bg33fjmLXiOxluejXfjgxzUm+RYUWmtWD+9/WNGa47ecHo991XkvxKhndNnjH78pbd/P4fmWTbAsdtT/LwqvrheSt9rXbz+/9LGe5yfNuq+uckt0ny+QzTLZ/ZWvvO7Etc5X+ZrQOvTfI7Gebf/laG+XnnVtVdV7KoFXB8hv+rq316xSSek2Gu+atXuI5lU1U3T7I1yTvnXYT04iT/0I+mrEXX0f8k+b8ZwtJ/ZZiPeJ/W2pq5C+gSfZ875keSPC/DL9E3zay4ldOS3HI0FznJd/9Qmpt7fuCKVDVlC33vW2tfzHCtyW0yTCn5WpK/T/InrbU/Xalal8Mi//cPzDA/fb4rRo83mkVts7BI/2+W5PpJ/ibD77y7Z/h9/9Ss4LuIRpD3Yq21E7ov/7Wqzs7wV+YzM/zFtV48PMkHW2sfWelCZqGq7pyhzz/XWluT6zCOLro6O8O0gZO67ccnuUOGOddr1mL97zwpyZ8kOSjJY5K8taruvppXM5gzRt/n7pL6txnegj16nbxz9JIMc03/uqp+N8k3M6zpf+ho/6qfmncdP/c3ybCywVVJ/neGEeRjkpxaVTtaa3+yAuVO3Tj/99ey6+j/xgwX4z2ltfbc0bbzR38sPqaqThtNOZkpAXkVaa19var+KckjV7qWWRlduX/rDFc2rxcvTfLKJJ+pqgNG2/ZNss/o6291t1ZfdarqhzLMrTssyV1GF2XOvXg+N0Mw3NH1fWOS/UZfX9Vau2bmRU/RYv3vtdY+mWEk7QNV9dZ87w/jY2dZ67SN0/fRqOlZGUaR7rte/jBurX2yqh6W5IUZVjJJhiX+zsywEsDnV6q2aVjie/+kJIckObi1NjeSen5V7ZPk9Kp6ZWvtyzMteMqW6P+VWXiU+MBu/6q2RP+/Mnp8x7xmb8+wus2RSf7fshc5jykWq9OaHFVcxIlJrsnw1st6cZsMLwpXdh9HZ7iI58oMV7uvSqPVKf4uw5qY95m3xuePZbhQ41n5/r4flO8tkXXfmRY8ZUv0f0GttauTfCTDUmGr1gR9f0mSByV58GhVj3Wjtfb3GUbNj0hyeGttS5IfTvLp1trlK1rcHhjje3/bJJd24XjO+zMsd7bW/+9vzxAC5zsiyeWrff7xmP2/Livy7okR5FWkqm6YYTmc1brU00Sqav8Ma4G+bZErYdequy2w7XkZLtp6bL43urSqjEYGX5fhrdP7LbB84ReycN9fn+Q/Miz5ttCFLKvCGP1frN31M/xiWbUXaI3b96r68yS/meTE1tqbZlfh3mN0QdLHku+uD/ygDNckrEpjfu+/kOQXq+pG80LyL4weP7vMZS6bMfv/5iQnVdVdWmsXjNrdMMN60Kt6cGjM/r8pw01i7pXhtX7OsUm+nRV63ReQV8jo7jJJsmX0eO+q+lKSL7XWLqiqJ2SYh3lehjtKHZzhbbafTPKwWdc7TUv1vTv0fhneYlpTF+ct1f/W2vkLtPlqkn0X2reKvDDDcj5nJLlqdNekOZ8ZveV2/vxGVfXtJF9c5X1Pxuh/Vb00w4U5/55h7eODk5ySYTWLE7J6jdP3Jyd5fIb1jj8+75gv9cufrUZjvObvl+HGGBdkWO7syAxLPG7PXrb81YTG+bl/SYbfa2+vqudkeMv9rhl+5/1ja+3Tsy15qsbp/5sz3DnvtaNVPK7M8L3fkOH/xGq2ZP9ba9uq6tVJto4C9cUZplj9Zoa7iK7ICLqAvHLeOO/rF40eL8jwwtCS/Oro40czvGC+N8kjW2urfQR5qb7POTFDWHjrDGqapXH7v9bce/T4lNFH7xkZbjG6lo3T/3/L8Evht5LcIMPI2b9l+LlfzbecHafvc8c8YvTROyvJbyxTbbOy1M/9riS3zHAXsQMy3CzprzLckneh5d9WiyW/9621C0cXJz8tyV8kuWGGW01vzer+4yAZr/87q+p+GVZseFGGC9bel+Ruq/yPg2T81/1HZXi9e2yGG+Z8KsnjW2t/sfwlLmzDrl3raTorAABcNxfpAQBAR0AGAICOgAwAAB0BGQAAOgIyAAB0BGQAAOhYBxlgL1ZVj8hw04CDk3yztXbAFJ/7kAzrC/91a+2T03pegNXOCDLAXmp0q+GXJfl/GW7Vevcpn+KQJE9PctiUnxdgVTOCDLD3umWSfZKc1Vp7z0oXM66q2tRa27HSdQDsLnfSA9gLVdWrM9xuvXdWa+03quq3kjwmSSX5RpKzkzyxtXZF1/6UJA8bHbMxyX8mOb219k+j/XdNct4Cp75ba+38qtqV5BmttdO65zwkyWVJTmqtvbqr8+5JHpjhtsA/m+RlrbXfq6qbJDk9yf2T/Nio7XNbay/brX8UgBkxxQJg73R6kt8dff6YJHdKcnpV/XGSFyZ5Z5LjkjwxybFJ3lZV+3TtD0nyigzB9UFJ/j3JW6vq2NH+i0fPm9F57jT6uHg3av3RJK9P8rdJ7p3kb6rqhknek+Q+SU5Lct8kb0ny4qp67G6cA2BmTLEA2Au11j5RVR8bffnR1tqFoxHcJ2YY2d06d2xVXZIhjN4/yZtG7Z/Q7d+Y5F1JbpXk0Un+pbX2P1X10dEhH2utXbgH5f5wkuNba2d353xqhgsLb9ta+/ho8zur6oAkT6+qF7fWrt2DcwIsGwEZYPW4R4Z3/l5XVf3r978l+XqSX84oIFfVliTPSHKHJDdJsmF0bFuGuq5J8tZ5244d1XXZvFrPSfKbSY5I8pFlqAVgjwnIAKvHj48eL11k/42TpKoOyjBi/NEkj01yeZJrM0zbuM0y1PWl1tp35m378SSHZwjPi9YKsDcSkAFWj6+MHu+Z5Mrr2H9shnnBv95a+8zczqq6/gTn2pFk/3nbFgu1C13t/ZUk/53k9xZpsxwj2QBTISADrB7vSLIzyU+31t5xHcfNBeHvjt5W1a2SHJ3kM91xc0ux/dACz/FfSTbP23bfCWr9l4xGr1tr/z1BO4AVJyADrBKjC/f+JMkLqqqSXJDk20kOyjA/+RWttfMyrHBxbZK/rqo/T3LTDPORL8/3r150yei4R1TVFRkCc2utfT3DqhSnVtVTklyY5M5JHjJBuWdmWD3jX6vqzAwjxjdIcuskd26tPWB3/g0AZsEybwCrSGvtj5L8VoYL8v5vhjWQn5xhysXHR8dsz7AG8sFJ3pzkSUn+IMm75z3XV5KckuR2GcL2B5JsGe1+dpIXjPa/KcPc5RMmqPNrSX4xyT+P6jsnyV8leUAWXn8ZYK/hRiEAANAxggwAAB0BGQAAOgIyAAB0BGQAAOgIyAAA0BGQAQCgIyADAEBHQAYAgI6ADAAAnf8Pjro4n5ou2VMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 720x720 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Get feature importance from random forest model\n",
    "importances, _ = extract_feature_importance(rf)\n",
    "\n",
    "# Rank features according to their importance\n",
    "ranked_features_id, ranked_features_name, ranked_importances = rank_feature_importance(importances, feature_names, n=10)\n",
    "\n",
    "# Plot feature importance\n",
    "plot_feature_importance(ranked_features_id, ranked_importances)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}