mainPipeline_step1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import sys\n",
    "from pathlib import Path\n",
    "import os\n",
    "import importlib\n",
    "import warnings\n",
    "\n",
    "warnings.simplefilter(action='ignore', category=FutureWarning)\n",
    "pd.set_option('mode.chained_assignment',  None) \n",
    "\n",
    "module_path='preprocessing/day_intervals_preproc'\n",
    "if module_path not in sys.path:\n",
    "    sys.path.append(module_path)\n",
    "    \n",
    "module_path='utils'\n",
    "if module_path not in sys.path:\n",
    "    sys.path.append(module_path)\n",
    "    \n",
    "module_path='preprocessing/hosp_module_preproc'\n",
    "if module_path not in sys.path:\n",
    "    sys.path.append(module_path)\n",
    "\n",
    "import cohort_selection\n",
    "from cohort_selection import *\n",
    "\n",
    "import feature_selection_eicu\n",
    "from feature_selection_eicu import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1. DATA EXTRACTION\n",
    "# The cohort will be svaed in **./data/cohort/**\n",
    "version = 'Version 2.0'\n",
    "root_dir = \"/Users/DAHS/Desktop/early_prediction_of_circ_scl\"\n",
    "\n",
    "version_path=\"eicu-crd/2.0\"\n",
    "version=='Version 2.0'\n",
    "local = \"/Users/DAHS/Desktop/early_prediction_of_circ_scl\"+\"/eicu-crd/preprocessing_data\"\n",
    "\n",
    "data_icu=True\n",
    "data_mort=True\n",
    "data_admn=False\n",
    "data_los=False\n",
    "\n",
    "# cohort_output = cohort_selection.extract_data(root_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "cohort_output = 'cohort_'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[EXTRACTING INGREDIENTS DATA]\n",
      "# of unique type of drug:  4799929\n",
      "# Admissions:   73435\n",
      "# Total rows 4799929\n",
      "[SUCCESSFULLY SAVED INGREDIENTS DATA]\n"
     ]
    }
   ],
   "source": [
    "### 2. FEATURE SELECTION 1\n",
    "\n",
    "# when you try first, replace False to True\n",
    "\n",
    "diag_flag  =  False\n",
    "out_flag   =  False\n",
    "chart_flag =  False\n",
    "proc_flag  =  False\n",
    "med_flag   =  False\n",
    "ing_flag   =  True\n",
    "lab_flag   =  False\n",
    "pe_flag   =  False\n",
    "microlab_flag = False\n",
    "vent_flag = False\n",
    "\n",
    "feature_icu(root_dir, cohort_output, version_path, diag_flag, out_flag,\n",
    "            chart_flag, proc_flag, med_flag, ing_flag, lab_flag, pe_flag, microlab_flag, vent_flag)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[FEATURE SELECTION INGREDIENTS DATA]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\DAHS\\Desktop\\eICU2.0_PREPROC\\preprocessing/hosp_module_preproc\\feature_selection_eicu.py:122: DtypeWarning: Columns (7) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  ing = pd.read_csv(local+'/'+\"features/preproc_ing_icu.csv.gz\", compression='gzip',header=0)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of rows 1407798\n",
      "[SUCCESSFULLY SAVED INGREDIENTS DATA]\n",
      "[FEATURE SELECTION PROCEDURES DATA]\n",
      "Total number of rows 256667\n",
      "[SUCCESSFULLY SAVED PROCEDURES DATA]\n"
     ]
    }
   ],
   "source": [
    "### 2. FEATURE SELECTION 2 \n",
    "\n",
    "select_diag=False\n",
    "select_med=False\n",
    "select_ing=True\n",
    "select_proc=True\n",
    "select_lab=False\n",
    "select_out=False\n",
    "select_chart=False\n",
    "select_microlab=False\n",
    "\n",
    "diag_flag  =  False\n",
    "out_flag   =  False\n",
    "chart_flag =  False\n",
    "proc_flag  =  True\n",
    "med_flag   =  False\n",
    "ing_flag   =  True\n",
    "lab_flag   =  False\n",
    "microlab_flag = False\n",
    "\n",
    "features_selection_icu(local, cohort_output,\n",
    "                       diag_flag, proc_flag, med_flag, ing_flag, out_flag, lab_flag,\n",
    "                       chart_flag, microlab_flag, select_diag, select_med, select_ing, select_proc,\n",
    "                       select_out, select_chart, select_lab, select_microlab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[PROCESSING LABS DATA]\n",
      "Total number of rows 17944364\n",
      "[SUCCESSFULLY SAVED LABS DATA]\n"
     ]
    }
   ],
   "source": [
    "### 4. CLEANING OF FEATURES\n",
    "# Below you will have option to to clean lab and chart events by performing outlier removal and unit conversion.\n",
    "\n",
    "# Outlier removal is performed to remove values higher than selected **right threshold** percentile and lower than selected **left threshold** percentile among all values for each itemid. \n",
    "\n",
    "# **Please run below cell to select preprocessing for diferent features**\n",
    "\n",
    "thresh=98\n",
    "clean_chart=False\n",
    "chart_flag =  False\n",
    "impute_outlier_chart=False\n",
    "left_thresh= 2\n",
    "\n",
    "imput_outlier_lab=False\n",
    "thresh_lab=98\n",
    "left_thresh_lab=2\n",
    "clean_labs=True\n",
    "lab_flag   =  True\n",
    "\n",
    "diag_flag = False\n",
    "group_diag = 'Convert ICD-9 to ICD-10 codes'\n",
    "        \n",
    "preprocess_feature_icu(chart_flag, clean_chart, impute_outlier_chart,thresh,left_thresh,\n",
    "                        lab_flag, imput_outlier_lab, thresh_lab, left_thresh_lab, clean_labs)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "MIMIC",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}