Cory-trial-scaffold.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0ec00416",
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import os\n",
    "import math\n",
    "import numpy as np\n",
    "import cv2\n",
    "import pandas as pd\n",
    "import pyautogui as pa\n",
    "\n",
    "from lib.auto_GUI.auto_DAT import AutoDAT\n",
    "from lib.auto_GUI.auto_PhotoZ import AutoPhotoZ\n",
    "\n",
    "from lib.file.ROI_reader import ROIFileReader\n",
    "from lib.analysis.laminar_dist import LaminarROI, Line\n",
    "from lib.analysis.align import ImageAlign\n",
    "from lib.utilities import *\n",
    "\n",
    "####################################################################################\n",
    "#    Scaffold for Cory's automatic data export\n",
    "#  Input:\n",
    "#     ZDA files, each with a corresponding ROI dat file\n",
    "#     A specified PhotoZ .pre file to be used for all ZDA\n",
    "#  Output:\n",
    "#     Half-amp latency, half-width, amp, SNR for every ROI/trial \n",
    "####################################################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5b23cb4f",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Fill all these variables to specify your input directory / format\n",
    "data_dir = \"C:/Users/jjudge3/Desktop/Data/mm_full_pipeline_targets/Kate's/hw_and_amp_scatter/\" # end this in a /\n",
    "photoZ_preference_file = None\n",
    "if photoZ_preference_file is None:\n",
    "    raise Exception(\"Please provide full path to photoZ preference file in the variable photoZ_preference_file\")\n",
    "\n",
    "n_trials_per_recording = 5\n",
    "roi_file_keyword = 'ROI'\n",
    "enable_photoZ_interaction = True\n",
    "initialize_photoZ = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f78e4f4d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# set up PhotoZ (open it manually)\n",
    "if enable_photoZ_interaction:\n",
    "    aPhz = AutoPhotoZ(data_dir=data_dir,\n",
    "                      pre_file=photoZ_preference_file)\n",
    "    if initialize_photoZ:\n",
    "        aPhz.prepare_photoZ()\n",
    "pa.alert(\"Load your ROIs before continuing.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f3a5126f",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "aDAT = AutoDAT()\n",
    "created_file_map = {}\n",
    "for subdir, dirs, files in os.walk(data_dir):\n",
    "    \n",
    "    # locate proper files in this subdir\n",
    "    zda_file, roi_filepath, file_prefix, zda_filepath = None, None, None, None\n",
    "    for file in files:\n",
    "        if file.endswith('.zda'):\n",
    "            zda_file = file\n",
    "            zda_filepath = subdir + \"/\" + zda_file\n",
    "            file_prefix = zda_file[:-4]\n",
    "\n",
    "        # parse out the rec no. of the zda file\n",
    "        rec_id = int(zda_file.split('_')[-1][:2])\n",
    "\n",
    "        # look for a corresponding ROI file containing the keyword, r<rec_id>, and ending in .dat\n",
    "        roi_filepath = None\n",
    "        for f in files:\n",
    "            if f.endswith('.dat') and roi_file_keyword in f and ('r' + str(rec_id)) in f:\n",
    "                roi_filepath = subdir + \"/\" + f\n",
    "\n",
    "        # filename validation\n",
    "        if any([x is None for x in [zda_file, file_prefix, zda_filepath]]):\n",
    "            print(\"File names not found or parsed properly. Check\")\n",
    "            print(\"zda_file, roi_filepath, file_prefix, zda_filepath\")\n",
    "            print(zda_file, roi_filepath, file_prefix, zda_filepath)\n",
    "            continue\n",
    "        \n",
    "        # interact with photoZ\n",
    "        if enable_photoZ_interaction:\n",
    "            aPhz = AutoPhotoZ(data_dir=subdir)\n",
    "            # open the PhotoZ file\n",
    "            aPhz.select_PhotoZ()\n",
    "\n",
    "            print(\"\\n\\nOpening\", zda_file)\n",
    "            aPhz.open_zda_file(zda_filepath)\n",
    "\n",
    "            # Open the ROI file for this recording\n",
    "            if roi_filepath is not None:\n",
    "                print(\"Opening new ROI file\", roi_filepath, \"for\", zda_file)\n",
    "                aPhz.select_roi_tab()\n",
    "                aPhz.open_roi_file(roi_filepath)\n",
    "            else:\n",
    "                print(\"No ROI file found for\", zda_file, 'Continuing with last ROI file')\n",
    "\n",
    "            # organized by subdir. Presumably you could replace this with date\n",
    "            #  (e.g. parse the date out of your subdir name)\n",
    "            created_file_map[subdir] = {}\n",
    "\n",
    "            for i_trial in range(1, n_trials_per_recording + 1):\n",
    "\n",
    "                aDAT.increment_trial()\n",
    "\n",
    "                # save SNR values from PhotoZ\n",
    "                snr_filename = subdir + \"/\" + file_prefix + \"SNR.dat\"\n",
    "                if enable_photoZ_interaction:\n",
    "                    aPhz.select_SNR_trace_value()\n",
    "                    aPhz.save_trace_values(snr_filename)\n",
    "\n",
    "                # save half-width values from PhotoZ\n",
    "                hw_filename = subdir + \"/\" + file_prefix + \"hw.dat\" \n",
    "                if enable_photoZ_interaction:\n",
    "                    aPhz.select_half_width_trace_value()\n",
    "                    aPhz.save_trace_values(hw_filename)\n",
    "\n",
    "                # save amp values from PhotoZ\n",
    "                amp_filename = subdir + \"/\" + file_prefix + \"amp.dat\"\n",
    "                if enable_photoZ_interaction:\n",
    "                    aPhz.select_maxamp_trace_value()\n",
    "                    aPhz.save_trace_values(amp_filename)\n",
    "\n",
    "                # save latency values from PhotoZ\n",
    "                latency_filename = subdir + \"/\" + file_prefix + \"latency.dat\"\n",
    "                if enable_photoZ_interaction:\n",
    "                    aPhz.select_latency_trace_value()\n",
    "                    aPhz.save_trace_values(latency_filename)\n",
    "\n",
    "                created_file_map[subdir][i_trial] = [\n",
    "                    snr_filename,\n",
    "                    hw_filename,\n",
    "                    amp_filename,\n",
    "                    latency_filename\n",
    "                ]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7145bca7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# read in the created files and store the data in a DataFrame\n",
    "def read_dat_trace_file(filename):\n",
    "    data_arr = pd.read_csv(filename,\n",
    "                                 sep='\\t',\n",
    "                                 header=None,\n",
    "                                 names=['ROI',  'Value'])\n",
    "    return data_arr\n",
    "    \n",
    "file_data = []\n",
    "for subdir in created_file_map:\n",
    "    for i_trial in created_file_map:\n",
    "        \n",
    "        snr_fn, hw_fn, amp_fn, latency_fn = created_file_map[subdir][i_trial]\n",
    "\n",
    "        amp = read_dat_trace_file(amp_fn)\n",
    "        hw = read_dat_trace_file(hw_fn)\n",
    "        df = read_dat_trace_file(snr_fn)\n",
    "        lat = read_dat_trace_file(latency_fun)\n",
    "\n",
    "        total_before_filter += len(amp['Value'])\n",
    "\n",
    "        df = df.join(amp, rsuffix='_Amp')\n",
    "        df = df.join(hw, rsuffix='_hw')\n",
    "        df = df.join(lat, rsuffix='_latency')\n",
    "        \n",
    "        tr_col = [i_trial for _ in range(len(df['ROI']))]\n",
    "        subdir_col = [subdir for _ in range(len(df['ROI']))]\n",
    "\n",
    "        df.insert(4, 'Trial', tr_col)\n",
    "        df.insert(5, 'Subdir/Date ID', subdir_col)\n",
    "\n",
    "        file_data.append(df)\n",
    "\n",
    "data_by_trial = pd.concat(file_data)\n",
    "\n",
    "# Drop some unneeded columns\n",
    "data_by_trial = data_by_trial.drop(columns=['Index_Amp',\n",
    "                                            'Index_hw',\n",
    "                                            'Index_latency'])\n",
    "# rename some columns\n",
    "data_by_trial = data_by_trial.rename(columns=\n",
    "                                    {'Value': 'SNR',\n",
    "                                     'Value_Amp': 'MaxAmp',\n",
    "                                     'Value_hw': 'HalfWidth',\n",
    "                                     'Value_latency': 'Latency'})\n",
    "\n",
    "data_by_trial"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ef366ed0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# save dataframe to a CSV file\n",
    "csv_filename = \"data_by_trial.csv\"\n",
    "df.to_csv(data_dir + csv_filename, index=False)\n",
    "print(\"Wrote data to file\", csv_filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5da88afa",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}