Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Sentence rep #60

Open
wants to merge 6 commits into
base: Global_local
Choose a base branch
from
Open

Sentence rep #60

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
847b8cd
added 'bad' option to BIDS_coding
Aaronearlerichardson May 24, 2024
98b296d
fixed by adding 'headerdata' to config usage
Aaronearlerichardson May 24, 2024
eadcadc
fixed by adding 'headerdata' to config usage
Aaronearlerichardson May 24, 2024
a733dfe
Merge remote-tracking branch 'origin/Sentence_rep' into Sentence_rep
Aaronearlerichardson May 24, 2024
4db7f31
fixed by adding 'headerdata' to config usage again and fixed configs
Aaronearlerichardson May 24, 2024
c7023ec
sentencerep gocue fixed
Aaronearlerichardson May 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 10 additions & 11 deletions BIDS_converter/BIDS_convert_wsl.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
#!/usr/bin/env bash


ORIG_DATA_DIR="$HOME/Library/CloudStorage/Box-Box/CoganLab"
ORIG_DATA_DIR="$HOME/Box/CoganLab"
# OUTPUT_DIR="$ORIG_DATA_DIR/BIDS-1.1_GlobalLocal"
OUTPUT_DIR="$ORIG_DATA_DIR/BIDS-1.2_GlobalLocal"
TASKS=("GlobalLocal")
OUTPUT_DIR="$HOME/Workspace/SentenceRep"
TASKS=("SentenceRep")
#(D3 D5 D6 D7 D8 D9 D12 D14 D15 D16 D17 D18 D20 D22 D23 D24 D26 D27 D28 D29 D30 D31 D32 D53
# D57 D59 D60 D61 D65 D66 D69 D70 D71 D72 D73)
#D3 D5 D6 D7 D8 D9 D18 D20 D22 D23 D24 D26 D27 D28 D29 D30 D31 D32 D53
Expand All @@ -16,7 +16,7 @@ TASKS=("GlobalLocal")
for TASK in "${TASKS[@]}"
do
mapfile -t SUB_IDS < <(find "$ORIG_DATA_DIR/D_Data/$TASK" -maxdepth 1 -type d -name "D*" -exec basename {} \;)
SUB_IDS=(D57)
SUB_IDS=(D5 D7 D8 D9 D15 D16 D17 D18 D20 D22 D23 D24 D26 D27 D28 D29 D30 D31)
# OUTPUT_DIR="$ORIG_DATA_DIR/$TASK"
BIDS_DIR="$OUTPUT_DIR/BIDS"
ZIP=false
Expand All @@ -34,8 +34,7 @@ for TASK in "${TASKS[@]}"
# mkdir -p $BIDS_DIR
# mkdir -p "$OUTPUT_DIR/stimuli"
# shellcheck disable=SC2038
# find "$ORIG_DATA_DIR/task_stimuli" -iname "sentence_rep" -type d -exec echo "{}/." \; | xargs -I{} cp -afv {} "$OUTPUT_DIR/stimuli/"
TASKLOWER=$(echo $TASK | tr '[:upper:]' '[:lower:]')
find "$ORIG_DATA_DIR/task_stimuli" -iname $TASK -type d -exec echo "{}/." \; | xargs -I{} cp -afv {} "$OUTPUT_DIR/stimuli/"
#echo "$ORIG_DATA_DIR/task_stimuli/$TASKLOWER/."
#cp -av "$ORIG_DATA_DIR/task_stimuli/$TASKLOWER/." "$BIDS_DIR/stimuli/"

Expand Down Expand Up @@ -68,15 +67,15 @@ for TASK in "${TASKS[@]}"
rsync -P "$ORIG_DATA_DIR/ECoG_Task_Data/response_coding/PhonemeSequencingStimStarts.txt" "$OUTPUT_DIR/$SUB_ID/${SUB_ID}_PhonemeSequencingStimStarts.txt"
fi
#eeg files
if ! find "$ORIG_DATA_DIR/D_Data/$TASK/" -iregex ".*$SUB_ID.*\.edf" -exec false {} + ; then #search for edf files
find "$ORIG_DATA_DIR/D_Data/$TASK/" -iregex ".*$SUB_ID.*\.edf" -exec cp -v -t "$OUTPUT_DIR/$SUB_ID/" {} +
if ! find "$ORIG_DATA_DIR/D_Data/$TASK/" -iregex ".*$SUB_ID .*\.edf" -exec false {} + ; then #search for edf files
find "$ORIG_DATA_DIR/D_Data/$TASK/" -iregex ".*$SUB_ID .*\.edf" -exec cp -v -t "$OUTPUT_DIR/$SUB_ID/" {} +
find "$OUTPUT_DIR/$SUB_ID/" -regex ".*\.EDF" | xargs -I{} basename -s ".EDF" {} | xargs -I{} mv -v "$OUTPUT_DIR/$SUB_ID/{}.EDF" "$OUTPUT_DIR/$SUB_ID/{}.edf"
if $ZIP ; then
find "$ORIG_DATA_DIR/D_Data/$TASK/$SUB_ID" -regex ".*\.\(EDF\)\|\(edf\)" | xargs -I{} basename {} | xargs -I{} cut -f 1 -d '.' {} | xargs -I{} echo "$OUTPUT_DIR/$SUB_ID/{}.edf" | xargs -I{} gzip -6 -v {}
find "$ORIG_DATA_DIR/D_Data/$TASK/$SUB_ID " -regex ".*\.\(EDF\)\|\(edf\)" | xargs -I{} basename {} | xargs -I{} cut -f 1 -d '.' {} | xargs -I{} echo "$OUTPUT_DIR/$SUB_ID/{}.edf" | xargs -I{} gzip -6 -v {}
fi
else #if no edf files find binary files
x=( $(find "$ORIG_DATA_DIR/D_Data/$TASK/$SUB_ID" -regex ".*$SUB_ID.*\.ieeg.dat" -type f | rev | sed -r "s|/|_|" | sed -r "s|/|noisseS/|" | rev | xargs -n 1 basename | sed -r "s|${SUB_ID}_||") )
y=( $(find "$ORIG_DATA_DIR/D_Data/$TASK/$SUB_ID" -regex ".*$SUB_ID.*\.ieeg.dat") )
x=( $(find "$ORIG_DATA_DIR/D_Data/$TASK/$SUB_ID " -regex ".*$SUB_ID.*\.ieeg.dat" -type f | rev | sed -r "s|/|_|" | sed -r "s|/|noisseS/|" | rev | xargs -n 1 basename | sed -r "s|${SUB_ID}_||") )
y=( $(find "$ORIG_DATA_DIR/D_Data/$TASK/$SUB_ID " -regex ".*$SUB_ID.*\.ieeg.dat") )
z=${#x[@]}
if $ZIP ; then
for((i=0;i<=$z-1;i+=1)); do gzip -c -6 -v ${y[$i]} > "$OUTPUT_DIR/$SUB_ID/${SUB_ID}_${x[$i]}.gz" ; done
Expand Down
25 changes: 8 additions & 17 deletions BIDS_converter/config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
"binaryEncoding": "float32",
"headerData": {
"experiment.mat": "channels.name",
"default": "~/Library/CloudStorage/Box-Box/CoganLab/ECoG_Task_Data/Timestamps (MASTER).xlsx",
"default": "~/Box/CoganLab/ECoG_Task_Data/Timestamps (MASTER).xlsx",
"channels": {
"experiment.mat": "channels"
},
Expand Down Expand Up @@ -113,8 +113,8 @@
"content": ["[0-9]{2}"]
},
"acq": {
"left": "((Session)|(Part)|(part))",
"right": "[_ \\.]",
"left": "((Session)|(Part)|(part)|(_))",
"right": "[_ \\.(\\.edf)]",
"content": ["[0-9]{1,3}"],
"fill": 2
},
Expand All @@ -132,24 +132,15 @@
},
"Timing": {
"start": "stimulus",
"end": "response"
"end": "go"
},
"SampleRate": 30000,
"AudioCorrection": "n/a",
"Events": [
{
"onset": "stimulus",
"duration": "0.5 * 30",
"trial_num": "trial",
"trial_type": "Stimulus + SPLITCHAR + congruency + percIncongruent + SPLITCHAR + switchType + percSwitch + SPLITCHAR + globalStim + SPLITCHAR + localStim + SPLITCHAR + task + SPLITCHAR + target + SPLITCHAR + partResponded + SPLITCHAR + partResp + SPLITCHAR + corrResp + SPLITCHAR + trialCount + SPLITCHAR + blockTrialCount"
},
{
"onset": "response",
"duration": "RT",
"trial_num": "trial",
"trial_type": "Response + SPLITCHAR + congruency + percIncongruent + SPLITCHAR + switchType + percSwitch + SPLITCHAR + globalStim + SPLITCHAR + localStim + SPLITCHAR + task + SPLITCHAR + target + SPLITCHAR + partResponded + SPLITCHAR + partResp + SPLITCHAR + corrResp + SPLITCHAR + trialCount + SPLITCHAR + blockTrialCount"
}
]
{"onset": "Start","duration": "(cueEnd - cueStart) * 30", "trial_num": "Trial","trial_type": "cue", "bad": "Noisy == 1"},
{"onset": "Auditory", "stim_file": "sound", "trial_num": "Trial", "trial_type": "Audio", "bad": "Noisy == 1"},
{"onset": "Go", "duration": "(goEnd - goStart) * 30", "trial_num": "Trial", "trial_type": "GoCue + SPLITCHAR + go", "bad": "Noisy == 1"},
{"onset": "ResponseStart", "duration": "ResponseEnd - ResponseStart", "trial_num": "Trial","trial_type": "Response", "bad": "Noisy == 1"}]
},
"split": {
"Sep": "all",
Expand Down
10 changes: 5 additions & 5 deletions BIDS_converter/data2bids.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -217,7 +217,7 @@ def chan_walk(self, root: PathLike, files: List[PathLike],
for i, file in enumerate(files):
src = op.join(root, file)
if any(f in op.basename(src) for f in
ieeg_conf["channels"].keys()):
ieeg_conf["headerData"]["channels"].keys()):
self._channels_file[part_match] = src

for name, var in ieeg_conf["headerData"].items():
Expand All @@ -231,7 +231,7 @@ def scan_chans(self, src: PathLike, var: str, part_match: str):
self.channels[part_match] = self.channels[part_match] + org.mat2df(
src, var).tolist()
self.sample_rate[part_match] = int(
org.mat2df(src, self._config['ieeg']['sampleRate']).iloc[0])
org.mat2df(src, self._config['ieeg']["headerData"]['sampleRate']).iloc[0])
self._ignore.append(src)
elif name.endswith((".txt", ".csv", ".tsv")):
f = open(name, 'r')
Expand Down Expand Up @@ -699,7 +699,7 @@ def read_edf(self, file_name: PathLike,
print("Reading " + file_name + "...")
[array, signal_headers, _] = highlevel.read_edf(
file_name, ch_nrs=chn_nums,
digital=self._config["ieeg"]["digital"], verbose=True)
digital=self._config["ieeg"]["headerData"]["digital"], verbose=True)
print("read it")
if extra_arrays:
array = array + extra_arrays
Expand Down Expand Up @@ -898,7 +898,7 @@ def write_edf(self, array: np.ndarray, signal_headers: List[dict],
exist_ok=True)
highlevel.write_edf(practice, np.split(array, [
0, start_nums[0][0]], axis=1)[1], signal_headers,
header, digital=self._config["ieeg"][
header, digital=self._config["ieeg"]["headerData"][
"digital"])
self.bidsignore("*practice*")
else:
Expand All @@ -916,7 +916,7 @@ def write_edf(self, array: np.ndarray, signal_headers: List[dict],
print(full_name + "(Samples[" + str(start) + ":" + str(
end) + "]) ---> " + edf_name)
highlevel.write_edf(edf_name, new_array, signal_headers, header,
digital=self._config["ieeg"]["digital"])
digital=self._config["ieeg"]["headerData"]["digital"])
# zero the timing so that each file starts at t=0
if i > 0:
org.reset_zero(tsv_name, start_nums[i - 1][1],
Expand Down
17 changes: 12 additions & 5 deletions BIDS_converter/utils/organize.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,8 @@ def frame2bids(df_in: pd.DataFrame, event_format: Dict[str, str],


def reframe_events(df_in: pd.DataFrame, events: Union[list, dict],
stim_dir: PathLike, mat_sample_rate,
audio_correction) -> pd.DataFrame:
stim_dir: PathLike, mat_sample_rate: int,
audio_correction=None) -> pd.DataFrame:
df = df_in.copy()
new_df = None
if isinstance(events, dict):
Expand Down Expand Up @@ -103,6 +103,13 @@ def reframe_events(df_in: pd.DataFrame, events: Union[list, dict],
float) / mat_sample_rate
else:
temp_df[key] = eval_df(df, value)

if "bad" in temp_df.columns:
temp_df["bad"].fillna(False)
func = lambda r: "bad " + r["trial_type"] if r["bad"] and not r["trial_type"].startswith("bad") else r["trial_type"]
temp_df["trial_type"] = temp_df.apply(func, axis=1)
temp_df.drop("bad", axis=1, inplace=True)

if "trial_num" not in temp_df.columns:
temp_df["trial_num"] = [1 + i for i in list(range(temp_df.shape[0]))]
temp_df.dropna(axis=0, how="all", subset=["onset", "duration"], inplace=True)
Expand Down Expand Up @@ -326,15 +333,15 @@ def sort_by_list(df: pd.DataFrame, ord: list[str], col: str) -> pd.DataFrame:
def prep_tsv(file_path: PathLike, task: str, pmatchz: str, ieeg_config: dict,
bids_dir: PathLike) -> (str, pd.DataFrame):
df = None
for name, var in ieeg_config["channels"].items():
for name, var in ieeg_config["headerData"]["channels"].items():
if name in file_path:
df = mat2df(file_path, var)
if "highpass_cutoff" in df.columns.to_list():
df = df.rename(columns={"highpass_cutoff": "high_cutoff"})
if "lowpass_cutoff" in df.columns.to_list():
df = df.rename(columns={"lowpass_cutoff": "low_cutoff"})
df["type"] = ieeg_config["type"]
df["units"] = ieeg_config["units"]
df["units"] = ieeg_config["headerData"]["units"]
new_row = pd.DataFrame(
{"name": ["Trigger"], "high_cutoff": [1], "low_cutoff": [1000],
"type": ["TRIG"], "units": ["uV"]})
Expand Down Expand Up @@ -397,7 +404,7 @@ def eval_df(df: pd.DataFrame, exp: str) -> pd.Series:
"""
if exp in df.columns:
return df[exp].squeeze()
fields = [i for i in re.split(r"[ +\-/*%]", exp) if i != '']
fields = [i for i in re.split(r"[ +\-/*%(==)><(>=)(<=)]", exp) if i != '']

df["SPLITCHAR"] = pd.Series(["/"] * df.shape[0], dtype="string")
for name in fields:
Expand Down