From b14cd4de943b0aa29785391aa5ea2e41bc881d1c Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Mon, 13 Jan 2025 17:02:25 -0600 Subject: [PATCH 1/9] add skeleton for funloc testing dataset --- .../tests/configs/config_funloc.py | 24 +++++++++++++++++++ mne_bids_pipeline/tests/datasets.py | 5 ++++ mne_bids_pipeline/tests/test_run.py | 1 + 3 files changed, 30 insertions(+) create mode 100644 mne_bids_pipeline/tests/configs/config_funloc.py diff --git a/mne_bids_pipeline/tests/configs/config_funloc.py b/mne_bids_pipeline/tests/configs/config_funloc.py new file mode 100644 index 000000000..75adbc8f8 --- /dev/null +++ b/mne_bids_pipeline/tests/configs/config_funloc.py @@ -0,0 +1,24 @@ +"""Funloc data.""" + +bids_root = "~/mne_data/funloc" +deriv_root = "~/mne_data/derivatives/mne-bids-pipeline/funloc" +task = "funloc" +ch_types = ["meg", "eeg"] + +# Preprocessing +l_freq = None +h_freq = 40.0 +use_maxwell_filter: bool = True + + +# Epochs +epochs_tmin = -0.08 +epochs_tmax = 0.18 +epochs_decim = 10 # 2000->200 Hz +baseline = (None, 0) +conditions = [ + "auditory/standard", + "auditory/deviant", + "visual/standard", + "visual/deviant", +] diff --git a/mne_bids_pipeline/tests/datasets.py b/mne_bids_pipeline/tests/datasets.py index 92e365cda..269a29c2d 100644 --- a/mne_bids_pipeline/tests/datasets.py +++ b/mne_bids_pipeline/tests/datasets.py @@ -118,4 +118,9 @@ class DATASET_OPTIONS_T(TypedDict, total=False): "MNE-phantom-KIT-data": { "mne": "phantom_kit", }, + "funloc": { + "mne": "funloc", + # "web": "https://osf.io/TODO/download?version=1", + # "hash": "sha256:TODO", # noqa: E501 + }, } diff --git a/mne_bids_pipeline/tests/test_run.py b/mne_bids_pipeline/tests/test_run.py index b0f34e62d..eab602c47 100644 --- a/mne_bids_pipeline/tests/test_run.py +++ b/mne_bids_pipeline/tests/test_run.py @@ -135,6 +135,7 @@ class _TestOptionsT(TypedDict, total=False): "MNE-phantom-KIT-data": { "config": "config_MNE_phantom_KIT_data.py", }, + "funloc": {}, } From 50b5be3bec942ea7ec3582166b73ed3578c2d747 Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Tue, 14 Jan 2025 14:29:22 -0600 Subject: [PATCH 2/9] allow .tar.gz downloads; adjust dataset name; add URL and hash --- mne_bids_pipeline/_download.py | 8 +++++--- mne_bids_pipeline/tests/configs/config_funloc.py | 4 ++-- mne_bids_pipeline/tests/datasets.py | 9 +++++---- mne_bids_pipeline/tests/test_run.py | 2 +- 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/mne_bids_pipeline/_download.py b/mne_bids_pipeline/_download.py index bf62b63a6..648c54a2b 100644 --- a/mne_bids_pipeline/_download.py +++ b/mne_bids_pipeline/_download.py @@ -43,16 +43,18 @@ def _download_from_web(*, ds_name: str, ds_path: Path) -> None: ds_path.mkdir(parents=True, exist_ok=True) path = ds_path.parent.resolve(strict=True) - fname = f"{ds_name}.zip" + ext = "tar.gz" if options.get("processor") == "untar" else "zip" + processor = pooch.Untar if options.get("processor") == "untar" else pooch.Unzip + fname = f"{ds_name}.{ext}" pooch.retrieve( url=url, path=path, fname=fname, - processor=pooch.Unzip(extract_dir="."), # relative to path + processor=processor(extract_dir="."), # relative to path progressbar=True, known_hash=known_hash, ) - (path / f"{ds_name}.zip").unlink() + (path / f"{ds_name}.{ext}").unlink() def _download_via_mne(*, ds_name: str, ds_path: Path) -> None: diff --git a/mne_bids_pipeline/tests/configs/config_funloc.py b/mne_bids_pipeline/tests/configs/config_funloc.py index 75adbc8f8..26b2ca64c 100644 --- a/mne_bids_pipeline/tests/configs/config_funloc.py +++ b/mne_bids_pipeline/tests/configs/config_funloc.py @@ -1,7 +1,7 @@ """Funloc data.""" -bids_root = "~/mne_data/funloc" -deriv_root = "~/mne_data/derivatives/mne-bids-pipeline/funloc" +bids_root = "~/mne_data/MNE-funloc-data" +deriv_root = "~/mne_data/derivatives/mne-bids-pipeline/MNE-funloc-data" task = "funloc" ch_types = ["meg", "eeg"] diff --git a/mne_bids_pipeline/tests/datasets.py b/mne_bids_pipeline/tests/datasets.py index 269a29c2d..22a97deff 100644 --- a/mne_bids_pipeline/tests/datasets.py +++ b/mne_bids_pipeline/tests/datasets.py @@ -15,6 +15,7 @@ class DATASET_OPTIONS_T(TypedDict, total=False): include: list[str] # [] exclude: list[str] # [] hash: str # "" + processor: str # "" DATASET_OPTIONS: dict[str, DATASET_OPTIONS_T] = { @@ -118,9 +119,9 @@ class DATASET_OPTIONS_T(TypedDict, total=False): "MNE-phantom-KIT-data": { "mne": "phantom_kit", }, - "funloc": { - "mne": "funloc", - # "web": "https://osf.io/TODO/download?version=1", - # "hash": "sha256:TODO", # noqa: E501 + "MNE-funloc-data": { + "web": "https://osf.io/2pemg/download?version=1", + "hash": "sha256:05a6b0e9d7a21ac3378236082de5e3b8f1a02315501b8e587179959bc5fafba2", # noqa: E501 + "processor": "untar", }, } diff --git a/mne_bids_pipeline/tests/test_run.py b/mne_bids_pipeline/tests/test_run.py index eab602c47..72d35a24b 100644 --- a/mne_bids_pipeline/tests/test_run.py +++ b/mne_bids_pipeline/tests/test_run.py @@ -135,7 +135,7 @@ class _TestOptionsT(TypedDict, total=False): "MNE-phantom-KIT-data": { "config": "config_MNE_phantom_KIT_data.py", }, - "funloc": {}, + "MNE-funloc-data": {"config": "config_funloc.py"}, } From c385f96fe489066633dd6ccbed378c201edc110f Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Wed, 15 Jan 2025 11:24:15 -0600 Subject: [PATCH 3/9] add dataset to circleCI and docs --- .circleci/config.yml | 55 ++++++++++++++++++++++++++++++++++++++++++++ docs/mkdocs.yml | 1 + 2 files changed, 56 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 049b271ef..2e29fad6f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -297,6 +297,25 @@ jobs: paths: - ~/mne_data/eeg_matchingpennies + cache_MNE-funloc-data: + <<: *imageconfig + steps: + - attach_workspace: + at: ~/ + - restore_cache: + keys: + - data-cache-MNE-funloc-data-1 + - bash_env + - run: + name: Get MNE-funloc-data + command: | + $DOWNLOAD_DATA MNE-funloc-data + - codecov/upload + - save_cache: + key: data-cache-MNE-funloc-data-1 + paths: + - ~/mne_data/MNE-funloc-data + cache_MNE-phantom-KIT-data: <<: *imageconfig steps: @@ -784,6 +803,32 @@ jobs: paths: - mne_data/derivatives/mne-bids-pipeline/eeg_matchingpennies/*/*/*.html + test_MNE-funloc-data: + <<: *imageconfig + steps: + - attach_workspace: + at: ~/ + - bash_env + - restore_cache: + keys: + - data-cache-MNE-funloc-data-1 + - run: + name: test MNE-funloc-data + command: $RUN_TESTS MNE-funloc-data + - codecov/upload + - store_test_results: + path: ./test-results + - store_artifacts: + path: ./test-results + destination: test-results + - store_artifacts: + path: /home/circleci/reports/MNE-funloc-data + destination: reports/MNE-funloc-data + - persist_to_workspace: + root: ~/ + paths: + - mne_data/derivatives/mne-bids-pipeline/MNE-funloc-data/*/*/*.html + test_MNE-phantom-KIT-data: <<: *imageconfig steps: @@ -1244,6 +1289,15 @@ workflows: - cache_eeg_matchingpennies <<: *filter_tags + - cache_MNE-funloc-data: + requires: + - setup_env + <<: *filter_tags + - test_MNE-funloc-data: + requires: + - cache_MNE-funloc-data + <<: *filter_tags + - cache_MNE-phantom-KIT-data: requires: - setup_env @@ -1304,6 +1358,7 @@ workflows: - test_ds003392 - test_ds004229 - test_eeg_matchingpennies + - test_MNE-funloc-data - test_MNE-phantom-KIT-data - test_ERP_CORE_N400 - test_ERP_CORE_ERN diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 29bce8604..a4721a185 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -121,6 +121,7 @@ nav: - examples/ds000248_no_mri.md - examples/ds003104.md - examples/eeg_matchingpennies.md + - examples/MNE-funloc-data.md - examples/MNE-phantom-KIT-data.md - examples/ds001810.md - examples/ds000117.md From bf013cff86b6719f43faebbcdba3663f12174d1b Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Wed, 15 Jan 2025 14:16:13 -0600 Subject: [PATCH 4/9] saner config --- .../tests/configs/config_funloc.py | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/mne_bids_pipeline/tests/configs/config_funloc.py b/mne_bids_pipeline/tests/configs/config_funloc.py index 26b2ca64c..92525fc87 100644 --- a/mne_bids_pipeline/tests/configs/config_funloc.py +++ b/mne_bids_pipeline/tests/configs/config_funloc.py @@ -1,20 +1,30 @@ """Funloc data.""" -bids_root = "~/mne_data/MNE-funloc-data" -deriv_root = "~/mne_data/derivatives/mne-bids-pipeline/MNE-funloc-data" +from pathlib import Path + +data_root = Path("~/mne_data").expanduser().resolve() +bids_root = data_root / "MNE-funloc-data" +deriv_root = data_root / "derivatives" / "mne-bids-pipeline" / "MNE-funloc-data" +subjects_dir = bids_root / "derivatives" / "freesurfer" / "subjects" task = "funloc" ch_types = ["meg", "eeg"] +data_type = "meg" -# Preprocessing +# filter l_freq = None -h_freq = 40.0 +h_freq = 50.0 +# maxfilter use_maxwell_filter: bool = True - +mf_st_duration = 60.0 +# SSP +n_proj_eog = dict(n_mag=1, n_grad=1, n_eeg=2) +n_proj_ecg = dict(n_mag=1, n_grad=1, n_eeg=0) # Epochs -epochs_tmin = -0.08 -epochs_tmax = 0.18 -epochs_decim = 10 # 2000->200 Hz +epochs_tmin = -0.2 +epochs_tmax = 0.5 +epochs_t_adjust = -4.0e-3 # TODO is there a way to do this in MBP? +epochs_decim = 5 # 1000 -> 200 Hz baseline = (None, 0) conditions = [ "auditory/standard", @@ -22,3 +32,5 @@ "visual/standard", "visual/deviant", ] +# contrasts +contrasts = [("auditory", "visual")] From 69f37f1440d2fbe6ad3ae7f8075e686acd16540a Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Tue, 14 Jan 2025 17:34:14 -0600 Subject: [PATCH 5/9] add test that monkeypatches funloc to emulate session-specific MRIs --- mne_bids_pipeline/tests/test_run.py | 102 +++++++++++++++++++++++++++- 1 file changed, 101 insertions(+), 1 deletion(-) diff --git a/mne_bids_pipeline/tests/test_run.py b/mne_bids_pipeline/tests/test_run.py index 72d35a24b..78657ac8e 100644 --- a/mne_bids_pipeline/tests/test_run.py +++ b/mne_bids_pipeline/tests/test_run.py @@ -9,7 +9,9 @@ from typing import Any, TypedDict import pytest +from mne_bids import BIDSPath, get_bids_path_from_fname +from mne_bids_pipeline._config_import import _import_config from mne_bids_pipeline._download import main as download_main from mne_bids_pipeline._main import main @@ -135,7 +137,10 @@ class _TestOptionsT(TypedDict, total=False): "MNE-phantom-KIT-data": { "config": "config_MNE_phantom_KIT_data.py", }, - "MNE-funloc-data": {"config": "config_funloc.py"}, + "MNE-funloc-data": { + "config": "config_funloc.py", + "steps": ["init", "preprocessing", "sensor", "source"], + }, } @@ -275,3 +280,98 @@ def test_missing_sessions( print() with context: main() + + +@pytest.mark.dataset_test +@pytest.mark.parametrize("dataset", ["MNE-funloc-data"]) +def test_session_specific_mri( + dataset: str, + monkeypatch: pytest.MonkeyPatch, + dataset_test: Any, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + """Test of (faked) session-specific MRIs.""" + test_options = TEST_SUITE[dataset] + config = test_options.get("config", f"config_{dataset}.py") + config_path = BIDS_PIPELINE_DIR / "tests" / "configs" / config + # copy the dataset to a tmpdir + config_obj = _import_config(config_path=config_path) + # make it seem like there's only one subj with different MRIs for different sessions + new_bids_path = BIDSPath(root=tmp_path / dataset, subject="01", session="a") + # sub-01/* → sub-01/ses-a/* + src_dir = config_obj.bids_root / "sub-01" + dst_dir = new_bids_path.root / "sub-01" / "ses-a" + for root, dirs, files in src_dir.walk(): + offset = root.relative_to(src_dir) + for _dir in dirs: + (dst_dir / offset / _dir).mkdir(parents=True) + for _file in files: + bp = get_bids_path_from_fname(root / _file) + bp.update(root=new_bids_path.root, session="a") + shutil.copyfile(src=root / _file, dst=dst_dir / offset / bp.basename) + # sub-02/* → sub-01/ses-b/* + src_dir = config_obj.bids_root / "sub-02" + dst_dir = new_bids_path.root / "sub-01" / "ses-b" + for root, dirs, files in src_dir.walk(): + offset = root.relative_to(src_dir) + for _dir in dirs: + (dst_dir / offset / _dir).mkdir(parents=True) + for _file in files: + bp = get_bids_path_from_fname(root / _file) + bp.update(root=new_bids_path.root, subject="01", session="b") + shutil.copyfile(src=root / _file, dst=dst_dir / offset / bp.basename) + # emptyroom + src_dir = config_obj.bids_root / "sub-emptyroom" + dst_dir = new_bids_path.root / "sub-emptyroom" + shutil.copytree(src=src_dir, dst=dst_dir) + # root-level files (dataset description, etc) + src_dir = config_obj.bids_root + dst_dir = new_bids_path.root + files = [f for f in src_dir.iterdir() if f.is_file()] + for _file in files: + shutil.copyfile(src=_file, dst=dst_dir / _file.name) + # now move derivatives (freesurfer files) + src_dir = config_obj.bids_root / "derivatives" / "freesurfer" / "subjects" + dst_dir = new_bids_path.root / "derivatives" / "freesurfer" / "subjects" + dst_dir.mkdir(parents=True) + for root, dirs, files in src_dir.walk(): + new_root = root + if "sub01" in root.parts: + new_root = Path( + *["sub-01_ses-a" if p == "sub01" else p for p in new_root.parts] + ) + elif "sub02" in root.parts: + new_root = Path( + *["sub-01_ses-b" if p == "sub02" else p for p in new_root.parts] + ) + offset = new_root.relative_to(src_dir) + for _dir in dirs: + if _dir == "sub01": + _dir = "sub-01_ses-a" + elif _dir == "sub02": + _dir = "sub-01_ses-b" + (dst_dir / offset / _dir).mkdir() + for _file in files: + dst_file = _file + if _file.startswith("sub01"): + dst_file = dst_file.replace("sub01", "sub-01_ses-a") + elif _file.startswith("sub02"): + dst_file = dst_file.replace("sub02", "sub-01_ses-b") + shutil.copyfile(src=root / _file, dst=dst_dir / offset / dst_file) + # print_dir_tree(new_bids_path.root) # for debugging + # hack in the new bids_root + extra_config = dict(bids_root=str(new_bids_path.root)) + extra_path = tmp_path / "extra_config.py" + extra_path.write_text(str(extra_config)) + monkeypatch.setenv("_MNE_BIDS_STUDY_TESTING_EXTRA_CONFIG", str(extra_path)) + # Run the tests. + steps = test_options.get("steps", ()) + command = ["mne_bids_pipeline", str(config_path), f"--steps={','.join(steps)}"] + if "--pdb" in sys.argv: + command.append("--n_jobs=1") + monkeypatch.setenv("_MNE_BIDS_STUDY_TESTING", "true") + monkeypatch.setattr(sys, "argv", command) + with capsys.disabled(): + print() + main() From 4757cb66fb0c35b05f35a11acd2b7406a5d2ea89 Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Fri, 17 Jan 2025 13:06:17 -0600 Subject: [PATCH 6/9] refactor _has_session_specific_anat to shared helper --- mne_bids_pipeline/_config_utils.py | 6 ++++++ mne_bids_pipeline/steps/freesurfer/_01_recon_all.py | 7 +------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/mne_bids_pipeline/_config_utils.py b/mne_bids_pipeline/_config_utils.py index 9c2776908..6dc0ebf1c 100644 --- a/mne_bids_pipeline/_config_utils.py +++ b/mne_bids_pipeline/_config_utils.py @@ -54,6 +54,12 @@ def get_fs_subject( return f"sub-{subject}" +def _has_session_specific_anat( + subject: str, session: str | None, subjects_dir: pathlib.Path +) -> bool: + return (subjects_dir / f"sub-{subject}_ses-{session}").exists() + + @functools.cache def _get_entity_vals_cached( *args: list[Any], diff --git a/mne_bids_pipeline/steps/freesurfer/_01_recon_all.py b/mne_bids_pipeline/steps/freesurfer/_01_recon_all.py index 960b1c6ff..c7c0496b0 100755 --- a/mne_bids_pipeline/steps/freesurfer/_01_recon_all.py +++ b/mne_bids_pipeline/steps/freesurfer/_01_recon_all.py @@ -14,6 +14,7 @@ from mne.utils import run_subprocess from mne_bids_pipeline._config_utils import ( + _has_session_specific_anat, get_fs_subjects_dir, get_sessions, get_subjects, @@ -78,12 +79,6 @@ def run_recon( run_subprocess(cmd, env=env, verbose=logger.level) -def _has_session_specific_anat( - subject: str, session: str | None, subjects_dir: Path -) -> bool: - return (subjects_dir / f"sub-{subject}_ses-{session}").exists() - - def main(*, config: SimpleNamespace) -> None: """Run freesurfer recon-all command on BIDS dataset. From 8fc715520b58303795e94364c68e86908d6fb58f Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Fri, 17 Jan 2025 13:09:18 -0600 Subject: [PATCH 7/9] WIP fix for make_bem_surfaces --- .../steps/source/_01_make_bem_surfaces.py | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/mne_bids_pipeline/steps/source/_01_make_bem_surfaces.py b/mne_bids_pipeline/steps/source/_01_make_bem_surfaces.py index e70636e81..d5e8487a9 100644 --- a/mne_bids_pipeline/steps/source/_01_make_bem_surfaces.py +++ b/mne_bids_pipeline/steps/source/_01_make_bem_surfaces.py @@ -12,10 +12,10 @@ from mne_bids_pipeline._config_utils import ( _bids_kwargs, _get_bem_conductivity, + _has_session_specific_anat, get_fs_subject, get_fs_subjects_dir, - get_sessions, - get_subjects, + get_subjects_sessions, ) from mne_bids_pipeline._logging import gen_log_kwargs, logger from mne_bids_pipeline._parallel import get_parallel_backend, parallel_func @@ -154,10 +154,23 @@ def main(*, config: SimpleNamespace) -> None: mne.datasets.fetch_fsaverage(get_fs_subjects_dir(config)) return + # check for session-specific MRIs within subject, and handle accordingly + subjects_dir = Path(get_fs_subjects_dir(config)) + subj_sess = list() + for _subj, sessions in get_subjects_sessions(config).items(): + for _sess in sessions: + session = ( + _sess + if _has_session_specific_anat(_subj, _sess, subjects_dir) + else None + ) + subj_sess.append((_subj, session)) + with get_parallel_backend(config.exec_params): parallel, run_func = parallel_func( make_bem_surfaces, exec_params=config.exec_params ) + 1 / 0 logs = parallel( run_func( cfg=get_config( @@ -170,7 +183,6 @@ def main(*, config: SimpleNamespace) -> None: session=session, force_run=config.recreate_bem, ) - for subject in get_subjects(config) - for session in get_sessions(config) + for subject, session in subj_sess ) save_logs(config=config, logs=logs) From cdc81e11df914f1e35fc3b22d110c9118fcc51f0 Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Fri, 17 Jan 2025 13:59:24 -0600 Subject: [PATCH 8/9] DRY; add comments [ci skip] --- mne_bids_pipeline/tests/test_run.py | 70 ++++++++++++----------------- 1 file changed, 29 insertions(+), 41 deletions(-) diff --git a/mne_bids_pipeline/tests/test_run.py b/mne_bids_pipeline/tests/test_run.py index 78657ac8e..4583b2592 100644 --- a/mne_bids_pipeline/tests/test_run.py +++ b/mne_bids_pipeline/tests/test_run.py @@ -295,32 +295,22 @@ def test_session_specific_mri( test_options = TEST_SUITE[dataset] config = test_options.get("config", f"config_{dataset}.py") config_path = BIDS_PIPELINE_DIR / "tests" / "configs" / config - # copy the dataset to a tmpdir config_obj = _import_config(config_path=config_path) - # make it seem like there's only one subj with different MRIs for different sessions + # copy the dataset to a tmpdir, and in the destination location (a tmpdir) make it + # seem like there's only one subj with different MRIs for different sessions new_bids_path = BIDSPath(root=tmp_path / dataset, subject="01", session="a") - # sub-01/* → sub-01/ses-a/* - src_dir = config_obj.bids_root / "sub-01" - dst_dir = new_bids_path.root / "sub-01" / "ses-a" - for root, dirs, files in src_dir.walk(): - offset = root.relative_to(src_dir) - for _dir in dirs: - (dst_dir / offset / _dir).mkdir(parents=True) - for _file in files: - bp = get_bids_path_from_fname(root / _file) - bp.update(root=new_bids_path.root, session="a") - shutil.copyfile(src=root / _file, dst=dst_dir / offset / bp.basename) - # sub-02/* → sub-01/ses-b/* - src_dir = config_obj.bids_root / "sub-02" - dst_dir = new_bids_path.root / "sub-01" / "ses-b" - for root, dirs, files in src_dir.walk(): - offset = root.relative_to(src_dir) - for _dir in dirs: - (dst_dir / offset / _dir).mkdir(parents=True) - for _file in files: - bp = get_bids_path_from_fname(root / _file) - bp.update(root=new_bids_path.root, subject="01", session="b") - shutil.copyfile(src=root / _file, dst=dst_dir / offset / bp.basename) + # sub-01/* → sub-01/ses-a/* ; sub-02/* → sub-01/ses-b/* + for src_subj, dst_sess in (("01", "a"), ("02", "b")): + src_dir = config_obj.bids_root / f"sub-{src_subj}" + dst_dir = new_bids_path.root / "sub-01" / f"ses-{dst_sess}" + for root, dirs, files in src_dir.walk(): + offset = root.relative_to(src_dir) + for _dir in dirs: + (dst_dir / offset / _dir).mkdir(parents=True) + for _file in files: + bp = get_bids_path_from_fname(root / _file) + bp.update(root=new_bids_path.root, subject="01", session=dst_sess) + shutil.copyfile(src=root / _file, dst=dst_dir / offset / bp.basename) # emptyroom src_dir = config_obj.bids_root / "sub-emptyroom" dst_dir = new_bids_path.root / "sub-emptyroom" @@ -335,30 +325,28 @@ def test_session_specific_mri( src_dir = config_obj.bids_root / "derivatives" / "freesurfer" / "subjects" dst_dir = new_bids_path.root / "derivatives" / "freesurfer" / "subjects" dst_dir.mkdir(parents=True) - for root, dirs, files in src_dir.walk(): - new_root = root - if "sub01" in root.parts: - new_root = Path( - *["sub-01_ses-a" if p == "sub01" else p for p in new_root.parts] - ) - elif "sub02" in root.parts: + freesurfer_subject_mapping = dict(sub01="sub-01_ses-a", sub02="sub-01_ses-b") + for walk_root, dirs, files in src_dir.walk(): + # change "root" so that in later steps of the walk when we're inside a subject's + # dir, the "offset" (folders between dst_dir and filename) will be correct + new_root = walk_root + if "sub01" in walk_root.parts or "sub02" in walk_root.parts: new_root = Path( - *["sub-01_ses-b" if p == "sub02" else p for p in new_root.parts] + *[freesurfer_subject_mapping.get(p, p) for p in new_root.parts] ) offset = new_root.relative_to(src_dir) + # the actual subject dirs need their names changed for _dir in dirs: - if _dir == "sub01": - _dir = "sub-01_ses-a" - elif _dir == "sub02": - _dir = "sub-01_ses-b" + _dir = freesurfer_subject_mapping.get(_dir, _dir) (dst_dir / offset / _dir).mkdir() + # for filenames that contain the subject identifier (e.g. BEM files), we need to + # change the filename too, not just parent folder name for _file in files: dst_file = _file - if _file.startswith("sub01"): - dst_file = dst_file.replace("sub01", "sub-01_ses-a") - elif _file.startswith("sub02"): - dst_file = dst_file.replace("sub02", "sub-01_ses-b") - shutil.copyfile(src=root / _file, dst=dst_dir / offset / dst_file) + fs_sub = dst_file[:5] + if fs_sub in freesurfer_subject_mapping: + dst_file = dst_file.replace(fs_sub, freesurfer_subject_mapping[fs_sub]) + shutil.copyfile(src=walk_root / _file, dst=dst_dir / offset / dst_file) # print_dir_tree(new_bids_path.root) # for debugging # hack in the new bids_root extra_config = dict(bids_root=str(new_bids_path.root)) From f6d2e6e9cacea20ce42351477b70c2770825340a Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Fri, 17 Jan 2025 17:01:33 -0600 Subject: [PATCH 9/9] use --root-dir instead of extra_config --- mne_bids_pipeline/tests/test_run.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/mne_bids_pipeline/tests/test_run.py b/mne_bids_pipeline/tests/test_run.py index 4583b2592..1c8019063 100644 --- a/mne_bids_pipeline/tests/test_run.py +++ b/mne_bids_pipeline/tests/test_run.py @@ -348,14 +348,11 @@ def test_session_specific_mri( dst_file = dst_file.replace(fs_sub, freesurfer_subject_mapping[fs_sub]) shutil.copyfile(src=walk_root / _file, dst=dst_dir / offset / dst_file) # print_dir_tree(new_bids_path.root) # for debugging - # hack in the new bids_root - extra_config = dict(bids_root=str(new_bids_path.root)) - extra_path = tmp_path / "extra_config.py" - extra_path.write_text(str(extra_config)) - monkeypatch.setenv("_MNE_BIDS_STUDY_TESTING_EXTRA_CONFIG", str(extra_path)) # Run the tests. steps = test_options.get("steps", ()) command = ["mne_bids_pipeline", str(config_path), f"--steps={','.join(steps)}"] + # hack in the new bids_root + command.append(f"--root-dir={new_bids_path.root}") if "--pdb" in sys.argv: command.append("--n_jobs=1") monkeypatch.setenv("_MNE_BIDS_STUDY_TESTING", "true")