From 47fc3fcbe34fd17f4b22ee182cabb7edd30087c6 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Mon, 12 Nov 2018 13:18:30 -0500 Subject: [PATCH 01/33] PIN: Nistats + pybids --- fitlins/__about__.py | 7 +++---- requirements.txt | 5 ++--- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/fitlins/__about__.py b/fitlins/__about__.py index cfa465ae..3c8128f4 100644 --- a/fitlins/__about__.py +++ b/fitlins/__about__.py @@ -40,15 +40,14 @@ 'nilearn>=0.4.0', 'pandas>=0.19', 'tables>=3.2.1', - 'nistats>=0.0.1a', - 'grabbit>=0.2.3', + 'nistats>=0.0.1b0', 'pybids>=0.6.5', 'jinja2', ] LINKS_REQUIRES = [ - 'git+https://github.com/nistats/nistats.git@' - 'ce3695e8f34c6f34323766dc96a60a53b69d2729#egg=nistats', + 'git+https://github.com/bids-standard/pybids.git@' + '54d4810a33be27c8ab340bf40c6bd1261dcb2c42#egg=pybids', ] TESTS_REQUIRES = [ diff --git a/requirements.txt b/requirements.txt index 330dfb98..2205fce0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,5 @@ numpy>=1.11 nilearn>=0.4.0 pandas>=0.19 nipype>=1.1.2 -grabbit>=0.2.3 -pybids>=0.6.5 -git+https://github.com/nistats/nistats.git@ce3695e8f34c6f34323766dc96a60a53b69d2729#egg=nistats +nistats>=0.0.1b0 +git+https://github.com/bids-standard/pybids.git@54d4810a33be27c8ab340bf40c6bd1261dcb2c42#egg=pybids From 63ef92038d917d27e8333c643d556eeda0098d56 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Mon, 12 Nov 2018 13:22:47 -0500 Subject: [PATCH 02/33] FIX: type->suffix, model.json->smdl.json --- fitlins/interfaces/bids.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/fitlins/interfaces/bids.py b/fitlins/interfaces/bids.py index f732d31c..dd5283de 100644 --- a/fitlins/interfaces/bids.py +++ b/fitlins/interfaces/bids.py @@ -99,10 +99,11 @@ def _run_interface(self, runtime): from bids.analysis import auto_model models = self.inputs.model if not isinstance(models, list): - layout = bids.BIDSLayout(self.inputs.bids_dir) + # model is not yet standardized, so validate=False + layout = bids.BIDSLayout(self.inputs.bids_dir, validate=False) if not isdefined(models): - models = layout.get(type='model') + models = layout.get(suffix='smdl') if not models: raise ValueError("No models found") elif models == 'default': @@ -200,9 +201,9 @@ def _load_level1(self, runtime, analysis): block.model['HRF_variables'], mode='sparse', force=True): info = {} - space = analysis.layout.get_spaces(type='preproc', + space = analysis.layout.get_spaces(suffix='bold', extensions=['.nii', '.nii.gz'])[0] - preproc_files = analysis.layout.get(type='preproc', + preproc_files = analysis.layout.get(suffix='bold', extensions=['.nii', '.nii.gz'], space=space, **ents) @@ -212,7 +213,7 @@ def _load_level1(self, runtime, analysis): fname = preproc_files[0].filename # Required field in seconds - TR = analysis.layout.get_metadata(fname, type='bold', + TR = analysis.layout.get_metadata(fname, suffix='bold', full_search=True)['RepetitionTime'] dense_vars = set(block.model['variables']) - set(block.model['HRF_variables']) From a8ba7b9921fcc3a4a2b7c1991cb2e8e5603a41f2 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Mon, 12 Nov 2018 15:22:45 -0500 Subject: [PATCH 03/33] FIX: BIDSLayout, derivatives syntax --- fitlins/interfaces/bids.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/fitlins/interfaces/bids.py b/fitlins/interfaces/bids.py index dd5283de..702e635f 100644 --- a/fitlins/interfaces/bids.py +++ b/fitlins/interfaces/bids.py @@ -162,23 +162,23 @@ class LoadBIDSModel(SimpleInterface): def _run_interface(self, runtime): import bids - bids.config.set_options(loop_preproc=True) include = self.inputs.include_pattern exclude = self.inputs.exclude_pattern + derivatives = self.inputs.preproc_dir if not isdefined(include): include = None if not isdefined(exclude): exclude = None + if not isdefined(derivatives): + exclude = False - paths = [(self.inputs.bids_dir, 'bids')] - if isdefined(self.inputs.preproc_dir): - paths.append((self.inputs.preproc_dir, ['bids', 'derivatives'])) - layout = bids.BIDSLayout(paths, include=include, exclude=exclude) + layout = bids.BIDSLayout(self.inputs.bids_dir, include=include, exclude=exclude, + derivatives=derivatives) selectors = self.inputs.selectors analysis = bids.Analysis(model=self.inputs.model, layout=layout) - analysis.setup(drop_na=False, **selectors) + analysis.setup(drop_na=False, desc='preproc', **selectors) self._load_level1(runtime, analysis) self._load_higher_level(runtime, analysis) @@ -388,10 +388,12 @@ class BIDSSelect(SimpleInterface): def _run_interface(self, runtime): import bids - paths = [(self.inputs.bids_dir, 'bids')] - if isdefined(self.inputs.preproc_dir): - paths.append((self.inputs.preproc_dir, ['bids', 'derivatives'])) - layout = bids.BIDSLayout(paths) + + derivatives = self.inputs.preproc_dir + if not isdefined(derivatives): + exclude = False + + layout = bids.BIDSLayout(self.inputs.bids_dir, derivatives=derivatives) bold_files = [] mask_files = [] From c54b0757d37b2cdf225ef2b02fdc56eb5fa175a4 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Mon, 26 Nov 2018 16:25:17 -0500 Subject: [PATCH 04/33] RF: Update paradigm/confounds to sparse/dense --- fitlins/interfaces/bids.py | 63 ++++++++++++++++---------------------- 1 file changed, 27 insertions(+), 36 deletions(-) diff --git a/fitlins/interfaces/bids.py b/fitlins/interfaces/bids.py index 702e635f..c44cb9cb 100644 --- a/fitlins/interfaces/bids.py +++ b/fitlins/interfaces/bids.py @@ -188,17 +188,16 @@ def _run_interface(self, runtime): return runtime def _load_level1(self, runtime, analysis): - block = analysis.blocks[0] - block_subdir = Path(runtime.cwd) / block.level - block_subdir.mkdir(parents=True, exist_ok=True) + step = analysis.steps[0] + step_subdir = Path(runtime.cwd) / step.level + step_subdir.mkdir(parents=True, exist_ok=True) entities = [] session_info = [] contrast_indices = [] contrast_info = [] warnings = [] - for paradigm, _, ents in block.get_design_matrix( - block.model['HRF_variables'], mode='sparse', force=True): + for sparse, dense, ents in step.get_design_matrix(mode='sparse', force=True): info = {} space = analysis.layout.get_spaces(suffix='bold', @@ -215,30 +214,23 @@ def _load_level1(self, runtime, analysis): # Required field in seconds TR = analysis.layout.get_metadata(fname, suffix='bold', full_search=True)['RepetitionTime'] - dense_vars = set(block.model['variables']) - set(block.model['HRF_variables']) - - _, confounds, _ = block.get_design_matrix(dense_vars, - mode='dense', - force=True, - sampling_rate=1/TR, - **ents)[0] ent_string = '_'.join('{}-{}'.format(key, val) for key, val in ents.items()) - events_file = block_subdir / '{}_events.h5'.format(ent_string) - paradigm.to_hdf(events_file, key='events') + sparse_file = step_subdir / '{}_sparse.h5'.format(ent_string) + sparse.to_hdf(sparse_file, key='sparse') imputed = [] - if confounds is not None: + if dense is not None: # Note that FMRIPREP includes CosineXX columns to accompany # t/aCompCor # We may want to add criteria to include HPF columns that are not # explicitly listed in the model - names = [col for col in confounds.columns - if col.startswith('NonSteadyStateOutlier') or - col in block.model['variables']] - confounds = confounds[names] + names = [col for col in dense.columns + if col.startswith('non_steady_state') or + col in step.model['variables']] + dense = dense[names] # These confounds are defined pairwise with the current volume # and its predecessor, and thus may be undefined (have value @@ -247,38 +239,37 @@ def _load_level1(self, runtime, analysis): # expected NaN only. # Any other NaNs must be handled by an explicit transform in # the BIDS model. - for imputable in ('FramewiseDisplacement', - 'stdDVARS', 'non-stdDVARS', - 'vx-wisestdDVARS'): - if imputable in confounds.columns: - vals = confounds[imputable].values + for imputable in ('framewise_displacement', + 'std_dvars', 'dvars'): + if imputable in dense.columns: + vals = dense[imputable].values if not np.isnan(vals[0]): continue # Impute the mean non-zero, non-NaN value - confounds[imputable][0] = np.nanmean(vals[vals != 0]) + dense[imputable][0] = np.nanmean(vals[vals != 0]) imputed.append(imputable) - if np.isnan(confounds.values).any(): - iflogger.warning('Unexpected NaNs found in confounds; ' + if np.isnan(dense.values).any(): + iflogger.warning('Unexpected NaNs found in design matrix; ' 'regression may fail.') - confounds_file = block_subdir / '{}_confounds.h5'.format(ent_string) - confounds.to_hdf(confounds_file, key='confounds') + dense_file = step_subdir / '{}_dense.h5'.format(ent_string) + dense.to_hdf(dense_file, key='dense') else: - confounds_file = None + dense_file = None - info['events'] = str(events_file) - info['confounds'] = str(confounds_file) + info['sparse'] = str(sparse_file) + info['dense'] = str(dense_file) info['repetition_time'] = TR # Transpose so each contrast gets a row of data instead of column - contrasts, index, _ = block.get_contrasts(**ents)[0] + contrasts, index, _ = step.get_contrasts(**ents)[0] contrast_type_map = defaultdict(lambda: 'T') contrast_type_map.update({contrast['name']: contrast['type'] - for contrast in block.contrasts}) + for contrast in step.contrasts}) contrast_type_list = [contrast_type_map[contrast] for contrast in contrasts.columns] @@ -286,11 +277,11 @@ def _load_level1(self, runtime, analysis): # Add test indicator column contrasts['type'] = contrast_type_list - contrasts_file = block_subdir / '{}_contrasts.h5'.format(ent_string) + contrasts_file = step_subdir / '{}_contrasts.h5'.format(ent_string) contrasts_file.parent.mkdir(parents=True, exist_ok=True) contrasts.to_hdf(contrasts_file, key='contrasts') - warning_file = block_subdir / '{}_warning.html'.format(ent_string) + warning_file = step_subdir / '{}_warning.html'.format(ent_string) with warning_file.open('w') as fobj: if imputed: fobj.write(IMPUTATION_SNIPPET.format(', '.join(imputed))) From 0f7ac984e76dc6508b01f1f73e6b1d11b52bcd81 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Fri, 30 Nov 2018 13:45:14 -0500 Subject: [PATCH 05/33] RF: filename->path and other renaming updates --- fitlins/interfaces/bids.py | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/fitlins/interfaces/bids.py b/fitlins/interfaces/bids.py index c44cb9cb..db6b66e7 100644 --- a/fitlins/interfaces/bids.py +++ b/fitlins/interfaces/bids.py @@ -109,7 +109,7 @@ def _run_interface(self, runtime): elif models == 'default': models = auto_model(layout) - models = [_ensure_model(m) for m in models] + models = [_ensure_model(m.path) for m in models] if self.inputs.selectors: # This is almost certainly incorrect @@ -197,7 +197,7 @@ def _load_level1(self, runtime, analysis): contrast_indices = [] contrast_info = [] warnings = [] - for sparse, dense, ents in step.get_design_matrix(mode='sparse', force=True): + for sparse, dense, ents in step.get_design_matrix(): info = {} space = analysis.layout.get_spaces(suffix='bold', @@ -209,7 +209,7 @@ def _load_level1(self, runtime, analysis): if len(preproc_files) != 1: raise ValueError('Too many BOLD files found') - fname = preproc_files[0].filename + fname = preproc_files[0].path # Required field in seconds TR = analysis.layout.get_metadata(fname, suffix='bold', @@ -218,8 +218,10 @@ def _load_level1(self, runtime, analysis): ent_string = '_'.join('{}-{}'.format(key, val) for key, val in ents.items()) - sparse_file = step_subdir / '{}_sparse.h5'.format(ent_string) - sparse.to_hdf(sparse_file, key='sparse') + sparse_file = None + if sparse is not None: + sparse_file = step_subdir / '{}_sparse.h5'.format(ent_string) + sparse.to_hdf(sparse_file, key='sparse') imputed = [] if dense is not None: @@ -229,7 +231,7 @@ def _load_level1(self, runtime, analysis): # explicitly listed in the model names = [col for col in dense.columns if col.startswith('non_steady_state') or - col in step.model['variables']] + col in step.model['x']] dense = dense[names] # These confounds are defined pairwise with the current volume @@ -260,12 +262,14 @@ def _load_level1(self, runtime, analysis): else: dense_file = None - info['sparse'] = str(sparse_file) - info['dense'] = str(dense_file) + if sparse_file is not None: + info['sparse'] = str(sparse_file) + if dense_file is not None: + info['dense'] = str(dense_file) info['repetition_time'] = TR # Transpose so each contrast gets a row of data instead of column - contrasts, index, _ = step.get_contrasts(**ents)[0] + contrasts = step.get_contrasts(**ents)[0] contrast_type_map = defaultdict(lambda: 'T') contrast_type_map.update({contrast['name']: contrast['type'] @@ -404,19 +408,18 @@ def _run_interface(self, runtime): "".format(self.inputs.bids_dir, selectors, "\n\t".join( '{} ({})'.format( - f.filename, - layout.files[f.filename].entities) + f.path, + layout.files[f.path].entities) for f in bold_file))) # Select exactly matching mask file (may be over-cautious) - bold_ents = layout.parse_file_entities( - bold_file[0].filename) + bold_ents = layout.parse_file_entities(bold_file[0].path) bold_ents['type'] = 'brainmask' mask_file = layout.get(extensions=['.nii', '.nii.gz'], **bold_ents) bold_ents.pop('type') - bold_files.append(bold_file[0].filename) - mask_files.append(mask_file[0].filename if mask_file else None) + bold_files.append(bold_file[0].path) + mask_files.append(mask_file[0].path if mask_file else None) entities.append(bold_ents) self._results['bold_files'] = bold_files From 45180ba71ea11847db0a2014202fef22fa41edb9 Mon Sep 17 00:00:00 2001 From: delavega4 Date: Fri, 30 Nov 2018 13:12:48 -0600 Subject: [PATCH 06/33] Extract model paths from BIDSFile objects --- fitlins/interfaces/bids.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fitlins/interfaces/bids.py b/fitlins/interfaces/bids.py index db6b66e7..951ce654 100644 --- a/fitlins/interfaces/bids.py +++ b/fitlins/interfaces/bids.py @@ -103,13 +103,13 @@ def _run_interface(self, runtime): layout = bids.BIDSLayout(self.inputs.bids_dir, validate=False) if not isdefined(models): - models = layout.get(suffix='smdl') + models = layout.get(suffix='smdl', return_type='file') if not models: raise ValueError("No models found") elif models == 'default': models = auto_model(layout) - models = [_ensure_model(m.path) for m in models] + models = [_ensure_model(m) for m in models] if self.inputs.selectors: # This is almost certainly incorrect From 487b62a31d12c4bc493734dc01bbeb4b0154c168 Mon Sep 17 00:00:00 2001 From: delavega4 Date: Fri, 30 Nov 2018 17:15:20 -0600 Subject: [PATCH 07/33] Change preproc_dir to more generic derivatives which complies with BIDSLayout API --- fitlins/cli/run.py | 16 +++++----------- fitlins/interfaces/bids.py | 19 ++++++++----------- fitlins/workflows/base.py | 11 +++-------- 3 files changed, 16 insertions(+), 30 deletions(-) diff --git a/fitlins/cli/run.py b/fitlins/cli/run.py index aa5c6959..9468b254 100755 --- a/fitlins/cli/run.py +++ b/fitlins/cli/run.py @@ -79,9 +79,9 @@ def get_parser(): 'removed)') g_bids.add_argument('-m', '--model', action='store', default='model.json', help='location of BIDS model description (default bids_dir/model.json)') - g_bids.add_argument('-p', '--preproc-dir', action='store', default='fmriprep', - help='location of preprocessed data (if relative path, search ' - 'bids_dir/derivatives, followed by output_dir)') + g_bids.add_argument('-d', '--derivatives', action='append', + help='location of derivatives (including preprocessed images).' + 'If none specified, indexes all derivatives under bids_dir/derivatives.') g_bids.add_argument('--derivative-label', action='store', type=str, help='execution label to append to derivative directory name') g_bids.add_argument('--space', action='store', @@ -139,13 +139,7 @@ def run_fitlins(argv=None): if opts.model in (None, 'default') and not op.exists(model): model = 'default' - preproc_dir = default_path(opts.preproc_dir, - op.join(opts.bids_dir, 'derivatives'), - 'fmriprep') - if not op.exists(preproc_dir): - preproc_dir = default_path(opts.preproc_dir, opts.output_dir, 'fmriprep') - if not op.exists(preproc_dir): - raise RuntimeError("Preprocessed data could not be found") + derivatives = True if not opts.derivatves else opts.derivatives pipeline_name = 'fitlins' if opts.derivative_label: @@ -158,7 +152,7 @@ def run_fitlins(argv=None): work_dir = mkdtemp() if opts.work_dir is None else opts.work_dir fitlins_wf = init_fitlins_wf( - opts.bids_dir, preproc_dir, deriv_dir, opts.space, model=model, + opts.bids_dir, derivatives, deriv_dir, opts.space, model=model, participants=subject_list, base_dir=work_dir, include_pattern=opts.include, exclude_pattern=opts.exclude ) diff --git a/fitlins/interfaces/bids.py b/fitlins/interfaces/bids.py index 951ce654..fd5b664b 100644 --- a/fitlins/interfaces/bids.py +++ b/fitlins/interfaces/bids.py @@ -136,8 +136,8 @@ class LoadBIDSModelInputSpec(BaseInterfaceInputSpec): bids_dir = Directory(exists=True, mandatory=True, desc='BIDS dataset root directory') - preproc_dir = Directory(exists=True, - desc='Optional preprocessed files directory') + derivatives = traits.Either(traits.Bool, InputMultiPath(Directory(exists=True)), + desc='Derivative folders') model = traits.Dict(desc='Model specification', mandatory=True) selectors = traits.Dict(desc='Limit collected sessions', usedefault=True) include_pattern = InputMultiPath( @@ -164,7 +164,7 @@ def _run_interface(self, runtime): import bids include = self.inputs.include_pattern exclude = self.inputs.exclude_pattern - derivatives = self.inputs.preproc_dir + derivatives = self.inputs.derivatives if not isdefined(include): include = None if not isdefined(exclude): @@ -172,8 +172,8 @@ def _run_interface(self, runtime): if not isdefined(derivatives): exclude = False - layout = bids.BIDSLayout(self.inputs.bids_dir, include=include, exclude=exclude, - derivatives=derivatives) + layout = bids.BIDSLayout(self.inputs.bids_dir, include=include, + exclude=exclude, derivatives=derivatives) selectors = self.inputs.selectors @@ -364,8 +364,8 @@ class BIDSSelectInputSpec(BaseInterfaceInputSpec): bids_dir = Directory(exists=True, mandatory=True, desc='BIDS dataset root directories') - preproc_dir = Directory(exists=True, - desc='Optional preprocessed files directory') + derivatives = traits.Either(True, InputMultiPath(Directory(exists=True)), + desc='Derivative folders') entities = InputMultiPath(traits.Dict(), mandatory=True) selectors = traits.Dict(desc='Additional selectors to be applied', usedefault=True) @@ -384,10 +384,7 @@ class BIDSSelect(SimpleInterface): def _run_interface(self, runtime): import bids - derivatives = self.inputs.preproc_dir - if not isdefined(derivatives): - exclude = False - + derivatives = self.inputs.derivatives layout = bids.BIDSLayout(self.inputs.bids_dir, derivatives=derivatives) bold_files = [] diff --git a/fitlins/workflows/base.py b/fitlins/workflows/base.py index afdf90f8..18eed266 100644 --- a/fitlins/workflows/base.py +++ b/fitlins/workflows/base.py @@ -9,7 +9,7 @@ from ..interfaces.utils import MergeAll -def init_fitlins_wf(bids_dir, preproc_dir, out_dir, space, exclude_pattern=None, +def init_fitlins_wf(bids_dir, derivatives, out_dir, space, exclude_pattern=None, include_pattern=None, model=None, participants=None, base_dir=None, name='fitlins_wf'): wf = pe.Workflow(name=name, base_dir=base_dir) @@ -31,12 +31,10 @@ def init_fitlins_wf(bids_dir, preproc_dir, out_dir, space, exclude_pattern=None, loader = pe.Node( LoadBIDSModel(bids_dir=bids_dir, - preproc_dir=preproc_dir, + derivatives=derivatives, selectors=selectors), name='loader') - if preproc_dir is not None: - loader.inputs.preproc_dir = preproc_dir if exclude_pattern is not None: loader.inputs.exclude_pattern = exclude_pattern if include_pattern is not None: @@ -53,13 +51,10 @@ def init_fitlins_wf(bids_dir, preproc_dir, out_dir, space, exclude_pattern=None, # Select preprocessed BOLD series to analyze getter = pe.Node( - BIDSSelect(bids_dir=bids_dir, + BIDSSelect(bids_dir=bids_dir, derivatives=derivatives, selectors={'type': 'preproc', 'space': space}), name='getter') - if preproc_dir is not None: - getter.inputs.preproc_dir = preproc_dir - select_l1_contrasts = pe.Node(niu.Select(index=0), name='select_l1_contrasts') # Run first level model From cb17b01e3efae6de6bf3db876b96612752941a7f Mon Sep 17 00:00:00 2001 From: delavega4 Date: Fri, 30 Nov 2018 17:52:55 -0600 Subject: [PATCH 08/33] Fix derivatives typos --- fitlins/cli/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fitlins/cli/run.py b/fitlins/cli/run.py index 9468b254..00f42cbf 100755 --- a/fitlins/cli/run.py +++ b/fitlins/cli/run.py @@ -139,7 +139,7 @@ def run_fitlins(argv=None): if opts.model in (None, 'default') and not op.exists(model): model = 'default' - derivatives = True if not opts.derivatves else opts.derivatives + derivatives = True if not opts.derivatives else opts.derivatives pipeline_name = 'fitlins' if opts.derivative_label: From 8079d958bbbce1f77b9a1daedde3d68c03a78240 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Mon, 3 Dec 2018 12:20:01 -0600 Subject: [PATCH 09/33] Store derivatives into a single list Co-Authored-By: adelavega --- fitlins/cli/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fitlins/cli/run.py b/fitlins/cli/run.py index 00f42cbf..cda961e1 100755 --- a/fitlins/cli/run.py +++ b/fitlins/cli/run.py @@ -79,7 +79,7 @@ def get_parser(): 'removed)') g_bids.add_argument('-m', '--model', action='store', default='model.json', help='location of BIDS model description (default bids_dir/model.json)') - g_bids.add_argument('-d', '--derivatives', action='append', + g_bids.add_argument('-d', '--derivatives', action='store', nargs='+', help='location of derivatives (including preprocessed images).' 'If none specified, indexes all derivatives under bids_dir/derivatives.') g_bids.add_argument('--derivative-label', action='store', type=str, From 377e5672a17f6d2606157ad13e060095966f8d18 Mon Sep 17 00:00:00 2001 From: delavega4 Date: Mon, 3 Dec 2018 16:20:37 -0600 Subject: [PATCH 10/33] Emulate previous contrast behavior --- fitlins/interfaces/bids.py | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/fitlins/interfaces/bids.py b/fitlins/interfaces/bids.py index fd5b664b..459237ba 100644 --- a/fitlins/interfaces/bids.py +++ b/fitlins/interfaces/bids.py @@ -188,6 +188,8 @@ def _run_interface(self, runtime): return runtime def _load_level1(self, runtime, analysis): + import pandas as pd + step = analysis.steps[0] step_subdir = Path(runtime.cwd) / step.level step_subdir.mkdir(parents=True, exist_ok=True) @@ -268,22 +270,15 @@ def _load_level1(self, runtime, analysis): info['dense'] = str(dense_file) info['repetition_time'] = TR - # Transpose so each contrast gets a row of data instead of column contrasts = step.get_contrasts(**ents)[0] - - contrast_type_map = defaultdict(lambda: 'T') - contrast_type_map.update({contrast['name']: contrast['type'] - for contrast in step.contrasts}) - contrast_type_list = [contrast_type_map[contrast] - for contrast in contrasts.columns] - - contrasts = contrasts.T - # Add test indicator column - contrasts['type'] = contrast_type_list + # Convert NamedTuples to matrix + contrast_matrix = pd.concat([c.weights for c in contrasts], sort=False) + contrast_matrix.index = [c.name for c in contrasts] + contrast_matrix['type'] = [c.type for c in contrasts] contrasts_file = step_subdir / '{}_contrasts.h5'.format(ent_string) contrasts_file.parent.mkdir(parents=True, exist_ok=True) - contrasts.to_hdf(contrasts_file, key='contrasts') + contrast_matrix.to_hdf(contrasts_file, key='contrasts') warning_file = step_subdir / '{}_warning.html'.format(ent_string) with warning_file.open('w') as fobj: @@ -292,7 +287,7 @@ def _load_level1(self, runtime, analysis): entities.append(ents) session_info.append(info) - contrast_indices.append(index.to_dict('records')) + contrast_indices.append([c.entities for c in contrasts]) contrast_info.append(str(contrasts_file)) warnings.append(str(warning_file)) From ab4f83af1acf057145dd03743158d34c0365b1b0 Mon Sep 17 00:00:00 2001 From: delavega4 Date: Mon, 3 Dec 2018 17:25:23 -0600 Subject: [PATCH 11/33] Begin converting higher level contrasts --- fitlins/interfaces/bids.py | 30 ++++++++++++------------------ 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/fitlins/interfaces/bids.py b/fitlins/interfaces/bids.py index 459237ba..2ba36b99 100644 --- a/fitlins/interfaces/bids.py +++ b/fitlins/interfaces/bids.py @@ -298,53 +298,47 @@ def _load_level1(self, runtime, analysis): self._results.setdefault('contrast_info', []).append(contrast_info) def _load_higher_level(self, runtime, analysis): + import pandas as pd + cwd = Path(runtime.cwd) - for block in analysis.blocks[1:]: + for block in analysis.steps[1:]: block_subdir = cwd / block.level block_subdir.mkdir(parents=True, exist_ok=True) entities = [] contrast_indices = [] contrast_info = [] - for contrasts, index, ents in block.get_contrasts(): - if contrasts.empty: + for contrasts in block.get_contrasts(): + if all([c.weights.empty for c in contrasts]): continue # The contrast index is the name of the input contrasts, # which will very frequently be non-unique # Hence, add the contrast to the index (table of entities) # and switch to a matching numeric index - index['contrast'] = contrasts.index - contrasts.index = index.index + contrasts.entities['contrast'] = contrasts.index - contrast_type_map = defaultdict(lambda: 'T') - contrast_type_map.update({contrast['name']: contrast['type'] - for contrast in block.contrasts}) - contrast_type_list = [contrast_type_map[contrast] - for contrast in contrasts.columns] + contrast_matrix = pd.concat([c.weights for c in contrasts], sort=False) + contrast_matrix.index = [c.name for c in contrasts] + contrast_matrix['type'] = [c.type for c in contrasts] - indices = index.to_dict('records') + indices = [c.entities for c in contrasts] # Entities for a given contrast matrix include the intersection of # entities of inputs, e.g., if this level is within-subject, the # subject should persist out_ents = reduce(dict_intersection, indices) # Explicit entities take precedence over derived - out_ents.update(ents) + out_ents.update(contrasts.entities) # Input-level contrasts will be overridden by the current level out_ents.pop('contrast', None) ent_string = '_'.join('{}-{}'.format(key, val) for key, val in out_ents.items()) - # Transpose so each contrast gets a row of data instead of column - contrasts = contrasts.T - # Add test indicator column - contrasts['type'] = contrast_type_list - contrasts_file = block_subdir / '{}_contrasts.h5'.format(ent_string) contrasts_file.parent.mkdir(parents=True, exist_ok=True) - contrasts.to_hdf(contrasts_file, key='contrasts') + contrast_matrix.to_hdf(contrasts_file, key='contrasts') entities.append(out_ents) contrast_indices.append(indices) From 414f613a10af4d87b81ec697c917e25909e871e5 Mon Sep 17 00:00:00 2001 From: delavega4 Date: Tue, 4 Dec 2018 18:37:55 -0600 Subject: [PATCH 12/33] Attempt to simplify contrasts --- fitlins/interfaces/bids.py | 19 ++--- fitlins/interfaces/nistats.py | 129 ++++++++-------------------------- fitlins/workflows/base.py | 5 +- 3 files changed, 37 insertions(+), 116 deletions(-) diff --git a/fitlins/interfaces/bids.py b/fitlins/interfaces/bids.py index 2ba36b99..8efad8e6 100644 --- a/fitlins/interfaces/bids.py +++ b/fitlins/interfaces/bids.py @@ -2,13 +2,12 @@ from functools import reduce from pathlib import Path from gzip import GzipFile +import pickle import json import shutil import numpy as np import nibabel as nb -from collections import defaultdict - from nipype import logging from nipype.utils.filemanip import makedirs, copyfile from nipype.interfaces.base import ( @@ -270,15 +269,9 @@ def _load_level1(self, runtime, analysis): info['dense'] = str(dense_file) info['repetition_time'] = TR - contrasts = step.get_contrasts(**ents)[0] - # Convert NamedTuples to matrix - contrast_matrix = pd.concat([c.weights for c in contrasts], sort=False) - contrast_matrix.index = [c.name for c in contrasts] - contrast_matrix['type'] = [c.type for c in contrasts] - - contrasts_file = step_subdir / '{}_contrasts.h5'.format(ent_string) - contrasts_file.parent.mkdir(parents=True, exist_ok=True) - contrast_matrix.to_hdf(contrasts_file, key='contrasts') + contrasts = [c._asdict() for c in step.get_contrasts(**ents)[0]] + for con in contrasts: + con['weights'] = con['weights'].to_dict('records') warning_file = step_subdir / '{}_warning.html'.format(ent_string) with warning_file.open('w') as fobj: @@ -287,14 +280,12 @@ def _load_level1(self, runtime, analysis): entities.append(ents) session_info.append(info) - contrast_indices.append([c.entities for c in contrasts]) - contrast_info.append(str(contrasts_file)) + contrast_info.append(contrasts) warnings.append(str(warning_file)) self._results['session_info'] = session_info self._results['warnings'] = warnings self._results.setdefault('entities', []).append(entities) - self._results.setdefault('contrast_indices', []).append(contrast_indices) self._results.setdefault('contrast_info', []).append(contrast_info) def _load_higher_level(self, runtime, analysis): diff --git a/fitlins/interfaces/nistats.py b/fitlins/interfaces/nistats.py index 883a0857..5d152397 100644 --- a/fitlins/interfaces/nistats.py +++ b/fitlins/interfaces/nistats.py @@ -9,7 +9,7 @@ from nipype.interfaces.base import ( LibraryBaseInterface, SimpleInterface, BaseInterfaceInputSpec, TraitedSpec, - OutputMultiObject, File, traits, isdefined + File, traits, isdefined ) from ..utils import dict_intersection @@ -19,56 +19,21 @@ class NistatsBaseInterface(LibraryBaseInterface): _pkg = 'nistats' -def build_contrast_matrix(contrast_spec, design_matrix, - identity=None): - """Construct contrast matrix and return contrast type - - Parameters - ---------- - contrast_spec : DataFrame - Weight matrix with contrasts as rows and regressors as columns - May have 'type' column indicating T/F test - design_matrix : DataFrame - GLM design matrix with regressors as columns and TR time as rows - identity : list of strings - Names of explanatory variables to ensure "identity" contrasts are - provided. - - Returns - ------- - contrast_matrix : DataFrame - Weight matrix with contrasts as columns and regressors as rows. - Regressors match columns (including order) of design matrix. - Identity contrasts are included. - contrast_types : Series - Series of 'T'/'F' indicating the type of test for each column in - the returned contrast matrix. - Identity contrasts use T-tests. - """ - # The basic spec is just a transposed matrix with an optional 'type' - # column - # We'll re-transpose and expand this matrix - init_matrix = contrast_spec.drop('type', axis='columns').T - init_types = contrast_spec['type'] if 'type' in contrast_spec \ - else pd.Series() - - if identity is None: - identity = [] - all_cols = init_matrix.columns.tolist() - all_cols.extend(set(identity) - set(all_cols)) - - contrast_matrix = pd.DataFrame(index=design_matrix.columns, - columns=all_cols, data=0) - contrast_matrix.loc[tuple(init_matrix.axes)] = init_matrix - - contrast_types = pd.Series(index=all_cols, data='T') - contrast_types[init_types.index] = init_types - - if identity: - contrast_matrix.loc[identity, identity] = np.eye(len(identity)) - contrast_types[identity] = 'T' - - return contrast_matrix, contrast_types +def prepare_contrasts(contrasts, design_matrix): + if not isdefined(contrasts): + contrasts = [] + all_regressors = design_matrix.columns + # Prepare contrast + for contrast in contrasts: + # Fill in zeros + contrast['weights'] = [ + [row[col] if col in row else 0 for col in all_regressors.columns] + for row in contrast['weights'] + ] + # Lower case T + contrast['type'] = {'T': 't', 'F': 'F'}[contrast['type']] + + return contrasts class FirstLevelModelInputSpec(BaseInterfaceInputSpec): @@ -82,7 +47,6 @@ class FirstLevelModelOutputSpec(TraitedSpec): contrast_maps = traits.List(File) contrast_metadata = traits.List(traits.Dict) design_matrix = File() - contrast_matrix = File() class FirstLevelModel(NistatsBaseInterface, SimpleInterface): @@ -106,12 +70,6 @@ def _run_interface(self, runtime): confound_names = None drift_model = 'cosine' - if isdefined(self.inputs.contrast_info): - contrast_spec = pd.read_hdf(self.inputs.contrast_info, - key='contrasts') - else: - contrast_spec = pd.DataFrame() - mat = dm.make_design_matrix( frame_times=np.arange(vols) * info['repetition_time'], paradigm=events.rename(columns={'condition': 'trial_type', @@ -121,20 +79,10 @@ def _run_interface(self, runtime): drift_model=drift_model, ) - # Assume that explanatory variables == HRF-convolved variables - exp_vars = events['condition'].unique().tolist() - - contrast_matrix, contrast_types = build_contrast_matrix(contrast_spec, - mat, exp_vars) - mat.to_csv('design.tsv', sep='\t') self._results['design_matrix'] = os.path.join(runtime.cwd, 'design.tsv') - contrast_matrix.to_csv('contrasts.tsv', sep='\t') - self._results['contrast_matrix'] = os.path.join( - runtime.cwd, 'contrasts.tsv') - mask_file = self.inputs.mask_file if not isdefined(mask_file): mask_file = None @@ -143,16 +91,16 @@ def _run_interface(self, runtime): contrast_maps = [] contrast_metadata = [] - stat_fmt = os.path.join(runtime.cwd, '{}.nii.gz').format - for contrast, ctype in zip(contrast_matrix, contrast_types): - es = flm.compute_contrast(contrast_matrix[contrast].values, - {'T': 't', 'F': 'F'}[ctype], + for contrast in prepare_contrasts(self.inputs_contrast_info): + es = flm.compute_contrast(contrast['weights'], + contrast['type'], output_type='effect_size') - es_fname = stat_fmt(contrast) + es_fname = os.path.join( + runtime.cwd, '{}.nii.gz').format(contrast) es.to_filename(es_fname) contrast_maps.append(es_fname) - contrast_metadata.append({'contrast': contrast, + contrast_metadata.append({'contrast': contrast['weights'], 'type': 'effect'}) self._results['contrast_maps'] = contrast_maps self._results['contrast_metadata'] = contrast_metadata @@ -191,8 +139,11 @@ class SecondLevelModel(NistatsBaseInterface, SimpleInterface): def _run_interface(self, runtime): model = level2.SecondLevelModel() files = [] - # Super inefficient... think more about this later - for idx in self.inputs.contrast_indices: + contrasts = prepare_contrasts(self.inputs_contrast_info) + + # Need a way to group appropriate files + for contrast in contrasts: + idx = contrasts['entities'] for fname, metadata in zip(_flatten(self.inputs.stat_files), _flatten(self.inputs.stat_metadata)): if _match(idx, metadata): @@ -202,37 +153,19 @@ def _run_interface(self, runtime): raise ValueError out_ents = reduce(dict_intersection, self.inputs.contrast_indices) - in_ents = [{key: val for key, val in index.items() if key not in out_ents} - for index in self.inputs.contrast_indices] - - contrast_spec = pd.read_hdf(self.inputs.contrast_info, - key='contrasts') - - contrast_matrix = contrast_spec.drop(columns=['type']).T - contrast_types = contrast_spec['type'] - - contrast_matrix.index = ['_'.join('{}-{}'.format(key, val) - for key, val in ents.items()) - for ents in in_ents] - contrast_matrix.to_csv('contrasts.tsv', sep='\t') - self._results['contrast_matrix'] = os.path.join( - runtime.cwd, 'contrasts.tsv') - out_ents['type'] = 'stat' contrast_maps = [] contrast_metadata = [] - stat_fmt = os.path.join(runtime.cwd, '{}.nii.gz').format - for contrast, ctype in zip(contrast_matrix, contrast_types): - intercept = contrast_matrix[contrast] + for contrast in contrasts: + intercept = contrast['weights'] data = np.array(files)[intercept != 0].tolist() intercept = intercept[intercept != 0] model.fit(data, design_matrix=pd.DataFrame({'intercept': intercept})) - stat_type = {'T': 't', 'F': 'F'}[ctype] - stat = model.compute_contrast(second_level_stat_type=stat_type) - stat_fname = stat_fmt(contrast) + stat = model.compute_contrast(second_level_stat_type=contrast['type']) + stat_fname = os.path.join(runtime.cwd, '{}.nii.gz').format(contrast) stat.to_filename(stat_fname) contrast_maps.append(stat_fname) diff --git a/fitlins/workflows/base.py b/fitlins/workflows/base.py index 18eed266..26d07a8a 100644 --- a/fitlins/workflows/base.py +++ b/fitlins/workflows/base.py @@ -78,14 +78,13 @@ def join_dict(base_dict, dict_list): name='collate_first_level') select_l2_entities = pe.Node(niu.Select(index=1), name='select_l2_entities') - select_l2_indices = pe.Node(niu.Select(index=1), name='select_l2_indices') select_l2_contrasts = pe.Node(niu.Select(index=1), name='select_l2_contrasts') # Run second-level model # TODO: Iterate over all higher levels l2_model = pe.MapNode( SecondLevelModel(), - iterfield=['contrast_info', 'contrast_indices'], + iterfield=['contrast_info'], name='l2_model') collate_second_level = pe.Node(MergeAll(['contrast_maps', 'contrast_metadata']), @@ -249,12 +248,10 @@ def _get_evs(info): (l1_metadata, collate_first_level, [('out', 'contrast_metadata')]), (loader, select_l2_entities, [('entities', 'inlist')]), - (loader, select_l2_indices, [('contrast_indices', 'inlist')]), (loader, select_l2_contrasts, [('contrast_info', 'inlist')]), (l1_model, l2_model, [('contrast_maps', 'stat_files')]), (l1_metadata, l2_model, [('out', 'stat_metadata')]), - (select_l2_indices, l2_model, [('out', 'contrast_indices')]), (select_l2_contrasts, l2_model, [('out', 'contrast_info')]), (l2_model, collate_second_level, [('contrast_maps', 'contrast_maps'), From 93c7dc184734c0d8612f4571b947b6f27f050911 Mon Sep 17 00:00:00 2001 From: delavega4 Date: Thu, 6 Dec 2018 12:33:52 -0600 Subject: [PATCH 13/33] Remove indices/matrix from higher level --- fitlins/interfaces/bids.py | 61 +++++++---------------------------- fitlins/interfaces/nistats.py | 4 +-- fitlins/workflows/base.py | 10 +++--- 3 files changed, 19 insertions(+), 56 deletions(-) diff --git a/fitlins/interfaces/bids.py b/fitlins/interfaces/bids.py index 8efad8e6..27cded5c 100644 --- a/fitlins/interfaces/bids.py +++ b/fitlins/interfaces/bids.py @@ -1,8 +1,6 @@ import os -from functools import reduce from pathlib import Path from gzip import GzipFile -import pickle import json import shutil import numpy as np @@ -17,7 +15,7 @@ ) from nipype.interfaces.io import IOBase -from ..utils import dict_intersection, snake_to_camel +from ..utils import snake_to_camel iflogger = logging.getLogger('nipype.interface') @@ -149,8 +147,7 @@ class LoadBIDSModelInputSpec(BaseInterfaceInputSpec): class LoadBIDSModelOutputSpec(TraitedSpec): session_info = traits.List(traits.Dict()) - contrast_info = traits.List(traits.List(File())) - contrast_indices = traits.List(traits.List(traits.List(traits.Dict))) + contrast_info = traits.List(traits.List(traits.List(traits.Dict()))) entities = traits.List(traits.List(traits.Dict())) warnings = traits.List(File) @@ -181,21 +178,15 @@ def _run_interface(self, runtime): self._load_level1(runtime, analysis) self._load_higher_level(runtime, analysis) - # Debug - remove, eventually - runtime.analysis = analysis - return runtime def _load_level1(self, runtime, analysis): - import pandas as pd - step = analysis.steps[0] step_subdir = Path(runtime.cwd) / step.level step_subdir.mkdir(parents=True, exist_ok=True) entities = [] session_info = [] - contrast_indices = [] contrast_info = [] warnings = [] for sparse, dense, ents in step.get_design_matrix(): @@ -269,7 +260,7 @@ def _load_level1(self, runtime, analysis): info['dense'] = str(dense_file) info['repetition_time'] = TR - contrasts = [c._asdict() for c in step.get_contrasts(**ents)[0]] + contrasts = [dict(c._asdict()) for c in step.get_contrasts(**ents)[0]] for con in contrasts: con['weights'] = con['weights'].to_dict('records') @@ -289,55 +280,27 @@ def _load_level1(self, runtime, analysis): self._results.setdefault('contrast_info', []).append(contrast_info) def _load_higher_level(self, runtime, analysis): - import pandas as pd - - cwd = Path(runtime.cwd) + # cwd = Path(runtime.cwd) for block in analysis.steps[1:]: - block_subdir = cwd / block.level - block_subdir.mkdir(parents=True, exist_ok=True) + # block_subdir = cwd / block.level + # block_subdir.mkdir(parents=True, exist_ok=True) entities = [] - contrast_indices = [] contrast_info = [] for contrasts in block.get_contrasts(): if all([c.weights.empty for c in contrasts]): continue - # The contrast index is the name of the input contrasts, - # which will very frequently be non-unique - # Hence, add the contrast to the index (table of entities) - # and switch to a matching numeric index - contrasts.entities['contrast'] = contrasts.index - - contrast_matrix = pd.concat([c.weights for c in contrasts], sort=False) - contrast_matrix.index = [c.name for c in contrasts] - contrast_matrix['type'] = [c.type for c in contrasts] - - indices = [c.entities for c in contrasts] - - # Entities for a given contrast matrix include the intersection of - # entities of inputs, e.g., if this level is within-subject, the - # subject should persist - out_ents = reduce(dict_intersection, indices) - # Explicit entities take precedence over derived - out_ents.update(contrasts.entities) - # Input-level contrasts will be overridden by the current level - out_ents.pop('contrast', None) - - ent_string = '_'.join('{}-{}'.format(key, val) - for key, val in out_ents.items()) - - contrasts_file = block_subdir / '{}_contrasts.h5'.format(ent_string) - contrasts_file.parent.mkdir(parents=True, exist_ok=True) - contrast_matrix.to_hdf(contrasts_file, key='contrasts') + entities.append(contrasts[0].entities) # Should all the same + contrasts = [dict(c._asdict()) for c in contrasts] + for contrast in contrasts: + contrast['weights'] = contrast['weights'].to_dict('records') + contrast.pop('entities') - entities.append(out_ents) - contrast_indices.append(indices) - contrast_info.append(str(contrasts_file)) + contrast_info.append(contrasts) self._results['entities'].append(entities) self._results['contrast_info'].append(contrast_info) - self._results['contrast_indices'].append(contrast_indices) class BIDSSelectInputSpec(BaseInterfaceInputSpec): diff --git a/fitlins/interfaces/nistats.py b/fitlins/interfaces/nistats.py index 5d152397..c71da322 100644 --- a/fitlins/interfaces/nistats.py +++ b/fitlins/interfaces/nistats.py @@ -40,7 +40,7 @@ class FirstLevelModelInputSpec(BaseInterfaceInputSpec): bold_file = File(exists=True, mandatory=True) mask_file = File(exists=True) session_info = traits.Dict() - contrast_info = File(exists=True) + contrast_info = traits.List(traits.List(traits.Dict)) class FirstLevelModelOutputSpec(TraitedSpec): @@ -111,7 +111,7 @@ def _run_interface(self, runtime): class SecondLevelModelInputSpec(BaseInterfaceInputSpec): stat_files = traits.List(traits.List(File(exists=True)), mandatory=True) stat_metadata = traits.List(traits.List(traits.Dict)) - contrast_info = File(exists=True) + contrast_info = traits.List(traits.List(traits.Dict)) contrast_indices = traits.List(traits.Dict) diff --git a/fitlins/workflows/base.py b/fitlins/workflows/base.py index 26d07a8a..1e20a53a 100644 --- a/fitlins/workflows/base.py +++ b/fitlins/workflows/base.py @@ -266,11 +266,11 @@ def _get_evs(info): (l1_model, plot_corr, [('design_matrix', 'data')]), (get_evs, plot_corr, [('out', 'explanatory_variables')]), - (l1_model, plot_l1_contrast_matrix, [('contrast_matrix', 'data')]), + # (l1_model, plot_l1_contrast_matrix, [('contrast_matrix', 'data')]), (collate_first_level, plot_l1_contrasts, [('contrast_maps', 'data')]), - (l2_model, plot_l2_contrast_matrix, [('contrast_matrix', 'data')]), + # (l2_model, plot_l2_contrast_matrix, [('contrast_matrix', 'data')]), (collate_second_level, plot_l2_contrasts, [('contrast_maps', 'data')]), @@ -297,16 +297,16 @@ def _get_evs(info): (select_l1_entities, ds_l1_contrasts, [('out', 'entities')]), - (plot_l1_contrast_matrix, ds_l1_contrasts, [('figure', 'in_file')]), + # (plot_l1_contrast_matrix, ds_l1_contrasts, [('figure', 'in_file')]), (collate_first_level, ds_l1_contrast_plots, [('contrast_metadata', 'entities')]), (plot_l1_contrasts, ds_l1_contrast_plots, [('figure', 'in_file')]), (select_l2_entities, ds_l2_contrasts, [('out', 'entities')]), - (plot_l2_contrast_matrix, ds_l2_contrasts, [('figure', 'in_file')]), + # (plot_l2_contrast_matrix, ds_l2_contrasts, [('figure', 'in_file')]), (collate_second_level, ds_l2_contrast_plots, [('contrast_metadata', 'entities')]), - (plot_l2_contrasts, ds_l2_contrast_plots, [('figure', 'in_file')]), + # (plot_l2_contrasts, ds_l2_contrast_plots, [('figure', 'in_file')]), ]) return wf From 8a61ee72246d4bfc1b2e3efe076acd98a05750ab Mon Sep 17 00:00:00 2001 From: delavega4 Date: Thu, 6 Dec 2018 13:55:31 -0600 Subject: [PATCH 14/33] Add dataset description to reportlet --- fitlins/workflows/base.py | 61 +++++++++++++++++++++++---------------- 1 file changed, 36 insertions(+), 25 deletions(-) diff --git a/fitlins/workflows/base.py b/fitlins/workflows/base.py index 1e20a53a..6bdd0657 100644 --- a/fitlins/workflows/base.py +++ b/fitlins/workflows/base.py @@ -1,4 +1,5 @@ from pathlib import Path +import json from nipype.pipeline import engine as pe from nipype.interfaces import utility as niu from ..interfaces.bids import ( @@ -113,20 +114,20 @@ def _get_evs(info): iterfield=['data', 'explanatory_variables'], name='plot_corr') - plot_l1_contrast_matrix = pe.MapNode( - ContrastMatrixPlot(image_type='svg'), - iterfield='data', - name='plot_l1_contrast_matrix') + # plot_l1_contrast_matrix = pe.MapNode( + # ContrastMatrixPlot(image_type='svg'), + # iterfield='data', + # name='plot_l1_contrast_matrix') plot_l1_contrasts = pe.MapNode( GlassBrainPlot(image_type='png'), iterfield='data', name='plot_l1_contrasts') - plot_l2_contrast_matrix = pe.MapNode( - ContrastMatrixPlot(image_type='svg'), - iterfield='data', - name='plot_l2_contrast_matrix') + # plot_l2_contrast_matrix = pe.MapNode( + # ContrastMatrixPlot(image_type='svg'), + # iterfield='data', + # name='plot_l2_contrast_matrix') plot_l2_contrasts = pe.MapNode( GlassBrainPlot(image_type='png'), @@ -140,6 +141,16 @@ def _get_evs(info): reportlet_dir = Path(base_dir) / 'reportlets' / 'fitlins' reportlet_dir.mkdir(parents=True, exist_ok=True) + with (reportlet_dir / 'dataset_description.json').open('w') as file: + json.dump( + { + 'Name': 'Fitlins reportlet', + 'BIDSVersion': '1.1.0', + 'PipelineDescription': { + 'Name': 'FitLins', + }, + }, + file) snippet_pattern = '[sub-{subject}/][ses-{session}/][sub-{subject}_]' \ '[ses-{session}_]task-{task}_[run-{run}_]snippet.html' @@ -190,21 +201,21 @@ def _get_evs(info): run_without_submitting=True, name='ds_corr') - ds_l1_contrasts = pe.MapNode( - BIDSDataSink(base_directory=out_dir, - fixed_entities={'type': 'contrasts'}, - path_patterns=image_pattern), - iterfield=['entities', 'in_file'], - run_without_submitting=True, - name='ds_l1_contrasts') - - ds_l2_contrasts = pe.MapNode( - BIDSDataSink(base_directory=out_dir, - fixed_entities={'type': 'contrasts'}, - path_patterns=image_pattern), - iterfield=['entities', 'in_file'], - run_without_submitting=True, - name='ds_l2_contrasts') + # ds_l1_contrasts = pe.MapNode( + # BIDSDataSink(base_directory=out_dir, + # fixed_entities={'type': 'contrasts'}, + # path_patterns=image_pattern), + # iterfield=['entities', 'in_file'], + # run_without_submitting=True, + # name='ds_l1_contrasts') + + # ds_l2_contrasts = pe.MapNode( + # BIDSDataSink(base_directory=out_dir, + # fixed_entities={'type': 'contrasts'}, + # path_patterns=image_pattern), + # iterfield=['entities', 'in_file'], + # run_without_submitting=True, + # name='ds_l2_contrasts') contrast_plot_pattern = '[sub-{subject}/][ses-{session}/]' \ '[sub-{subject}_][ses-{session}_]task-{task}[_acq-{acquisition}]' \ @@ -296,13 +307,13 @@ def _get_evs(info): (plot_corr, ds_corr, [('figure', 'in_file')]), - (select_l1_entities, ds_l1_contrasts, [('out', 'entities')]), + # (select_l1_entities, ds_l1_contrasts, [('out', 'entities')]), # (plot_l1_contrast_matrix, ds_l1_contrasts, [('figure', 'in_file')]), (collate_first_level, ds_l1_contrast_plots, [('contrast_metadata', 'entities')]), (plot_l1_contrasts, ds_l1_contrast_plots, [('figure', 'in_file')]), - (select_l2_entities, ds_l2_contrasts, [('out', 'entities')]), + # (select_l2_entities, ds_l2_contrasts, [('out', 'entities')]), # (plot_l2_contrast_matrix, ds_l2_contrasts, [('figure', 'in_file')]), (collate_second_level, ds_l2_contrast_plots, [('contrast_metadata', 'entities')]), From 6f36d5af5db05892051495f7202e0601ba240b49 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Thu, 6 Dec 2018 15:54:17 -0500 Subject: [PATCH 15/33] TMP: Do not produce correlation matrix plot --- fitlins/workflows/base.py | 45 ++++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/fitlins/workflows/base.py b/fitlins/workflows/base.py index 6bdd0657..4ca8c409 100644 --- a/fitlins/workflows/base.py +++ b/fitlins/workflows/base.py @@ -100,19 +100,20 @@ def join_dict(base_dict, dict_list): iterfield='data', name='plot_design') - def _get_evs(info): - import pandas as pd - events = pd.read_hdf(info['events'], key='events') - return len(events['condition'].unique()) + # TODO: get_evs should be based on contrasts, EVs are any used in contrasts, others are confounds + # def _get_evs(info): + # import pandas as pd + # events = pd.read_hdf(info['events'], key='events') + # return len(events['condition'].unique()) - # Number of explanatory variables is used to mark off sections of the - # correlation matrix - get_evs = pe.MapNode(niu.Function(function=_get_evs), iterfield='info', name='get_evs') + # # Number of explanatory variables is used to mark off sections of the + # # correlation matrix + # get_evs = pe.MapNode(niu.Function(function=_get_evs), iterfield='info', name='get_evs') - plot_corr = pe.MapNode( - DesignCorrelationPlot(image_type='svg'), - iterfield=['data', 'explanatory_variables'], - name='plot_corr') + # plot_corr = pe.MapNode( + # DesignCorrelationPlot(image_type='svg'), + # iterfield=['data', 'explanatory_variables'], + # name='plot_corr') # plot_l1_contrast_matrix = pe.MapNode( # ContrastMatrixPlot(image_type='svg'), @@ -194,12 +195,12 @@ def _get_evs(info): run_without_submitting=True, name='ds_design') - ds_corr = pe.MapNode( - BIDSDataSink(base_directory=out_dir, fixed_entities={'type': 'corr'}, - path_patterns=image_pattern), - iterfield=['entities', 'in_file'], - run_without_submitting=True, - name='ds_corr') + # ds_corr = pe.MapNode( + # BIDSDataSink(base_directory=out_dir, fixed_entities={'type': 'corr'}, + # path_patterns=image_pattern), + # iterfield=['entities', 'in_file'], + # run_without_submitting=True, + # name='ds_corr') # ds_l1_contrasts = pe.MapNode( # BIDSDataSink(base_directory=out_dir, @@ -273,9 +274,9 @@ def _get_evs(info): # (l1_model, plot_design, [('design_matrix', 'data')]), - (loader, get_evs, [('session_info', 'info')]), - (l1_model, plot_corr, [('design_matrix', 'data')]), - (get_evs, plot_corr, [('out', 'explanatory_variables')]), + # (loader, get_evs, [('session_info', 'info')]), + # (l1_model, plot_corr, [('design_matrix', 'data')]), + # (get_evs, plot_corr, [('out', 'explanatory_variables')]), # (l1_model, plot_l1_contrast_matrix, [('contrast_matrix', 'data')]), @@ -303,8 +304,8 @@ def _get_evs(info): (select_l1_entities, ds_design, [('out', 'entities')]), (plot_design, ds_design, [('figure', 'in_file')]), - (select_l1_entities, ds_corr, [('out', 'entities')]), - (plot_corr, ds_corr, [('figure', 'in_file')]), + # (select_l1_entities, ds_corr, [('out', 'entities')]), + # (plot_corr, ds_corr, [('figure', 'in_file')]), # (select_l1_entities, ds_l1_contrasts, [('out', 'entities')]), From e1314913bcf0bee73af5d84b99ebc829cefcc23e Mon Sep 17 00:00:00 2001 From: delavega4 Date: Thu, 6 Dec 2018 17:53:01 -0600 Subject: [PATCH 16/33] Merge remove get_ev --- fitlins/viz/reports.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fitlins/viz/reports.py b/fitlins/viz/reports.py index d686a000..ddd89148 100644 --- a/fitlins/viz/reports.py +++ b/fitlins/viz/reports.py @@ -29,8 +29,9 @@ def deroot(val, root): def parse_directory(deriv_dir, work_dir, analysis): fl_layout = bids.BIDSLayout( - (deriv_dir, ['bids', 'derivatives', - pkgr.resource_filename('fitlins', 'data/fitlins.json')])) + deriv_dir, + config=['bids', 'derivatives', + pkgr.resource_filename('fitlins', 'data/fitlins.json')]) wd_layout = bids.BIDSLayout(str(Path(work_dir) / 'reportlets' / 'fitlins')) contrast_svgs = fl_layout.get(extensions='.svg', type='contrasts') From 9b58cb6c9144f731e7da64e95f8838a93a4bce98 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Fri, 7 Dec 2018 11:55:49 -0500 Subject: [PATCH 17/33] PIN: pybids, nipype, nistats --- requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 2205fce0..27f2671a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,6 @@ seaborn>=0.7.1 numpy>=1.11 nilearn>=0.4.0 pandas>=0.19 -nipype>=1.1.2 -nistats>=0.0.1b0 -git+https://github.com/bids-standard/pybids.git@54d4810a33be27c8ab340bf40c6bd1261dcb2c42#egg=pybids +git+https://github.com/nistats/nistats.git@009ce3fddb3dd01e82bca3ad7d2cdbeece0138f2#egg=nistats +git+https://github.com/bids-standard/pybids.git@8fd04f3a0443da48cb07357bba0f618fb7f04838#egg=pybids +git+https://github.com/effigies/nipype.git@5c4e266323a158cf4233eb71f388a0d4b326e51c#egg=nipype From 16ff0fed6fd49d963fa8c1cf8d89243c44909758 Mon Sep 17 00:00:00 2001 From: delavega4 Date: Fri, 7 Dec 2018 16:57:35 -0600 Subject: [PATCH 18/33] Set bids config for custom fitlins entities, and disable validation for output dirs --- fitlins/interfaces/bids.py | 2 +- fitlins/viz/reports.py | 14 +++++++++----- fitlins/workflows/base.py | 11 ----------- 3 files changed, 10 insertions(+), 17 deletions(-) diff --git a/fitlins/interfaces/bids.py b/fitlins/interfaces/bids.py index 27cded5c..a4f08a89 100644 --- a/fitlins/interfaces/bids.py +++ b/fitlins/interfaces/bids.py @@ -427,7 +427,7 @@ def _list_outputs(self): os.makedirs(base_dir, exist_ok=True) - layout = bids.BIDSLayout(base_dir) + layout = bids.BIDSLayout(base_dir, validate=False) path_patterns = self.inputs.path_patterns if not isdefined(path_patterns): path_patterns = None diff --git a/fitlins/viz/reports.py b/fitlins/viz/reports.py index ddd89148..d55d96a1 100644 --- a/fitlins/viz/reports.py +++ b/fitlins/viz/reports.py @@ -11,6 +11,11 @@ 'model-{model}.html' ] +bids.config.set_option( + 'config_paths', + {'fitlins': pkgr.resource_filename('fitlins', 'data/fitlins.json'), + **bids.config.get_option('config_paths')}) + def deroot(val, root): if isinstance(val, str): @@ -30,9 +35,9 @@ def deroot(val, root): def parse_directory(deriv_dir, work_dir, analysis): fl_layout = bids.BIDSLayout( deriv_dir, - config=['bids', 'derivatives', - pkgr.resource_filename('fitlins', 'data/fitlins.json')]) - wd_layout = bids.BIDSLayout(str(Path(work_dir) / 'reportlets' / 'fitlins')) + config=['bids', 'derivatives', 'fitlins']) + wd_layout = bids.BIDSLayout(str(Path(work_dir) / 'reportlets' / 'fitlins'), + validate=False) contrast_svgs = fl_layout.get(extensions='.svg', type='contrasts') analyses = [] @@ -75,8 +80,7 @@ def parse_directory(deriv_dir, work_dir, analysis): def write_report(level, report_dicts, run_context, deriv_dir): fl_layout = bids.BIDSLayout( - (deriv_dir, ['bids', 'derivatives', - pkgr.resource_filename('fitlins', 'data/fitlins.json')])) + deriv_dir, config=['bids', 'derivatives', 'fitlins']) env = jinja2.Environment( loader=jinja2.FileSystemLoader( diff --git a/fitlins/workflows/base.py b/fitlins/workflows/base.py index 4ca8c409..b89d2e65 100644 --- a/fitlins/workflows/base.py +++ b/fitlins/workflows/base.py @@ -142,17 +142,6 @@ def join_dict(base_dict, dict_list): reportlet_dir = Path(base_dir) / 'reportlets' / 'fitlins' reportlet_dir.mkdir(parents=True, exist_ok=True) - with (reportlet_dir / 'dataset_description.json').open('w') as file: - json.dump( - { - 'Name': 'Fitlins reportlet', - 'BIDSVersion': '1.1.0', - 'PipelineDescription': { - 'Name': 'FitLins', - }, - }, - file) - snippet_pattern = '[sub-{subject}/][ses-{session}/][sub-{subject}_]' \ '[ses-{session}_]task-{task}_[run-{run}_]snippet.html' ds_model_warnings = pe.MapNode( From 132abf5924f05879f23c686f7c6315550f757a85 Mon Sep 17 00:00:00 2001 From: delavega4 Date: Fri, 7 Dec 2018 17:30:54 -0600 Subject: [PATCH 19/33] Select only bold files --- fitlins/workflows/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fitlins/workflows/base.py b/fitlins/workflows/base.py index b89d2e65..0dcf31a2 100644 --- a/fitlins/workflows/base.py +++ b/fitlins/workflows/base.py @@ -53,7 +53,7 @@ def init_fitlins_wf(bids_dir, derivatives, out_dir, space, exclude_pattern=None, # Select preprocessed BOLD series to analyze getter = pe.Node( BIDSSelect(bids_dir=bids_dir, derivatives=derivatives, - selectors={'type': 'preproc', 'space': space}), + selectors={'type': 'preproc', 'suffix': 'bold', 'space': space}), name='getter') select_l1_contrasts = pe.Node(niu.Select(index=0), name='select_l1_contrasts') From b52675efec5b78e9e68bd2eb40e84bf76eaf9ccf Mon Sep 17 00:00:00 2001 From: delavega4 Date: Fri, 7 Dec 2018 17:59:23 -0600 Subject: [PATCH 20/33] Fix l1_model input types, and except sparse and dense inputs --- fitlins/interfaces/nistats.py | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/fitlins/interfaces/nistats.py b/fitlins/interfaces/nistats.py index c71da322..53f921d5 100644 --- a/fitlins/interfaces/nistats.py +++ b/fitlins/interfaces/nistats.py @@ -40,7 +40,7 @@ class FirstLevelModelInputSpec(BaseInterfaceInputSpec): bold_file = File(exists=True, mandatory=True) mask_file = File(exists=True) session_info = traits.Dict() - contrast_info = traits.List(traits.List(traits.Dict)) + contrast_info = traits.List(traits.Dict) class FirstLevelModelOutputSpec(TraitedSpec): @@ -59,23 +59,29 @@ def _run_interface(self, runtime): img = nb.load(self.inputs.bold_file) vols = img.shape[3] - events = pd.read_hdf(info['events'], key='events') + if info['sparse'] is not None and info['sparse'] != 'None': + sparse = pd.read_hdf(info['sparse'], key='sparse') + sparse.rename(columns={ + 'condition': 'trial_type', + 'amplitude': 'modulation'}) + else: + sparse = None - if info['confounds'] is not None and info['confounds'] != 'None': - confounds = pd.read_hdf(info['confounds'], key='confounds') - confound_names = confounds.columns.tolist() - drift_model = None if 'Cosine00' in confound_names else 'cosine' + # I'm stll assuming dirft model is in sparse events only + if info['dense'] is not None and info['dense'] != 'None': + dense = pd.read_hdf(info['dense'], key='dense') + dense_names = dense.columns.tolist() + drift_model = None if 'Cosine00' in dense_names else 'cosine' else: - confounds = None - confound_names = None + dense = None + dense_names = None drift_model = 'cosine' mat = dm.make_design_matrix( frame_times=np.arange(vols) * info['repetition_time'], - paradigm=events.rename(columns={'condition': 'trial_type', - 'amplitude': 'modulation'}), - add_regs=confounds, - add_reg_names=confound_names, + paradigm=sparse, + add_regs=dense, + add_reg_names=dense_names, drift_model=drift_model, ) From 1bb261749b08275c34194762b35173d7a9b03c91 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Fri, 7 Dec 2018 19:26:52 -0500 Subject: [PATCH 21/33] FIX: Update to recent nistats syntax, always set sparse/dense --- fitlins/interfaces/bids.py | 6 ++---- fitlins/interfaces/nistats.py | 32 +++++++++++++++----------------- 2 files changed, 17 insertions(+), 21 deletions(-) diff --git a/fitlins/interfaces/bids.py b/fitlins/interfaces/bids.py index a4f08a89..b3d12df8 100644 --- a/fitlins/interfaces/bids.py +++ b/fitlins/interfaces/bids.py @@ -254,10 +254,8 @@ def _load_level1(self, runtime, analysis): else: dense_file = None - if sparse_file is not None: - info['sparse'] = str(sparse_file) - if dense_file is not None: - info['dense'] = str(dense_file) + info['sparse'] = str(sparse_file) if sparse_file else None + info['dense'] = str(dense_file) if dense_file else None info['repetition_time'] = TR contrasts = [dict(c._asdict()) for c in step.get_contrasts(**ents)[0]] diff --git a/fitlins/interfaces/nistats.py b/fitlins/interfaces/nistats.py index 53f921d5..cb575f87 100644 --- a/fitlins/interfaces/nistats.py +++ b/fitlins/interfaces/nistats.py @@ -59,31 +59,29 @@ def _run_interface(self, runtime): img = nb.load(self.inputs.bold_file) vols = img.shape[3] - if info['sparse'] is not None and info['sparse'] != 'None': - sparse = pd.read_hdf(info['sparse'], key='sparse') - sparse.rename(columns={ - 'condition': 'trial_type', - 'amplitude': 'modulation'}) - else: - sparse = None - - # I'm stll assuming dirft model is in sparse events only - if info['dense'] is not None and info['dense'] != 'None': + sparse = None + if info['sparse'] not in (None, 'None'): + sparse = pd.read_hdf(info['sparse'], key='sparse').rename( + columns={'condition': 'trial_type', + 'amplitude': 'modulation'}) + + dense = None + if info['dense'] not in (None, 'None'): dense = pd.read_hdf(info['dense'], key='dense') - dense_names = dense.columns.tolist() - drift_model = None if 'Cosine00' in dense_names else 'cosine' + column_names = dense.columns.tolist() + drift_model = None if 'cosine_00' in column_names else 'cosine' else: dense = None - dense_names = None + column_names = None drift_model = 'cosine' - mat = dm.make_design_matrix( + mat = dm.make_first_level_design_matrix( frame_times=np.arange(vols) * info['repetition_time'], - paradigm=sparse, + events=sparse, add_regs=dense, - add_reg_names=dense_names, + add_reg_names=column_names, drift_model=drift_model, - ) + ) mat.to_csv('design.tsv', sep='\t') self._results['design_matrix'] = os.path.join(runtime.cwd, From a5cafac381f92ae84cda1b8ffbbd1d59979dc5eb Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Fri, 7 Dec 2018 19:40:48 -0500 Subject: [PATCH 22/33] FIX: Mask file selection --- fitlins/interfaces/bids.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fitlins/interfaces/bids.py b/fitlins/interfaces/bids.py index b3d12df8..aff2d2e3 100644 --- a/fitlins/interfaces/bids.py +++ b/fitlins/interfaces/bids.py @@ -352,9 +352,11 @@ def _run_interface(self, runtime): # Select exactly matching mask file (may be over-cautious) bold_ents = layout.parse_file_entities(bold_file[0].path) - bold_ents['type'] = 'brainmask' + bold_ents['suffix'] = 'mask' + bold_ents['desc'] = 'brain' mask_file = layout.get(extensions=['.nii', '.nii.gz'], **bold_ents) - bold_ents.pop('type') + bold_ents.pop('suffix') + bold_ents.pop('desc') bold_files.append(bold_file[0].path) mask_files.append(mask_file[0].path if mask_file else None) From a6f9f8c9fe43674f872512d0a95db02d8c38421d Mon Sep 17 00:00:00 2001 From: delavega4 Date: Sun, 9 Dec 2018 17:46:43 -0600 Subject: [PATCH 23/33] Create new contrast dictionary in prepare_contrasts --- fitlins/interfaces/nistats.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/fitlins/interfaces/nistats.py b/fitlins/interfaces/nistats.py index cb575f87..164882b0 100644 --- a/fitlins/interfaces/nistats.py +++ b/fitlins/interfaces/nistats.py @@ -19,21 +19,23 @@ class NistatsBaseInterface(LibraryBaseInterface): _pkg = 'nistats' -def prepare_contrasts(contrasts, design_matrix): +def prepare_contrasts(contrasts, all_regressors): if not isdefined(contrasts): contrasts = [] - all_regressors = design_matrix.columns # Prepare contrast + out_contrasts = [] for contrast in contrasts: # Fill in zeros - contrast['weights'] = [ - [row[col] if col in row else 0 for col in all_regressors.columns] + out = {**contrast} + out['weights'] = [ + [row[col] if col in row else 0 for col in all_regressors] for row in contrast['weights'] ] # Lower case T - contrast['type'] = {'T': 't', 'F': 'F'}[contrast['type']] + out['type'] = {'T': 't', 'F': 'F'}[contrast['type']] + out_contrasts.append(out) - return contrasts + return out_contrasts class FirstLevelModelInputSpec(BaseInterfaceInputSpec): @@ -59,13 +61,14 @@ def _run_interface(self, runtime): img = nb.load(self.inputs.bold_file) vols = img.shape[3] - sparse = None if info['sparse'] not in (None, 'None'): sparse = pd.read_hdf(info['sparse'], key='sparse').rename( columns={'condition': 'trial_type', 'amplitude': 'modulation'}) + sparse = sparse.dropna(subset=['modulation']) # Drop NAs + else: + sparse = None - dense = None if info['dense'] not in (None, 'None'): dense = pd.read_hdf(info['dense'], key='dense') column_names = dense.columns.tolist() @@ -95,17 +98,19 @@ def _run_interface(self, runtime): contrast_maps = [] contrast_metadata = [] - for contrast in prepare_contrasts(self.inputs_contrast_info): + for contrast in prepare_contrasts( + self.inputs.contrast_info, mat.columns.tolist()): es = flm.compute_contrast(contrast['weights'], contrast['type'], output_type='effect_size') es_fname = os.path.join( - runtime.cwd, '{}.nii.gz').format(contrast) + runtime.cwd, '{}.nii.gz').format(contrast['name']) es.to_filename(es_fname) contrast_maps.append(es_fname) contrast_metadata.append({'contrast': contrast['weights'], 'type': 'effect'}) + self._results['contrast_maps'] = contrast_maps self._results['contrast_metadata'] = contrast_metadata From 7b878583e2a2f7e46282452e01fa0fa75c9d4758 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Mon, 10 Dec 2018 13:43:09 -0500 Subject: [PATCH 24/33] FIX: Final cleanups before merge --- fitlins/interfaces/nistats.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/fitlins/interfaces/nistats.py b/fitlins/interfaces/nistats.py index 164882b0..16ec311b 100644 --- a/fitlins/interfaces/nistats.py +++ b/fitlins/interfaces/nistats.py @@ -31,8 +31,8 @@ def prepare_contrasts(contrasts, all_regressors): [row[col] if col in row else 0 for col in all_regressors] for row in contrast['weights'] ] - # Lower case T - out['type'] = {'T': 't', 'F': 'F'}[contrast['type']] + # Pybids returns lowercase contrast types, nistats uses capital F + out['type'] = {'t': 't', 'f': 'F'}[contrast['type']] out_contrasts.append(out) return out_contrasts @@ -98,8 +98,7 @@ def _run_interface(self, runtime): contrast_maps = [] contrast_metadata = [] - for contrast in prepare_contrasts( - self.inputs.contrast_info, mat.columns.tolist()): + for contrast in prepare_contrasts(self.inputs.contrast_info, mat.columns.tolist()): es = flm.compute_contrast(contrast['weights'], contrast['type'], output_type='effect_size') @@ -148,7 +147,7 @@ class SecondLevelModel(NistatsBaseInterface, SimpleInterface): def _run_interface(self, runtime): model = level2.SecondLevelModel() files = [] - contrasts = prepare_contrasts(self.inputs_contrast_info) + contrasts = prepare_contrasts(self.inputs.contrast_info) # Need a way to group appropriate files for contrast in contrasts: From 96c5ac3863aaaf54d7fe4b289874924f11b54e3d Mon Sep 17 00:00:00 2001 From: delavega4 Date: Mon, 10 Dec 2018 14:47:12 -0600 Subject: [PATCH 25/33] Update add_config_paths syntax --- fitlins/viz/reports.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/fitlins/viz/reports.py b/fitlins/viz/reports.py index d55d96a1..a874b915 100644 --- a/fitlins/viz/reports.py +++ b/fitlins/viz/reports.py @@ -2,7 +2,7 @@ from pathlib import Path import jinja2 import pkg_resources as pkgr -import bids +from bids.layout import add_config_paths, BIDSLayout from ..utils import snake_to_camel @@ -11,10 +11,7 @@ 'model-{model}.html' ] -bids.config.set_option( - 'config_paths', - {'fitlins': pkgr.resource_filename('fitlins', 'data/fitlins.json'), - **bids.config.get_option('config_paths')}) +add_config_paths(fitlins=pkgr.resource_filename('fitlins', 'data/fitlins.json')) def deroot(val, root): @@ -33,11 +30,12 @@ def deroot(val, root): def parse_directory(deriv_dir, work_dir, analysis): - fl_layout = bids.BIDSLayout( + fl_layout = BIDSLayout( deriv_dir, config=['bids', 'derivatives', 'fitlins']) - wd_layout = bids.BIDSLayout(str(Path(work_dir) / 'reportlets' / 'fitlins'), - validate=False) + wd_layout = BIDSLayout( + str(Path(work_dir) / 'reportlets' / 'fitlins'), + validate=False) contrast_svgs = fl_layout.get(extensions='.svg', type='contrasts') analyses = [] @@ -79,7 +77,7 @@ def parse_directory(deriv_dir, work_dir, analysis): def write_report(level, report_dicts, run_context, deriv_dir): - fl_layout = bids.BIDSLayout( + fl_layout = BIDSLayout( deriv_dir, config=['bids', 'derivatives', 'fitlins']) env = jinja2.Environment( From d0740ac3638b9b1133b4a02e9bcf17370f254639 Mon Sep 17 00:00:00 2001 From: delavega4 Date: Mon, 10 Dec 2018 16:02:35 -0600 Subject: [PATCH 26/33] Remove capitalizaton of F contrast-- pybids returns correct case --- fitlins/interfaces/nistats.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/fitlins/interfaces/nistats.py b/fitlins/interfaces/nistats.py index 16ec311b..4f73c468 100644 --- a/fitlins/interfaces/nistats.py +++ b/fitlins/interfaces/nistats.py @@ -31,8 +31,6 @@ def prepare_contrasts(contrasts, all_regressors): [row[col] if col in row else 0 for col in all_regressors] for row in contrast['weights'] ] - # Pybids returns lowercase contrast types, nistats uses capital F - out['type'] = {'t': 't', 'f': 'F'}[contrast['type']] out_contrasts.append(out) return out_contrasts From 528c6ef8ca5e511f99ff34a9aaf566adb3ca9d49 Mon Sep 17 00:00:00 2001 From: delavega4 Date: Mon, 10 Dec 2018 16:42:06 -0600 Subject: [PATCH 27/33] Contrast metadata passes name, contrast_info unnest --- fitlins/interfaces/nistats.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fitlins/interfaces/nistats.py b/fitlins/interfaces/nistats.py index 4f73c468..cc5c975f 100644 --- a/fitlins/interfaces/nistats.py +++ b/fitlins/interfaces/nistats.py @@ -105,7 +105,7 @@ def _run_interface(self, runtime): es.to_filename(es_fname) contrast_maps.append(es_fname) - contrast_metadata.append({'contrast': contrast['weights'], + contrast_metadata.append({'contrast': contrast['name'], 'type': 'effect'}) self._results['contrast_maps'] = contrast_maps @@ -117,7 +117,7 @@ def _run_interface(self, runtime): class SecondLevelModelInputSpec(BaseInterfaceInputSpec): stat_files = traits.List(traits.List(File(exists=True)), mandatory=True) stat_metadata = traits.List(traits.List(traits.Dict)) - contrast_info = traits.List(traits.List(traits.Dict)) + contrast_info = traits.List(traits.Dict) contrast_indices = traits.List(traits.Dict) From decbdb148fc12555be11c03b54ab6cd31bb51ed3 Mon Sep 17 00:00:00 2001 From: delavega4 Date: Tue, 11 Dec 2018 18:06:34 -0600 Subject: [PATCH 28/33] Refactor prepare_contrasts, and second level model --- fitlins/interfaces/bids.py | 9 --- fitlins/interfaces/nistats.py | 107 ++++++++++++++++++---------------- fitlins/workflows/base.py | 8 +-- 3 files changed, 59 insertions(+), 65 deletions(-) diff --git a/fitlins/interfaces/bids.py b/fitlins/interfaces/bids.py index aff2d2e3..0486da4a 100644 --- a/fitlins/interfaces/bids.py +++ b/fitlins/interfaces/bids.py @@ -278,26 +278,17 @@ def _load_level1(self, runtime, analysis): self._results.setdefault('contrast_info', []).append(contrast_info) def _load_higher_level(self, runtime, analysis): - # cwd = Path(runtime.cwd) for block in analysis.steps[1:]: - # block_subdir = cwd / block.level - # block_subdir.mkdir(parents=True, exist_ok=True) - - entities = [] contrast_info = [] for contrasts in block.get_contrasts(): if all([c.weights.empty for c in contrasts]): continue - entities.append(contrasts[0].entities) # Should all the same contrasts = [dict(c._asdict()) for c in contrasts] for contrast in contrasts: contrast['weights'] = contrast['weights'].to_dict('records') - contrast.pop('entities') - contrast_info.append(contrasts) - self._results['entities'].append(entities) self._results['contrast_info'].append(contrast_info) diff --git a/fitlins/interfaces/nistats.py b/fitlins/interfaces/nistats.py index cc5c975f..9fb6a40e 100644 --- a/fitlins/interfaces/nistats.py +++ b/fitlins/interfaces/nistats.py @@ -1,5 +1,4 @@ import os -from functools import reduce import numpy as np import pandas as pd import nibabel as nb @@ -12,28 +11,29 @@ File, traits, isdefined ) -from ..utils import dict_intersection - class NistatsBaseInterface(LibraryBaseInterface): _pkg = 'nistats' def prepare_contrasts(contrasts, all_regressors): + """ Make mutable copy of contrast list, and generate contrast design_matrix + from dictionary weight mapping + """ if not isdefined(contrasts): - contrasts = [] - # Prepare contrast - out_contrasts = [] - for contrast in contrasts: - # Fill in zeros - out = {**contrast} - out['weights'] = [ - [row[col] if col in row else 0 for col in all_regressors] - for row in contrast['weights'] - ] - out_contrasts.append(out) + return [] + else: + out_contrasts = [] + for contrast in contrasts: + # Fill in zeros + weights = np.array([ + [row[col] if col in row else 0 for col in all_regressors] + for row in contrast['weights'] + ]) + out_contrasts.append( + (contrast['name'], weights, contrast['type'])) - return out_contrasts + return out_contrasts class FirstLevelModelInputSpec(BaseInterfaceInputSpec): @@ -55,7 +55,6 @@ class FirstLevelModel(NistatsBaseInterface, SimpleInterface): def _run_interface(self, runtime): info = self.inputs.session_info - img = nb.load(self.inputs.bold_file) vols = img.shape[3] @@ -96,16 +95,17 @@ def _run_interface(self, runtime): contrast_maps = [] contrast_metadata = [] - for contrast in prepare_contrasts(self.inputs.contrast_info, mat.columns.tolist()): - es = flm.compute_contrast(contrast['weights'], - contrast['type'], + for name, weights, type in prepare_contrasts( + self.inputs.contrast_info, mat.columns.tolist()): + es = flm.compute_contrast(weights, + type, output_type='effect_size') es_fname = os.path.join( - runtime.cwd, '{}.nii.gz').format(contrast['name']) + runtime.cwd, '{}.nii.gz').format(name) es.to_filename(es_fname) contrast_maps.append(es_fname) - contrast_metadata.append({'contrast': contrast['name'], + contrast_metadata.append({'contrast': name, 'type': 'effect'}) self._results['contrast_maps'] = contrast_maps @@ -116,9 +116,8 @@ def _run_interface(self, runtime): class SecondLevelModelInputSpec(BaseInterfaceInputSpec): stat_files = traits.List(traits.List(File(exists=True)), mandatory=True) - stat_metadata = traits.List(traits.List(traits.Dict)) - contrast_info = traits.List(traits.Dict) - contrast_indices = traits.List(traits.Dict) + stat_metadata = traits.List(traits.List(traits.Dict), mandatory=True) + contrast_info = traits.List(traits.Dict, mandatory=True) class SecondLevelModelOutputSpec(TraitedSpec): @@ -144,39 +143,45 @@ class SecondLevelModel(NistatsBaseInterface, SimpleInterface): def _run_interface(self, runtime): model = level2.SecondLevelModel() - files = [] - contrasts = prepare_contrasts(self.inputs.contrast_info) - - # Need a way to group appropriate files - for contrast in contrasts: - idx = contrasts['entities'] - for fname, metadata in zip(_flatten(self.inputs.stat_files), - _flatten(self.inputs.stat_metadata)): - if _match(idx, metadata): - files.append(fname) - break - else: - raise ValueError - - out_ents = reduce(dict_intersection, self.inputs.contrast_indices) - out_ents['type'] = 'stat' + # Flatten and join files and metadata into single list of tuples + inputs = [(self.inputs.stat_files[i][j], item) + for i, sublist in enumerate(self.inputs.stat_metadata) + for j, item in enumerate(sublist)] contrast_maps = [] contrast_metadata = [] - for contrast in contrasts: - intercept = contrast['weights'] - data = np.array(files)[intercept != 0].tolist() - intercept = intercept[intercept != 0] - - model.fit(data, design_matrix=pd.DataFrame({'intercept': intercept})) - stat = model.compute_contrast(second_level_stat_type=contrast['type']) - stat_fname = os.path.join(runtime.cwd, '{}.nii.gz').format(contrast) + ents = self.inputs.contrast_info[0]['entities'] + files = [] + names = [] + for file, md in inputs: + # If file matches all contrast entities + if not sum([1 for e, v in ents.items() if md[e] != v]): + files.append(file) + names.append(md['contrast']) + files = np.array(files) + + for name, weights, type in prepare_contrasts( + self.inputs.contrast_info, names): + # Need to add F-test support for intercept (more than one column) + # Currently only taking 0th column as intercept (t-test) + dm = weights[0] + + # Filter input files [intercept != 0] + model.fit(files[dm != 0].tolist(), + design_matrix=pd.DataFrame({'intercept': dm[dm != 0]})) + + stat = model.compute_contrast( + second_level_stat_type=type) + stat_fname = os.path.join( + runtime.cwd, '{}.nii.gz').format(name) stat.to_filename(stat_fname) - contrast_maps.append(stat_fname) - metadata = out_ents.copy() - metadata['contrast'] = contrast + + metadata = { + 'type': 'stat', + 'contrast': name + } contrast_metadata.append(metadata) self._results['contrast_maps'] = contrast_maps diff --git a/fitlins/workflows/base.py b/fitlins/workflows/base.py index 0dcf31a2..2656b4d1 100644 --- a/fitlins/workflows/base.py +++ b/fitlins/workflows/base.py @@ -78,7 +78,6 @@ def join_dict(base_dict, dict_list): collate_first_level = pe.Node(MergeAll(['contrast_maps', 'contrast_metadata']), name='collate_first_level') - select_l2_entities = pe.Node(niu.Select(index=1), name='select_l2_entities') select_l2_contrasts = pe.Node(niu.Select(index=1), name='select_l2_contrasts') # Run second-level model @@ -100,7 +99,8 @@ def join_dict(base_dict, dict_list): iterfield='data', name='plot_design') - # TODO: get_evs should be based on contrasts, EVs are any used in contrasts, others are confounds + # TODO: get_evs should be based on contrasts, EVs are any used in contrasts, + # others are confounds # def _get_evs(info): # import pandas as pd # events = pd.read_hdf(info['events'], key='events') @@ -248,7 +248,6 @@ def join_dict(base_dict, dict_list): (l1_model, collate_first_level, [('contrast_maps', 'contrast_maps')]), (l1_metadata, collate_first_level, [('out', 'contrast_metadata')]), - (loader, select_l2_entities, [('entities', 'inlist')]), (loader, select_l2_contrasts, [('contrast_info', 'inlist')]), (l1_model, l2_model, [('contrast_maps', 'stat_files')]), @@ -303,11 +302,10 @@ def join_dict(base_dict, dict_list): (collate_first_level, ds_l1_contrast_plots, [('contrast_metadata', 'entities')]), (plot_l1_contrasts, ds_l1_contrast_plots, [('figure', 'in_file')]), - # (select_l2_entities, ds_l2_contrasts, [('out', 'entities')]), # (plot_l2_contrast_matrix, ds_l2_contrasts, [('figure', 'in_file')]), (collate_second_level, ds_l2_contrast_plots, [('contrast_metadata', 'entities')]), - # (plot_l2_contrasts, ds_l2_contrast_plots, [('figure', 'in_file')]), + (plot_l2_contrasts, ds_l2_contrast_plots, [('figure', 'in_file')]), ]) return wf From e11a38f9861a2a58684dbd1faee5a5930a678303 Mon Sep 17 00:00:00 2001 From: delavega4 Date: Tue, 11 Dec 2018 18:30:29 -0600 Subject: [PATCH 29/33] Second level running --- fitlins/interfaces/nistats.py | 40 ++++++++++++++--------------------- 1 file changed, 16 insertions(+), 24 deletions(-) diff --git a/fitlins/interfaces/nistats.py b/fitlins/interfaces/nistats.py index 9fb6a40e..6a74320d 100644 --- a/fitlins/interfaces/nistats.py +++ b/fitlins/interfaces/nistats.py @@ -143,46 +143,38 @@ class SecondLevelModel(NistatsBaseInterface, SimpleInterface): def _run_interface(self, runtime): model = level2.SecondLevelModel() - - # Flatten and join files and metadata into single list of tuples - inputs = [(self.inputs.stat_files[i][j], item) - for i, sublist in enumerate(self.inputs.stat_metadata) - for j, item in enumerate(sublist)] contrast_maps = [] contrast_metadata = [] - ents = self.inputs.contrast_info[0]['entities'] + entities = self.inputs.contrast_info[0]['entities'] # Same for all + out_ents = {'type': 'stat', + **entities} + files = [] names = [] - for file, md in inputs: - # If file matches all contrast entities - if not sum([1 for e, v in ents.items() if md[e] != v]): - files.append(file) - names.append(md['contrast']) - files = np.array(files) + # Flatten list of lists, only keeping files that match entities + for i, sublist in enumerate(self.inputs.stat_metadata): + for j, metadata in enumerate(sublist): + if not sum([1 for e, v in entities.items() if metadata[e] != v]): + files.append(self.inputs.stat_files[i][j]) + names.append(metadata['contrast']) for name, weights, type in prepare_contrasts( self.inputs.contrast_info, names): # Need to add F-test support for intercept (more than one column) # Currently only taking 0th column as intercept (t-test) - dm = weights[0] + weights = weights[0] + data = (np.array(files)[weights != 0]).tolist() + design_matrix = pd.DataFrame({'intercept': weights[weights != 0]}) - # Filter input files [intercept != 0] - model.fit(files[dm != 0].tolist(), - design_matrix=pd.DataFrame({'intercept': dm[dm != 0]})) + model.fit(data, design_matrix=design_matrix) - stat = model.compute_contrast( - second_level_stat_type=type) + stat = model.compute_contrast(second_level_stat_type=type) stat_fname = os.path.join( runtime.cwd, '{}.nii.gz').format(name) stat.to_filename(stat_fname) contrast_maps.append(stat_fname) - - metadata = { - 'type': 'stat', - 'contrast': name - } - contrast_metadata.append(metadata) + contrast_metadata.append({'contrast': name, **out_ents}) self._results['contrast_maps'] = contrast_maps self._results['contrast_metadata'] = contrast_metadata From b4c877f0fcd1ef313ff4bdef2a07bf299f2097fa Mon Sep 17 00:00:00 2001 From: delavega4 Date: Tue, 11 Dec 2018 19:01:21 -0600 Subject: [PATCH 30/33] Minor clear up --- fitlins/interfaces/nistats.py | 43 +++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/fitlins/interfaces/nistats.py b/fitlins/interfaces/nistats.py index 6a74320d..86679457 100644 --- a/fitlins/interfaces/nistats.py +++ b/fitlins/interfaces/nistats.py @@ -17,8 +17,8 @@ class NistatsBaseInterface(LibraryBaseInterface): def prepare_contrasts(contrasts, all_regressors): - """ Make mutable copy of contrast list, and generate contrast design_matrix - from dictionary weight mapping + """ Make mutable copy of contrast list, and + generate contrast design_matrix from dictionary weight mapping """ if not isdefined(contrasts): return [] @@ -95,18 +95,21 @@ def _run_interface(self, runtime): contrast_maps = [] contrast_metadata = [] + out_ents = self.inputs.contrast_info[0]['entities'] for name, weights, type in prepare_contrasts( self.inputs.contrast_info, mat.columns.tolist()): - es = flm.compute_contrast(weights, - type, - output_type='effect_size') + es = flm.compute_contrast( + weights, type, output_type='effect_size') es_fname = os.path.join( runtime.cwd, '{}.nii.gz').format(name) es.to_filename(es_fname) contrast_maps.append(es_fname) - contrast_metadata.append({'contrast': name, - 'type': 'effect'}) + contrast_metadata.append( + {'contrast': name, + 'type': 'effect', + **out_ents} + ) self._results['contrast_maps'] = contrast_maps self._results['contrast_metadata'] = contrast_metadata @@ -147,32 +150,32 @@ def _run_interface(self, runtime): contrast_metadata = [] entities = self.inputs.contrast_info[0]['entities'] # Same for all - out_ents = {'type': 'stat', - **entities} + out_ents = {'type': 'stat', **entities} - files = [] + # Only keep files which match all entities for contrast + stat_metadata = _flatten(self.inputs.stat_metadata) + stat_files = _flatten(self.inputs.stat_files) + filtered_files = [] names = [] - # Flatten list of lists, only keeping files that match entities - for i, sublist in enumerate(self.inputs.stat_metadata): - for j, metadata in enumerate(sublist): - if not sum([1 for e, v in entities.items() if metadata[e] != v]): - files.append(self.inputs.stat_files[i][j]) - names.append(metadata['contrast']) + for m, f in zip(stat_metadata, stat_files): + if _match(entities, m): + filtered_files.append(f) + names.append(m['contrast']) for name, weights, type in prepare_contrasts( self.inputs.contrast_info, names): # Need to add F-test support for intercept (more than one column) # Currently only taking 0th column as intercept (t-test) weights = weights[0] - data = (np.array(files)[weights != 0]).tolist() + input = (np.array(filtered_files)[weights != 0]).tolist() design_matrix = pd.DataFrame({'intercept': weights[weights != 0]}) - model.fit(data, design_matrix=design_matrix) + model.fit(input, design_matrix=design_matrix) stat = model.compute_contrast(second_level_stat_type=type) - stat_fname = os.path.join( - runtime.cwd, '{}.nii.gz').format(name) + stat_fname = os.path.join(runtime.cwd, '{}.nii.gz').format(name) stat.to_filename(stat_fname) + contrast_maps.append(stat_fname) contrast_metadata.append({'contrast': name, **out_ents}) From 4e0d0de893f26b748459e207d710f6be4b7e7094 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Wed, 12 Dec 2018 14:27:17 -0500 Subject: [PATCH 31/33] PIN: Update pybids, nipype pins --- fitlins/__about__.py | 4 ++-- requirements.txt | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/fitlins/__about__.py b/fitlins/__about__.py index 3c8128f4..16082d78 100644 --- a/fitlins/__about__.py +++ b/fitlins/__about__.py @@ -34,7 +34,7 @@ REQUIRES = [ 'nibabel>=2.0', - 'nipype>=1.1.2', + 'nipype>=1.1.6', 'seaborn>=0.7.1', 'numpy>=1.11', 'nilearn>=0.4.0', @@ -47,7 +47,7 @@ LINKS_REQUIRES = [ 'git+https://github.com/bids-standard/pybids.git@' - '54d4810a33be27c8ab340bf40c6bd1261dcb2c42#egg=pybids', + 'dd8c9bc6478bb63e09f6c95566a11677ce12ded7#egg=pybids', ] TESTS_REQUIRES = [ diff --git a/requirements.txt b/requirements.txt index 27f2671a..91570e63 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,6 @@ seaborn>=0.7.1 numpy>=1.11 nilearn>=0.4.0 pandas>=0.19 +nipype>=1.1.6 git+https://github.com/nistats/nistats.git@009ce3fddb3dd01e82bca3ad7d2cdbeece0138f2#egg=nistats -git+https://github.com/bids-standard/pybids.git@8fd04f3a0443da48cb07357bba0f618fb7f04838#egg=pybids -git+https://github.com/effigies/nipype.git@5c4e266323a158cf4233eb71f388a0d4b326e51c#egg=nipype +git+https://github.com/bids-standard/pybids.git@dd8c9bc6478bb63e09f6c95566a11677ce12ded7#egg=pybids From 27226a1550177e841cb27a1050cbb54571a4c513 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Wed, 12 Dec 2018 14:29:15 -0500 Subject: [PATCH 32/33] STY: Minor cleanups --- fitlins/interfaces/nistats.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/fitlins/interfaces/nistats.py b/fitlins/interfaces/nistats.py index 86679457..1065f6e8 100644 --- a/fitlins/interfaces/nistats.py +++ b/fitlins/interfaces/nistats.py @@ -22,18 +22,18 @@ def prepare_contrasts(contrasts, all_regressors): """ if not isdefined(contrasts): return [] - else: - out_contrasts = [] - for contrast in contrasts: - # Fill in zeros - weights = np.array([ - [row[col] if col in row else 0 for col in all_regressors] - for row in contrast['weights'] - ]) - out_contrasts.append( - (contrast['name'], weights, contrast['type'])) - return out_contrasts + out_contrasts = [] + for contrast in contrasts: + # Fill in zeros + weights = np.array([ + [row[col] if col in row else 0 for col in all_regressors] + for row in contrast['weights'] + ]) + out_contrasts.append( + (contrast['name'], weights, contrast['type'])) + + return out_contrasts class FirstLevelModelInputSpec(BaseInterfaceInputSpec): @@ -162,8 +162,7 @@ def _run_interface(self, runtime): filtered_files.append(f) names.append(m['contrast']) - for name, weights, type in prepare_contrasts( - self.inputs.contrast_info, names): + for name, weights, type in prepare_contrasts(self.inputs.contrast_info, names): # Need to add F-test support for intercept (more than one column) # Currently only taking 0th column as intercept (t-test) weights = weights[0] From 1f11b09e105b2d5a2c397d4448ac4f1907962446 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Wed, 12 Dec 2018 14:29:39 -0500 Subject: [PATCH 33/33] ENH: Disable multiple models for now; STY: unused import --- fitlins/workflows/base.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/fitlins/workflows/base.py b/fitlins/workflows/base.py index 2656b4d1..577b8ef7 100644 --- a/fitlins/workflows/base.py +++ b/fitlins/workflows/base.py @@ -1,5 +1,4 @@ from pathlib import Path -import json from nipype.pipeline import engine as pe from nipype.interfaces import utility as niu from ..interfaces.bids import ( @@ -23,6 +22,9 @@ def init_fitlins_wf(bids_dir, derivatives, out_dir, space, exclude_pattern=None, all_models = specs.run().outputs.model_spec if not all_models: raise RuntimeError("Unable to find or construct models") + if isinstance(all_models, list): + raise RuntimeError("Currently unable to run multiple models in parallel - " + "please specify model") # # Load and run the model @@ -33,7 +35,8 @@ def init_fitlins_wf(bids_dir, derivatives, out_dir, space, exclude_pattern=None, loader = pe.Node( LoadBIDSModel(bids_dir=bids_dir, derivatives=derivatives, - selectors=selectors), + selectors=selectors, + model=all_models), name='loader') if exclude_pattern is not None: @@ -41,11 +44,6 @@ def init_fitlins_wf(bids_dir, derivatives, out_dir, space, exclude_pattern=None, if include_pattern is not None: loader.inputs.include_pattern = include_pattern - if isinstance(all_models, list): - loader.iterables = ('model', all_models) - else: - loader.inputs.model = all_models - # Because pybids generates the entire model in one go, we will need # various helper nodes to select the correct portions of the model select_l1_entities = pe.Node(niu.Select(index=0), name='select_l1_entities')