FIX: Restore report generation

fitlins/cli/run.py

@@ -77,8 +77,8 @@ def get_parser():
     g_bids.add_argument('--participant-label', action='store', nargs='+', default=None,
                         help='one or more participant identifiers (the sub- prefix can be '
                              'removed)')
-    g_bids.add_argument('-m', '--model', action='store', default='model.json',
-                        help='location of BIDS model description (default bids_dir/model.json)')
+    g_bids.add_argument('-m', '--model', action='store',
+                        help='location of BIDS model description')
     g_bids.add_argument('-d', '--derivatives', action='store', nargs='+',
                         help='location of derivatives (including preprocessed images).'
                         'If none specified, indexes all derivatives under bids_dir/derivatives.')
@@ -135,7 +135,7 @@ def run_fitlins(argv=None):
         subject_list=subject_list)
     )
 
-    model = default_path(opts.model, opts.bids_dir, 'model.json')
+    model = default_path(opts.model, opts.bids_dir, 'model-default_smdl.json')
     if opts.model in (None, 'default') and not op.exists(model):
         model = 'default'
 

fitlins/interfaces/bids.py

@@ -232,9 +232,8 @@ def _load_level1(self, runtime, analysis):
                 # t/aCompCor
                 # We may want to add criteria to include HPF columns that are not
                 # explicitly listed in the model
-                names = [col for col in dense.columns
-                         if col.startswith('non_steady_state') or
-                         col in step.model['x']]
+                names = [var for var in step.model['x'] if var in dense.columns]
+                names.extend(col for col in dense.columns if col.startswith('non_steady_state'))
                 dense = dense[names]
 
                 # These confounds are defined pairwise with the current volume

fitlins/interfaces/nistats.py

@@ -107,7 +107,7 @@ def _run_interface(self, runtime):
             contrast_maps.append(es_fname)
             contrast_metadata.append(
                 {'contrast': name,
-                 'type': 'effect',
+                 'suffix': 'effect',
                  **out_ents}
                 )
 
@@ -150,7 +150,7 @@ def _run_interface(self, runtime):
         contrast_metadata = []
 
         entities = self.inputs.contrast_info[0]['entities']  # Same for all
-        out_ents = {'type': 'stat', **entities}
+        out_ents = {'suffix': 'stat', **entities}
 
         # Only keep files which match all entities for contrast
         stat_metadata = _flatten(self.inputs.stat_metadata)

fitlins/interfaces/visualizations.py

@@ -61,20 +61,28 @@ def _visualize(self, data, out_name):
 
 
 class DesignCorrelationPlotInputSpec(VisualizationInputSpec):
-    explanatory_variables = traits.Range(
-        low=0, desc='Number of explanatory variables')
+    contrast_info = traits.List(traits.Dict)
 
 
 class DesignCorrelationPlot(Visualization):
     input_spec = DesignCorrelationPlotInputSpec
 
     def _visualize(self, data, out_name):
+        contrast_matrix = pd.DataFrame({c['name']: c['weights'][0]
+                                        for c in self.inputs.contrast_info})
+        all_cols = list(data.columns)
+        evs = set(contrast_matrix.index)
+        if set(contrast_matrix.index) != all_cols[:len(evs)]:
+            ev_cols = [col for col in all_cols if col in evs]
+            confound_cols = [col for col in all_cols if col not in evs]
+            data = data[ev_cols + confound_cols]
         plot_and_save(out_name, plot_corr_matrix,
                       data.drop(columns='constant').corr(),
-                      self.inputs.explanatory_variables)
+                      len(evs))
 
 
 class ContrastMatrixPlotInputSpec(VisualizationInputSpec):
+    contrast_info = traits.List(traits.Dict)
     orientation = traits.Enum('horizontal', 'vertical', usedefault=True,
                               desc='Display orientation of contrast matrix')
 
@@ -83,9 +91,12 @@ class ContrastMatrixPlot(Visualization):
     input_spec = ContrastMatrixPlotInputSpec
 
     def _visualize(self, data, out_name):
-        if 'constant' in data.index:
-            data = data.drop(index='constant')
-        plot_and_save(out_name, plot_contrast_matrix, data,
+        contrast_matrix = pd.DataFrame({c['name']: c['weights'][0]
+                                        for c in self.inputs.contrast_info},
+                                       index=data.columns)
+        if 'constant' in contrast_matrix.index:
+            contrast_matrix = contrast_matrix.drop(index='constant')
+        plot_and_save(out_name, plot_contrast_matrix, contrast_matrix,
                       ornt=self.inputs.orientation)
 
 

fitlins/viz/reports.py

@@ -8,7 +8,7 @@
 
 PATH_PATTERNS = [
     '[sub-{subject}/][ses-{session}/][sub-{subject}_][ses-{session}_]'
-    'model-{model}.html'
+    'model-{model}[_run-{run}].html'
 ]
 
 add_config_paths(fitlins=pkgr.resource_filename('fitlins', 'data/fitlins.json'))
@@ -37,40 +37,40 @@ def parse_directory(deriv_dir, work_dir, analysis):
     wd_layout = BIDSLayout(
         str(Path(work_dir) / 'reportlets' / 'fitlins'),
         validate=False)
-    contrast_svgs = fl_layout.get(extensions='.svg', type='contrasts')
+    contrast_svgs = fl_layout.get(extensions='.svg', suffix='contrasts')
 
     analyses = []
     for contrast_svg in contrast_svgs:
-        ents = fl_layout.parse_file_entities(contrast_svg.filename)
-        ents.pop('type')
+        ents = contrast_svg.entities
+        ents.pop('suffix')
         ents.setdefault('subject', None)
-        correlation_matrix = fl_layout.get(extensions='.svg', type='corr',
+        correlation_matrix = fl_layout.get(extensions='.svg', suffix='corr',
                                            **ents)
-        design_matrix = fl_layout.get(extensions='.svg', type='design', **ents)
+        design_matrix = fl_layout.get(extensions='.svg', suffix='design', **ents)
         job_desc = {
             'ents': {k: v for k, v in ents.items() if v is not None},
             'dataset': analysis.layout.root,
             'model_name': analysis.model['name'],
-            'contrasts_svg': contrast_svg.filename,
+            'contrasts_svg': contrast_svg.path,
             }
         if ents.get('subject'):
             job_desc['subject_id'] = ents.get('subject')
         if correlation_matrix:
-            job_desc['correlation_matrix_svg'] = correlation_matrix[0].filename
+            job_desc['correlation_matrix_svg'] = correlation_matrix[0].path
         if design_matrix:
-            job_desc['design_matrix_svg'] = design_matrix[0].filename
+            job_desc['design_matrix_svg'] = design_matrix[0].path
 
-        snippet = wd_layout.get(extensions='.html', type='snippet', **ents)
+        snippet = wd_layout.get(extensions='.html', suffix='snippet', **ents)
         if snippet:
-            with open(snippet[0].filename) as fobj:
+            with open(snippet[0].path) as fobj:
                 job_desc['warning'] = fobj.read()
 
-        contrasts = fl_layout.get(extensions='.png', type='ortho', **ents)
+        contrasts = fl_layout.get(extensions='.png', suffix='ortho', **ents)
         # TODO: Split contrasts from estimates
-        job_desc['contrasts'] = [{'image_file': c.filename,
+        job_desc['contrasts'] = [{'image_file': c.path,
                                   'name':
                                       fl_layout.parse_file_entities(
-                                          c.filename)['contrast']}
+                                          c.path)['contrast']}
                                  for c in contrasts]
         analyses.append(job_desc)
 

fitlins/workflows/base.py

@@ -9,10 +9,6 @@
 from ..interfaces.utils import MergeAll
 
 
-def join_dict(base_dict, dict_list):
-    return [{**base_dict, **iter_dict} for iter_dict in dict_list]
-
-
 def init_fitlins_wf(bids_dir, derivatives, out_dir, space, exclude_pattern=None,
                     include_pattern=None, model=None, participants=None,
                     base_dir=None, name='fitlins_wf'):
@@ -55,13 +51,6 @@ def init_fitlins_wf(bids_dir, derivatives, out_dir, space, exclude_pattern=None,
                 'type': 'preproc', 'suffix': 'bold', 'space': space}),
         name='getter')
 
-    # Accumulate metadata
-    l1_metadata = pe.MapNode(
-        niu.Function(function=join_dict),
-        iterfield=['base_dict', 'dict_list'],
-        name='l1_metadata',
-        run_without_submitting=True)
-
     l1_model = pe.MapNode(
         FirstLevelModel(),
         iterfield=['session_info', 'contrast_info', 'bold_file', 'mask_file'],
@@ -71,15 +60,15 @@ def init_fitlins_wf(bids_dir, derivatives, out_dir, space, exclude_pattern=None,
     image_pattern = '[sub-{subject}/][ses-{session}/]' \
         '[sub-{subject}_][ses-{session}_]task-{task}[_acq-{acquisition}]' \
         '[_rec-{reconstruction}][_run-{run}][_echo-{echo}]_bold_' \
-        '{type<design|corr|contrasts>}.svg'
+        '{suffix<design|corr|contrasts>}.svg'
     contrast_plot_pattern = '[sub-{subject}/][ses-{session}/]' \
         '[sub-{subject}_][ses-{session}_]task-{task}[_acq-{acquisition}]' \
         '[_rec-{reconstruction}][_run-{run}][_echo-{echo}]_bold' \
         '[_space-{space}]_contrast-{contrast}_ortho.png'
     contrast_pattern = '[sub-{subject}/][ses-{session}/]' \
         '[sub-{subject}_][ses-{session}_]task-{task}[_acq-{acquisition}]' \
         '[_rec-{reconstruction}][_run-{run}][_echo-{echo}]_bold' \
-        '[_space-{space}]_contrast-{contrast}_{type<effect|stat>}.nii.gz'
+        '[_space-{space}]_contrast-{contrast}_{suffix<effect|stat>}.nii.gz'
 
     # Set up general interfaces
     #
@@ -103,13 +92,37 @@ def init_fitlins_wf(bids_dir, derivatives, out_dir, space, exclude_pattern=None,
         iterfield='data',
         name='plot_design')
 
+    plot_corr = pe.MapNode(
+        DesignCorrelationPlot(image_type='svg'),
+        iterfield=['data', 'contrast_info'],
+        name='plot_corr')
+
+    plot_l1_contrast_matrix = pe.MapNode(
+        ContrastMatrixPlot(image_type='svg'),
+        iterfield=['data', 'contrast_info'],
+        name='plot_l1_contrast_matrix')
+
     ds_design = pe.MapNode(
-        BIDSDataSink(base_directory=out_dir, fixed_entities={'type': 'design'},
+        BIDSDataSink(base_directory=out_dir, fixed_entities={'suffix': 'design'},
                      path_patterns=image_pattern),
         iterfield=['entities', 'in_file'],
         run_without_submitting=True,
         name='ds_design')
 
+    ds_corr = pe.MapNode(
+        BIDSDataSink(base_directory=out_dir, fixed_entities={'suffix': 'corr'},
+                     path_patterns=image_pattern),
+        iterfield=['entities', 'in_file'],
+        run_without_submitting=True,
+        name='ds_corr')
+
+    ds_l1_contrasts = pe.MapNode(
+        BIDSDataSink(base_directory=out_dir, fixed_entities={'suffix': 'contrasts'},
+                     path_patterns=image_pattern),
+        iterfield=['entities', 'in_file'],
+        run_without_submitting=True,
+        name='ds_l1_contrasts')
+
     #
     # General Connections
     #
@@ -118,12 +131,11 @@ def init_fitlins_wf(bids_dir, derivatives, out_dir, space, exclude_pattern=None,
         (loader, l1_model, [('session_info', 'session_info')]),
         (getter, l1_model, [('bold_files', 'bold_file'),
                             ('mask_files', 'mask_file')]),
-        (getter, l1_metadata, [('entities', 'base_dict')]),
-        (l1_model, l1_metadata, [('contrast_metadata', 'dict_list')]),
         (l1_model, plot_design, [('design_matrix', 'data')]),
         ])
 
-    models = []
+    stage = None
+    model = l1_model
     for ix, step in enumerate(step['Level'] for step in model_dict['Steps']):
         # Set up elements common across levels
 
@@ -178,14 +190,20 @@ def init_fitlins_wf(bids_dir, derivatives, out_dir, space, exclude_pattern=None,
             name='ds_{}_contrast_plots'.format(level))
 
         if ix == 0:
-            model = l1_model
             wf.connect([
                 (loader, select_entities, [('entities', 'inlist')]),
                 (select_entities, getter,  [('out', 'entities')]),
                 (select_entities, ds_model_warnings,  [('out', 'entities')]),
-                (l1_metadata, collate, [('out', 'contrast_metadata')]),
                 (select_entities, ds_design, [('out', 'entities')]),
                 (plot_design, ds_design, [('figure', 'in_file')]),
+                (select_contrasts, plot_l1_contrast_matrix,  [('out', 'contrast_info')]),
+                (select_contrasts, plot_corr,  [('out', 'contrast_info')]),
+                (model, plot_l1_contrast_matrix,  [('design_matrix', 'data')]),
+                (model, plot_corr,  [('design_matrix', 'data')]),
+                (select_entities, ds_l1_contrasts, [('out', 'entities')]),
+                (select_entities, ds_corr, [('out', 'entities')]),
+                (plot_l1_contrast_matrix, ds_l1_contrasts,  [('figure', 'in_file')]),
+                (plot_corr, ds_corr,  [('figure', 'in_file')]),
             ])
 
         #  Set up higher levels
@@ -196,15 +214,15 @@ def init_fitlins_wf(bids_dir, derivatives, out_dir, space, exclude_pattern=None,
                 name='{}_model'.format(level))
 
             wf.connect([
-                (models[-1], model, [('contrast_maps', 'stat_files')]),
-                (l1_metadata, model, [('out', 'stat_metadata')]),
-                (model, collate, [('contrast_metadata', 'contrast_metadata')]),
+                (stage, model, [('contrast_maps', 'stat_files'),
+                                ('contrast_metadata', 'stat_metadata')]),
             ])
 
         wf.connect([
             (loader, select_contrasts, [('contrast_info', 'inlist')]),
             (select_contrasts, model,  [('out', 'contrast_info')]),
-            (model, collate, [('contrast_maps', 'contrast_maps')]),
+            (model, collate, [('contrast_maps', 'contrast_maps'),
+                              ('contrast_metadata', 'contrast_metadata')]),
             (collate, plot_contrasts, [('contrast_maps', 'data')]),
             (collate, ds_contrast_maps, [('contrast_maps', 'in_file'),
                                          ('contrast_metadata', 'entities')]),
@@ -213,6 +231,6 @@ def init_fitlins_wf(bids_dir, derivatives, out_dir, space, exclude_pattern=None,
 
             ])
 
-        models.append(model)
+        stage = model
 
     return wf