Merge pull request #9 from gagnonanthony/dev

ENH: Move freesurfer/fastsurfer output directory alongside output dir for easier reusability

CHANGELOG.md

@@ -3,6 +3,12 @@
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unreleased] - [2025-01-08]
+
+### `Changed`
+
+- Fastsurfer and freesurfer outputs are now in their own dedicated output folder.
+
 ## [Unreleased] - [2024-12-23]
 
 ### `Added`

conf/modules.config

@@ -16,30 +16,52 @@ process {
     withName: 'NF_PEDIATRIC:PEDIATRIC:FREESURFERFLOW:FASTSURFER' {
         ext.acq3T = params.acq3T
         publishDir = [
-            path: { "${params.outdir}/${meta.id}/anat/"},
+            [
+                path: { params.fs_output_dir ?: "${params.outdir}/../fastsurfer-v2.3.3/" },
+                mode: params.publish_dir_mode,
+                saveAs: {
+                    filename ->
+                    if ( filename.contains("fastsurfer") ) { "${meta.id}" }
+                    else if ( filename.contains("versions.yml") ) { null }
+                    else { params.lean_output ? null : filename }
+                }
+            ],
+            [
+                path: { "${params.outdir}/${meta.id}/anat/"},
+                mode: params.publish_dir_mode,
+                saveAs: {
+                    filename ->
+                    if ( filename.contains("__final_t1.nii.gz") ) { "${meta.id}_desc-preproc_T1w.nii.gz" }
+                    else if ( filename.contains("versions.yml") ) { null }
+                    else { params.lean_output ? null : filename }
+                }
+            ]
+        ]
+    }
+
+    withName: 'NF_PEDIATRIC:PEDIATRIC:FREESURFERFLOW:RECONALL' {
+        publishDir = [
+        [
+            path: { params.fs_output_dir ?: "${params.outdir}/../freesurfer-7.4.1/" },
             mode: params.publish_dir_mode,
             saveAs: {
                 filename ->
-                if ( filename.contains("fastsurfer") ) { "${meta.id}_fastsurfer" }
-                else if ( filename.contains("__final_t1.nii.gz") ) { "${meta.id}_desc-preproc_T1w.nii.gz" }
+                if ( filename.contains("recon_all") ) { "${meta.id}" }
                 else if ( filename.contains("versions.yml") ) { null }
                 else { params.lean_output ? null : filename }
             }
-        ]
-    }
-
-    withName: 'NF_PEDIATRIC:PEDIATRIC:FREESURFERFLOW:RECONALL' {
-        publishDir = [
+        ],
+        [
             path: { "${params.outdir}/${meta.id}/anat/" },
             mode: params.publish_dir_mode,
             saveAs: {
                 filename ->
-                if ( filename.contains("recon_all") ) { "${meta.id}_reconall"}
-                else if ( filename.contains("__final_t1.nii.gz") ) { "${meta.id}_desc-preproc_T1w.nii.gz" }
+                if ( filename.contains("__final_t1.nii.gz") ) { "${meta.id}_desc-preproc_T1w.nii.gz" }
                 else if ( filename.contains("versions.yml") ) { null }
                 else { params.lean_output ? null : filename }
             }
         ]
+        ]
     }
 
     withName: 'NF_PEDIATRIC:PEDIATRIC:FREESURFERFLOW:BRAINNETOMECHILD' {
@@ -729,7 +751,7 @@ process {
             mode: params.publish_dir_mode,
             saveAs: {
                 filename ->
-                if ( filename.contains("_warped.nii.gz") ) { "${meta.id}_space-DWI_seg-BrainnetomeChild_dseg.nii.gz" }
+                if ( filename.contains("_warped.nii.gz") ) { "${meta.id}_space-DWI_seg-${params.atlas_name}_dseg.nii.gz" }
                 else if ( filename.contains("versions.yml") ) { null }
                 else { params.lean_output ? null : filename }
             }

docs/output.md

@@ -24,6 +24,20 @@ The directories listed below will be created in the results directory after the
 
 The pipeline will output only the final preprocessed files by default. This behavior is used to limit the number of files generated by the pipeline by omitting the publishing of intermediate files. It is particularly useful when running the pipeline on clusters where file quota are often rapidly met. To opt-out from the lean output version, set the `--lean_output` parameter to false when launching the pipeline (`--lean_output false`).
 
+If you decided to run FreeSurfer or FastSurfer as part of `nf-pediatric`, your output will be located (by default) alongside the `nf-pediatric` output folder. You can select the destination folder by using the `--fs_output_dir` parameter. If you do not specify and output folder destination, it will look like this:
+
+```bash
+<your-nf-pediatric-outdir>
+  |-- multiqc
+  |-- sub-0001
+  |-- sub-0002
+  <...>
+fastsurfer-v2.3.3/
+  |-- sub-0001
+  |-- sub-0002
+  <...>
+```
+
 ## Pipeline overview
 
 The pipeline is built using [Nextflow](https://www.nextflow.io/) and data processing steps can be grouped. The final output files are listed below for each of those steps. If you used `--lean_output false`, you will find additional files than the ones described here.

nextflow.config

@@ -17,6 +17,7 @@ params {
     use_fastsurfer                          = true
     fs_license                              = null
     acq3T                                   = true
+    fs_output_dir                           = null
 
     // Atlas options
     utils_folder                            = "${projectDir}/assets/FS_BN_GL_SF_utils/"

nextflow_schema.json

@@ -76,6 +76,14 @@
                     "fa_icon": "fas fa-magnet",
                     "default": true,
                     "hidden": true
+                },
+                "fs_output_dir": {
+                    "type": "string",
+                    "description": "Path to FreeSurfer output directory.",
+                    "fa_icon": "fas fa-folder-open",
+                    "help_text": "Path to FreeSurfer output directory. This is where FreeSurfer will save its output files.",
+                    "format": "directory-path",
+                    "hidden": false
                 }
             }
         },

tests/chained.nf.test

@@ -12,6 +12,7 @@ nextflow_pipeline {
 
                 params.input = "$projectDir/tests/data/samplesheet_testconn.csv"
                 params.outdir = "$outputDir"
+                use_fastsurfer = true
 
                 params.connectomics = true
                 params.freesurfer = true
@@ -23,6 +24,8 @@ nextflow_pipeline {
         then {
             // stable name: All files + folders in ${params.outdir}/ with a stable name.
             def stable_name = getAllFilesFromDir(params.outdir, relative: true, includeDir: true, ignore:  ['pipeline_info/*.{html,json,txt}'])
+            // All fastsurfer output.
+            def fastsurfer_output = getAllFilesFromDir("$outputDir/../fastsurfer-v2.3.3/", relative: true, includeDir: true)
             assertAll {
                 assert workflow.success
                 assert snapshot(
@@ -31,7 +34,9 @@ nextflow_pipeline {
                     // Remove the nextflow version from the versions.yml because we test it using different nextflow versions.
                     removeNextflowVersion("$outputDir/pipeline_info/pipeline_software_mqc_versions.yml"),
                     // All stable name.
-                    stable_name
+                    stable_name,
+                    // All fastsurfer output.
+                    fastsurfer_output
                 ).match()
             }
         }
@@ -88,6 +93,8 @@ nextflow_pipeline {
         then {
             // stable name: All files + folders in ${params.outdir}/ with a stable name.
             def stable_name = getAllFilesFromDir(params.outdir, relative: true, includeDir: true, ignore:  ['pipeline_info/*.{html,json,txt}'])
+            // All fastsurfer output.
+            def fastsurfer_output = getAllFilesFromDir("$outputDir/../fastsurfer-v2.3.3/", relative: true, includeDir: true)
             assertAll {
                 assert workflow.success
                 assert snapshot(
@@ -96,7 +103,9 @@ nextflow_pipeline {
                     // Remove the nextflow version from the versions.yml because we test it using different nextflow versions.
                     removeNextflowVersion("$outputDir/pipeline_info/pipeline_software_mqc_versions.yml"),
                     // All stable name.
-                    stable_name
+                    stable_name,
+                    // All fastsurfer output.
+                    fastsurfer_output
                 ).match()
             }
         }

tests/chained.nf.test.snap

@@ -351,7 +351,6 @@
                 "sub-test",
                 "sub-test/anat",
                 "sub-test/anat/sub-test_desc-preproc_T1w.nii.gz",
-                "sub-test/anat/sub-test_fastsurfer",
                 "sub-test/anat/sub-test_from-T1w_to-dwi_affine.mat",
                 "sub-test/anat/sub-test_from-T1w_to-dwi_warp.nii.gz",
                 "sub-test/anat/sub-test_from-dwi_to-T1w_warp.nii.gz",
@@ -429,13 +428,17 @@
                 "sub-test/multiqc/sub-test_multiqc_plots",
                 "sub-test/multiqc/sub-test_multiqc_report.html",
                 "subcortical_volumes.tsv"
+            ],
+            [
+                "",
+                "sub-test"
             ]
         ],
         "meta": {
             "nf-test": "0.9.2",
             "nextflow": "24.10.1"
         },
-        "timestamp": "2025-01-03T09:26:58.519601"
+        "timestamp": "2025-01-08T10:10:09.573439"
     },
     "Freesurfer + connectomics profiles - should run successfully": {
         "content": [
@@ -482,7 +485,6 @@
                 "sub-test",
                 "sub-test/anat",
                 "sub-test/anat/sub-test_desc-preproc_T1w.nii.gz",
-                "sub-test/anat/sub-test_fastsurfer",
                 "sub-test/anat/sub-test_seg-BrainnetomeChild_desc-labels.json",
                 "sub-test/anat/sub-test_seg-BrainnetomeChild_desc-labels.txt",
                 "sub-test/anat/sub-test_seg-BrainnetomeChild_dseg.nii.gz",
@@ -512,13 +514,17 @@
                 "sub-test/multiqc/sub-test_multiqc_plots",
                 "sub-test/multiqc/sub-test_multiqc_report.html",
                 "subcortical_volumes.tsv"
+            ],
+            [
+                "",
+                "sub-test"
             ]
         ],
         "meta": {
             "nf-test": "0.9.2",
             "nextflow": "24.10.1"
         },
-        "timestamp": "2025-01-03T09:22:33.330752"
+        "timestamp": "2025-01-08T10:05:54.243678"
     },
     "Connectomics + tracking profiles - should run successfully": {
         "content": [

tests/freesurfer.nf.test

@@ -13,6 +13,7 @@ nextflow_pipeline {
                 input = "$projectDir/tests/data/samplesheet_testtracking.csv"
                 outdir = "$outputDir"
 
+                use_fastsurfer = false
                 freesurfer = true
                 fs_license = "https://www.dropbox.com/scl/fi/0s8lp6lydyd0rxawxb4jm/license.txt?rlkey=hz54oc0d4sor69avqphtrjvgn&st=9e0yij97&dl=0"
 
@@ -22,6 +23,8 @@ nextflow_pipeline {
         then {
             // stable name: All files + folders in ${params.outdir}/ with a stable name.
             def stable_name = getAllFilesFromDir(params.outdir, relative: true, includeDir: true, ignore:  ['pipeline_info/*.{html,json,txt}'])
+            // All freesurfer output.
+            def freesurfer_output = getAllFilesFromDir("$outputDir/../freesurfer-7.4.1/", relative: true, includeDir: true)
             assertAll {
                 assert workflow.success
                 assert snapshot(
@@ -30,7 +33,9 @@ nextflow_pipeline {
                     // Remove the nextflow version from the versions.yml because we test it using different nextflow versions.
                     removeNextflowVersion("$outputDir/pipeline_info/pipeline_software_mqc_versions.yml"),
                     // All stable name.
-                    stable_name
+                    stable_name,
+                    // All freesurfer output.
+                    freesurfer_output
                 ).match()
             }
         }
@@ -43,6 +48,7 @@ nextflow_pipeline {
 
                 params.input = "$projectDir/tests/data/samplesheet_testtracking.csv"
                 params.outdir = "$outputDir"
+                params.fs_output_dir = "$outputDir/../onedirdown/fastsurfer-v2.3.3"
 
                 params.freesurfer = true
                 params.use_fastsurfer = true
@@ -54,6 +60,8 @@ nextflow_pipeline {
         then {
             // stable name: All files + folders in ${params.outdir}/ with a stable name.
             def stable_name = getAllFilesFromDir(params.outdir, relative: true, includeDir: true, ignore:  ['pipeline_info/*.{html,json,txt}'])
+            // All fastsurfer output.
+            def fastsurfer_output = getAllFilesFromDir(params.fs_output_dir, relative: true, includeDir: true)
             assertAll {
                 assert workflow.success
                 assert snapshot(
@@ -62,7 +70,9 @@ nextflow_pipeline {
                     // Remove the nextflow version from the versions.yml because we test it using different nextflow versions.
                     removeNextflowVersion("$outputDir/pipeline_info/pipeline_software_mqc_versions.yml"),
                     // All stable name.
-                    stable_name
+                    stable_name,
+                    // All fastsurfer output.
+                    fastsurfer_output
                 ).match()
             }
         }

tests/freesurfer.nf.test.snap

@@ -26,7 +26,6 @@
                 "sub-test",
                 "sub-test/anat",
                 "sub-test/anat/sub-test_desc-preproc_T1w.nii.gz",
-                "sub-test/anat/sub-test_fastsurfer",
                 "sub-test/anat/sub-test_seg-BrainnetomeChild_desc-labels.json",
                 "sub-test/anat/sub-test_seg-BrainnetomeChild_desc-labels.txt",
                 "sub-test/anat/sub-test_seg-BrainnetomeChild_dseg.nii.gz",
@@ -45,20 +44,24 @@
                 "sub-test/multiqc/sub-test_multiqc_plots",
                 "sub-test/multiqc/sub-test_multiqc_report.html",
                 "subcortical_volumes.tsv"
+            ],
+            [
+                "",
+                "sub-test"
             ]
         ],
         "meta": {
             "nf-test": "0.9.2",
             "nextflow": "24.10.1"
         },
-        "timestamp": "2025-01-02T23:01:18.627052"
+        "timestamp": "2025-01-08T09:57:43.80135"
     },
     "Freesurfer profile - freesurfer - should run successfully": {
         "content": [
             6,
             {
-                "FASTSURFER": {
-                    "fastsurfer": "2.3.3+0000000"
+                "RECONALL": {
+                    "freesurfer": "7.4.1"
                 },
                 "Workflow": {
                     "nf/pediatric": "v1.0dev"
@@ -80,7 +83,6 @@
                 "sub-test",
                 "sub-test/anat",
                 "sub-test/anat/sub-test_desc-preproc_T1w.nii.gz",
-                "sub-test/anat/sub-test_fastsurfer",
                 "sub-test/anat/sub-test_seg-BrainnetomeChild_desc-labels.json",
                 "sub-test/anat/sub-test_seg-BrainnetomeChild_desc-labels.txt",
                 "sub-test/anat/sub-test_seg-BrainnetomeChild_dseg.nii.gz",
@@ -99,12 +101,16 @@
                 "sub-test/multiqc/sub-test_multiqc_plots",
                 "sub-test/multiqc/sub-test_multiqc_report.html",
                 "subcortical_volumes.tsv"
+            ],
+            [
+                "",
+                "sub-test"
             ]
         ],
         "meta": {
             "nf-test": "0.9.2",
             "nextflow": "24.10.1"
         },
-        "timestamp": "2025-01-02T22:41:26.12873"
+        "timestamp": "2025-01-08T10:01:23.448734"
     }
 }

tests/multisubjects.nf.test

@@ -12,6 +12,7 @@ nextflow_pipeline {
 
                 params.input = "$projectDir/tests/data/samplesheet_multisubject.csv"
                 params.outdir = "$outputDir"
+                params.use_fastsurfer = true
 
                 params.connectomics = true
                 params.tracking = true
@@ -24,6 +25,8 @@ nextflow_pipeline {
         then {
             // stable name: All files + folders in ${params.outdir}/ with a stable name.
             def stable_name = getAllFilesFromDir(params.outdir, relative: true, includeDir: true, ignore:  ['pipeline_info/*.{html,json,txt}'])
+            // All fastsurfer output.
+            def fastsurfer_output = getAllFilesFromDir("$outputDir/../fastsurfer-v2.3.3/", relative: true, includeDir: true)
             assertAll {
                 assert workflow.success
                 assert snapshot(
@@ -32,7 +35,9 @@ nextflow_pipeline {
                     // Remove the nextflow version from the versions.yml because we test it using different nextflow versions.
                     removeNextflowVersion("$outputDir/pipeline_info/pipeline_software_mqc_versions.yml"),
                     // All stable name.
-                    stable_name
+                    stable_name,
+                    // All fastsurfer output.
+                    fastsurfer_output
                 ).match()
             }
         }