Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add nftests custom modules #7201

Open
wants to merge 20 commits into
base: master
Choose a base branch
from
+513 −414
Open

Add nftests custom modules #7201

Show file tree
Hide file tree
Changes from 18 commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
8f16c73
Added nf-tests
nschcolnicov Dec 11, 2024
22db11c
Add missing tags
nschcolnicov Dec 11, 2024
e54ce38
Fix conda tests
nschcolnicov Dec 11, 2024
3fe76eb
Merge branch 'master' of https://github.com/nf-core/modules into add_…
nschcolnicov Dec 12, 2024
3ae8c7c
Update versions.yml file
nschcolnicov Dec 12, 2024
35b031d
Update version generation
nschcolnicov Dec 12, 2024
96ee087
Update versions command
nschcolnicov Dec 12, 2024
b459baf
Removed quotes from versions
nschcolnicov Dec 12, 2024
816ff3b
add modules to pytest-workflow excludes
nschcolnicov Dec 12, 2024
b63fed2
Addressing PR comments
nschcolnicov Dec 13, 2024
75a029c
Addressing PR comments
nschcolnicov Dec 17, 2024
50e2138
Pull from master
nschcolnicov Dec 17, 2024
5c2e614
Addressing PR comments
nschcolnicov Dec 19, 2024
455c0fa
Update container for matrix
nschcolnicov Dec 19, 2024
452a354
Merge branch 'master' of https://github.com/nf-core/modules into add_…
nschcolnicov Dec 20, 2024
bb0f593
Update snaps
nschcolnicov Dec 20, 2024
2b8efa7
Merge branch 'master' into add_nftests_custom_modules
famosab Jan 7, 2025
68a4758
Merge branch 'master' into add_nftests_custom_modules
SPPearce Jan 8, 2025
2bb1baa
Fix linting
nschcolnicov Jan 9, 2025
d9b9443
Merge branch 'master' into add_nftests_custom_modules
famosab Jan 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions modules/nf-core/cnvkit/export/tests/main.nf.test.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -218,7 +218,7 @@
"id": "test",
"single_end": false
},
"test.vcf:md5,0a0b4a00192be12f67bb895888026f00"
"test.vcf:md5,07fa974547dae22ff2af4b043466c12d"
]
],
"1": [
Expand All @@ -230,7 +230,7 @@
"id": "test",
"single_end": false
},
"test.vcf:md5,0a0b4a00192be12f67bb895888026f00"
"test.vcf:md5,07fa974547dae22ff2af4b043466c12d"
]
],
"versions": [
Expand All @@ -242,6 +242,6 @@
"nf-test": "0.9.2",
"nextflow": "24.10.3"
},
"timestamp": "2024-12-18T17:51:27.269813231"
"timestamp": "2024-12-20T21:13:12.582014142"
}
}
21 changes: 17 additions & 4 deletions modules/nf-core/custom/matrixfilter/main.nf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@ process CUSTOM_MATRIXFILTER {
tag "$meta"
label 'process_single'
conda "${moduleDir}/environment.yml"
container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
'https://depot.galaxyproject.org/singularity/r-base:4.2.1' :
'biocontainers/r-base:4.2.1' }"
container "${workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container
? 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/48/483e9d9b3b07e5658792d579e230ad40ed18daf7b9ebfb4323c08570f92fd1d5/data'
: 'community.wave.seqera.io/library/r-base:4.2.1--b0b5476e2e7a0872'}"

input:
tuple val(meta), path(abundance)
Expand All @@ -13,7 +13,7 @@ process CUSTOM_MATRIXFILTER {
output:
tuple val(meta), path("*.filtered.tsv") , emit: filtered
tuple val(meta), path("*.tests.tsv") , emit: tests
tuple val(meta), path("R_sessionInfo.log") , emit: session_info
tuple val(meta), path("*R_sessionInfo.log") , emit: session_info
famosab marked this conversation as resolved.
Show resolved Hide resolved
path "versions.yml" , emit: versions

when:
Expand All @@ -27,4 +27,17 @@ process CUSTOM_MATRIXFILTER {
// (new variables defined here don't seem to be available in templates, so
// we have to access $task directly)
template 'matrixfilter.R'
nschcolnicov marked this conversation as resolved.
Show resolved Hide resolved

stub:
def prefix = task.ext.prefix ?: "${meta.id}"
"""
touch ${prefix}.filtered.tsv
touch ${prefix}.tests.tsv
touch ${prefix}.R_sessionInfo.log
cat <<-END_VERSIONS > versions.yml
"${task.process}":
r-base: \$(echo \$(R --version 2>&1) | sed 's/^.*R version //; s/ .*\$//')
END_VERSIONS
"""
}
2 changes: 1 addition & 1 deletion modules/nf-core/custom/matrixfilter/templates/matrixfilter.R
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -207,7 +207,7 @@ keep <- apply(boolean_matrix, 1, all)
# Write out the matrix retaining the specified rows and re-prepending the
# column with the feature identifiers

prefix = ifelse('$task.ext.prefix' == 'null', '', '$task.ext.prefix')
prefix = ifelse('$task.ext.prefix' == 'null', '$meta.id', '$task.ext.prefix')

write.table(
data.frame(rownames(abundance_matrix)[keep], abundance_matrix[keep,,drop = FALSE]),
Expand Down
92 changes: 92 additions & 0 deletions modules/nf-core/custom/matrixfilter/tests/main.nf.test
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
nextflow_process {

name "Test Process CUSTOM_MATRIXFILTER"
script "../main.nf"
process "CUSTOM_MATRIXFILTER"
config "./nextflow.config"
tag "modules"
tag "modules_nfcore"
tag "custom_matrixfilter"
tag "custom"
tag "custom/matrixfilter"

test("test_custom_matrixfilter") {

when {
params {
observations_id_col = 'sample'
filtering_min_samples = 1.0
filtering_min_abundance = 10
filtering_min_proportion = null
filtering_grouping_var = null
filtering_min_proportion_not_na = 0.5
filtering_min_samples_not_na = null

module_args = [
"--sample_id_col \"${params.observations_id_col}\"",
"--minimum_samples ${params.filtering_min_samples}",
"--minimum_abundance ${params.filtering_min_abundance}",
(params.filtering_min_proportion ? "--minimum_proportion ${params.filtering_min_proportion}" : ''),
(params.filtering_grouping_var ? "--grouping_variable \"${params.filtering_grouping_var}\"" : ''),
(params.filtering_min_proportion_not_na ? "--minimum_proportion_not_na \"${params.filtering_min_proportion_not_na}\"" : ''),
(params.filtering_min_samples_not_na ? "--minimum_samples_not_na \"${params.filtering_min_samples_not_na}\"" : '')
].join(' ').trim()
}
process {
"""
input[0] = [ [id:"test"], file("https://github.com/nf-core/test-datasets/raw/refs/heads/differentialabundance/modules_testdata/SRP254919.salmon.merged.gene_counts.top1000cov.assay.tsv",checkIfExists: true) ]
input[1] = [ [id:"test"], file("https://github.com/nf-core/test-datasets/raw/refs/heads/differentialabundance/modules_testdata/SRP254919.samplesheet.sample_metadata.tsv",checkIfExists: true) ]
"""

}
}

then {
assertAll (
{ assert process.success },
{ assert snapshot(process.out.filtered,
process.out.tests,
process.out.versions).match() }
)
}
}

test("test_custom_matrixfilter - stub") {

options "-stub"

when {
params {
observations_id_col = 'sample'
filtering_min_samples = 1.0
filtering_min_abundance = 10
filtering_min_proportion = null
filtering_grouping_var = null
filtering_min_proportion_not_na = 0.5
filtering_min_samples_not_na = null
module_args = [
"--sample_id_col \"${params.observations_id_col}\"",
"--minimum_samples ${params.filtering_min_samples}",
"--minimum_abundance ${params.filtering_min_abundance}",
(params.filtering_min_proportion ? "--minimum_proportion ${params.filtering_min_proportion}" : ''),
(params.filtering_grouping_var ? "--grouping_variable \"${params.filtering_grouping_var}\"" : ''),
(params.filtering_min_proportion_not_na ? "--minimum_proportion_not_na \"${params.filtering_min_proportion_not_na}\"" : ''),
(params.filtering_min_samples_not_na ? "--minimum_samples_not_na \"${params.filtering_min_samples_not_na}\"" : '')
].join(' ').trim()
}
process {
"""
input[0] = [ [id:"test"], file("https://github.com/nf-core/test-datasets/raw/refs/heads/differentialabundance/modules_testdata/SRP254919.salmon.merged.gene_counts.top1000cov.assay.tsv",checkIfExists: true) ]
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

One more (small) thing - I would ask you to update your links so that they follow the convention and look like this:

file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/vcf/test.vcf.gz', checkIfExists: true)

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That also means that all the test-datasets that you need must be present on the modules branch of the test-datasets repository. I know that there will be some restructuring around the test-datasets so maybe it makes sense to have them present there so they do not get lost and the tests stop working. I can review a PR over at the test-datasets repo when you add them.

input[1] = [ [id:"test"], file("https://github.com/nf-core/test-datasets/raw/refs/heads/differentialabundance/modules_testdata/SRP254919.samplesheet.sample_metadata.tsv",checkIfExists: true) ]
"""
}
}

then {
assertAll (
{ assert process.success },
{ assert snapshot(process.out).match() }
)
}
}
}
95 changes: 95 additions & 0 deletions modules/nf-core/custom/matrixfilter/tests/main.nf.test.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
{
"test_custom_matrixfilter": {
"content": [
[
[
{
"id": "test"
},
"test.filtered.tsv:md5,ecc3ca4c6538f9248dce844108474a66"
]
],
[
[
{
"id": "test"
},
"test.tests.tsv:md5,9a0f14988df8c1b8975039eed91b85e2"
]
],
[
"versions.yml:md5,275e382be9ca5bdd30ff5d6fc4340616"
]
],
"meta": {
"nf-test": "0.9.2",
"nextflow": "24.10.2"
},
"timestamp": "2024-12-13T21:21:11.862199262"
},
nschcolnicov marked this conversation as resolved.
Show resolved Hide resolved
"test_custom_matrixfilter - stub": {
"content": [
{
"0": [
[
{
"id": "test"
},
"test.filtered.tsv:md5,d41d8cd98f00b204e9800998ecf8427e"
]
],
"1": [
[
{
"id": "test"
},
"test.tests.tsv:md5,d41d8cd98f00b204e9800998ecf8427e"
]
],
"2": [
[
{
"id": "test"
},
"test.R_sessionInfo.log:md5,d41d8cd98f00b204e9800998ecf8427e"
]
],
"3": [
"versions.yml:md5,275e382be9ca5bdd30ff5d6fc4340616"
],
"filtered": [
[
{
"id": "test"
},
"test.filtered.tsv:md5,d41d8cd98f00b204e9800998ecf8427e"
]
],
"session_info": [
[
{
"id": "test"
},
"test.R_sessionInfo.log:md5,d41d8cd98f00b204e9800998ecf8427e"
]
],
"tests": [
[
{
"id": "test"
},
"test.tests.tsv:md5,d41d8cd98f00b204e9800998ecf8427e"
]
],
"versions": [
"versions.yml:md5,275e382be9ca5bdd30ff5d6fc4340616"
]
}
],
"meta": {
"nf-test": "0.9.2",
"nextflow": "24.10.2"
},
"timestamp": "2024-12-13T21:22:31.144793503"
}
}
3 changes: 3 additions & 0 deletions modules/nf-core/custom/matrixfilter/tests/nextflow.config
View file
Open in desktop
nschcolnicov marked this conversation as resolved.
Show resolved Hide resolved
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

there was a new convention recently, maybe you can update to adhere to that: https://nf-co.re/docs/guidelines/components/modules#configuration-of-extargs-in-tests

Copy link
Contributor Author

@nschcolnicov nschcolnicov Dec 18, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just to make sure, you are requesting me to pass the values that I'm providing via ext.args using params instead of the actual value? Didn't you correct me to do it this way on a previous comment? https://github.com/nf-core/modules/pull/7201/files/816ff3b787af7e12e75a7d95f76b631f474853ee#r1888548704 Want to make sure I'm doing it the exact way you are suggesting

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If I did then I am sorry. But what is meant is that the params that you need to pass are written directly in the nf.test file which makes its directly understandable for other people that look at your tests and then they do not have to go through multiple nextflow.config files.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No, sorry, you are right, I was wrong both times, I just wasn't understanding how you wanted me to structure the config, I think I have now understood and addressed the issues, let me know what you think @famosab 😄

Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
process {
ext.args = params.module_args
}
13 changes: 12 additions & 1 deletion modules/nf-core/custom/tabulartogseacls/main.nf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,18 @@ process CUSTOM_TABULARTOGSEACLS {

cat <<-END_VERSIONS > versions.yml
"${task.process}":
bash: \$(echo \$(bash --version | grep -Eo 'version [[:alnum:].]+' | sed 's/version //'))
awk: \$(mawk -W version | head -n 1 | awk '{print \$2}')
END_VERSIONS
"""

stub:
def prefix = task.ext.prefix ?: "${meta.id}"
"""
touch ${prefix}.cls
cat <<-END_VERSIONS > versions.yml
"${task.process}":
awk: \$(mawk -W version | head -n 1 | awk '{print \$2}')
END_VERSIONS
"""

}
57 changes: 57 additions & 0 deletions modules/nf-core/custom/tabulartogseacls/tests/main.nf.test
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
nextflow_process {

name "Test Process CUSTOM_TABULARTOGSEACLS"
script "../main.nf"
process "CUSTOM_TABULARTOGSEACLS"
config "./nextflow.config"
tag "modules"
tag "modules_nfcore"
tag "custom_tabulartogseacls"
tag "custom"
tag "custom/tabulartogseacls"

test("test_custom_tabulartogseacls") {

when {
params {
module_args = [ "separator": "\t", "variable": "treatment" ]
}
process {
"""
input[0] = [ [id:'treatment_mCherry_hND6_sample_number', variable:'treatment', reference:'mCherry', target:'hND6', blocking:'sample_number'], file("https://github.com/nf-core/test-datasets/raw/refs/heads/differentialabundance/modules_testdata/SRP254919.samplesheet.sample_metadata.tsv",checkIfExists: true) ]
"""

}
}

then {
assertAll (
{ assert process.success },
{ assert snapshot(process.out).match() }
)
}
}

test("test_custom_tabulartogseacls - stub") {

options "-stub"

when {
params {
module_args = { [ "separator": "\t", "variable": "treatment" ] }
}
process {
"""
input[0] = [ [id:'treatment_mCherry_hND6_sample_number', variable:'treatment', reference:'mCherry', target:'hND6', blocking:'sample_number'], file("https://github.com/nf-core/test-datasets/raw/refs/heads/differentialabundance/modules_testdata/SRP254919.samplesheet.sample_metadata.tsv",checkIfExists: true) ]
"""
}
}

then {
assertAll (
{ assert process.success },
{ assert snapshot(process.out).match() }
)
}
}
}
Loading
Loading