From 626f706c215ed8531d0415dabdcfb9af72642995 Mon Sep 17 00:00:00 2001 From: Cristian Goina Date: Thu, 9 May 2024 15:48:26 -0400 Subject: [PATCH] add neuron annotations and options to filter by neuron annotations --- .../dto/AbstractNeuronMetadata.java | 10 +++++ .../colormipsearch/dto/EMNeuronMetadata.java | 9 ---- .../model/AbstractNeuronEntity.java | 11 +++++ .../colormipsearch/model/EMNeuronEntity.java | 13 +----- .../colormipsearch/model/LMNeuronEntity.java | 1 + .../colormipsearch/dao/NeuronSelector.java | 43 ++++++++++++++++++- .../dao/mongo/NeuronSelectionHelper.java | 6 +++ .../dataio/DataSourceParam.java | 29 ++++++++++--- .../dataio/NeuronMatchesReader.java | 16 +++++++ .../dataio/db/DBCDMIPsReader.java | 4 +- .../dataio/db/DBNeuronMatchesReader.java | 24 +++++++++-- .../dataio/fs/JSONNeuronMatchesReader.java | 8 ++++ .../dataio/fs/JSONReadWriteTest.java | 4 +- .../cmd/AbstractGradientScoresArgs.java | 20 +++++++++ .../cmd/CalculateGradientScoresCmd.java | 6 +++ .../cmd/ColorDepthSearchCmd.java | 28 ++++++++++++ .../colormipsearch/cmd/ExportData4NBCmd.java | 21 +++++++++ .../cmd/ImportV2CDMatchesCmd.java | 4 +- .../cmd/NormalizeGradientScoresCmd.java | 6 ++- .../dataexport/AbstractCDMatchesExporter.java | 9 ++++ .../cmd/dataexport/EMCDMatchesExporter.java | 15 ++++--- .../cmd/dataexport/EMPPPMatchesExporter.java | 4 ++ .../cmd/dataexport/LMCDMatchesExporter.java | 29 +++++++++---- .../JSONV2Em2LmMatchesReader.java | 8 +++- 24 files changed, 273 insertions(+), 55 deletions(-) diff --git a/colormipsearch-api/src/main/java/org/janelia/colormipsearch/dto/AbstractNeuronMetadata.java b/colormipsearch-api/src/main/java/org/janelia/colormipsearch/dto/AbstractNeuronMetadata.java index 3bd2875e..eed633a2 100644 --- a/colormipsearch-api/src/main/java/org/janelia/colormipsearch/dto/AbstractNeuronMetadata.java +++ b/colormipsearch-api/src/main/java/org/janelia/colormipsearch/dto/AbstractNeuronMetadata.java @@ -1,6 +1,7 @@ package org.janelia.colormipsearch.dto; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.function.BiFunction; @@ -47,6 +48,7 @@ public abstract class AbstractNeuronMetadata { private String alignmentSpace; private String anatomicalArea; private Gender gender; + private List terms; private boolean unpublished; // neuronFiles holds S3 files used by the NeuronBridge app private final Map neuronFiles = new HashMap<>(); @@ -154,6 +156,14 @@ public void setGender(Gender gender) { this.gender = gender; } + public List getTerms() { + return terms; + } + + public void setTerms(List terms) { + this.terms = terms; + } + @JsonIgnore public boolean isUnpublished() { return unpublished; diff --git a/colormipsearch-api/src/main/java/org/janelia/colormipsearch/dto/EMNeuronMetadata.java b/colormipsearch-api/src/main/java/org/janelia/colormipsearch/dto/EMNeuronMetadata.java index 783bea3e..0bf650ed 100644 --- a/colormipsearch-api/src/main/java/org/janelia/colormipsearch/dto/EMNeuronMetadata.java +++ b/colormipsearch-api/src/main/java/org/janelia/colormipsearch/dto/EMNeuronMetadata.java @@ -10,7 +10,6 @@ public class EMNeuronMetadata extends AbstractNeuronMetadata { private String neuronType; private String neuronInstance; private String state; - private List neuronTerms; @Override public String getTypeDiscriminator() { @@ -49,12 +48,4 @@ public String getState() { public void setState(String state) { this.state = state; } - - public List getNeuronTerms() { - return neuronTerms; - } - - public void setNeuronTerms(List neuronTerms) { - this.neuronTerms = neuronTerms; - } } diff --git a/colormipsearch-api/src/main/java/org/janelia/colormipsearch/model/AbstractNeuronEntity.java b/colormipsearch-api/src/main/java/org/janelia/colormipsearch/model/AbstractNeuronEntity.java index b85db437..e051bb37 100644 --- a/colormipsearch-api/src/main/java/org/janelia/colormipsearch/model/AbstractNeuronEntity.java +++ b/colormipsearch-api/src/main/java/org/janelia/colormipsearch/model/AbstractNeuronEntity.java @@ -40,6 +40,7 @@ public abstract class AbstractNeuronEntity extends AbstractBaseEntity { // This will be used to identify the matched neurons for PPP since the color depth MIPs are not // part of the PPP match process at all. private String sourceRefId; + private List neuronTerms; // computeFileData holds local files used either for precompute or upload private final Map computeFiles = new HashMap<>(); // processed tags holds the corresponding processing tag used for the ColorDepthSearch or PPPM import @@ -123,6 +124,14 @@ public String getSourceRefIdOnly() { } } + public List getNeuronTerms() { + return neuronTerms; + } + + public void setNeuronTerms(List neuronTerms) { + this.neuronTerms = neuronTerms; + } + @JsonProperty public Map getComputeFiles() { return computeFiles; @@ -246,6 +255,7 @@ public List> updateableFieldValues() { fieldList.add(new EntityField<>("libraryName", false, libraryName)); fieldList.add(new EntityField<>("publishedName", false, publishedName)); fieldList.add(new EntityField<>("sourceRefId", false, sourceRefId)); + fieldList.add(new EntityField<>("neuronTerms", false, neuronTerms)); fieldList.add(new EntityField<>("updatedDate", false, getUpdatedDate())); // datasetLabels is a collection but will always be replaced instead of appended to existing values fieldList.add(new EntityField<>("datasetLabels", false, datasetLabels)); @@ -309,6 +319,7 @@ protected void copyFrom(N that) { this.libraryName = that.getLibraryName(); this.publishedName = that.getPublishedName(); this.sourceRefId = that.getSourceRefId(); + this.neuronTerms = that.getNeuronTerms(); this.computeFiles.clear(); this.computeFiles.putAll(that.getComputeFiles()); this.addAllTags(that.getTags()); diff --git a/colormipsearch-api/src/main/java/org/janelia/colormipsearch/model/EMNeuronEntity.java b/colormipsearch-api/src/main/java/org/janelia/colormipsearch/model/EMNeuronEntity.java index b357a5dc..e1bb386c 100644 --- a/colormipsearch-api/src/main/java/org/janelia/colormipsearch/model/EMNeuronEntity.java +++ b/colormipsearch-api/src/main/java/org/janelia/colormipsearch/model/EMNeuronEntity.java @@ -12,7 +12,6 @@ public class EMNeuronEntity extends AbstractNeuronEntity { // neuronType and the neuronInstance are only for reference purposes here private String neuronType; private String neuronInstance; - private List neuronTerms; @Override public String getNeuronId() { @@ -35,20 +34,11 @@ public void setNeuronInstance(String neuronInstance) { this.neuronInstance = neuronInstance; } - public List getNeuronTerms() { - return neuronTerms; - } - - public void setNeuronTerms(List neuronTerms) { - this.neuronTerms = neuronTerms; - } - @Override public List> updateableFieldValues() { List> fieldList = new ArrayList<>(super.updateableFieldValues()); fieldList.add(new EntityField<>("neuronType", false, neuronType)); fieldList.add(new EntityField<>("neuronInstance", false, neuronInstance)); - fieldList.add(new EntityField<>("neuronTerms", false, neuronTerms)); return fieldList; } @@ -58,7 +48,6 @@ public EMNeuronEntity duplicate() { n.copyFrom(this); n.neuronType = this.getNeuronType(); n.neuronInstance = this.getNeuronInstance(); - n.neuronTerms = this.getNeuronTerms(); return n; } @@ -73,7 +62,7 @@ public EMNeuronMetadata metadata() { n.setPublishedName(getPublishedName()); n.setNeuronType(getNeuronType()); n.setNeuronInstance(getNeuronInstance()); - n.setNeuronTerms(getNeuronTerms()); + n.setTerms(getNeuronTerms()); getComputeFiles().forEach((ft, fd) -> n.setNeuronComputeFile(ft, fd.getFileName())); getProcessedTags().forEach(n::putProcessedTags); return n; diff --git a/colormipsearch-api/src/main/java/org/janelia/colormipsearch/model/LMNeuronEntity.java b/colormipsearch-api/src/main/java/org/janelia/colormipsearch/model/LMNeuronEntity.java index 35cad67f..f5f5a145 100644 --- a/colormipsearch-api/src/main/java/org/janelia/colormipsearch/model/LMNeuronEntity.java +++ b/colormipsearch-api/src/main/java/org/janelia/colormipsearch/model/LMNeuronEntity.java @@ -137,6 +137,7 @@ public LMNeuronMetadata metadata() { n.setAnatomicalArea(anatomicalArea); n.setGender(gender); n.setObjective(objective); + n.setTerms(getNeuronTerms()); getComputeFiles().forEach((ft, fd) -> n.setNeuronComputeFile(ft, fd.getFileName())); getProcessedTags().forEach(n::putProcessedTags); return n; diff --git a/colormipsearch-persist/src/main/java/org/janelia/colormipsearch/dao/NeuronSelector.java b/colormipsearch-persist/src/main/java/org/janelia/colormipsearch/dao/NeuronSelector.java index d9516d91..37e6eb20 100644 --- a/colormipsearch-persist/src/main/java/org/janelia/colormipsearch/dao/NeuronSelector.java +++ b/colormipsearch-persist/src/main/java/org/janelia/colormipsearch/dao/NeuronSelector.java @@ -24,6 +24,8 @@ public class NeuronSelector { private final Set entityIds = new HashSet<>(); // matching internal entity IDs private final Set tags = new HashSet<>(); // matching tags private final Set excludedTags = new HashSet<>(); + private final Set annotations = new HashSet<>(); // matching annotations + private final Set excludedAnnotations = new HashSet<>(); private final List>> processedTagsSelections = new ArrayList<>(); public String getNeuronClassname() { return neuronClassname; @@ -204,6 +206,42 @@ public boolean hasExcludedTags() { return CollectionUtils.isNotEmpty(excludedTags); } + public Set getAnnotations() { + return annotations; + } + + public NeuronSelector addAnnotation(String annotation) { + if (StringUtils.isNotBlank(annotation)) this.annotations.add(annotation); + return this; + } + + public NeuronSelector addAnnotations(Collection annotations) { + if (annotations != null) annotations.forEach(this::addAnnotation); + return this; + } + + public boolean hasAnnotations() { + return CollectionUtils.isNotEmpty(annotations); + } + + public Set getExcludedAnnotations() { + return excludedAnnotations; + } + + public NeuronSelector addExcludedAnnotation(String annotation) { + if (StringUtils.isNotBlank(annotation)) this.excludedAnnotations.add(annotation); + return this; + } + + public NeuronSelector addExcludedAnnotations(Collection annotations) { + if (annotations != null) annotations.forEach(this::addExcludedAnnotation); + return this; + } + + public boolean hasExcludedAnnotations() { + return CollectionUtils.isNotEmpty(excludedAnnotations); + } + public List>> getProcessedTagsSelections() { return processedTagsSelections; } @@ -275,7 +313,10 @@ public boolean isEmpty() { && !hasExcludedTags() && !hasDatasetLabels() && !hasSourceRefIds() - && !hasProcessedTags(); + && !hasProcessedTags() + && !hasAnnotations() + && !hasExcludedAnnotations() + ; } public boolean isNotEmpty() { diff --git a/colormipsearch-persist/src/main/java/org/janelia/colormipsearch/dao/mongo/NeuronSelectionHelper.java b/colormipsearch-persist/src/main/java/org/janelia/colormipsearch/dao/mongo/NeuronSelectionHelper.java index 3eca53db..0a597de1 100644 --- a/colormipsearch-persist/src/main/java/org/janelia/colormipsearch/dao/mongo/NeuronSelectionHelper.java +++ b/colormipsearch-persist/src/main/java/org/janelia/colormipsearch/dao/mongo/NeuronSelectionHelper.java @@ -64,6 +64,12 @@ static Bson getNeuronFilter(String fieldQualifier, NeuronSelector neuronSelector if (neuronSelector.hasExcludedTags()) { filter.add(Filters.nin(qualifier + "tags", neuronSelector.getExcludedTags())); } + if (neuronSelector.hasAnnotations()) { + filter.add(Filters.in(qualifier + "neuronTerms", neuronSelector.getAnnotations())); + } + if (neuronSelector.hasExcludedAnnotations()) { + filter.add(Filters.nin(qualifier + "neuronTerms", neuronSelector.getExcludedAnnotations())); + } if (neuronSelector.hasProcessedTags()) { // all filters from a selection are "and"-ed // and all selections are "or"-ed diff --git a/colormipsearch-persist/src/main/java/org/janelia/colormipsearch/dataio/DataSourceParam.java b/colormipsearch-persist/src/main/java/org/janelia/colormipsearch/dataio/DataSourceParam.java index aff3d874..fb823ce5 100644 --- a/colormipsearch-persist/src/main/java/org/janelia/colormipsearch/dataio/DataSourceParam.java +++ b/colormipsearch-persist/src/main/java/org/janelia/colormipsearch/dataio/DataSourceParam.java @@ -17,8 +17,10 @@ public class DataSourceParam { private Collection mipIDs = new HashSet<>(); private Collection names = new HashSet<>(); private Collection tags = new HashSet<>(); - private Collection datasets = new HashSet<>(); private Collection excludedTags = new HashSet<>(); + private Collection datasets = new HashSet<>(); + private Collection annotations = new HashSet<>(); + private Collection excludedAnnotations = new HashSet<>(); private long offset; private int size; @@ -82,6 +84,15 @@ public DataSourceParam addTags(Collection tags) { return this; } + public Collection getExcludedTags() { + return excludedTags; + } + + public DataSourceParam addExcludedTags(Collection excludedTags) { + if (excludedTags != null) excludedTags.stream().filter(StringUtils::isNotBlank).forEach(t -> this.excludedTags.add(t)); + return this; + } + public Collection getDatasets() { return datasets; } @@ -89,15 +100,23 @@ public Collection getDatasets() { public DataSourceParam addDatasets(Collection datasets) { if (datasets != null) datasets.stream().filter(StringUtils::isNotBlank).forEach(ds -> this.datasets.add(ds)); return this; + } + public Collection getAnnotations() { + return annotations; } - public Collection getExcludedTags() { - return excludedTags; + public DataSourceParam addAnnotations(Collection annotations) { + if (annotations != null) annotations.stream().filter(StringUtils::isNotBlank).forEach(a -> this.annotations.add(a)); + return this; } - public DataSourceParam addExcludedTags(Collection excludedTags) { - if (excludedTags != null) excludedTags.stream().filter(StringUtils::isNotBlank).forEach(t -> this.excludedTags.add(t)); + public Collection getExcludedAnnotations() { + return excludedAnnotations; + } + + public DataSourceParam addExcludedAnnotations(Collection annotations) { + if (annotations != null) annotations.stream().filter(StringUtils::isNotBlank).forEach(a -> this.excludedAnnotations.add(a)); return this; } diff --git a/colormipsearch-persist/src/main/java/org/janelia/colormipsearch/dataio/NeuronMatchesReader.java b/colormipsearch-persist/src/main/java/org/janelia/colormipsearch/dataio/NeuronMatchesReader.java index bcc079b2..5befd91c 100644 --- a/colormipsearch-persist/src/main/java/org/janelia/colormipsearch/dataio/NeuronMatchesReader.java +++ b/colormipsearch-persist/src/main/java/org/janelia/colormipsearch/dataio/NeuronMatchesReader.java @@ -30,12 +30,16 @@ public interface NeuronMatchesReader readMatchesByMask(String alignmentSpace, Collection maskDatasets, Collection maskTags, Collection maskExcludedTags, + Collection maskAnnotations, + Collection excludedMaskAnnotations, Collection targetLibraries, Collection targetPublishedNames, Collection targetMipIds, Collection targetDatasets, Collection targetTags, Collection targetExcludedTags, + Collection targetAnnotations, + Collection excludedTargetAnnotations, Collection matchTags, Collection matchExcludedTags, ScoresFilter matchScoresFilter, @@ -70,12 +78,16 @@ List readMatchesByMask(String alignmentSpace, * @param maskDatasets * @param maskTags * @param maskExcludedTags + * @param maskAnnotations + * @param excludedMaskAnnotations * @param targetLibraries * @param targetPublishedNames * @param targetMipIds * @param targetDatasets * @param targetTags * @param targetExcludedTags + * @param targetAnnotations + * @param excludedTargetAnnotations * @param matchTags * @param matchExcludedTags * @param matchScoresFilter @@ -89,12 +101,16 @@ List readMatchesByTarget(String alignmentSpace, Collection maskDatasets, Collection maskTags, Collection maskExcludedTags, + Collection maskAnnotations, + Collection excludedMaskAnnotations, Collection targetLibraries, Collection targetPublishedNames, Collection targetMipIds, Collection targetDatasets, Collection targetTags, Collection targetExcludedTags, + Collection targetAnnotations, + Collection excludedTargetAnnotations, Collection matchTags, Collection matchExcludedTags, ScoresFilter matchScoresFilter, diff --git a/colormipsearch-persist/src/main/java/org/janelia/colormipsearch/dataio/db/DBCDMIPsReader.java b/colormipsearch-persist/src/main/java/org/janelia/colormipsearch/dataio/db/DBCDMIPsReader.java index 9eb36061..ed481444 100644 --- a/colormipsearch-persist/src/main/java/org/janelia/colormipsearch/dataio/db/DBCDMIPsReader.java +++ b/colormipsearch-persist/src/main/java/org/janelia/colormipsearch/dataio/db/DBCDMIPsReader.java @@ -27,7 +27,9 @@ public List readMIPs(DataSourceParam mipsDataSou .addMipIDs(mipsDataSource.getMipIDs()) .addDatasetLabels(mipsDataSource.getDatasets()) .addTags(mipsDataSource.getTags()) - .addExcludedTags(mipsDataSource.getExcludedTags()), + .addExcludedTags(mipsDataSource.getExcludedTags()) + .addAnnotations(mipsDataSource.getAnnotations()) + .addExcludedAnnotations(mipsDataSource.getExcludedAnnotations()), new PagedRequest().setPageSize(mipsDataSource.getSize()).setFirstPageOffset(mipsDataSource.getOffset()) ).getResultList(); } diff --git a/colormipsearch-persist/src/main/java/org/janelia/colormipsearch/dataio/db/DBNeuronMatchesReader.java b/colormipsearch-persist/src/main/java/org/janelia/colormipsearch/dataio/db/DBNeuronMatchesReader.java index 2a7bbf0d..3a0599e4 100644 --- a/colormipsearch-persist/src/main/java/org/janelia/colormipsearch/dataio/db/DBNeuronMatchesReader.java +++ b/colormipsearch-persist/src/main/java/org/janelia/colormipsearch/dataio/db/DBNeuronMatchesReader.java @@ -73,12 +73,16 @@ public List readMatchesByMask(String alignmentSpace, Collection maskDatasets, Collection maskTags, Collection maskExcludedTags, + Collection maskAnnotations, + Collection excludedMaskAnnotations, Collection targetLibraries, Collection targetPublishedNames, Collection targetMipIds, Collection targetDatasets, Collection targetTags, Collection targetExcludedTags, + Collection targetAnnotations, + Collection excludedTargetAnnotations, Collection matchTags, Collection matchExcludedTags, ScoresFilter matchScoresFilter, @@ -90,7 +94,9 @@ public List readMatchesByMask(String alignmentSpace, .addMipIDs(maskMipIds) .addDatasetLabels(maskDatasets) .addTags(maskTags) - .addExcludedTags(maskExcludedTags); + .addExcludedTags(maskExcludedTags) + .addAnnotations(maskAnnotations) + .addExcludedAnnotations(excludedMaskAnnotations); NeuronSelector targetSelector = new NeuronSelector() .setAlignmentSpace(alignmentSpace) .addLibraries(targetLibraries) @@ -98,7 +104,9 @@ public List readMatchesByMask(String alignmentSpace, .addMipIDs(targetMipIds) .addDatasetLabels(targetDatasets) .addTags(targetTags) - .addExcludedTags(targetExcludedTags); + .addExcludedTags(targetExcludedTags) + .addAnnotations(targetAnnotations) + .addExcludedAnnotations(excludedTargetAnnotations); List maskEntityIds = getNeuronEntityIds(maskSelector); NeuronsMatchFilter neuronsMatchFilter = new NeuronsMatchFilter() .setScoresFilter(matchScoresFilter) @@ -117,12 +125,16 @@ public List readMatchesByTarget(String alignmentSpace, Collection maskDatasets, Collection maskTags, Collection maskExcludedTags, + Collection maskAnnotations, + Collection excludedMaskAnnotations, Collection targetLibraries, Collection targetPublishedNames, Collection targetMipIds, Collection targetDatasets, Collection targetTags, Collection targetExcludedTags, + Collection targetAnnotations, + Collection excludedTargetAnnotations, Collection matchTags, Collection matchExcludedTags, ScoresFilter matchScoresFilter, @@ -134,7 +146,9 @@ public List readMatchesByTarget(String alignmentSpace, .addMipIDs(maskMipIds) .addDatasetLabels(maskDatasets) .addTags(maskTags) - .addExcludedTags(maskExcludedTags); + .addExcludedTags(maskExcludedTags) + .addAnnotations(maskAnnotations) + .addExcludedAnnotations(excludedMaskAnnotations); NeuronSelector targetSelector = new NeuronSelector() .setAlignmentSpace(alignmentSpace) .addLibraries(targetLibraries) @@ -142,7 +156,9 @@ public List readMatchesByTarget(String alignmentSpace, .addMipIDs(targetMipIds) .addDatasetLabels(targetDatasets) .addTags(targetTags) - .addExcludedTags(targetExcludedTags); + .addExcludedTags(targetExcludedTags) + .addAnnotations(targetAnnotations) + .addExcludedAnnotations(excludedTargetAnnotations); List targetEntityIds = getNeuronEntityIds(targetSelector); NeuronsMatchFilter neuronsMatchFilter = new NeuronsMatchFilter() .setScoresFilter(matchScoresFilter) diff --git a/colormipsearch-persist/src/main/java/org/janelia/colormipsearch/dataio/fs/JSONNeuronMatchesReader.java b/colormipsearch-persist/src/main/java/org/janelia/colormipsearch/dataio/fs/JSONNeuronMatchesReader.java index c49e993a..2cc276dc 100644 --- a/colormipsearch-persist/src/main/java/org/janelia/colormipsearch/dataio/fs/JSONNeuronMatchesReader.java +++ b/colormipsearch-persist/src/main/java/org/janelia/colormipsearch/dataio/fs/JSONNeuronMatchesReader.java @@ -60,12 +60,16 @@ public List readMatchesByMask(String alignmentSpace, Collection maskDatasets, Collection maskTags, Collection maskExcludedTags, + Collection maskAnnotations, + Collection excludedMaskAnnotations, Collection targetLibraries, Collection targetPublishedNames, Collection targetMipIds, Collection targetDatasets, Collection targetTags, Collection targetExcludedTags, + Collection targetAnnotations, + Collection excludedTargetAnnotations, Collection matchTags, Collection matchExcludedTags, ScoresFilter matchScoresFilter, @@ -88,12 +92,16 @@ public List readMatchesByTarget(String alignmentSpace, Collection maskDatasets, Collection maskTags, Collection maskExcludedTags, + Collection maskAnnotations, + Collection excludedMaskAnnotations, Collection targetLibraries, Collection targetPublishedNames, Collection targetMipIds, Collection targetDatasets, Collection targetTags, Collection targetExcludedTags, + Collection targetAnnotations, + Collection excludedTargetAnnotations, Collection matchTags, Collection matchExcludedTags, ScoresFilter matchScoresFilter, diff --git a/colormipsearch-persist/src/test/java/org/janelia/colormipsearch/dataio/fs/JSONReadWriteTest.java b/colormipsearch-persist/src/test/java/org/janelia/colormipsearch/dataio/fs/JSONReadWriteTest.java index d115afb4..fce9d3e2 100644 --- a/colormipsearch-persist/src/test/java/org/janelia/colormipsearch/dataio/fs/JSONReadWriteTest.java +++ b/colormipsearch-persist/src/test/java/org/janelia/colormipsearch/dataio/fs/JSONReadWriteTest.java @@ -99,13 +99,13 @@ public void readWriteCDSResults() { /* maskDatasets */null, /* maskTags */null, /* maskExcludedTags */null, - /* targetLibraries */null, + /* targetLibraries */null, null, null, /* targetPublishedNames */null, /* targetMIPIDs */null, /* targetDatasets*/null, /* targetTags */null, /* targetExcludedTags */null, - /* matchTags */null, + /* matchTags */null, null, null, /* matchExcludedTags */null, /* matchScoresFilter */null, /* sortCriteria */null); diff --git a/colormipsearch-tools/src/main/java/org/janelia/colormipsearch/cmd/AbstractGradientScoresArgs.java b/colormipsearch-tools/src/main/java/org/janelia/colormipsearch/cmd/AbstractGradientScoresArgs.java index bf2b8fbd..599298d8 100644 --- a/colormipsearch-tools/src/main/java/org/janelia/colormipsearch/cmd/AbstractGradientScoresArgs.java +++ b/colormipsearch-tools/src/main/java/org/janelia/colormipsearch/cmd/AbstractGradientScoresArgs.java @@ -36,6 +36,16 @@ class AbstractGradientScoresArgs extends AbstractColorDepthMatchArgs { variableArity = true) List maskTags = new ArrayList<>(); + @Parameter(names = {"--masks-terms"}, description = "Terms associated with the mask of the match to be scored", + listConverter = ListValueAsFileArgConverter.class, + variableArity = true) + List maskAnnotations = new ArrayList<>(); + + @Parameter(names = {"--excluded-masks-terms"}, description = "Terms associated with the mask of the match to NOT be scored", + listConverter = ListValueAsFileArgConverter.class, + variableArity = true) + List excludedMaskAnnotations = new ArrayList<>(); + @Parameter(names = {"--targets-datasets"}, description = "Datasets associated with the target of the match to be scored", listConverter = ListValueAsFileArgConverter.class, variableArity = true) @@ -61,6 +71,16 @@ class AbstractGradientScoresArgs extends AbstractColorDepthMatchArgs { variableArity = true) List targetsMIPIDs; + @Parameter(names = {"--targets-terms"}, description = "Terms associated with the target of the match to be scored", + listConverter = ListValueAsFileArgConverter.class, + variableArity = true) + List targetAnnotations = new ArrayList<>(); + + @Parameter(names = {"--excluded-targets-terms"}, description = "Terms associated with the target of the match to NOT be scored", + listConverter = ListValueAsFileArgConverter.class, + variableArity = true) + List excludedTargetAnnotations = new ArrayList<>(); + @Parameter(names = {"--match-tags"}, description = "Match tags to be scored", listConverter = ListValueAsFileArgConverter.class, variableArity = true) diff --git a/colormipsearch-tools/src/main/java/org/janelia/colormipsearch/cmd/CalculateGradientScoresCmd.java b/colormipsearch-tools/src/main/java/org/janelia/colormipsearch/cmd/CalculateGradientScoresCmd.java index d6e2f5af..9cf963a8 100644 --- a/colormipsearch-tools/src/main/java/org/janelia/colormipsearch/cmd/CalculateGradientScoresCmd.java +++ b/colormipsearch-tools/src/main/java/org/janelia/colormipsearch/cmd/CalculateGradientScoresCmd.java @@ -142,6 +142,8 @@ private void calculateAllGradientScores() { .addMipIDs(args.masksMIPIDs) .addDatasets(args.maskDatasets) .addTags(args.maskTags) + .addAnnotations(args.maskAnnotations) + .addExcludedAnnotations(args.excludedMaskAnnotations) .setOffset(larg.offset) .setSize(larg.length)) .collect(Collectors.toList())); @@ -351,12 +353,16 @@ List> getCDMatchesForMask(NeuronMatchesReader masksTags; + @Parameter(names = {"--masks-terms"}, description = "Masks MIPs annotations (terms) to be selected for CDS", + listConverter = ListValueAsFileArgConverter.class, + variableArity = true) + List masksAnnotations; + + @Parameter(names = {"--excluded-masks-terms"}, description = "Masks MIPs annotations (terms) to be excluded for CDS", + listConverter = ListValueAsFileArgConverter.class, + variableArity = true) + List excludedMasksAnnotations; + @Parameter(names = {"--masks-datasets"}, description = "Masks MIPs datasets to be selected for CDS", listConverter = ListValueAsFileArgConverter.class, variableArity = true) @@ -103,6 +113,16 @@ static class ColorDepthSearchArgs extends AbstractColorDepthMatchArgs { variableArity = true) List targetsTags; + @Parameter(names = {"--targets-terms"}, description = "Targets MIPs annotations (terms) to be selected for CDS", + listConverter = ListValueAsFileArgConverter.class, + variableArity = true) + List targetsAnnotations; + + @Parameter(names = {"--excluded-targets-terms"}, description = "Targets MIPs annotations (terms) to be excluded for CDS", + listConverter = ListValueAsFileArgConverter.class, + variableArity = true) + List excludedTargetsAnnotations; + @Parameter(names = {"--targets-datasets"}, description = "Targets MIPs datasets to be selected for CDS", listConverter = ListValueAsFileArgConverter.class, variableArity = true) @@ -191,6 +211,8 @@ private void ru args.masksPublishedNames, args.masksDatasets, args.masksTags, + args.masksAnnotations, + args.excludedMasksAnnotations, args.masksStartIndex, args.masksLength, args.maskMIPsFilter); LOG.info("Read {} masks", maskMips.size()); @@ -200,6 +222,8 @@ private void ru args.targetsPublishedNames, args.targetsDatasets, args.targetsTags, + args.targetsAnnotations, + args.excludedTargetsAnnotations, args.targetsStartIndex, args.targetsLength, args.libraryMIPsFilter); LOG.info("Read {} targets", targetMips.size()); @@ -368,6 +392,8 @@ private List readMIPs(CDMIPsReader mipsReader, List mipsPublishedNames, List mipsDatasets, List mipsTags, + List mipsAnnotations, + List excludedMipsAnnotations, long startIndexArg, int length, Set filter) { long startIndex = startIndexArg > 0 ? startIndexArg : 0; @@ -379,6 +405,8 @@ private List readMIPs(CDMIPsReader mipsReader, .addNames(mipsPublishedNames) .addDatasets(mipsDatasets) .addTags(mipsTags) + .addAnnotations(mipsAnnotations) + .addExcludedAnnotations(excludedMipsAnnotations) .setOffset(libraryInput.offset) .setSize(libraryInput.length)).stream()) .filter(neuronMetadata -> CollectionUtils.isEmpty(filter) || diff --git a/colormipsearch-tools/src/main/java/org/janelia/colormipsearch/cmd/ExportData4NBCmd.java b/colormipsearch-tools/src/main/java/org/janelia/colormipsearch/cmd/ExportData4NBCmd.java index 6f87ba58..febdeb21 100644 --- a/colormipsearch-tools/src/main/java/org/janelia/colormipsearch/cmd/ExportData4NBCmd.java +++ b/colormipsearch-tools/src/main/java/org/janelia/colormipsearch/cmd/ExportData4NBCmd.java @@ -123,6 +123,12 @@ static class ExportMatchesCmdArgs extends AbstractCmdArgs { @Parameter(names = {"--excluded-neuron-tags"}, description = "Neuron tags to be excluded from export", variableArity = true) List excludedNeuronTags = new ArrayList<>(); + @Parameter(names = {"--neuron-terms"}, description = "Neuron annotations to be exported", variableArity = true) + List neuronAnnotations = new ArrayList<>(); + + @Parameter(names = {"--excluded-neuron-terms"}, description = "Neuron terms to be excluded from export", variableArity = true) + List excludedNeuronAnnotations = new ArrayList<>(); + @Parameter(names = {"--target-tags"}, description = "Target neuron tags to be exported", variableArity = true) List targetNeuronTags = new ArrayList<>(); @@ -131,6 +137,14 @@ static class ExportMatchesCmdArgs extends AbstractCmdArgs { variableArity = true) List excludedTargetNeuronTags = new ArrayList<>(); + @Parameter(names = {"--target-terms"}, description = "Target neuron annotations to be exported", + variableArity = true) + List targetNeuronAnnotations = new ArrayList<>(); + + @Parameter(names = {"--excluded-target-terms"}, description = "Target neuron annotations to be excluded from export", + variableArity = true) + List excludedTargetNeuronAnnotations = new ArrayList<>(); + @Parameter(names = {"--excluded-matches-tags"}, description = "Matches tags to be excluded from export", variableArity = true) List excludedMatchesTags = new ArrayList<>(); @@ -234,6 +248,8 @@ private DataExporter getDataExporter() { .addLibraries(args.libraries) .addTags(args.neuronTags) .addExcludedTags(args.excludedNeuronTags) + .addAnnotations(args.neuronAnnotations) + .addExcludedAnnotations(args.excludedNeuronAnnotations) .addNames(args.exportedNames) .setOffset(args.offset) .setSize(args.size); @@ -255,6 +271,8 @@ private DataExporter getDataExporter() { args.targetLibraries, args.targetNeuronTags, args.excludedTargetNeuronTags, + args.targetNeuronAnnotations, + args.excludedTargetNeuronAnnotations, args.excludedMatchesTags, getCDScoresFilter(), urlTransformer, @@ -275,7 +293,10 @@ private DataExporter getDataExporter() { dataHelper, dataSource, args.targetLibraries, + args.targetNeuronTags, args.excludedTargetNeuronTags, + args.targetNeuronAnnotations, + args.excludedTargetNeuronAnnotations, args.excludedMatchesTags, getCDScoresFilter(), urlTransformer, diff --git a/colormipsearch-tools/src/main/java/org/janelia/colormipsearch/cmd/ImportV2CDMatchesCmd.java b/colormipsearch-tools/src/main/java/org/janelia/colormipsearch/cmd/ImportV2CDMatchesCmd.java index 5429cb96..d454d69b 100644 --- a/colormipsearch-tools/src/main/java/org/janelia/colormipsearch/cmd/ImportV2CDMatchesCmd.java +++ b/colormipsearch-tools/src/main/java/org/janelia/colormipsearch/cmd/ImportV2CDMatchesCmd.java @@ -207,13 +207,13 @@ List> getCDMatchesForMask(NeuronMatchesReader> getCDMatchesForMask(NeuronMatchesReader targetLibraries; + final List targetTags; final List targetExcludedTags; + final List targetAnnotations; + final List targetExcludedAnnotations; final List matchesExcludedTags; final ScoresFilter scoresFilter; final NeuronMatchesReader> neuronMatchesReader; @@ -49,7 +52,10 @@ public abstract class AbstractCDMatchesExporter extends AbstractDataExporter { protected AbstractCDMatchesExporter(CachedDataHelper jacsDataHelper, DataSourceParam dataSourceParam, List targetLibraries, + List targetTags, List targetExcludedTags, + List targetAnnotations, + List targetExcludedAnnotations, List matchesExcludedTags, ScoresFilter scoresFilter, URLTransformer urlTransformer, @@ -62,8 +68,11 @@ protected AbstractCDMatchesExporter(CachedDataHelper jacsDataHelper, int processingPartitionSize) { super(jacsDataHelper, dataSourceParam, urlTransformer, imageStoreMapping, outputDir, executor); this.targetLibraries = targetLibraries; + this.targetTags = targetTags; this.targetExcludedTags = targetExcludedTags; this.matchesExcludedTags = matchesExcludedTags; + this.targetAnnotations = targetAnnotations; + this.targetExcludedAnnotations = targetExcludedAnnotations; this.scoresFilter = scoresFilter; this.neuronMatchesReader = neuronMatchesReader; this.neuronMetadataDao = neuronMetadataDao; diff --git a/colormipsearch-tools/src/main/java/org/janelia/colormipsearch/cmd/dataexport/EMCDMatchesExporter.java b/colormipsearch-tools/src/main/java/org/janelia/colormipsearch/cmd/dataexport/EMCDMatchesExporter.java index c0b10076..0579b235 100644 --- a/colormipsearch-tools/src/main/java/org/janelia/colormipsearch/cmd/dataexport/EMCDMatchesExporter.java +++ b/colormipsearch-tools/src/main/java/org/janelia/colormipsearch/cmd/dataexport/EMCDMatchesExporter.java @@ -37,15 +37,14 @@ public class EMCDMatchesExporter extends AbstractCDMatchesExporter { private static final Logger LOG = LoggerFactory.getLogger(EMCDMatchesExporter.class); - private final List targetLibraries; - private final List targetTags; - private final List targetExcludedTags; public EMCDMatchesExporter(CachedDataHelper jacsDataHelper, DataSourceParam dataSourceParam, List targetLibraries, List targetTags, List targetExcludedTags, + List targetAnnotations, + List targetExcludedAnnotations, List matchesExcludedTags, ScoresFilter scoresFilter, URLTransformer urlTransformer, @@ -59,7 +58,10 @@ public EMCDMatchesExporter(CachedDataHelper jacsDataHelper, super(jacsDataHelper, dataSourceParam, targetLibraries, + targetTags, targetExcludedTags, + targetAnnotations, + targetExcludedAnnotations, matchesExcludedTags, scoresFilter, urlTransformer, @@ -70,9 +72,6 @@ public EMCDMatchesExporter(CachedDataHelper jacsDataHelper, neuronMetadataDao, resultMatchesWriter, processingPartitionSize); - this.targetLibraries = targetLibraries; - this.targetTags = targetTags; - this.targetExcludedTags = targetExcludedTags; } @Override @@ -103,12 +102,16 @@ private void runExportForMaskIds(int jobId, List maskMipIds) { /* maskDatasets */dataSourceParam.getDatasets(), /* maskTags */dataSourceParam.getTags(), // use the tags for selecting the masks but not for selecting the matches /* maskExcludedTags */dataSourceParam.getExcludedTags(), + /* maskAnnotations */dataSourceParam.getAnnotations(), + /* maskExcludedAnnotations */dataSourceParam.getExcludedAnnotations(), /* targetLibraries */targetLibraries, /* targetPublishedNames */null, /* targetMIPIDs */null, /* targetDatasets */null, /* targetTags */targetTags, /* targetExcludedTags */targetExcludedTags, + targetAnnotations, + targetExcludedAnnotations, /* matchTags */null, /* matchExcludedTags */matchesExcludedTags, scoresFilter, diff --git a/colormipsearch-tools/src/main/java/org/janelia/colormipsearch/cmd/dataexport/EMPPPMatchesExporter.java b/colormipsearch-tools/src/main/java/org/janelia/colormipsearch/cmd/dataexport/EMPPPMatchesExporter.java index af6ba79d..4cd59080 100644 --- a/colormipsearch-tools/src/main/java/org/janelia/colormipsearch/cmd/dataexport/EMPPPMatchesExporter.java +++ b/colormipsearch-tools/src/main/java/org/janelia/colormipsearch/cmd/dataexport/EMPPPMatchesExporter.java @@ -103,12 +103,16 @@ private void runExportForMaskIds(int jobId, List maskIds) { /* maskDatasets */null, /* maskTags */null, /* maskExcludedTags */null, + dataSourceParam.getAnnotations(), + dataSourceParam.getExcludedAnnotations(), /* targetLibraries */null, /* targetPublishedNames */null, /* targetMIPIDs */null, /* targetDatasets */null, /* targetTags */null, /* targetExcludedTags */null, + /* targetAnnotations */null, + /* targetExcludedAnnotations */null, /* matchTags */null, // use the tags for selecting the masks but not for selecting the matches /* matchExcludedTags */null, scoresFilter, diff --git a/colormipsearch-tools/src/main/java/org/janelia/colormipsearch/cmd/dataexport/LMCDMatchesExporter.java b/colormipsearch-tools/src/main/java/org/janelia/colormipsearch/cmd/dataexport/LMCDMatchesExporter.java index 940688c2..2da1ca66 100644 --- a/colormipsearch-tools/src/main/java/org/janelia/colormipsearch/cmd/dataexport/LMCDMatchesExporter.java +++ b/colormipsearch-tools/src/main/java/org/janelia/colormipsearch/cmd/dataexport/LMCDMatchesExporter.java @@ -41,7 +41,10 @@ public class LMCDMatchesExporter extends AbstractCDMatchesExporter { public LMCDMatchesExporter(CachedDataHelper jacsDataHelper, DataSourceParam dataSourceParam, List maskLibraries, + List maskTags, List maskExcludedTags, + List maskAnnotations, + List maskExcludedAnnotations, List matchesExcludedTags, ScoresFilter scoresFilter, URLTransformer urlTransformer, @@ -54,8 +57,13 @@ public LMCDMatchesExporter(CachedDataHelper jacsDataHelper, int processingPartitionSize) { super(jacsDataHelper, dataSourceParam, - maskLibraries, maskExcludedTags, - matchesExcludedTags, scoresFilter, + maskLibraries, + maskTags, + maskExcludedTags, + maskAnnotations, + maskExcludedAnnotations, + matchesExcludedTags, + scoresFilter, urlTransformer, imageStoreMapping, outputDir, @@ -95,16 +103,19 @@ private void runExportForTargetIds(int jobId, List targetMipIds) { /* maskDatasets */null, /* maskTags */null, /* maskExcludedTags */targetExcludedTags, + targetAnnotations, + targetExcludedAnnotations, /* targetLibraries */null, /* targetPublishedNames */null, - /* targetMIPIds */Collections.singletonList(targetMipId), - /* targetDatasets */dataSourceParam.getDatasets(), - /* targetTags */dataSourceParam.getTags(), - /* targetExcludedTags */dataSourceParam.getExcludedTags(), // use the tags for selecting the targets but not for selecting the matches - /* matchTags */null, + /* targetMIPIds */null, + /* targetDatasets */Collections.singletonList(targetMipId), + /* targetTags */dataSourceParam.getDatasets(), + /* targetExcludedTags */dataSourceParam.getTags(), // use the tags for selecting the targets but not for selecting the matches + /* targetAnnotations */dataSourceParam.getAnnotations(), + /* targtExcludedAnnotations */dataSourceParam.getExcludedAnnotations(), + /* matchTags */dataSourceParam.getExcludedTags(), /* matchExcludedTags */matchesExcludedTags, - scoresFilter, - null/* no sorting yet because it uses too much memory on the server */); + scoresFilter, /* no sorting yet because it uses too much memory on the server */null); LOG.info("Found {} color depth matches for mip {}", allMatchesForTarget.size(), targetMipId); List> selectedMatchesForTarget; if (allMatchesForTarget.isEmpty()) { diff --git a/colormipsearch-tools/src/main/java/org/janelia/colormipsearch/cmd/v2dataimport/JSONV2Em2LmMatchesReader.java b/colormipsearch-tools/src/main/java/org/janelia/colormipsearch/cmd/v2dataimport/JSONV2Em2LmMatchesReader.java index 90939b86..3858a88c 100644 --- a/colormipsearch-tools/src/main/java/org/janelia/colormipsearch/cmd/v2dataimport/JSONV2Em2LmMatchesReader.java +++ b/colormipsearch-tools/src/main/java/org/janelia/colormipsearch/cmd/v2dataimport/JSONV2Em2LmMatchesReader.java @@ -103,12 +103,16 @@ public List> readMatchesByMask(Str Collection maskDatasets, Collection maskTags, Collection maskExcludedTags, + Collection maskAnnotations, + Collection excludedMaskAnnotations, Collection targetLibraries, Collection targetPublishedNames, Collection targetMipIds, Collection targetDatasets, Collection targetTags, Collection targetExcludedTags, + Collection targetAnnotations, + Collection excludedTargetAnnotations, Collection matchTags, Collection matchExcludedTags, ScoresFilter matchScoresFilter, @@ -130,13 +134,13 @@ public List> readMatchesByTarget(S Collection maskDatasets, Collection maskTags, Collection maskExcludedTags, - Collection targetLibraries, + Collection maskAnnotations, Collection excludedMaskAnnotations, Collection targetLibraries, Collection targetPublishedNames, Collection targetMipIds, Collection targetDatasets, Collection targetTags, Collection targetExcludedTags, - Collection matchTags, + Collection targetAnnotations, Collection excludedTargetAnnotations, Collection matchTags, Collection matchExcludedTags, ScoresFilter matchScoresFilter, List sortCriteriaList) {