add neuron annotations and options to filter by neuron annotations

JaneliaSciComp · May 9, 2024 · 626f706 · 626f706
1 parent ae6eb7d
commit 626f706
Show file tree

Hide file tree

Showing 24 changed files with 273 additions and 55 deletions.
diff --git a/colormipsearch-api/src/main/java/org/janelia/colormipsearch/dto/AbstractNeuronMetadata.java b/colormipsearch-api/src/main/java/org/janelia/colormipsearch/dto/AbstractNeuronMetadata.java
@@ -1,6 +1,7 @@
 package org.janelia.colormipsearch.dto;
 
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.function.BiFunction;
@@ -47,6 +48,7 @@ public abstract class AbstractNeuronMetadata {
     private String alignmentSpace;
     private String anatomicalArea;
     private Gender gender;
+    private List<String> terms;
     private boolean unpublished;
     // neuronFiles holds S3 files used by the NeuronBridge app
     private final Map<FileType, String> neuronFiles = new HashMap<>();
@@ -154,6 +156,14 @@ public void setGender(Gender gender) {
         this.gender = gender;
     }
 
+    public List<String> getTerms() {
+        return terms;
+    }
+
+    public void setTerms(List<String> terms) {
+        this.terms = terms;
+    }
+
     @JsonIgnore
     public boolean isUnpublished() {
         return unpublished;

diff --git a/colormipsearch-api/src/main/java/org/janelia/colormipsearch/dto/EMNeuronMetadata.java b/colormipsearch-api/src/main/java/org/janelia/colormipsearch/dto/EMNeuronMetadata.java
@@ -10,7 +10,6 @@ public class EMNeuronMetadata extends AbstractNeuronMetadata {
     private String neuronType;
     private String neuronInstance;
     private String state;
-    private List<String> neuronTerms;
 
     @Override
     public String getTypeDiscriminator() {
@@ -49,12 +48,4 @@ public String getState() {
     public void setState(String state) {
         this.state = state;
     }
-
-    public List<String> getNeuronTerms() {
-        return neuronTerms;
-    }
-
-    public void setNeuronTerms(List<String> neuronTerms) {
-        this.neuronTerms = neuronTerms;
-    }
 }
diff --git a/colormipsearch-api/src/main/java/org/janelia/colormipsearch/model/AbstractNeuronEntity.java b/colormipsearch-api/src/main/java/org/janelia/colormipsearch/model/AbstractNeuronEntity.java
@@ -40,6 +40,7 @@ public abstract class AbstractNeuronEntity extends AbstractBaseEntity {
     // This will be used to identify the matched neurons for PPP since the color depth MIPs are not
     // part of the PPP match process at all.
     private String sourceRefId;
+    private List<String> neuronTerms;
     // computeFileData holds local files used either for precompute or upload
     private final Map<ComputeFileType, FileData> computeFiles = new HashMap<>();
     // processed tags holds the corresponding processing tag used for the ColorDepthSearch or PPPM import
@@ -123,6 +124,14 @@ public String getSourceRefIdOnly() {
         }
     }
 
+    public List<String> getNeuronTerms() {
+        return neuronTerms;
+    }
+
+    public void setNeuronTerms(List<String> neuronTerms) {
+        this.neuronTerms = neuronTerms;
+    }
+
     @JsonProperty
     public Map<ComputeFileType, FileData> getComputeFiles() {
         return computeFiles;
@@ -246,6 +255,7 @@ public List<EntityField<?>> updateableFieldValues() {
         fieldList.add(new EntityField<>("libraryName", false, libraryName));
         fieldList.add(new EntityField<>("publishedName", false, publishedName));
         fieldList.add(new EntityField<>("sourceRefId", false, sourceRefId));
+        fieldList.add(new EntityField<>("neuronTerms", false, neuronTerms));
         fieldList.add(new EntityField<>("updatedDate", false, getUpdatedDate()));
         // datasetLabels is a collection but will always be replaced instead of appended to existing values
         fieldList.add(new EntityField<>("datasetLabels", false, datasetLabels));
@@ -309,6 +319,7 @@ protected <N extends AbstractNeuronEntity> void copyFrom(N that) {
         this.libraryName = that.getLibraryName();
         this.publishedName = that.getPublishedName();
         this.sourceRefId = that.getSourceRefId();
+        this.neuronTerms = that.getNeuronTerms();
         this.computeFiles.clear();
         this.computeFiles.putAll(that.getComputeFiles());
         this.addAllTags(that.getTags());

diff --git a/colormipsearch-api/src/main/java/org/janelia/colormipsearch/model/EMNeuronEntity.java b/colormipsearch-api/src/main/java/org/janelia/colormipsearch/model/EMNeuronEntity.java
@@ -12,7 +12,6 @@ public class EMNeuronEntity extends AbstractNeuronEntity {
     // neuronType and the neuronInstance are only for reference purposes here
     private String neuronType;
     private String neuronInstance;
-    private List<String> neuronTerms;
 
     @Override
     public String getNeuronId() {
@@ -35,20 +34,11 @@ public void setNeuronInstance(String neuronInstance) {
         this.neuronInstance = neuronInstance;
     }
 
-    public List<String> getNeuronTerms() {
-        return neuronTerms;
-    }
-
-    public void setNeuronTerms(List<String> neuronTerms) {
-        this.neuronTerms = neuronTerms;
-    }
-
     @Override
     public List<EntityField<?>> updateableFieldValues() {
         List<EntityField<?>> fieldList = new ArrayList<>(super.updateableFieldValues());
         fieldList.add(new EntityField<>("neuronType", false, neuronType));
         fieldList.add(new EntityField<>("neuronInstance", false, neuronInstance));
-        fieldList.add(new EntityField<>("neuronTerms", false, neuronTerms));
         return fieldList;
     }
 
@@ -58,7 +48,6 @@ public EMNeuronEntity duplicate() {
         n.copyFrom(this);
         n.neuronType = this.getNeuronType();
         n.neuronInstance = this.getNeuronInstance();
-        n.neuronTerms = this.getNeuronTerms();
         return n;
     }
 
@@ -73,7 +62,7 @@ public EMNeuronMetadata metadata() {
         n.setPublishedName(getPublishedName());
         n.setNeuronType(getNeuronType());
         n.setNeuronInstance(getNeuronInstance());
-        n.setNeuronTerms(getNeuronTerms());
+        n.setTerms(getNeuronTerms());
         getComputeFiles().forEach((ft, fd) -> n.setNeuronComputeFile(ft, fd.getFileName()));
         getProcessedTags().forEach(n::putProcessedTags);
         return n;

diff --git a/colormipsearch-api/src/main/java/org/janelia/colormipsearch/model/LMNeuronEntity.java b/colormipsearch-api/src/main/java/org/janelia/colormipsearch/model/LMNeuronEntity.java
@@ -137,6 +137,7 @@ public LMNeuronMetadata metadata() {
         n.setAnatomicalArea(anatomicalArea);
         n.setGender(gender);
         n.setObjective(objective);
+        n.setTerms(getNeuronTerms());
         getComputeFiles().forEach((ft, fd) -> n.setNeuronComputeFile(ft, fd.getFileName()));
         getProcessedTags().forEach(n::putProcessedTags);
         return n;

diff --git a/colormipsearch-persist/src/main/java/org/janelia/colormipsearch/dao/NeuronSelector.java b/colormipsearch-persist/src/main/java/org/janelia/colormipsearch/dao/NeuronSelector.java
@@ -24,6 +24,8 @@ public class NeuronSelector {
     private final Set<Number> entityIds = new HashSet<>(); // matching internal entity IDs
     private final Set<String> tags = new HashSet<>(); // matching tags
     private final Set<String> excludedTags = new HashSet<>();
+    private final Set<String> annotations = new HashSet<>(); // matching annotations
+    private final Set<String> excludedAnnotations = new HashSet<>();
     private final List<Map<String, Set<String>>> processedTagsSelections = new ArrayList<>();
     public String getNeuronClassname() {
         return neuronClassname;
@@ -204,6 +206,42 @@ public boolean hasExcludedTags() {
         return CollectionUtils.isNotEmpty(excludedTags);
     }
 
+    public Set<String> getAnnotations() {
+        return annotations;
+    }
+
+    public NeuronSelector addAnnotation(String annotation) {
+        if (StringUtils.isNotBlank(annotation)) this.annotations.add(annotation);
+        return this;
+    }
+
+    public NeuronSelector addAnnotations(Collection<String> annotations) {
+        if (annotations != null) annotations.forEach(this::addAnnotation);
+        return this;
+    }
+
+    public boolean hasAnnotations() {
+        return CollectionUtils.isNotEmpty(annotations);
+    }
+
+    public Set<String> getExcludedAnnotations() {
+        return excludedAnnotations;
+    }
+
+    public NeuronSelector addExcludedAnnotation(String annotation) {
+        if (StringUtils.isNotBlank(annotation)) this.excludedAnnotations.add(annotation);
+        return this;
+    }
+
+    public NeuronSelector addExcludedAnnotations(Collection<String> annotations) {
+        if (annotations != null) annotations.forEach(this::addExcludedAnnotation);
+        return this;
+    }
+
+    public boolean hasExcludedAnnotations() {
+        return CollectionUtils.isNotEmpty(excludedAnnotations);
+    }
+
     public List<Map<String, Set<String>>> getProcessedTagsSelections() {
         return processedTagsSelections;
     }
@@ -275,7 +313,10 @@ public boolean isEmpty() {
                 && !hasExcludedTags()
                 && !hasDatasetLabels()
                 && !hasSourceRefIds()
-                && !hasProcessedTags();
+                && !hasProcessedTags()
+                && !hasAnnotations()
+                && !hasExcludedAnnotations()
+                ;
     }
 
     public boolean isNotEmpty() {

diff --git a/...rch-persist/src/main/java/org/janelia/colormipsearch/dao/mongo/NeuronSelectionHelper.java b/...rch-persist/src/main/java/org/janelia/colormipsearch/dao/mongo/NeuronSelectionHelper.java
@@ -64,6 +64,12 @@ static Bson getNeuronFilter(String fieldQualifier, NeuronSelector neuronSelector
         if (neuronSelector.hasExcludedTags()) {
             filter.add(Filters.nin(qualifier + "tags", neuronSelector.getExcludedTags()));
         }
+        if (neuronSelector.hasAnnotations()) {
+            filter.add(Filters.in(qualifier + "neuronTerms", neuronSelector.getAnnotations()));
+        }
+        if (neuronSelector.hasExcludedAnnotations()) {
+            filter.add(Filters.nin(qualifier + "neuronTerms", neuronSelector.getExcludedAnnotations()));
+        }
         if (neuronSelector.hasProcessedTags()) {
             // all filters from a selection are "and"-ed
             // and all selections are "or"-ed

diff --git a/colormipsearch-persist/src/main/java/org/janelia/colormipsearch/dataio/DataSourceParam.java b/colormipsearch-persist/src/main/java/org/janelia/colormipsearch/dataio/DataSourceParam.java
@@ -17,8 +17,10 @@ public class DataSourceParam {
     private Collection<String> mipIDs = new HashSet<>();
     private Collection<String> names = new HashSet<>();
     private Collection<String> tags = new HashSet<>();
-    private Collection<String> datasets = new HashSet<>();
     private Collection<String> excludedTags = new HashSet<>();
+    private Collection<String> datasets = new HashSet<>();
+    private Collection<String> annotations = new HashSet<>();
+    private Collection<String> excludedAnnotations = new HashSet<>();
     private long offset;
     private int size;
 
@@ -82,22 +84,39 @@ public DataSourceParam addTags(Collection<String> tags) {
         return this;
     }
 
+    public Collection<String> getExcludedTags() {
+        return excludedTags;
+    }
+
+    public DataSourceParam addExcludedTags(Collection<String> excludedTags) {
+        if (excludedTags != null) excludedTags.stream().filter(StringUtils::isNotBlank).forEach(t -> this.excludedTags.add(t));
+        return this;
+    }
+
     public Collection<String> getDatasets() {
         return datasets;
     }
 
     public DataSourceParam addDatasets(Collection<String> datasets) {
         if (datasets != null) datasets.stream().filter(StringUtils::isNotBlank).forEach(ds -> this.datasets.add(ds));
         return this;
+    }
 
+    public Collection<String> getAnnotations() {
+        return annotations;
     }
 
-    public Collection<String> getExcludedTags() {
-        return excludedTags;
+    public DataSourceParam addAnnotations(Collection<String> annotations) {
+        if (annotations != null) annotations.stream().filter(StringUtils::isNotBlank).forEach(a -> this.annotations.add(a));
+        return this;
     }
 
-    public DataSourceParam addExcludedTags(Collection<String> excludedTags) {
-        if (excludedTags != null) excludedTags.stream().filter(StringUtils::isNotBlank).forEach(t -> this.excludedTags.add(t));
+    public Collection<String> getExcludedAnnotations() {
+        return excludedAnnotations;
+    }
+
+    public DataSourceParam addExcludedAnnotations(Collection<String> annotations) {
+        if (annotations != null) annotations.stream().filter(StringUtils::isNotBlank).forEach(a -> this.excludedAnnotations.add(a));
         return this;
     }
 

diff --git a/...ipsearch-persist/src/main/java/org/janelia/colormipsearch/dataio/NeuronMatchesReader.java b/...ipsearch-persist/src/main/java/org/janelia/colormipsearch/dataio/NeuronMatchesReader.java
@@ -30,12 +30,16 @@ public interface NeuronMatchesReader<R extends AbstractMatchEntity<? extends Abs
      * @param maskDatasets
      * @param maskTags
      * @param maskExcludedTags
+     * @param maskAnnotations
+     * @param excludedMaskAnnotations
      * @param targetLibraries
      * @param targetPublishedNames
      * @param targetMipIds
      * @param targetDatasets
      * @param targetTags
      * @param targetExcludedTags
+     * @param targetAnnotations
+     * @param excludedTargetAnnotations
      * @param matchTags
      * @param matchExcludedTags
      * @param matchScoresFilter
@@ -49,12 +53,16 @@ List<R> readMatchesByMask(String alignmentSpace,
                               Collection<String> maskDatasets,
                               Collection<String> maskTags,
                               Collection<String> maskExcludedTags,
+                              Collection<String> maskAnnotations,
+                              Collection<String> excludedMaskAnnotations,
                               Collection<String> targetLibraries,
                               Collection<String> targetPublishedNames,
                               Collection<String> targetMipIds,
                               Collection<String> targetDatasets,
                               Collection<String> targetTags,
                               Collection<String> targetExcludedTags,
+                              Collection<String> targetAnnotations,
+                              Collection<String> excludedTargetAnnotations,
                               Collection<String> matchTags,
                               Collection<String> matchExcludedTags,
                               ScoresFilter matchScoresFilter,
@@ -70,12 +78,16 @@ List<R> readMatchesByMask(String alignmentSpace,
      * @param maskDatasets
      * @param maskTags
      * @param maskExcludedTags
+     * @param maskAnnotations
+     * @param excludedMaskAnnotations
      * @param targetLibraries
      * @param targetPublishedNames
      * @param targetMipIds
      * @param targetDatasets
      * @param targetTags
      * @param targetExcludedTags
+     * @param targetAnnotations
+     * @param excludedTargetAnnotations
      * @param matchTags
      * @param matchExcludedTags
      * @param matchScoresFilter
@@ -89,12 +101,16 @@ List<R> readMatchesByTarget(String alignmentSpace,
                                 Collection<String> maskDatasets,
                                 Collection<String> maskTags,
                                 Collection<String> maskExcludedTags,
+                                Collection<String> maskAnnotations,
+                                Collection<String> excludedMaskAnnotations,
                                 Collection<String> targetLibraries,
                                 Collection<String> targetPublishedNames,
                                 Collection<String> targetMipIds,
                                 Collection<String> targetDatasets,
                                 Collection<String> targetTags,
                                 Collection<String> targetExcludedTags,
+                                Collection<String> targetAnnotations,
+                                Collection<String> excludedTargetAnnotations,
                                 Collection<String> matchTags,
                                 Collection<String> matchExcludedTags,
                                 ScoresFilter matchScoresFilter,

diff --git a/...rmipsearch-persist/src/main/java/org/janelia/colormipsearch/dataio/db/DBCDMIPsReader.java b/...rmipsearch-persist/src/main/java/org/janelia/colormipsearch/dataio/db/DBCDMIPsReader.java
@@ -27,7 +27,9 @@ public List<? extends AbstractNeuronEntity> readMIPs(DataSourceParam mipsDataSou
                         .addMipIDs(mipsDataSource.getMipIDs())
                         .addDatasetLabels(mipsDataSource.getDatasets())
                         .addTags(mipsDataSource.getTags())
-                        .addExcludedTags(mipsDataSource.getExcludedTags()),
+                        .addExcludedTags(mipsDataSource.getExcludedTags())
+                        .addAnnotations(mipsDataSource.getAnnotations())
+                        .addExcludedAnnotations(mipsDataSource.getExcludedAnnotations()),
                 new PagedRequest().setPageSize(mipsDataSource.getSize()).setFirstPageOffset(mipsDataSource.getOffset())
         ).getResultList();
     }