use the exact same queries which have been used for the clinical data…

… counts endpoint
onursumer · Jul 2, 2024 · 19603fc · 19603fc
1 parent d7b6f15
commit 19603fc
Show file tree

Hide file tree

Showing 9 changed files with 50 additions and 103 deletions.
diff --git a/src/main/java/org/cbioportal/persistence/StudyViewRepository.java b/src/main/java/org/cbioportal/persistence/StudyViewRepository.java
@@ -8,6 +8,7 @@
 import org.cbioportal.model.CopyNumberCountByGene;
 import org.cbioportal.model.GenomicDataCount;
 import org.cbioportal.model.Sample;
+import org.cbioportal.web.parameter.ClinicalDataType;
 import org.cbioportal.web.parameter.StudyViewFilter;
 
 import java.util.List;
@@ -21,10 +22,6 @@ public interface StudyViewRepository {
 
     List<ClinicalData> getPatientClinicalData(StudyViewFilter studyViewFilter, List<String> attributeIds);
 
-    List<ClinicalDataCount>getSampleClinicalDataCountsForBinning(StudyViewFilter studyViewFilter, List<String> attributeIds);
-
-    List<ClinicalDataCount>getPatientClinicalDataCountsForBinning(StudyViewFilter studyViewFilter, List<String> attributeIds);
-
     List<AlterationCountByGene> getMutatedGenes(StudyViewFilter studyViewFilter);
 
     List<AlterationCountByGene> getStructuralVariantGenes(StudyViewFilter studyViewFilter);
@@ -36,6 +33,8 @@ public interface StudyViewRepository {
 
     List<ClinicalAttribute> getClinicalAttributes();
 
+    Map<String, ClinicalDataType> getClinicalAttributeDatatypeMap();
+
     List<CaseListDataCount> getCaseListDataCounts(StudyViewFilter studyViewFilter);
 
     Map<String, AlterationCountByGene> getTotalProfiledCounts(StudyViewFilter studyViewFilter, String alterationType);

diff --git a/src/main/java/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.java b/src/main/java/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.java
@@ -32,12 +32,6 @@ List<CopyNumberCountByGene> getCnaGenes(StudyViewFilter studyViewFilter, Categor
     List<AlterationCountByGene> getStructuralVariantGenes(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter,
                                                 boolean applyPatientIdFilters, AlterationFilterHelper alterationFilterHelper);
 
-    List<ClinicalDataCount> getPatientClinicalDataCounts(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter,
-                                                         boolean applyPatientIdFilters,  List<String> attributeIds, List<String> filteredAttributeValues);
-
-    List<ClinicalDataCount> getSampleClinicalDataCounts(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter,
-                                                        boolean applyPatientIdFilters, List<String> attributeIds, List<String> filteredAttributeValues );
-
     List<ClinicalDataCount> getClinicalDataCounts(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter,
                                                   boolean applyPatientIdFilters, List<String> attributeIds, List<String> filteredAttributeValues);
 
@@ -49,10 +43,6 @@ List<ClinicalDataCount> getClinicalDataCounts(StudyViewFilter studyViewFilter, C
 
     List<ClinicalData> getPatientClinicalDataFromStudyViewFilter(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter, boolean applyPatientIdFilters, List<String> attributeIds);
 
-    List<ClinicalDataCount> getSampleClinicalDataCountsForBinning(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter, boolean applyPatientIdFilters, List<String> attributeIds);
-
-    List<ClinicalDataCount> getPatientClinicalDataCountsForBinning(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter, boolean applyPatientIdFilters, List<String> attributeIds);
-
     @MapKey("hugoGeneSymbol")
     Map<String, AlterationCountByGene> getTotalProfiledCounts(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter, boolean applyPatientIdFilters, String alterationType);
 

diff --git a/src/main/java/org/cbioportal/persistence/mybatisclickhouse/StudyViewMyBatisRepository.java b/src/main/java/org/cbioportal/persistence/mybatisclickhouse/StudyViewMyBatisRepository.java
@@ -12,6 +12,7 @@
 import org.cbioportal.persistence.enums.ClinicalAttributeDataSource;
 import org.cbioportal.persistence.helper.AlterationFilterHelper;
 import org.cbioportal.web.parameter.CategorizedClinicalDataCountFilter;
+import org.cbioportal.web.parameter.ClinicalDataType;
 import org.cbioportal.web.parameter.StudyViewFilter;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Repository;
@@ -83,6 +84,25 @@ public List<ClinicalAttribute> getClinicalAttributes() {
         return mapper.getClinicalAttributes();
     }
 
+    @Override
+    public Map<String, ClinicalDataType> getClinicalAttributeDatatypeMap() {
+        if (clinicalAttributesMap.isEmpty()) {
+            buildClinicalAttributeNameMap();
+        }
+
+        Map<String, ClinicalDataType> attributeDatatypeMap = new HashMap<>();
+
+        clinicalAttributesMap
+            .get(ClinicalAttributeDataSource.SAMPLE)
+            .forEach(attribute -> attributeDatatypeMap.put(attribute.getAttrId(), ClinicalDataType.SAMPLE));
+
+        clinicalAttributesMap
+            .get(ClinicalAttributeDataSource.PATIENT)
+            .forEach(attribute -> attributeDatatypeMap.put(attribute.getAttrId(), ClinicalDataType.PATIENT));
+
+        return attributeDatatypeMap;
+    }
+
     @Override
     public List<CaseListDataCount> getCaseListDataCounts(StudyViewFilter studyViewFilter) {
         CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter = extractClinicalDataCountFilters(studyViewFilter);
@@ -100,16 +120,6 @@ public List<ClinicalData> getSampleClinicalData(StudyViewFilter studyViewFilter,
         CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter = extractClinicalDataCountFilters(studyViewFilter);
         return mapper.getSampleClinicalDataFromStudyViewFilter(studyViewFilter, categorizedClinicalDataCountFilter, shouldApplyPatientIdFilters(categorizedClinicalDataCountFilter), attributeIds);
     }
-
-    public List<ClinicalDataCount> getSampleClinicalDataCountsForBinning(StudyViewFilter studyViewFilter, List<String> attributeIds) {
-        CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter = extractClinicalDataCountFilters(studyViewFilter);
-        return mapper.getSampleClinicalDataCountsForBinning(studyViewFilter, categorizedClinicalDataCountFilter, shouldApplyPatientIdFilters(categorizedClinicalDataCountFilter), attributeIds);
-    }
-
-    public List<ClinicalDataCount> getPatientClinicalDataCountsForBinning(StudyViewFilter studyViewFilter, List<String> attributeIds) {
-        CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter = extractClinicalDataCountFilters(studyViewFilter);
-        return mapper.getPatientClinicalDataCountsForBinning(studyViewFilter, categorizedClinicalDataCountFilter, shouldApplyPatientIdFilters(categorizedClinicalDataCountFilter), attributeIds);
-    }
 
     @Override
     public List<ClinicalData> getPatientClinicalData(StudyViewFilter studyViewFilter, List<String> attributeIds) {

diff --git a/src/main/java/org/cbioportal/service/StudyViewColumnarService.java b/src/main/java/org/cbioportal/service/StudyViewColumnarService.java
@@ -3,14 +3,15 @@
 import org.cbioportal.model.AlterationCountByGene;
 import org.cbioportal.model.CaseListDataCount;
 import org.cbioportal.model.ClinicalData;
-import org.cbioportal.model.ClinicalDataCount;
 import org.cbioportal.model.ClinicalDataCountItem;
 import org.cbioportal.model.GenomicDataCount;
 import org.cbioportal.model.CopyNumberCountByGene;
 import org.cbioportal.model.Sample;
+import org.cbioportal.web.parameter.ClinicalDataType;
 import org.cbioportal.web.parameter.StudyViewFilter;
 
 import java.util.List;
+import java.util.Map;
 
 public interface StudyViewColumnarService {
 
@@ -20,6 +21,8 @@ public interface StudyViewColumnarService {
     List<CopyNumberCountByGene> getCnaGenes(StudyViewFilter interceptedStudyViewFilter);
     List<AlterationCountByGene> getStructuralVariantGenes(StudyViewFilter studyViewFilter);
 
+    Map<String, ClinicalDataType> getClinicalAttributeDatatypeMap();
+
     List<ClinicalDataCountItem> getClinicalDataCounts(StudyViewFilter studyViewFilter, List<String> filteredAttributes);
 
     List<CaseListDataCount> getCaseListDataCounts(StudyViewFilter studyViewFilter);
@@ -28,10 +31,6 @@ public interface StudyViewColumnarService {
 
     List<ClinicalData> getSampleClinicalData(StudyViewFilter studyViewFilter, List<String> attributeIds);
 
-    List<ClinicalDataCount> getSampleClinicalDataCountsForBinning(StudyViewFilter studyViewFilter, List<String> attributeIds);
-
-    List<ClinicalDataCount> getPatientClinicalDataCountsForBinning(StudyViewFilter studyViewFilter, List<String> attributeIds);
-
     List<GenomicDataCount> getGenomicDataCounts(StudyViewFilter studyViewFilter);
 
 }
diff --git a/src/main/java/org/cbioportal/service/impl/StudyViewColumnarServiceImpl.java b/src/main/java/org/cbioportal/service/impl/StudyViewColumnarServiceImpl.java
@@ -11,12 +11,14 @@
 import org.cbioportal.persistence.StudyViewRepository;
 import org.cbioportal.service.AlterationCountService;
 import org.cbioportal.service.StudyViewColumnarService;
+import org.cbioportal.web.parameter.ClinicalDataType;
 import org.cbioportal.web.parameter.StudyViewFilter;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.cache.annotation.Cacheable;
 import org.springframework.stereotype.Service;
 
 import java.util.List;
+import java.util.Map;
 import java.util.stream.Collectors;
 
 @Service
@@ -58,6 +60,11 @@ public List<AlterationCountByGene> getStructuralVariantGenes(StudyViewFilter stu
         return alterationCountService.getStructuralVariantGenes(studyViewFilter);
     }
 
+    @Override
+    public Map<String, ClinicalDataType> getClinicalAttributeDatatypeMap() {
+        return studyViewRepository.getClinicalAttributeDatatypeMap();
+    }
+
     @Override
     public List<ClinicalDataCountItem> getClinicalDataCounts(StudyViewFilter studyViewFilter, List<String> filteredAttributes) {
         return studyViewRepository.getClinicalDataCounts(studyViewFilter, filteredAttributes)
@@ -85,15 +92,4 @@ public List<ClinicalData> getPatientClinicalData(StudyViewFilter studyViewFilter
     public List<ClinicalData> getSampleClinicalData(StudyViewFilter studyViewFilter, List<String> attributeIds) {
         return studyViewRepository.getSampleClinicalData(studyViewFilter, attributeIds);
     }
-
-    @Override
-    public List<ClinicalDataCount> getSampleClinicalDataCountsForBinning(StudyViewFilter studyViewFilter, List<String> attributeIds) {
-        return studyViewRepository.getSampleClinicalDataCountsForBinning(studyViewFilter, attributeIds);
-    }
-
-    @Override
-    public List<ClinicalDataCount> getPatientClinicalDataCountsForBinning(StudyViewFilter studyViewFilter, List<String> attributeIds) {
-        return studyViewRepository.getPatientClinicalDataCountsForBinning(studyViewFilter, attributeIds);
-    }
-
 }
diff --git a/src/main/java/org/cbioportal/web/columnar/ClinicalDataBinner.java b/src/main/java/org/cbioportal/web/columnar/ClinicalDataBinner.java
@@ -11,7 +11,6 @@
 
 import java.util.*;
 import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 @Component
 public class ClinicalDataBinner {
@@ -63,36 +62,19 @@ public List<ClinicalDataBin> fetchClinicalDataBinCounts(
         partialFilter.setSampleIdentifiers(studyViewFilter.getSampleIdentifiers());
 
         // we need the clinical data for the partial filter in order to generate the bins for initial state
-        // we use the filtered data to calculate the counts for each bin, we do not regenerate bins for the filtered data 
-
-        // TODO investigate if we can directly use studyViewColumnarService.getClinicalDataCounts instead of adding new SQL
-        List<ClinicalDataCount> unfilteredClinicalDataCountsForSamples = studyViewColumnarService.getSampleClinicalDataCountsForBinning(partialFilter, attributeIds);
-        List<ClinicalDataCount> filteredClinicalDataCountsForSamples = studyViewColumnarService.getSampleClinicalDataCountsForBinning(studyViewFilter, attributeIds);
-        List<ClinicalDataCount> unfilteredClinicalDataCountsForPatients = studyViewColumnarService.getPatientClinicalDataCountsForBinning(partialFilter, attributeIds);
-        List<ClinicalDataCount> filteredClinicalDataCountsForPatients = studyViewColumnarService.getPatientClinicalDataCountsForBinning(studyViewFilter, attributeIds);
+        // we use the filtered data to calculate the counts for each bin, we do not regenerate bins for the filtered data
+        List<ClinicalDataCountItem> unfilteredClinicalDataCounts = studyViewColumnarService.getClinicalDataCounts(partialFilter, attributeIds);
+        List<ClinicalDataCountItem> filteredClinicalDataCounts = studyViewColumnarService.getClinicalDataCounts(studyViewFilter, attributeIds);
 
-        List<ClinicalData> unfilteredClinicalDataForSamples = convertCountsToData(unfilteredClinicalDataCountsForSamples);
-        List<ClinicalData> filteredClinicalDataForSamples = convertCountsToData(filteredClinicalDataCountsForSamples);
-        List<ClinicalData> unfilteredClinicalDataForPatients = convertCountsToData(unfilteredClinicalDataCountsForPatients);
-        List<ClinicalData> filteredClinicalDataForPatients = convertCountsToData(filteredClinicalDataCountsForPatients);
-
-        Map<String, ClinicalDataType> attributeDatatypeMap = NewClinicalDataBinUtil.toAttributeDatatypeMap(
-            unfilteredClinicalDataCountsForSamples.stream().map(ClinicalDataCount::getAttributeId).collect(Collectors.toList()),
-            unfilteredClinicalDataCountsForPatients.stream().map(ClinicalDataCount::getAttributeId).collect(Collectors.toList()),
-            Collections.emptyList() // TODO ignoring conflictingPatientAttributeIds for now
+        // TODO ignoring conflictingPatientAttributeIds for now
+        List<ClinicalData> unfilteredClinicalData = convertCountsToData(
+            unfilteredClinicalDataCounts.stream().flatMap(c -> c.getCounts().stream()).toList()
+        );
+        List<ClinicalData> filteredClinicalData = convertCountsToData(
+            filteredClinicalDataCounts.stream().flatMap(c -> c.getCounts().stream()).toList()
         );
 
-        List<Binnable> unfilteredClinicalData = Stream.of(
-            unfilteredClinicalDataForSamples,
-            unfilteredClinicalDataForPatients
-            // unfilteredClinicalDataForConflictingPatientAttributes /// TODO ignoring conflictingPatientAttributeIds for now
-        ).flatMap(Collection::stream).collect(Collectors.toList());
-
-        List<Binnable> filteredClinicalData = Stream.of(
-            filteredClinicalDataForSamples,
-            filteredClinicalDataForPatients
-            // filteredClinicalDataForConflictingPatientAttributes // TODO ignoring conflictingPatientAttributeIds for now
-        ).flatMap(Collection::stream).collect(Collectors.toList());
+        Map<String, ClinicalDataType> attributeDatatypeMap = studyViewColumnarService.getClinicalAttributeDatatypeMap();
 
         Map<String, List<Binnable>> unfilteredClinicalDataByAttributeId =
             unfilteredClinicalData.stream().collect(Collectors.groupingBy(Binnable::getAttrId));

diff --git a/...khouse/StudyViewClinicalDataBinMapper.xml → ...lickhouse/StudyViewClinicalDataMapper.xml b/...khouse/StudyViewClinicalDataBinMapper.xml → ...lickhouse/StudyViewClinicalDataMapper.xml
@@ -35,34 +35,6 @@
             </foreach>
         </if>
     </sql>
-
-    <select id="getPatientClinicalDataCountsForBinning">
-        SELECT
-            attribute_name as attributeId,
-            <include refid="normalizeAttributeValueNA">
-                <property name="attribute_value" value="attribute_value"/>
-                <property name="as_value" value="value"/>
-            </include>,
-            count(value) as count
-        FROM clinical_data_derived
-        <include refid="patientClinicalDataFromStudyViewFilter" />
-        AND type = 'patient'
-        GROUP BY attribute_name, value
-    </select>
-
-    <select id="getSampleClinicalDataCountsForBinning">
-        SELECT
-            attribute_name as attributeId,
-            <include refid="normalizeAttributeValueNA">
-                <property name="attribute_value" value="attribute_value"/>
-                <property name="as_value" value="value"/>
-            </include>,
-            count(value) as count
-        FROM clinical_data_derived
-        <include refid="sampleClinicalDataFromStudyViewFilter" />
-        AND type = 'sample'
-        GROUP BY attribute_name, value
-    </select>
 
     <select id="getSampleClinicalDataFromStudyViewFilter" resultType="org.cbioportal.model.ClinicalData">
         SELECT

diff --git a/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.xml b/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.xml
@@ -76,12 +76,11 @@
 
     <!-- for /clinical-data-counts/fetch (returns ClinicalData) which will then be converted to clinicalDataCountItems -->
     <select id="getClinicalDataCounts" resultType="org.cbioportal.model.ClinicalDataCount">
-        <include refid="getCategoricalClinicalDataCountsQuerySample" />
+        <include refid="getClinicalDataCountsQuerySample" />
         UNION ALL
-        <include refid="getCategoricalClinicalDataCountsQueryPatient" />
+        <include refid="getClinicalDataCountsQueryPatient" />
     </select>
 
-
     <!-- for /molecular-profile-sample-counts/fetch (returns GenomicDataCount) which will then be converted to clinicalDataCountItems -->
     <select id="getGenomicDataCounts" resultType="org.cbioportal.model.GenomicDataCount">
             --we need to derive the alteration type from the stable_id by removing cancer study id
@@ -115,7 +114,7 @@
     </select>
 
 
-    <sql id="getCategoricalClinicalDataCountsQuerySample">
+    <sql id="getClinicalDataCountsQuerySample">
         SELECT
         attribute_name as attributeId,
         <include refid="normalizeAttributeValueNA">
@@ -148,7 +147,7 @@
         value
     </sql>
 
-    <sql id="getCategoricalClinicalDataCountsQueryPatient">
+    <sql id="getClinicalDataCountsQueryPatient">
         SELECT
         attribute_name as attributeId,
         <include refid="normalizeAttributeValueNA">

diff --git a/src/test/java/org/cbioportal/persistence/mybatisclickhouse/config/MyBatisConfig.java b/src/test/java/org/cbioportal/persistence/mybatisclickhouse/config/MyBatisConfig.java
@@ -35,9 +35,9 @@ public SqlSessionFactoryBean sqlColumnarSessionFactory(ResourceLoader resourceLo
         var studyViewMapperResource = resourceLoader.getResource("classpath:org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.xml") ;
         var studyViewFilterMapperResource = resourceLoader.getResource("classpath:org/cbioportal/persistence/mybatisclickhouse/StudyViewFilterMapper.xml");
         var alterationFilterMapperResource = resourceLoader.getResource("classpath:org/cbioportal/persistence/mybatisclickhouse/StudyViewAlterationFilterMapper.xml");
-        var clinicalDataBinMapperResource = resourceLoader.getResource("classpath:org/cbioportal/persistence/mybatisclickhouse/StudyViewAlterationFilterMapper.xml");
+        var clinicalDataMapperResource = resourceLoader.getResource("classpath:org/cbioportal/persistence/mybatisclickhouse/StudyViewClinicalDataMapper.xml");
         sessionFactory.setMapperLocations(
-            studyViewMapperResource,studyViewFilterMapperResource, alterationFilterMapperResource, clinicalDataBinMapperResource
+            studyViewMapperResource,studyViewFilterMapperResource, alterationFilterMapperResource, clinicalDataMapperResource
         );
         return sessionFactory;
     }