Skip to content

Commit

Permalink
very wip - adding tests
Browse files Browse the repository at this point in the history
  • Loading branch information
cbioportal import user committed Dec 27, 2024
1 parent 50b052b commit 0723054
Show file tree
Hide file tree
Showing 10 changed files with 530 additions and 15 deletions.
2 changes: 1 addition & 1 deletion docker/web-and-data/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ WORKDIR /cbioportal
RUN mvn dependency:go-offline --fail-never

COPY $PWD /cbioportal
RUN mvn install package -DskipTests -q
RUN mvn install package -DskipTests -Dfed.mode=FEDERATOR -q
RUN mkdir -p target/dependency && (cd target/dependency; jar -xf ../*-exec.jar)

FROM eclipse-temurin:21
Expand Down
6 changes: 3 additions & 3 deletions src/main/java/org/cbioportal/PortalApplication.java
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package org.cbioportal;

import org.cbioportal.persistence.fedapi.FederatedDataSourceConfig;
import org.cbioportal.persistence.fedapi.FederatorConfig;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration;
Expand All @@ -19,9 +19,9 @@
@PropertySource(ignoreResourceNotFound = true, value = "classpath:maven.properties"),
@PropertySource(ignoreResourceNotFound = true, value = "classpath:git.properties"),
@PropertySource(ignoreResourceNotFound = true, value = "classpath:springdoc.properties"),
@PropertySource(ignoreResourceNotFound = true, value = "classpath:fed-sources.properties")
@PropertySource(ignoreResourceNotFound = true, value = "classpath:federator.properties")
})
@EnableConfigurationProperties(FederatedDataSourceConfig.class)
@EnableConfigurationProperties(FederatorConfig.class)
public class PortalApplication {
public static void main(String[] args) {
SpringApplication.run(PortalApplication.class, args);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@

@Configuration
@ConfigurationProperties(prefix = "fed")
public class FederatedDataSourceConfig {
public class FederatorConfig {

private List<FederatedDataSourceInfo> sources = new ArrayList<>();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
import org.cbioportal.model.ClinicalDataCountItem;
import org.cbioportal.persistence.fedapi.FederatedDataSource;
import org.cbioportal.persistence.fedapi.FederatedDataSourceImpl;
import org.cbioportal.persistence.fedapi.FederatedDataSourceConfig;
import org.cbioportal.persistence.fedapi.FederatorConfig;
import org.cbioportal.service.ClinicalAttributeService;
import org.cbioportal.service.ClinicalDataService;
import org.cbioportal.service.FederatedViewService;
Expand All @@ -20,6 +20,8 @@

import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.function.Function;
import java.util.stream.Collectors;

enum FederationMode {
Expand All @@ -39,7 +41,7 @@ public class FederatedViewServiceImpl implements FederatedViewService {
private List<String> dataSourceStudies;

@Autowired
private FederatedDataSourceConfig federatedDataSourceConfig;
private FederatorConfig federatorConfig;

@Autowired
private ClinicalDataService clinicalDataService;
Expand All @@ -60,8 +62,9 @@ public class FederatedViewServiceImpl implements FederatedViewService {
public List<ClinicalAttribute> fetchClinicalAttributes() throws FederationException {
if (federationMode == FederationMode.FEDERATOR) {
try {
FederatedDataSource federatedDataSource = new FederatedDataSourceImpl(federatedDataSourceConfig.getSources().get(0));
return federatedDataSource.fetchClinicalAttributes().get();
return aggResultsFromDifferentSources(
s -> s.fetchClinicalAttributes()
);
} catch (Exception e) {
throw new FederationException("Failed to fetch clinical attributes", e);
}
Expand All @@ -72,19 +75,53 @@ public List<ClinicalAttribute> fetchClinicalAttributes() throws FederationExcept
}
}

private <T> List<T> aggResultsFromDifferentSources(
Function<FederatedDataSource, CompletableFuture<List<T>>> apiFunc
) throws Exception {
// Each list = results from one data source
// WhenAll's over the list of tasks to get a CF<List<List<CA>>>
// Awaits this to get a List<List<CA>>
// Once we have the list of lists... for now just flatten to merge them (assume each source has unique studies)

List<FederatedDataSource> sources = federatorConfig.getSources()
.stream()
.<FederatedDataSource>map(inf -> new FederatedDataSourceImpl(inf))
.toList();

List<CompletableFuture<List<T>>> futures = sources.stream()
.map(apiFunc::apply)
.toList();

CompletableFuture<List<List<T>>> combinedFuture =
CompletableFuture.allOf(futures.toArray(new CompletableFuture[0]))
.thenApply(v ->
futures.stream()
.map(CompletableFuture::join) // Join each future result
.collect(Collectors.toList()) // Collect into List<List<ClinicalAttribute>>
);

List<List<T>> results = combinedFuture.get();
List<T> flattened = results.stream()
.flatMap(List::stream) // Flatten each sublist
.collect(Collectors.toList());
return flattened;
}

@Override
public List<ClinicalDataCountItem> fetchClinicalDataCounts(
ClinicalDataCountFilter filter
) throws FederationException {
if (federationMode == FederationMode.FEDERATOR) {
try {
FederatedDataSource federatedDataSource = new FederatedDataSourceImpl(federatedDataSourceConfig.getSources().get(0));
return federatedDataSource.fetchClinicalDataCounts(filter).get();
return aggResultsFromDifferentSources(
s -> s.fetchClinicalDataCounts(filter)
);
} catch (Exception e) {
throw new FederationException("Failed to fetch clinical data counts", e);
}
} else if (federationMode == FederationMode.DATASOURCE) {
// TODO: replicate the logic for cacheableClinicalDataCounts here
filter.getStudyViewFilter().setStudyIds(dataSourceStudies);
return cachedClinicalDataCounts(filter);
} else {
throw new FederationException("Federation is disabled");
Expand Down Expand Up @@ -118,13 +155,15 @@ public List<ClinicalDataBin> fetchClinicalDataBinCounts(
) throws FederationException {
if (federationMode == FederationMode.FEDERATOR) {
try {
FederatedDataSource federatedDataSource = new FederatedDataSourceImpl(federatedDataSourceConfig.getSources().get(0));
return federatedDataSource.fetchClinicalDataBinCounts(filter).get();
return aggResultsFromDifferentSources(
s -> s.fetchClinicalDataBinCounts(filter)
);
} catch (Exception e) {
throw new FederationException("Failed to fetch clinical data bin counts", e);
}
} else if (federationMode == FederationMode.DATASOURCE) {
// TODO: replicate the logic for cacheableClinicalDataBinCounts here
filter.getStudyViewFilter().setStudyIds(dataSourceStudies);
return cachedFetchClinicalDataBinCounts(filter);
} else {
throw new FederationException("Federation is disabled");
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
fed.sources[0].name=Test cBioPortal 1
fed.sources[0].base-url=https://test.cbioportal.dev.aws.mskcc.org/api-fed

fed.sources[0].name=Test cBioPortal 2
fed.sources[0].base-url=https://test2.cbioportal.dev.aws.mskcc.org/api-fed
fed.sources[1].name=Test cBioPortal 2
fed.sources[1].base-url=https://test2.cbioportal.dev.aws.mskcc.org/api-fed
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ public class BaseServiceImplTest {
public static final String SORT = "sort";
public static final String DIRECTION = "direction";
public static final String STUDY_ID = "study_id";
public static final String STUDY_ID_2 = "study_id_2";
public static final String MOLECULAR_PROFILE_ID = "molecular_profile_id";
public static final String MOLECULAR_PROFILE_ID_A = "molecular_profile_id_a";
public static final String MOLECULAR_PROFILE_ID_B = "molecular_profile_id_b";
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,182 @@
package org.cbioportal.service.impl;

import org.cbioportal.model.ClinicalAttribute;
import org.cbioportal.model.Sample;
import org.cbioportal.persistence.ClinicalAttributeRepository;
import org.cbioportal.persistence.ClinicalDataRepository;
import org.cbioportal.persistence.SampleRepository;
import org.cbioportal.service.ClinicalAttributeService;
import org.cbioportal.service.ClinicalDataService;
import org.cbioportal.service.exception.FederationException;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.Spy;
import org.mockito.junit.MockitoJUnitRunner;
import org.springframework.test.util.ReflectionTestUtils;

import java.util.List;

@RunWith(MockitoJUnitRunner.class)
public class FederatedViewServiceImplTest extends BaseServiceImplTest {

@InjectMocks
private FederatedViewServiceImpl federatedViewService;

@Mock
private List<String> dataSourceStudies = List.of(
STUDY_ID,
STUDY_ID_2
);

@Mock
private ClinicalAttributeService clinicalAttributeService;

@Mock
private ClinicalDataService clinicalDataService;


private List<ClinicalAttribute> mockClinicalAttributes() {

var attrib11 = new ClinicalAttribute();
attrib11.setDisplayName("Sex");
attrib11.setDescription("Sex");
attrib11.setDatatype("STRING");
attrib11.setPatientAttribute(false);
attrib11.setPriority("1");
attrib11.setAttrId("SEX");
attrib11.setCancerStudyIdentifier(STUDY_ID);

var attrib21 = new ClinicalAttribute();
attrib21.setDisplayName("Age");
attrib21.setDescription("Age");
attrib21.setDatatype("NUMBER");
attrib21.setPatientAttribute(false);
attrib21.setPriority("1");
attrib21.setAttrId("AGE");
attrib21.setCancerStudyIdentifier(STUDY_ID);

var attrib12 = new ClinicalAttribute();
attrib12.setDisplayName("Sex");
attrib12.setDescription("Sex");
attrib12.setDatatype("STRING");
attrib12.setPatientAttribute(false);
attrib12.setPriority("1");
attrib12.setAttrId("SEX");
attrib12.setCancerStudyIdentifier(STUDY_ID_2);

var attrib22 = new ClinicalAttribute();
attrib22.setDisplayName("Age");
attrib22.setDescription("Age");
attrib22.setDatatype("NUMBER");
attrib22.setPatientAttribute(false);
attrib22.setPriority("1");
attrib22.setAttrId("AGE");
attrib22.setCancerStudyIdentifier(STUDY_ID_2);

return List.of(attrib11, attrib21, attrib12, attrib22);
}

private List<Sample> mockSamples() {
var sample1 = new Sample();
sample1.setStableId(SAMPLE_ID1);
sample1.setPatientStableId(PATIENT_ID_1);
sample1.setCancerStudyIdentifier(STUDY_ID);

var sample2 = new Sample();
sample2.setStableId(SAMPLE_ID2);
sample2.setPatientStableId(PATIENT_ID_2);
sample2.setCancerStudyIdentifier(STUDY_ID);

var sample3 = new Sample();
sample3.setStableId(SAMPLE_ID3);
sample3.setPatientStableId(PATIENT_ID_3);
sample3.setCancerStudyIdentifier(STUDY_ID_2);

return List.of(sample1, sample2, sample3);
}

@Before
public void setup() {
ReflectionTestUtils.setField(federatedViewService, "federationMode", FederationMode.DATASOURCE);

List<ClinicalAttribute> attribs = mockClinicalAttributes();

// Methods used by fetchClinicalAttributes
}

@Test
public void fetchClinicalAttributes() throws FederationException {
// Arrange
var expected = mockClinicalAttributes();

// Act
var result = federatedViewService.fetchClinicalAttributes();

// Assert
Assert.assertEquals(expected, result);
}

// @Test
// public void fetchClinicalDataCountsNoFilter() throws FederationException {
// // Arrange
// // Mock out: the cohort, ie.
// // - 1 person is this race & this age & this study
// // - .. 2 other people ..
// var filter = new ClinicalDataCountFilter();
// // First we get the cohort
//// Mockito.when(
//// studyViewFilterApplier.apply()
//// ).thenReturn(
////
//// );
// // Then we extract the study and sample IDs from this cohort
// Mockito.when().thenReturn();
// // Then we call clinicalDataService
// Mockito.when(
// clinicalDataService.fetchClinicalDataCounts(
// studyIds,
// sampleIds,
// attributeIds
// )
// ).thenReturn(
// /* ... */
// );
//
// // Act
// var result = federatedViewService.fetchClinicalDataCounts(filter);
//
// // Assert
// // - Returns some list of SampleIdentifiers
// // - Extracts them into study ID + sample ID list
// // - Calls fetchClinicalDataCounts()
// }
//
// // TODO test with filter
//
// @Test
// public void fetchClinicalDataBinCountsNoFilter() throws FederationException {
// // Arrange
// var filter = new ClinicalDataBinCountFilter();
// Mockito.when(
// clinicalDataBinUtil.fetchClinicalDataBinCounts(
// DataBinMethod.STATIC,
// filter,
// false
// )
// ).thenReturn(
// /* ... */
// );
//
// // Act
// var result = federatedViewService.fetchClinicalDataBinCounts();
//
// // Assert ...
// }
//
// // TODO test with filter
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
package org.cbioportal.test.integration;

import org.cbioportal.model.ClinicalAttribute;
import org.cbioportal.model.Sample;
import org.cbioportal.persistence.ClinicalAttributeRepository;
import org.cbioportal.persistence.ClinicalDataRepository;
import org.cbioportal.persistence.SampleRepository;
import org.cbioportal.service.ClinicalDataService;
import org.cbioportal.service.FederatedViewService;
import org.cbioportal.web.FederatedViewController;
import org.cbioportal.web.config.TestConfig;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc;
import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.http.MediaType;
import org.springframework.security.test.context.support.WithMockUser;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import org.springframework.test.web.servlet.MockMvc;
import org.springframework.test.web.servlet.request.MockMvcRequestBuilders;

import java.util.List;

import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.when;
import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;

@RunWith(SpringJUnit4ClassRunner.class)
//@WebMvcTest
//@AutoConfigureMockMvc // load the whole Spring Boot application context
@ContextConfiguration(classes = {
FederatedViewController.class,
FederatedViewService.class,
ClinicalDataService.class,
TestConfig.class
})
public class FederatedApiIntegrationTest {
}
Loading

0 comments on commit 0723054

Please sign in to comment.