From a03ec54679191a62ef6506d5d00671fec6b2313f Mon Sep 17 00:00:00 2001 From: Neel Kuila Date: Tue, 18 Jun 2024 16:19:22 -0400 Subject: [PATCH] RFC80:density plot endpoint(#10837) * working poc * refactor logic into service, so clean * refactor for parameters builder, simplify min max logic, streamline service call * remove unused services and imports * remove more unused imports --- .../ClinicalDataDensityPlotService.java | 12 ++ .../ClinicalDataDensityPlotServiceImpl.java | 138 ++++++++++++++++++ .../StudyViewColumnStoreController.java | 81 +++++++++- .../web/util/DensityPlotParameters.java | 118 +++++++++++++++ 4 files changed, 345 insertions(+), 4 deletions(-) create mode 100644 src/main/java/org/cbioportal/service/ClinicalDataDensityPlotService.java create mode 100644 src/main/java/org/cbioportal/service/impl/ClinicalDataDensityPlotServiceImpl.java create mode 100644 src/main/java/org/cbioportal/web/util/DensityPlotParameters.java diff --git a/src/main/java/org/cbioportal/service/ClinicalDataDensityPlotService.java b/src/main/java/org/cbioportal/service/ClinicalDataDensityPlotService.java new file mode 100644 index 00000000000..641ea3dbeb4 --- /dev/null +++ b/src/main/java/org/cbioportal/service/ClinicalDataDensityPlotService.java @@ -0,0 +1,12 @@ +package org.cbioportal.service; + +import org.cbioportal.model.ClinicalData; +import org.cbioportal.model.DensityPlotData; +import org.cbioportal.web.util.DensityPlotParameters; + +import java.math.BigDecimal; +import java.util.List; + +public interface ClinicalDataDensityPlotService { + DensityPlotData getDensityPlotData(List filteredClinicalData, DensityPlotParameters densityPlotParameters); +} diff --git a/src/main/java/org/cbioportal/service/impl/ClinicalDataDensityPlotServiceImpl.java b/src/main/java/org/cbioportal/service/impl/ClinicalDataDensityPlotServiceImpl.java new file mode 100644 index 00000000000..cfbb8c5fdd0 --- /dev/null +++ b/src/main/java/org/cbioportal/service/impl/ClinicalDataDensityPlotServiceImpl.java @@ -0,0 +1,138 @@ +package org.cbioportal.service.impl; + +import org.apache.commons.lang3.math.NumberUtils; +import org.apache.commons.math3.stat.correlation.PearsonsCorrelation; +import org.apache.commons.math3.stat.correlation.SpearmansCorrelation; +import org.cbioportal.model.ClinicalData; +import org.cbioportal.model.DensityPlotBin; +import org.cbioportal.model.DensityPlotData; +import org.cbioportal.service.ClinicalDataDensityPlotService; +import org.cbioportal.web.columnar.StudyViewColumnStoreController; +import org.cbioportal.web.util.DensityPlotParameters; +import org.springframework.stereotype.Service; + +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +@Service +public class ClinicalDataDensityPlotServiceImpl implements ClinicalDataDensityPlotService { + @Override + public DensityPlotData getDensityPlotData(List sampleClinicalData, DensityPlotParameters densityPlotParameters) { + DensityPlotData result = new DensityPlotData(); + result.setBins(new ArrayList<>()); + + Map> clinicalDataGroupedBySampleId = sampleClinicalData.stream(). + collect(Collectors.groupingBy(ClinicalData::getSampleId)); + + List extractedXYClinicalData = clinicalDataGroupedBySampleId.entrySet().stream() + .filter(entry -> entry.getValue().size() == 2 && + NumberUtils.isCreatable(entry.getValue().get(0).getAttrValue()) && + NumberUtils.isCreatable(entry.getValue().get(1).getAttrValue()) + ).flatMap(entry -> entry.getValue().stream()) + .toList(); + + if (extractedXYClinicalData.isEmpty()) { + return result; + } + + Map> partition = extractedXYClinicalData.stream().collect( + Collectors.partitioningBy(c -> c.getAttrId().equals(densityPlotParameters.getXAxisAttributeId()))); + + boolean useXLogScale = densityPlotParameters.getXAxisLogScale() && ClinicalDataDensityPlotServiceImpl.isLogScalePossibleForAttribute(densityPlotParameters.getXAxisAttributeId()); + boolean useYLogScale = densityPlotParameters.getYAxisLogScale() && ClinicalDataDensityPlotServiceImpl.isLogScalePossibleForAttribute(densityPlotParameters.getYAxisAttributeId()); + + double[] xValues = partition.get(true).stream().mapToDouble( + useXLogScale ? ClinicalDataDensityPlotServiceImpl::parseValueLog : ClinicalDataDensityPlotServiceImpl::parseValueLinear + ).toArray(); + double[] yValues = partition.get(false).stream().mapToDouble( + useYLogScale ? ClinicalDataDensityPlotServiceImpl::parseValueLog : ClinicalDataDensityPlotServiceImpl::parseValueLinear + ).toArray(); + double[] xValuesCopy = Arrays.copyOf(xValues, xValues.length); + double[] yValuesCopy = Arrays.copyOf(yValues, yValues.length); // Why copy these? + Arrays.sort(xValuesCopy); + Arrays.sort(yValuesCopy); + + double xAxisStartValue = densityPlotParameters.getXAxisStart() == null ? xValuesCopy[0] : + (useXLogScale ? ClinicalDataDensityPlotServiceImpl.logScale(densityPlotParameters.getXAxisStart().doubleValue()) : densityPlotParameters.getXAxisStart().doubleValue()); + double xAxisEndValue = densityPlotParameters.getXAxisEnd() == null ? xValuesCopy[xValuesCopy.length - 1] : + (useXLogScale ? ClinicalDataDensityPlotServiceImpl.logScale(densityPlotParameters.getXAxisEnd().doubleValue()) : densityPlotParameters.getXAxisEnd().doubleValue()); + double yAxisStartValue = densityPlotParameters.getYAxisStart() == null ? yValuesCopy[0] : + (useYLogScale ? ClinicalDataDensityPlotServiceImpl.logScale(densityPlotParameters.getYAxisStart().doubleValue()) : densityPlotParameters.getYAxisStart().doubleValue()); + double yAxisEndValue = densityPlotParameters.getYAxisEnd() == null ? yValuesCopy[yValuesCopy.length - 1] : + (useYLogScale ? ClinicalDataDensityPlotServiceImpl.logScale(densityPlotParameters.getYAxisEnd().doubleValue()) : densityPlotParameters.getYAxisEnd().doubleValue()); + double xAxisBinInterval = (xAxisEndValue - xAxisStartValue) / densityPlotParameters.getXAxisBinCount(); + double yAxisBinInterval = (yAxisEndValue - yAxisStartValue) / densityPlotParameters.getYAxisBinCount(); + List bins = result.getBins(); + for (int i = 0; i < densityPlotParameters.getXAxisBinCount(); i++) { + for (int j = 0; j < densityPlotParameters.getYAxisBinCount(); j++) { + DensityPlotBin densityPlotBin = new DensityPlotBin(); + densityPlotBin.setBinX(BigDecimal.valueOf(xAxisStartValue + (i * xAxisBinInterval))); + densityPlotBin.setBinY(BigDecimal.valueOf(yAxisStartValue + (j * yAxisBinInterval))); + densityPlotBin.setCount(0); + bins.add(densityPlotBin); + } + } + + for (int i = 0; i < xValues.length; i++) { + double xValue = xValues[i]; + double yValue = yValues[i]; + int xBinIndex = (int) ((xValue - xAxisStartValue) / xAxisBinInterval); + int yBinIndex = (int) ((yValue - yAxisStartValue) / yAxisBinInterval); + int index = (int) (((xBinIndex - (xBinIndex == densityPlotParameters.getXAxisBinCount() ? 1 : 0)) * densityPlotParameters.getYAxisBinCount()) + + (yBinIndex - (yBinIndex == densityPlotParameters.getYAxisBinCount() ? 1 : 0))); + DensityPlotBin densityPlotBin = bins.get(index); + densityPlotBin.setCount(densityPlotBin.getCount() + 1); + BigDecimal xValueBigDecimal = BigDecimal.valueOf(xValue); + BigDecimal yValueBigDecimal = BigDecimal.valueOf(yValue); + + // Set new min and max as needed + if (densityPlotBin.getMinX() == null || densityPlotBin.getMinX().compareTo(xValueBigDecimal) > 0){ + densityPlotBin.setMinX(xValueBigDecimal); + } + if (densityPlotBin.getMaxX() == null || densityPlotBin.getMaxX().compareTo(xValueBigDecimal) < 0){ + densityPlotBin.setMaxX(xValueBigDecimal); + } + if (densityPlotBin.getMinY() == null || densityPlotBin.getMinY().compareTo(yValueBigDecimal) > 0){ + densityPlotBin.setMinY(yValueBigDecimal); + } + if (densityPlotBin.getMaxY() == null || densityPlotBin.getMaxY().compareTo(yValueBigDecimal) < 0){ + densityPlotBin.setMaxY(yValueBigDecimal); + } + } + + if (xValues.length > 1) { + // need at least 2 entries in each to compute correlation + result.setPearsonCorr(new PearsonsCorrelation().correlation(xValues, yValues)); + result.setSpearmanCorr(new SpearmansCorrelation().correlation(xValues, yValues)); + } else { + // if less than 1 entry, just set 0 correlation + result.setSpearmanCorr(0.0); + result.setPearsonCorr(0.0); + } + + // filter out empty bins + result.setBins(result.getBins().stream().filter((bin)->(bin.getCount() > 0)).collect(Collectors.toList())); + return result; + } + + + private static boolean isLogScalePossibleForAttribute(String clinicalAttributeId) { + return clinicalAttributeId.equals("MUTATION_COUNT"); + } + + private static double logScale(double val) { + return Math.log(1+val); + } + + private static double parseValueLog(ClinicalData c) { + return ClinicalDataDensityPlotServiceImpl.logScale(Double.parseDouble(c.getAttrValue())); + } + + private static double parseValueLinear(ClinicalData c) { + return Double.parseDouble(c.getAttrValue()); + } +} diff --git a/src/main/java/org/cbioportal/web/columnar/StudyViewColumnStoreController.java b/src/main/java/org/cbioportal/web/columnar/StudyViewColumnStoreController.java index cf65ed4cc78..dffc99a9797 100644 --- a/src/main/java/org/cbioportal/web/columnar/StudyViewColumnStoreController.java +++ b/src/main/java/org/cbioportal/web/columnar/StudyViewColumnStoreController.java @@ -1,12 +1,20 @@ package org.cbioportal.web.columnar; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import jakarta.validation.Valid; import org.cbioportal.model.AlterationCountByGene; import org.cbioportal.model.AlterationFilter; +import org.cbioportal.model.ClinicalData; import org.cbioportal.model.ClinicalDataBin; import org.cbioportal.model.ClinicalDataCountItem; +import org.cbioportal.model.DensityPlotData; import org.cbioportal.model.Sample; +import org.cbioportal.service.ClinicalDataDensityPlotService; import org.cbioportal.service.StudyViewColumnarService; -import org.cbioportal.service.StudyViewService; import org.cbioportal.service.exception.StudyNotFoundException; import org.cbioportal.web.columnar.util.NewStudyViewFilterUtil; import org.cbioportal.web.config.annotation.InternalApi; @@ -15,6 +23,7 @@ import org.cbioportal.web.parameter.ClinicalDataFilter; import org.cbioportal.web.parameter.DataBinMethod; import org.cbioportal.web.parameter.StudyViewFilter; +import org.cbioportal.web.util.DensityPlotParameters; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; @@ -29,7 +38,9 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; +import java.math.BigDecimal; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.stream.Collectors; @@ -41,16 +52,21 @@ public class StudyViewColumnStoreController { private final StudyViewColumnarService studyViewColumnarService; - private final StudyViewService studyViewService; private final ClinicalDataBinner clinicalDataBinner; + private final ClinicalDataDensityPlotService clinicalDataDensityPlotService; @Autowired - public StudyViewColumnStoreController(StudyViewColumnarService studyViewColumnarService, StudyViewService studyViewService, ClinicalDataBinner clinicalDataBinner) { + public StudyViewColumnStoreController(StudyViewColumnarService studyViewColumnarService, + ClinicalDataBinner clinicalDataBinner, + ClinicalDataDensityPlotService clinicalDataDensityPlotService + ) { this.studyViewColumnarService = studyViewColumnarService; - this.studyViewService = studyViewService; this.clinicalDataBinner = clinicalDataBinner; + this.clinicalDataDensityPlotService = clinicalDataDensityPlotService; } + + @PreAuthorize("hasPermission(#involvedCancerStudies, 'Collection', T(org.cbioportal.utils.security.AccessLevel).READ)") @PostMapping(value = "/column-store/filtered-samples/fetch", consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE) @@ -118,4 +134,61 @@ public ResponseEntity> fetchClinicalDataBinCounts( ); return new ResponseEntity<>(clinicalDataBins, HttpStatus.OK); } + + @PreAuthorize("hasPermission(#involvedCancerStudies, 'Collection', T(org.cbioportal.utils.security.AccessLevel).READ)") + @RequestMapping(value = "/column-store/clinical-data-density-plot/fetch", method = RequestMethod.POST, + consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE) + @Operation(description = "Fetch clinical data density plot bins by study view filter") + @ApiResponse(responseCode = "200", description = "OK", + content = @Content(schema = @Schema(implementation = DensityPlotData.class))) + @Validated + public ResponseEntity fetchClinicalDataDensityPlot( + @Parameter(required = true, description = "Clinical Attribute ID of the X axis") + @RequestParam String xAxisAttributeId, + @Parameter(description = "Number of the bins in X axis") + @RequestParam(defaultValue = "50") Integer xAxisBinCount, + @Parameter(description = "Starting point of the X axis, if different than smallest value") + @RequestParam(required = false) BigDecimal xAxisStart, + @Parameter(description = "Starting point of the X axis, if different than largest value") + @RequestParam(required = false) BigDecimal xAxisEnd, + @Parameter(required = true, description = "Clinical Attribute ID of the Y axis") + @RequestParam String yAxisAttributeId, + @Parameter(description = "Number of the bins in Y axis") + @RequestParam(defaultValue = "50") Integer yAxisBinCount, + @Parameter(description = "Starting point of the Y axis, if different than smallest value") + @RequestParam(required = false) BigDecimal yAxisStart, + @Parameter(description = "Starting point of the Y axis, if different than largest value") + @RequestParam(required = false) BigDecimal yAxisEnd, + @Parameter(description="Use log scale for X axis") + @RequestParam(required = false, defaultValue = "false") Boolean xAxisLogScale, + @Schema(defaultValue = "false") + @Parameter(description="Use log scale for Y axis") + @RequestParam(required = false, defaultValue = "false") Boolean yAxisLogScale, + @Parameter(hidden = true) // prevent reference to this attribute in the swagger-ui interface + @RequestAttribute(required = false, value = "involvedCancerStudies") Collection involvedCancerStudies, + @Parameter(hidden = true) // prevent reference to this attribute in the swagger-ui interface. this attribute is needed for the @PreAuthorize tag above. + @Valid @RequestAttribute(required = false, value = "interceptedStudyViewFilter") StudyViewFilter interceptedStudyViewFilter, + @Parameter(required = true, description = "Study view filter") + @RequestBody(required = false) StudyViewFilter studyViewFilter) { + + List xyAttributeId = new ArrayList<>(Arrays.asList(xAxisAttributeId, yAxisAttributeId)); + DensityPlotParameters densityPlotParameters = + new DensityPlotParameters.Builder() + .xAxisAttributeId(xAxisAttributeId) + .yAxisAttributeId(yAxisAttributeId) + .xAxisBinCount(xAxisBinCount) + .yAxisBinCount(yAxisBinCount) + .xAxisStart(xAxisStart) + .yAxisStart(yAxisStart) + .xAxisEnd(xAxisEnd) + .yAxisEnd(yAxisEnd) + .xAxisLogScale(xAxisLogScale) + .yAxisLogScale(yAxisLogScale) + .build(); + + List sampleClinicalDataList = studyViewColumnarService.getSampleClinicalData(interceptedStudyViewFilter, xyAttributeId); + DensityPlotData result = clinicalDataDensityPlotService.getDensityPlotData(sampleClinicalDataList, densityPlotParameters); + + return new ResponseEntity<>(result, HttpStatus.OK); + } } diff --git a/src/main/java/org/cbioportal/web/util/DensityPlotParameters.java b/src/main/java/org/cbioportal/web/util/DensityPlotParameters.java new file mode 100644 index 00000000000..47e65543680 --- /dev/null +++ b/src/main/java/org/cbioportal/web/util/DensityPlotParameters.java @@ -0,0 +1,118 @@ +package org.cbioportal.web.util; + +import java.math.BigDecimal; + +public class DensityPlotParameters { + private final Integer xAxisBinCount; + private final Integer yAxisBinCount; + private final BigDecimal xAxisStart; + private final BigDecimal xAxisEnd; + private final BigDecimal yAxisStart; + private final BigDecimal yAxisEnd; + private final Boolean xAxisLogScale; + private final Boolean yAxisLogScale; + private final String xAxisAttributeId; + private final String yAxisAttributeId; + + DensityPlotParameters(Builder builder) { + this.xAxisBinCount = builder.xAxisBinCount; + this.yAxisBinCount = builder.yAxisBinCount; + this.xAxisStart = builder.xAxisStart; + this.xAxisEnd = builder.xAxisEnd; + this.yAxisStart = builder.yAxisStart; + this.yAxisEnd = builder.yAxisEnd; + this.xAxisAttributeId = builder.xAxisAttributeId; + this.yAxisAttributeId = builder.yAxisAttributeId; + this.xAxisLogScale = builder.xAxisLogScale; + this.yAxisLogScale = builder.yAxisLogScale; + } + + public Integer getXAxisBinCount() { + return xAxisBinCount; + } + public Integer getYAxisBinCount() { + return yAxisBinCount; + } + public BigDecimal getXAxisStart() { + return xAxisStart; + } + public BigDecimal getXAxisEnd() { + return xAxisEnd; + } + public BigDecimal getYAxisStart() { + return yAxisStart; + } + public BigDecimal getYAxisEnd() { + return yAxisEnd; + } + public Boolean getXAxisLogScale() { + return xAxisLogScale; + } + public Boolean getYAxisLogScale() { + return yAxisLogScale; + } + public String getXAxisAttributeId() { + return xAxisAttributeId; + } + public String getYAxisAttributeId() { + return yAxisAttributeId; + } + + public static class Builder { + private Integer xAxisBinCount; + private Integer yAxisBinCount; + private BigDecimal xAxisStart; + private BigDecimal xAxisEnd; + private BigDecimal yAxisStart; + private BigDecimal yAxisEnd; + private Boolean xAxisLogScale; + private Boolean yAxisLogScale; + private String xAxisAttributeId; + private String yAxisAttributeId; + + public Builder xAxisBinCount(Integer xAxisBinCount) { + this.xAxisBinCount = xAxisBinCount; + return this; + } + public Builder yAxisBinCount(Integer yAxisBinCount) { + this.yAxisBinCount = yAxisBinCount; + return this; + } + public Builder xAxisStart(BigDecimal xAxisStart) { + this.xAxisStart = xAxisStart; + return this; + } + public Builder xAxisEnd(BigDecimal xAxisEnd) { + this.xAxisEnd = xAxisEnd; + return this; + } + public Builder yAxisStart(BigDecimal yAxisStart) { + this.yAxisStart = yAxisStart; + return this; + } + public Builder yAxisEnd(BigDecimal yAxisEnd) { + this.yAxisEnd = yAxisEnd; + return this; + } + public Builder xAxisLogScale(Boolean xAxisLogScale) { + this.xAxisLogScale = xAxisLogScale; + return this; + } + public Builder yAxisLogScale(Boolean yAxisLogScale) { + this.yAxisLogScale = yAxisLogScale; + return this; + } + public Builder xAxisAttributeId(String xAxisAttributeId) { + this.xAxisAttributeId = xAxisAttributeId; + return this; + } + public Builder yAxisAttributeId(String yAxisAttributeId) { + this.yAxisAttributeId = yAxisAttributeId; + return this; + } + public DensityPlotParameters build() { + return new DensityPlotParameters(this); + + } + } +}