Fix alteration count services (#10911)

* Fix alteration count services

* Only add genes that are protein coding to gene_panel_to_gene table for WES

* fix test

* fix sonar issues

* Add documentation for filtering out WES

src/main/java/org/cbioportal/persistence/StudyViewRepository.java

@@ -38,13 +38,15 @@ public interface StudyViewRepository {
 
     List<CaseListDataCount> getCaseListDataCounts(StudyViewFilter studyViewFilter);
 
-    Map<String, AlterationCountByGene> getTotalProfiledCounts(StudyViewFilter studyViewFilter, String alterationType);
+    Map<String, Integer> getTotalProfiledCounts(StudyViewFilter studyViewFilter, String alterationType);
 
     int getFilteredSamplesCount(StudyViewFilter studyViewFilter);
 
     Map<String, Set<String>> getMatchingGenePanelIds(StudyViewFilter studyViewFilter, String alterationType);
 
     int getTotalProfiledCountsByAlterationType(StudyViewFilter studyViewFilter, String alterationType);
 
+    int getSampleProfileCountWithoutPanelData(StudyViewFilter studyViewFilter, String alterationType);
+
     List<ClinicalEventTypeCount> getClinicalEventTypeCounts(StudyViewFilter studyViewFilter);
 }

src/main/java/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.java

@@ -44,14 +44,15 @@ List<ClinicalDataCount> getClinicalDataCounts(StudyViewFilter studyViewFilter, C
 
     List<ClinicalData> getPatientClinicalDataFromStudyViewFilter(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter, boolean applyPatientIdFilters, List<String> attributeIds);
 
-    @MapKey("hugoGeneSymbol")
-    Map<String, AlterationCountByGene> getTotalProfiledCounts(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter, boolean applyPatientIdFilters, String alterationType);
+    List<AlterationCountByGene> getTotalProfiledCounts(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter, boolean applyPatientIdFilters, String alterationType);
 
     int getFilteredSamplesCount(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter, boolean applyPatientIdFilters);
 
     List<GenePanelToGene> getMatchingGenePanelIds(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter, boolean applyPatientIdFilters, String alterationType);
 
     int getTotalProfiledCountByAlterationType(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter, boolean applyPatientIdFilters, String alterationType);
+
+    int getSampleProfileCountWithoutPanelData(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter, boolean applyPatientIdFilters, String alterationType);
 
     List<ClinicalEventTypeCount> getClinicalEventTypeCounts(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter, boolean applyPatientIdFilters);
 }

src/main/java/org/cbioportal/persistence/mybatisclickhouse/StudyViewMyBatisRepository.java

@@ -132,10 +132,13 @@ public List<ClinicalData> getPatientClinicalData(StudyViewFilter studyViewFilter
     }
 
     @Override
-    public Map<String, AlterationCountByGene> getTotalProfiledCounts(StudyViewFilter studyViewFilter, String alterationType) {
+    public Map<String, Integer> getTotalProfiledCounts(StudyViewFilter studyViewFilter, String alterationType) {
         CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter = extractClinicalDataCountFilters(studyViewFilter);
         return mapper.getTotalProfiledCounts(studyViewFilter, categorizedClinicalDataCountFilter,
-            shouldApplyPatientIdFilters(studyViewFilter,categorizedClinicalDataCountFilter), alterationType);
+            shouldApplyPatientIdFilters(studyViewFilter,categorizedClinicalDataCountFilter), alterationType)
+            .stream()
+            .collect(Collectors.groupingBy(AlterationCountByGene::getHugoGeneSymbol,
+                Collectors.mapping(AlterationCountByGene::getNumberOfProfiledCases, Collectors.summingInt(Integer::intValue))));
     }
 
     @Override
@@ -162,6 +165,14 @@ public int getTotalProfiledCountsByAlterationType(StudyViewFilter studyViewFilte
            shouldApplyPatientIdFilters(studyViewFilter,categorizedClinicalDataCountFilter), alterationType); 
     }
 
+    @Override
+    public int getSampleProfileCountWithoutPanelData(StudyViewFilter studyViewFilter, String alterationType) {
+        CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter = extractClinicalDataCountFilters(studyViewFilter);
+        return mapper.getSampleProfileCountWithoutPanelData(studyViewFilter, categorizedClinicalDataCountFilter,
+            shouldApplyPatientIdFilters(studyViewFilter,categorizedClinicalDataCountFilter), alterationType);
+    }
+
+
     @Override
     public List<ClinicalEventTypeCount> getClinicalEventTypeCounts(StudyViewFilter studyViewFilter) {
         CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter = extractClinicalDataCountFilters(studyViewFilter);

src/main/java/org/cbioportal/service/impl/AlterationCountServiceImpl.java

@@ -273,37 +273,28 @@ public List<AlterationCountByGene> getStructuralVariantGenes(StudyViewFilter stu
     private < T extends AlterationCountByGene> List<T> populateAlterationCounts(@NonNull List<T> alterationCounts,
                                                                                 @NonNull StudyViewFilter studyViewFilter,
                                                                                 @NonNull AlterationType alterationType) {
-        var updatedAlterationCounts = alterationCounts.stream().map(SerializationUtils::clone).toList();
-        var profiledCountsMap = studyViewRepository.getTotalProfiledCounts(studyViewFilter,
-            alterationType.toString());
-        var profiledCountWithoutGenePanelData = studyViewRepository.getTotalProfiledCountsByAlterationType(studyViewFilter, alterationType.toString());
-        var matchingGenePanelIdsMap = studyViewRepository.getMatchingGenePanelIds(studyViewFilter, alterationType.toString());
+        final int profiledCountWithoutGenePanelData = studyViewRepository.getTotalProfiledCountsByAlterationType(studyViewFilter, alterationType.toString());
+        var profiledCountsMap = studyViewRepository.getTotalProfiledCounts(studyViewFilter, alterationType.toString());
+        final var matchingGenePanelIdsMap = studyViewRepository.getMatchingGenePanelIds(studyViewFilter, alterationType.toString());
+        final int sampleProfileCountWithoutGenePanelData = studyViewRepository.getSampleProfileCountWithoutPanelData(studyViewFilter, alterationType.toString());
 
-        updatedAlterationCounts.parallelStream()
+        alterationCounts.parallelStream()
             .forEach(alterationCountByGene ->  {
                 String hugoGeneSymbol = alterationCountByGene.getHugoGeneSymbol();
                 Set<String> matchingGenePanelIds = matchingGenePanelIdsMap.get(hugoGeneSymbol) != null ?
                     matchingGenePanelIdsMap.get(hugoGeneSymbol) : Collections.emptySet();
-
-                int totalProfiledCount = getTotalProfiledCount(hugoGeneSymbol,
-                    profiledCountsMap, profiledCountWithoutGenePanelData, matchingGenePanelIds);
+
+                int totalProfiledCount = hasGenePanelData(matchingGenePanelIds) 
+                    ? profiledCountsMap.getOrDefault(hugoGeneSymbol, 0) + sampleProfileCountWithoutGenePanelData
+                    : profiledCountWithoutGenePanelData;
 
                 alterationCountByGene.setNumberOfProfiledCases(totalProfiledCount);
 
                 alterationCountByGene.setMatchingGenePanelIds(matchingGenePanelIds);
             }); 
-        return updatedAlterationCounts;
+        return alterationCounts;
     }
 
-    private int getTotalProfiledCount(@NonNull String hugoGeneSymbol, @NonNull Map<String, AlterationCountByGene> profiledCountsMap,
-                                      int profiledCountWithoutGenePanelData, @NonNull Set<String> matchingGenePanelIds) {
-        int totalProfiledCount = profiledCountWithoutGenePanelData;
-
-        if (hasGenePanelData(matchingGenePanelIds) && profiledCountsMap.containsKey(hugoGeneSymbol)) {
-                totalProfiledCount  = profiledCountsMap.get(hugoGeneSymbol).getNumberOfProfiledCases();
-        }
-        return totalProfiledCount;
-    }
 
     private boolean hasGenePanelData(@NonNull Set<String> matchingGenePanelIds) {
         return matchingGenePanelIds.contains(WHOLE_EXOME_SEQUENCING) 

src/main/resources/db-scripts/clickhouse/clickhouse.sql

@@ -47,7 +47,7 @@ SELECT
     'WES' AS gene_panel_id,
     gene.hugo_gene_symbol AS gene
 FROM gene
-WHERE gene.entrez_gene_id > 0;
+WHERE gene.entrez_gene_id > 0 AND gene.type = 'protein-coding';
 
 CREATE TABLE sample_derived
 (

src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.xml

@@ -232,23 +232,41 @@
         JOIN cancer_study cs on cs.cancer_study_id = cam.cancer_study_id
     </select>
 
-    <!-- Grab Total Profiled Counts -->
-    <select id="getTotalProfiledCounts">
-        SELECT
+    <!-- Grab Total Profiled Counts. Currently, this query filters out all samples associated with a Gene Panel WES before
+    doing a join on gene_panel_to_gene_derived table. This is to prevent unnecessary stress on the db. For every sample
+    associated with a gene panel WES we multiply that row by the number of genes in the gene table. This could be greater
+    than 20K genes. This can create a huge join table that can slow down CH tremendously. 
+    We compute the WES counts by using the query getSampleProfileCountWithoutPanelData and adding this value to the 
+    totalProfiled Count per gene in java.
+     -->
+    <select id="getTotalProfiledCounts" resultType="org.cbioportal.model.AlterationCountByGene">
+        SELECT 
         gene as hugoGeneSymbol,
         COUNT(*) as numberOfProfiledCases
         FROM sample_to_gene_panel_derived stgp
             INNER JOIN gene_panel_to_gene_derived gptg on stgp.gene_panel_id = gptg.gene_panel_id 
         <where>
-            stgp.alteration_type = '${alterationType}' 
+            stgp.alteration_type = '${alterationType}'
+            AND stgp.gene_panel_id != 'WES'
             AND
             <include refid="applyStudyViewFilter">
                 <property name="filter_type" value="'SAMPLE_ID_ONLY'"/>
             </include>
         </where>
         GROUP BY gptg.gene;
     </select>
-
+
+    <select id="getSampleProfileCountWithoutPanelData" resultType="int">
+       SELECT COUNT(*)
+        FROM sample_to_gene_panel_derived
+        <where>
+            alteration_type = '${alterationType}' AND gene_panel_id = 'WES'
+            AND
+            <include refid="applyStudyViewFilter">
+                <property name="filter_type" value="'SAMPLE_ID_ONLY'"/>
+            </include>
+        </where>
+    </select>
     <!-- Helper query to get counts for totalProfiledCount when no genePanel data is available. 
     When no genePanel data is available we assume the whole genome was sequenced AKA WES (Whole Exome Sequencing). If this 
     is the case we use the current sample count from the cohort as the totalProfiledCount -->

src/test/java/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapperTest.java

@@ -114,8 +114,9 @@ public void getTotalProfiledCountsByGene() {
 
        assertEquals(3, totalProfiledCountsMap.size());
 
-       var akt2TotalProfiledCounts = totalProfiledCountsMap.get("akt2");
-       assertEquals(4, akt2TotalProfiledCounts.getNumberOfProfiledCases().intValue());
+       var akt2TotalProfiledCounts = totalProfiledCountsMap.stream().filter(c -> c.getHugoGeneSymbol().equals("akt2")).findFirst();
+       assertTrue(akt2TotalProfiledCounts.isPresent());
+       assertEquals(4, akt2TotalProfiledCounts.get().getNumberOfProfiledCases().intValue());
    } 
 
    @Test