diff --git a/client-api/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/query/Query.java b/client-api/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/query/Query.java index eb7ff4d7..8e28e74e 100644 --- a/client-api/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/query/Query.java +++ b/client-api/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/query/Query.java @@ -37,6 +37,7 @@ public Query(Query query) { private List fields = new ArrayList<>(); private List requiredFields = new ArrayList<>(); private List anyRecordOf = new ArrayList<>(); + private List> anyRecordOfMulti = new ArrayList<>(); private Map numericFilters = new HashMap<>(); private Map categoryFilters = new HashMap<>(); private List variantInfoFilters = new ArrayList<>(); @@ -62,6 +63,14 @@ public List getRequiredFields() { public List getAnyRecordOf() { return anyRecordOf; } + public List> getAnyRecordOfMulti() { + return anyRecordOfMulti; + } + public List> getAllAnyRecordOf() { + List> anyRecordOfMultiCopy = new ArrayList<>(anyRecordOfMulti); + anyRecordOfMultiCopy.add(anyRecordOf); + return anyRecordOfMultiCopy; + } public Map getNumericFilters() { return numericFilters; @@ -98,6 +107,9 @@ public void setRequiredFields(Collection requiredFields) { public void setAnyRecordOf(Collection anyRecordOf) { this.anyRecordOf = anyRecordOf != null ? new ArrayList<>(anyRecordOf) : new ArrayList<>(); } + public void setAnyRecordOfMulti(Collection> anyRecordOfMulti) { + this.anyRecordOfMulti = anyRecordOfMulti != null ? new ArrayList<>(anyRecordOfMulti) : new ArrayList<>(); + } public void setNumericFilters(Map numericFilters) { this.numericFilters = numericFilters != null ? new HashMap<>(numericFilters) : new HashMap<>(); @@ -191,7 +203,7 @@ public String toString() { writePartFormat("Numeric filters", numericFilters, builder); writePartFormat("Category filters", categoryFilters, builder); writePartFormat("Variant Info filters", variantInfoFilters, builder, false); - writePartFormat("Any-Record-Of filters", anyRecordOf, builder, true); + writePartFormat("Any-Record-Of filters", getAllAnyRecordOf(), builder, true); return builder.toString(); } @@ -234,7 +246,7 @@ private static void showTopLevelValues(Collection varList, StringBuilder builder Integer count = countMap.get(firstLevel); if(count == null) { - count = new Integer(1); + count = 1; } else { count = count + 1; } diff --git a/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/AbstractProcessor.java b/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/AbstractProcessor.java index e2f6c44a..e67d24cb 100644 --- a/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/AbstractProcessor.java +++ b/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/AbstractProcessor.java @@ -6,6 +6,7 @@ import java.util.Map.Entry; import java.util.concurrent.*; import java.util.stream.Collectors; +import java.util.stream.Stream; import java.util.zip.GZIPInputStream; import com.google.common.util.concurrent.UncheckedExecutionException; @@ -195,24 +196,27 @@ protected Set applyBooleanLogic(List> filteredIdSets) { * @return */ protected List> idSetsForEachFilter(Query query) { - ArrayList> filteredIdSets = new ArrayList>(); + final ArrayList> filteredIdSets = new ArrayList<>(); try { - addIdSetsForAnyRecordOf(query, filteredIdSets); + query.getAllAnyRecordOf().forEach(anyRecordOfFilterList -> { + addIdSetsForAnyRecordOf(anyRecordOfFilterList, filteredIdSets); + }); addIdSetsForRequiredFields(query, filteredIdSets); addIdSetsForNumericFilters(query, filteredIdSets); addIdSetsForCategoryFilters(query, filteredIdSets); } catch (InvalidCacheLoadException e) { log.warn("Invalid query supplied: " + e.getLocalizedMessage()); - filteredIdSets.add(new HashSet()); // if an invalid path is supplied, no patients should match. + filteredIdSets.add(new HashSet<>()); // if an invalid path is supplied, no patients should match. } //AND logic to make sure all patients match each filter if(filteredIdSets.size()>1) { - filteredIdSets = new ArrayList>(List.of(applyBooleanLogic(filteredIdSets))); + List> processedFilteredIdSets = new ArrayList<>(List.of(applyBooleanLogic(filteredIdSets))); + return addIdSetsForVariantInfoFilters(query, processedFilteredIdSets); + } else { + return addIdSetsForVariantInfoFilters(query, filteredIdSets); } - - return addIdSetsForVariantInfoFilters(query, filteredIdSets); } /** @@ -260,22 +264,19 @@ private void addIdSetsForRequiredFields(Query query, ArrayList> fil } } - private void addIdSetsForAnyRecordOf(Query query, ArrayList> filteredIdSets) { - if(!query.getAnyRecordOf().isEmpty()) { - Set patientsInScope = new ConcurrentSkipListSet(); - VariantBucketHolder bucketCache = new VariantBucketHolder(); - query.getAnyRecordOf().parallelStream().forEach(path->{ - if(patientsInScope.size() anyRecordOfFilters, ArrayList> filteredIdSets) { + if(!anyRecordOfFilters.isEmpty()) { + VariantBucketHolder bucketCache = new VariantBucketHolder<>(); + Set anyRecordOfPatientSet = anyRecordOfFilters.parallelStream().flatMap(path -> { + if (VariantUtils.pathIsVariantSpec(path)) { + TreeSet patientsInScope = new TreeSet<>(); + addIdSetsForVariantSpecCategoryFilters(new String[]{"0/1", "1/1"}, path, patientsInScope, bucketCache); + return patientsInScope.stream(); + } else { + return (Stream) getCube(path).keyBasedIndex().stream(); } - }); - filteredIdSets.add(patientsInScope); + }).collect(Collectors.toSet()); + filteredIdSets.add(anyRecordOfPatientSet); } } @@ -289,9 +290,9 @@ private void addIdSetsForNumericFilters(Query query, ArrayList> fil private void addIdSetsForCategoryFilters(Query query, ArrayList> filteredIdSets) { if(!query.getCategoryFilters().isEmpty()) { - VariantBucketHolder bucketCache = new VariantBucketHolder(); - Set> idsThatMatchFilters = (Set>)query.getCategoryFilters().entrySet().parallelStream().map(entry->{ - Set ids = new TreeSet(); + VariantBucketHolder bucketCache = new VariantBucketHolder<>(); + Set> idsThatMatchFilters = query.getCategoryFilters().entrySet().parallelStream().map(entry->{ + Set ids = new TreeSet<>(); if(VariantUtils.pathIsVariantSpec(entry.getKey())) { addIdSetsForVariantSpecCategoryFilters(entry.getValue(), entry.getKey(), ids, bucketCache); } else {