fix(snomed): further improvements to reduce the load from external...

...ES clusters with HTTP restrictions Return only the necessary fields for the current query and algorithm.
b2ihealthcare · Apr 27, 2022 · cca6f87 · cca6f87
1 parent 4a46daa
commit cca6f87
Showing 1 changed file with 39 additions and 16 deletions.
diff --git a/.../b2international/snowowl/snomed/datastore/index/change/SnomedRepositoryPreCommitHook.java b/.../b2international/snowowl/snomed/datastore/index/change/SnomedRepositoryPreCommitHook.java
@@ -134,26 +134,46 @@ protected Collection<ChangeSetProcessor> getChangeSetProcessors(StagingArea stag
 		final LongSet inferredConceptIds = PrimitiveSets.newLongOpenHashSet();
 
 		if (!statedDestinationIds.isEmpty()) {
-			for (SnomedConceptDocument statedDestinationConcept : index.get(SnomedConceptDocument.class, statedDestinationIds)) {
-				statedConceptIds.add(Long.parseLong(statedDestinationConcept.getId()));
-				if (statedDestinationConcept.getStatedParents() != null) {
-					statedConceptIds.addAll(statedDestinationConcept.getStatedParents());
-				}
-				if (statedDestinationConcept.getStatedAncestors() != null) {
-					statedConceptIds.addAll(statedDestinationConcept.getStatedAncestors());
-				}
+			for (List<String> statedDestinationIdsPartition : Iterables.partition(statedDestinationIds, maxTermsCount)) {
+				Query.select(SnomedConceptDocument.class)
+					// make sure we only load the necessary parent arrays, not everything
+					.fields(SnomedConceptDocument.Fields.ID, SnomedConceptDocument.Fields.STATED_PARENTS, SnomedConceptDocument.Fields.STATED_ANCESTORS)
+					.where(SnomedConceptDocument.Expressions.ids(statedDestinationIdsPartition))
+					.limit(50_000)
+					.build()
+					.stream(index)
+					.flatMap(Hits::stream)
+					.forEach(statedDestinationConcept -> {
+						statedConceptIds.add(Long.parseLong(statedDestinationConcept.getId()));
+						if (statedDestinationConcept.getStatedParents() != null) {
+							statedConceptIds.addAll(statedDestinationConcept.getStatedParents());
+						}
+						if (statedDestinationConcept.getStatedAncestors() != null) {
+							statedConceptIds.addAll(statedDestinationConcept.getStatedAncestors());
+						}
+					});
 			}
 		}
 
 		if (!inferredDestinationIds.isEmpty()) {
-			for (SnomedConceptDocument inferredDestinationConcept : index.get(SnomedConceptDocument.class, inferredDestinationIds)) {
-				inferredConceptIds.add(Long.parseLong(inferredDestinationConcept.getId()));
-				if (inferredDestinationConcept.getParents() != null) {
-					inferredConceptIds.addAll(inferredDestinationConcept.getParents());
-				}
-				if (inferredDestinationConcept.getAncestors() != null) {
-					inferredConceptIds.addAll(inferredDestinationConcept.getAncestors());
-				}
+			for (List<String> inferredDestinationIdsPartition : Iterables.partition(inferredDestinationIds, maxTermsCount)) {
+				Query.select(SnomedConceptDocument.class)
+					// make sure we only load the necessary parent arrays, not everything
+					.fields(SnomedConceptDocument.Fields.ID, SnomedConceptDocument.Fields.PARENTS, SnomedConceptDocument.Fields.ANCESTORS)
+					.where(SnomedConceptDocument.Expressions.ids(inferredDestinationIdsPartition))
+					.limit(50_000)
+					.build()
+					.stream(index)
+					.flatMap(Hits::stream)
+					.forEach(inferredDestinationConcept -> {
+						inferredConceptIds.add(Long.parseLong(inferredDestinationConcept.getId()));
+						if (inferredDestinationConcept.getParents() != null) {
+							inferredConceptIds.addAll(inferredDestinationConcept.getParents());
+						}
+						if (inferredDestinationConcept.getAncestors() != null) {
+							inferredConceptIds.addAll(inferredDestinationConcept.getAncestors());
+						}
+					});
 			}
 		}
 
@@ -183,6 +203,8 @@ protected Collection<ChangeSetProcessor> getChangeSetProcessors(StagingArea stag
 		if (!statedSourceIds.isEmpty()) {
 			for (List<String> statedSourceIdsPartition : Iterables.partition(statedSourceIds, maxTermsCount)) {
 				Query.select(SnomedConceptDocument.class)
+					// make sure we only load the necessary parent arrays, not everything
+					.fields(SnomedConceptDocument.Fields.ID, SnomedConceptDocument.Fields.STATED_PARENTS, SnomedConceptDocument.Fields.STATED_ANCESTORS)
 					.where(Expressions.bool()
 							.should(SnomedConceptDocument.Expressions.ids(statedSourceIdsPartition))
 							.should(SnomedConceptDocument.Expressions.statedParents(statedSourceIdsPartition))
@@ -207,6 +229,7 @@ protected Collection<ChangeSetProcessor> getChangeSetProcessors(StagingArea stag
 		if (!inferredSourceIds.isEmpty()) {
 			for (List<String> inferredSourceIdsPartition : Iterables.partition(inferredSourceIds, maxTermsCount)) {
 				Query.select(SnomedConceptDocument.class)
+					.fields(SnomedConceptDocument.Fields.ID, SnomedConceptDocument.Fields.PARENTS, SnomedConceptDocument.Fields.ANCESTORS)
 					.where(Expressions.bool()
 							.should(SnomedConceptDocument.Expressions.ids(inferredSourceIdsPartition))
 							.should(SnomedConceptDocument.Expressions.parents(inferredSourceIdsPartition))