diff --git a/snomed/com.b2international.snowowl.snomed.datastore.tests/src/com/b2international/snowowl/snomed/datastore/AllSnomedDatastoreTests.java b/snomed/com.b2international.snowowl.snomed.datastore.tests/src/com/b2international/snowowl/snomed/datastore/AllSnomedDatastoreTests.java index 8c0103a5241..d4d308c9205 100644 --- a/snomed/com.b2international.snowowl.snomed.datastore.tests/src/com/b2international/snowowl/snomed/datastore/AllSnomedDatastoreTests.java +++ b/snomed/com.b2international.snowowl.snomed.datastore.tests/src/com/b2international/snowowl/snomed/datastore/AllSnomedDatastoreTests.java @@ -31,6 +31,7 @@ import com.b2international.snowowl.snomed.datastore.internal.id.reservations.SnomedIdentifierReservationServiceImplTest; import com.b2international.snowowl.snomed.datastore.request.SnomedOWLExpressionConverterTest; import com.b2international.snowowl.snomed.datastore.request.SnomedOWLRelationshipConverterTest; +import com.b2international.snowowl.snomed.datastore.request.dsv.SnomedSimpleTypeRefSetDSVExporterTest; import com.b2international.snowowl.snomed.validation.SnomedQueryValidationRuleEvaluatorTest; /** @@ -74,6 +75,8 @@ SnomedOWLRelationshipConverterTest.class, // SnomedRefSetUtilTest.class, + // + SnomedSimpleTypeRefSetDSVExporterTest.class, }) public class AllSnomedDatastoreTests { diff --git a/snomed/com.b2international.snowowl.snomed.datastore.tests/src/com/b2international/snowowl/snomed/datastore/request/dsv/SnomedSimpleTypeRefSetDSVExporterTest.java b/snomed/com.b2international.snowowl.snomed.datastore.tests/src/com/b2international/snowowl/snomed/datastore/request/dsv/SnomedSimpleTypeRefSetDSVExporterTest.java new file mode 100644 index 00000000000..f92f8dc0929 --- /dev/null +++ b/snomed/com.b2international.snowowl.snomed.datastore.tests/src/com/b2international/snowowl/snomed/datastore/request/dsv/SnomedSimpleTypeRefSetDSVExporterTest.java @@ -0,0 +1,734 @@ +/* + * Copyright 2024 B2i Healthcare, https://b2ihealthcare.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.b2international.snowowl.snomed.datastore.request.dsv; + +import static org.junit.Assert.assertEquals; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.time.LocalDate; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.eclipse.core.runtime.NullProgressMonitor; +import org.junit.BeforeClass; +import org.junit.Test; + +import com.b2international.snowowl.core.ApplicationContext; +import com.b2international.snowowl.core.config.SnowOwlConfiguration; +import com.b2international.snowowl.snomed.common.SnomedConstants.Concepts; +import com.b2international.snowowl.snomed.common.SnomedRf2Headers; +import com.b2international.snowowl.snomed.core.domain.*; +import com.b2international.snowowl.snomed.core.domain.refset.DataType; +import com.b2international.snowowl.snomed.core.domain.refset.SnomedReferenceSetMember; +import com.b2international.snowowl.snomed.core.domain.refset.SnomedReferenceSetMembers; +import com.b2international.snowowl.snomed.datastore.SnomedRefSetUtil; +import com.b2international.snowowl.snomed.datastore.config.SnomedCoreConfiguration; +import com.b2international.snowowl.snomed.datastore.internal.rf2.*; +import com.b2international.snowowl.snomed.datastore.request.dsv.SnomedSimpleTypeRefSetDSVExporter.AncestorCollector; +import com.b2international.snowowl.snomed.datastore.request.dsv.SnomedSimpleTypeRefSetDSVExporter.ConceptStreamFactory; + +public class SnomedSimpleTypeRefSetDSVExporterTest { + + private static final String LS = System.lineSeparator(); + + private static String getContentsAndDelete(final SnomedSimpleTypeRefSetDSVExporter exporter) throws IOException { + final File exportFile = exporter.executeDSVExport(new NullProgressMonitor()); + final String exportContents = Files.readString(exportFile.toPath()); + exportFile.delete(); + return exportContents; + } + + @BeforeClass + public static void populateCoreConfig() { + final SnowOwlConfiguration configuration = new SnowOwlConfiguration(); + final SnomedCoreConfiguration coreConfiguration = configuration.getModuleConfig(SnomedCoreConfiguration.class); + coreConfiguration.setIntegerDatatypeRefsetIdentifier("rs1"); + coreConfiguration.setBooleanDatatypeRefsetIdentifier("rs2"); + + ApplicationContext.getInstance().registerService(SnowOwlConfiguration.class, configuration); + } + + @Test + public void exportEmptyFile() throws IOException { + ConceptStreamFactory conceptStreamFactory = (expand, locales, context, includeInactiveMembers, refSetId) -> Stream.empty(); + AncestorCollector ancestorCollector = (locales, ancestorId, context) -> new SnomedConcepts(0, 0); + + SnomedRefSetDSVExportModel exportSetting = new SnomedRefSetDSVExportModel(); + exportSetting.setDelimiter("\t"); + + final var exporter = new SnomedSimpleTypeRefSetDSVExporter(null, conceptStreamFactory, ancestorCollector, exportSetting); + final String exportContents = getContentsAndDelete(exporter); + assertEquals("Export file should have a single line separator", LS, exportContents); + } + + @Test + public void exportSingleConcept() throws IOException { + SnomedConcept concept = new SnomedConcept("c1"); + + SnomedConcepts chunk = new SnomedConcepts(List.of(concept), null, 1, 1); + ConceptStreamFactory conceptStreamFactory = (expand, locales, context, includeInactiveMembers, refSetId) -> Stream.of(chunk); + AncestorCollector ancestorCollector = (locales, ancestorId, context) -> new SnomedConcepts(0, 0); + + SnomedRefSetDSVExportModel exportSetting = new SnomedRefSetDSVExportModel(); + exportSetting.setDelimiter("\t"); + + final var exporter = new SnomedSimpleTypeRefSetDSVExporter(null, conceptStreamFactory, ancestorCollector, exportSetting); + final String exportContents = getContentsAndDelete(exporter); + assertEquals("Export file should have two line separators (header and data row)", LS + LS, exportContents); + } + + @Test + public void exportSingleConceptSimpleFields() throws IOException { + SnomedConcept concept = new SnomedConcept("c1"); + concept.setActive(false); + concept.setModuleId("m1"); + concept.setEffectiveTime(LocalDate.of(2024, 11, 27)); + concept.setDefinitionStatusId(Concepts.FULLY_DEFINED); + + SnomedConcepts chunk = new SnomedConcepts(List.of(concept), null, 1, 1); + ConceptStreamFactory conceptStreamFactory = (expand, locales, context, includeInactiveMembers, refSetId) -> Stream.of(chunk); + AncestorCollector ancestorCollector = (locales, ancestorId, context) -> new SnomedConcepts(0, 0); + + SnomedRefSetDSVExportModel exportSetting = new SnomedRefSetDSVExportModel(); + exportSetting.setDelimiter("\t"); + exportSetting.addExportItem(new SimpleSnomedDsvExportItem(SnomedDsvExportItemType.CONCEPT_ID)); + exportSetting.addExportItem(new SimpleSnomedDsvExportItem(SnomedDsvExportItemType.STATUS_LABEL)); + exportSetting.addExportItem(new SimpleSnomedDsvExportItem(SnomedDsvExportItemType.MODULE)); + exportSetting.addExportItem(new SimpleSnomedDsvExportItem(SnomedDsvExportItemType.EFFECTIVE_TIME)); + exportSetting.addExportItem(new SimpleSnomedDsvExportItem(SnomedDsvExportItemType.DEFINITION_STATUS)); + + final String expectedHeader = exportSetting.getExportItems() + .stream() + .map(i -> i.getDisplayName()) + .collect(Collectors.joining(exportSetting.getDelimiter())); + + final String expectedData = String.join(exportSetting.getDelimiter(), + concept.getId(), + "Inactive", + concept.getModuleId(), + "2024-11-27", + concept.getDefinitionStatusId()); + + final var exporter = new SnomedSimpleTypeRefSetDSVExporter(null, conceptStreamFactory, ancestorCollector, exportSetting); + final String exportContents = getContentsAndDelete(exporter); + assertEquals("Export file should have a header and a data row with simple item values", String.join(LS, expectedHeader, expectedData) + LS, exportContents); + } + + @Test + public void exportSingleConceptPreferredTerm() throws IOException { + SnomedDescription pt = new SnomedDescription("d1"); + pt.setTerm("PT term"); + + SnomedConcept concept = new SnomedConcept("c1"); + concept.setPt(pt); + + SnomedConcepts chunk = new SnomedConcepts(List.of(concept), null, 1, 1); + ConceptStreamFactory conceptStreamFactory = (expand, locales, context, includeInactiveMembers, refSetId) -> Stream.of(chunk); + AncestorCollector ancestorCollector = (locales, ancestorId, context) -> new SnomedConcepts(0, 0); + + SnomedRefSetDSVExportModel exportSetting = new SnomedRefSetDSVExportModel(); + exportSetting.setDelimiter("\t"); + exportSetting.addExportItem(new SimpleSnomedDsvExportItem(SnomedDsvExportItemType.CONCEPT_ID)); + exportSetting.addExportItem(new SimpleSnomedDsvExportItem(SnomedDsvExportItemType.PREFERRED_TERM)); + + String expectedHeader = exportSetting.getExportItems() + .stream() + .map(i -> i.getDisplayName()) + .collect(Collectors.joining(exportSetting.getDelimiter())); + + String expectedData = String.join(exportSetting.getDelimiter(), + concept.getId(), + pt.getTerm()); + + var exporter = new SnomedSimpleTypeRefSetDSVExporter(null, conceptStreamFactory, ancestorCollector, exportSetting); + String exportContents = getContentsAndDelete(exporter); + assertEquals("Export file should have a header and a data row with PT term", String.join(LS, expectedHeader, expectedData) + LS, exportContents); + + // Second run: include the SCTID for the preferred term + exportSetting.setIncludeDescriptionId(true); + + // As a result of the above setting, "Preferred term" appears twice in the header (once for the ID and once for the description term) + expectedHeader = String.join(exportSetting.getDelimiter(), "Concept ID", "Preferred term", "Preferred term"); + String expectedDetails = String.join(exportSetting.getDelimiter(), "", "ID", "Term"); + expectedData = String.join(exportSetting.getDelimiter(), concept.getId(), pt.getId(), pt.getTerm()); + + exporter = new SnomedSimpleTypeRefSetDSVExporter(null, conceptStreamFactory, ancestorCollector, exportSetting); + exportContents = getContentsAndDelete(exporter); + assertEquals("Export file should have a header and a data row with PT SCTID and term", String.join(LS, expectedHeader, expectedDetails, expectedData) + LS, exportContents); + } + + @Test + public void exportSingleConceptFsn() throws IOException { + SnomedDescription fsn = new SnomedDescription("d1"); + fsn.setTerm("FSN term"); + fsn.setTypeId(Concepts.FULLY_SPECIFIED_NAME); + + SnomedDescription decoy = new SnomedDescription("d2"); + decoy.setTerm("Should not appear in the output"); + decoy.setTypeId(Concepts.SYNONYM); + + SnomedConcept concept = new SnomedConcept("c1"); + concept.setDescriptions(new SnomedDescriptions(List.of(fsn, decoy), null, 2, 2)); + + SnomedConcepts chunk = new SnomedConcepts(List.of(concept), null, 1, 1); + ConceptStreamFactory conceptStreamFactory = (expand, locales, context, includeInactiveMembers, refSetId) -> Stream.of(chunk); + AncestorCollector ancestorCollector = (locales, ancestorId, context) -> new SnomedConcepts(0, 0); + + SnomedRefSetDSVExportModel exportSetting = new SnomedRefSetDSVExportModel(); + exportSetting.setDelimiter("\t"); + exportSetting.addExportItem(new SimpleSnomedDsvExportItem(SnomedDsvExportItemType.CONCEPT_ID)); + exportSetting.addExportItem(new ComponentIdSnomedDsvExportItem(SnomedDsvExportItemType.DESCRIPTION, Concepts.FULLY_SPECIFIED_NAME, "Fully specified name")); + + String expectedHeader = exportSetting.getExportItems() + .stream() + .map(i -> i.getDisplayName()) + .collect(Collectors.joining(exportSetting.getDelimiter())); + + String expectedData = String.join(exportSetting.getDelimiter(), + concept.getId(), + fsn.getTerm()); + + var exporter = new SnomedSimpleTypeRefSetDSVExporter(null, conceptStreamFactory, ancestorCollector, exportSetting); + String exportContents = getContentsAndDelete(exporter); + assertEquals("Export file should have a header and a data row with FSN term", String.join(LS, expectedHeader, expectedData) + LS, exportContents); + + // Second run: include the SCTID for the FSN + exportSetting.setIncludeDescriptionId(true); + + // As a result of the above setting, "Fully specified name" appears twice in the header (once for the ID and once for the description term) + expectedHeader = String.join(exportSetting.getDelimiter(), "Concept ID", "Fully specified name", "Fully specified name"); + String expectedDetails = String.join(exportSetting.getDelimiter(), "", "ID", "Term"); + expectedData = String.join(exportSetting.getDelimiter(), concept.getId(), fsn.getId(), fsn.getTerm()); + + exporter = new SnomedSimpleTypeRefSetDSVExporter(null, conceptStreamFactory, ancestorCollector, exportSetting); + exportContents = getContentsAndDelete(exporter); + assertEquals("Export file should have a header and a data row with FSN SCTID and term", String.join(LS, expectedHeader, expectedDetails, expectedData) + LS, exportContents); + } + + @Test + public void exportTwoConceptsFsn() throws IOException { + SnomedDescription fsn1 = new SnomedDescription("d1"); + fsn1.setTerm("FSN term 1"); + fsn1.setTypeId(Concepts.FULLY_SPECIFIED_NAME); + + SnomedDescription decoy1 = new SnomedDescription("d2"); + decoy1.setTerm("Should not appear in the output"); + decoy1.setTypeId(Concepts.SYNONYM); + + SnomedConcept concept1 = new SnomedConcept("c1"); + concept1.setDescriptions(new SnomedDescriptions(List.of(fsn1, decoy1), null, 2, 2)); + + // ------------------------------------ + + SnomedDescription fsn2 = new SnomedDescription("d3"); + fsn2.setTerm("FSN term 2.1"); + fsn2.setTypeId(Concepts.FULLY_SPECIFIED_NAME); + + SnomedDescription fsn3 = new SnomedDescription("d4"); + fsn3.setTerm("FSN term 2.2"); + fsn3.setTypeId(Concepts.FULLY_SPECIFIED_NAME); + + SnomedDescription decoy2 = new SnomedDescription("d5"); + decoy2.setTerm("Should not appear in the output either"); + decoy2.setTypeId(Concepts.SYNONYM); + + SnomedConcept concept2 = new SnomedConcept("c2"); + concept2.setDescriptions(new SnomedDescriptions(List.of(fsn2, fsn3, decoy2), null, 3, 3)); + + SnomedConcepts chunk = new SnomedConcepts(List.of(concept1, concept2), null, 2, 2); + ConceptStreamFactory conceptStreamFactory = (expand, locales, context, includeInactiveMembers, refSetId) -> Stream.of(chunk); + AncestorCollector ancestorCollector = (locales, ancestorId, context) -> new SnomedConcepts(0, 0); + + SnomedRefSetDSVExportModel exportSetting = new SnomedRefSetDSVExportModel(); + exportSetting.setDelimiter("\t"); + exportSetting.addExportItem(new SimpleSnomedDsvExportItem(SnomedDsvExportItemType.CONCEPT_ID)); + exportSetting.addExportItem(new ComponentIdSnomedDsvExportItem(SnomedDsvExportItemType.DESCRIPTION, Concepts.FULLY_SPECIFIED_NAME, "Fully specified name")); + + // We get numbered columns because c2 has two FSNs + String expectedHeader = String.join(exportSetting.getDelimiter(), "Concept ID", "Fully specified name (1)", "Fully specified name (2)"); + String expectedData1 = String.join(exportSetting.getDelimiter(), concept1.getId(), fsn1.getTerm(), ""); + String expectedData2 = String.join(exportSetting.getDelimiter(), concept2.getId(), fsn2.getTerm(), fsn3.getTerm()); + + var exporter = new SnomedSimpleTypeRefSetDSVExporter(null, conceptStreamFactory, ancestorCollector, exportSetting); + String exportContents = getContentsAndDelete(exporter); + assertEquals("Export file should have a header and two data rows with FSN terms", String.join(LS, expectedHeader, expectedData1, expectedData2) + LS, exportContents); + + // Second run: include the SCTID for the FSN + exportSetting.setIncludeDescriptionId(true); + + // As a result of the above setting, "Fully specified name" appears four times in the header (once for the ID and once for the description term) + expectedHeader = String.join(exportSetting.getDelimiter(), "Concept ID", "Fully specified name (1)", "Fully specified name (1)", "Fully specified name (2)", "Fully specified name (2)"); + String expectedDetails = String.join(exportSetting.getDelimiter(), "", "ID", "Term", "ID", "Term"); + expectedData1 = String.join(exportSetting.getDelimiter(), concept1.getId(), fsn1.getId(), fsn1.getTerm(), "", ""); + expectedData2 = String.join(exportSetting.getDelimiter(), concept2.getId(), fsn2.getId(), fsn2.getTerm(), fsn3.getId(), fsn3.getTerm()); + + exporter = new SnomedSimpleTypeRefSetDSVExporter(null, conceptStreamFactory, ancestorCollector, exportSetting); + exportContents = getContentsAndDelete(exporter); + assertEquals("Export file should have a header and two data rows with FSN SCTIDs and terms", String.join(LS, expectedHeader, expectedDetails, expectedData1, expectedData2) + LS, exportContents); + } + + @Test + public void exportSingleConceptRelationship() throws IOException { + SnomedDescription handStructurePt = new SnomedDescription("d1"); + handStructurePt.setTerm("Hand structure"); + + SnomedConcept handStructure = new SnomedConcept("c1"); + handStructure.setPt(handStructurePt); + + SnomedDescription legStructurePt = new SnomedDescription("d2"); + legStructurePt.setTerm("Leg structure"); + + SnomedConcept legStructure = new SnomedConcept("c2"); + legStructure.setPt(legStructurePt); + + SnomedRelationship findingSite = new SnomedRelationship("r1"); + findingSite.setRelationshipGroup(0); + findingSite.setTypeId(Concepts.FINDING_SITE); + findingSite.setDestination(handStructure); + findingSite.setCharacteristicTypeId(Concepts.INFERRED_RELATIONSHIP); + + // Grouped additional relationships are ignored + SnomedRelationship decoy1 = new SnomedRelationship("r2"); + decoy1.setRelationshipGroup(1); + decoy1.setTypeId(Concepts.FINDING_SITE); + decoy1.setDestination(legStructure); + decoy1.setCharacteristicTypeId(Concepts.ADDITIONAL_RELATIONSHIP); + + // Relationships with a different type are also ignored (because no export item is sent in the request for it) + SnomedRelationship decoy2 = new SnomedRelationship("r3"); + decoy2.setRelationshipGroup(0); + decoy2.setTypeId(Concepts.HAS_ACTIVE_INGREDIENT); + decoy2.setDestination(legStructure); + decoy2.setCharacteristicTypeId(Concepts.INFERRED_RELATIONSHIP); + + SnomedConcept concept = new SnomedConcept("c3"); + concept.setRelationships(new SnomedRelationships(List.of(findingSite, decoy1, decoy2), null, 3, 3)); + + SnomedConcepts chunk = new SnomedConcepts(List.of(concept), null, 1, 1); + ConceptStreamFactory conceptStreamFactory = (expand, locales, context, includeInactiveMembers, refSetId) -> Stream.of(chunk); + AncestorCollector ancestorCollector = (locales, ancestorId, context) -> new SnomedConcepts(0, 0); + + SnomedRefSetDSVExportModel exportSetting = new SnomedRefSetDSVExportModel(); + exportSetting.setDelimiter("\t"); + exportSetting.addExportItem(new SimpleSnomedDsvExportItem(SnomedDsvExportItemType.CONCEPT_ID)); + exportSetting.addExportItem(new ComponentIdSnomedDsvExportItem(SnomedDsvExportItemType.RELATIONSHIP, Concepts.FINDING_SITE, "Finding site")); + + String expectedHeader = exportSetting.getExportItems() + .stream() + .map(i -> i.getDisplayName()) + .collect(Collectors.joining(exportSetting.getDelimiter())); + + String expectedData = String.join(exportSetting.getDelimiter(), + concept.getId(), + handStructurePt.getTerm()); + + var exporter = new SnomedSimpleTypeRefSetDSVExporter(null, conceptStreamFactory, ancestorCollector, exportSetting); + String exportContents = getContentsAndDelete(exporter); + assertEquals("Export file should have a header and a data row with relationship destination term", String.join(LS, expectedHeader, expectedData) + LS, exportContents); + + // Second run: include the SCTID for destination concepts + exportSetting.setIncludeRelationshipTargetId(true); + + // As a result of the above setting, "Finding site" appears twice in the header (once for the ID and once for the description term) + expectedHeader = String.join(exportSetting.getDelimiter(), "Concept ID", "Finding site", "Finding site"); + String expectedDetails = String.join(exportSetting.getDelimiter(), "", "ID", "Destination"); + expectedData = String.join(exportSetting.getDelimiter(), concept.getId(), findingSite.getDestinationId(), handStructurePt.getTerm()); + + exporter = new SnomedSimpleTypeRefSetDSVExporter(null, conceptStreamFactory, ancestorCollector, exportSetting); + exportContents = getContentsAndDelete(exporter); + assertEquals("Export file should have a header and a data row with relationship destination SCTID and term", String.join(LS, expectedHeader, expectedDetails, expectedData) + LS, exportContents); + } + + @Test + public void exportSingleConceptGroupedRelationship() throws IOException { + SnomedDescription handStructurePt = new SnomedDescription("d1"); + handStructurePt.setTerm("Hand structure"); + + SnomedConcept handStructure = new SnomedConcept("c1"); + handStructure.setPt(handStructurePt); + + SnomedDescription legStructurePt = new SnomedDescription("d2"); + legStructurePt.setTerm("Leg structure"); + + SnomedConcept legStructure = new SnomedConcept("c2"); + legStructure.setPt(legStructurePt); + + SnomedRelationship findingSite = new SnomedRelationship("r1"); + findingSite.setRelationshipGroup(2); + findingSite.setTypeId(Concepts.FINDING_SITE); + findingSite.setDestination(handStructure); + findingSite.setCharacteristicTypeId(Concepts.INFERRED_RELATIONSHIP); + + SnomedConcept concept = new SnomedConcept("c3"); + concept.setRelationships(new SnomedRelationships(List.of(findingSite), null, 1, 1)); + + SnomedConcepts chunk = new SnomedConcepts(List.of(concept), null, 1, 1); + ConceptStreamFactory conceptStreamFactory = (expand, locales, context, includeInactiveMembers, refSetId) -> Stream.of(chunk); + AncestorCollector ancestorCollector = (locales, ancestorId, context) -> new SnomedConcepts(0, 0); + + SnomedRefSetDSVExportModel exportSetting = new SnomedRefSetDSVExportModel(); + exportSetting.setDelimiter("\t"); + exportSetting.addExportItem(new SimpleSnomedDsvExportItem(SnomedDsvExportItemType.CONCEPT_ID)); + exportSetting.addExportItem(new ComponentIdSnomedDsvExportItem(SnomedDsvExportItemType.RELATIONSHIP, Concepts.FINDING_SITE, "Finding site")); + + String expectedHeader = String.join(exportSetting.getDelimiter(), "Concept ID", "Finding site (AG2)"); + String expectedData = String.join(exportSetting.getDelimiter(), concept.getId(), handStructurePt.getTerm()); + + var exporter = new SnomedSimpleTypeRefSetDSVExporter(null, conceptStreamFactory, ancestorCollector, exportSetting); + String exportContents = getContentsAndDelete(exporter); + assertEquals("Export file should have a header (with group number) and a data row with relationship destination term", + String.join(LS, expectedHeader, expectedData) + LS, exportContents); + + // Second run: include the SCTID for destination concepts + exportSetting.setIncludeRelationshipTargetId(true); + + // As a result of the above setting, "Finding site" appears twice in the header (once for the ID and once for the description term) + expectedHeader = String.join(exportSetting.getDelimiter(), "Concept ID", "Finding site (AG2)", "Finding site (AG2)"); + String expectedDetails = String.join(exportSetting.getDelimiter(), "", "ID", "Destination"); + expectedData = String.join(exportSetting.getDelimiter(), concept.getId(), findingSite.getDestinationId(), handStructurePt.getTerm()); + + exporter = new SnomedSimpleTypeRefSetDSVExporter(null, conceptStreamFactory, ancestorCollector, exportSetting); + exportContents = getContentsAndDelete(exporter); + assertEquals("Export file should have a header (with group number) and a data row with relationship destination SCTID and term", + String.join(LS, expectedHeader, expectedDetails, expectedData) + LS, exportContents); + } + + @Test + public void exportTwoConceptsNonOverlappingGroupedRelationships() throws IOException { + SnomedDescription handStructurePt = new SnomedDescription("d1"); + handStructurePt.setTerm("Hand structure"); + + SnomedConcept handStructure = new SnomedConcept("c1"); + handStructure.setPt(handStructurePt); + + SnomedDescription legStructurePt = new SnomedDescription("d2"); + legStructurePt.setTerm("Leg structure"); + + SnomedConcept legStructure = new SnomedConcept("c2"); + legStructure.setPt(legStructurePt); + + SnomedRelationship findingSiteAG2 = new SnomedRelationship("r1"); + findingSiteAG2.setRelationshipGroup(2); + findingSiteAG2.setTypeId(Concepts.FINDING_SITE); + findingSiteAG2.setDestination(handStructure); + findingSiteAG2.setCharacteristicTypeId(Concepts.INFERRED_RELATIONSHIP); + + SnomedRelationship findingSiteAG3 = new SnomedRelationship("r2"); + findingSiteAG3.setRelationshipGroup(3); + findingSiteAG3.setTypeId(Concepts.FINDING_SITE); + findingSiteAG3.setDestination(legStructure); + findingSiteAG3.setCharacteristicTypeId(Concepts.INFERRED_RELATIONSHIP); + + SnomedConcept concept1 = new SnomedConcept("c3"); + concept1.setRelationships(new SnomedRelationships(List.of(findingSiteAG2), null, 1, 1)); + + SnomedConcept concept2 = new SnomedConcept("c4"); + concept2.setRelationships(new SnomedRelationships(List.of(findingSiteAG3), null, 1, 1)); + + SnomedConcepts chunk = new SnomedConcepts(List.of(concept1, concept2), null, 2, 2); + ConceptStreamFactory conceptStreamFactory = (expand, locales, context, includeInactiveMembers, refSetId) -> Stream.of(chunk); + AncestorCollector ancestorCollector = (locales, ancestorId, context) -> new SnomedConcepts(0, 0); + + SnomedRefSetDSVExportModel exportSetting = new SnomedRefSetDSVExportModel(); + exportSetting.setDelimiter("\t"); + exportSetting.addExportItem(new SimpleSnomedDsvExportItem(SnomedDsvExportItemType.CONCEPT_ID)); + exportSetting.addExportItem(new ComponentIdSnomedDsvExportItem(SnomedDsvExportItemType.RELATIONSHIP, Concepts.FINDING_SITE, "Finding site")); + + String expectedHeader = String.join(exportSetting.getDelimiter(), "Concept ID", "Finding site (AG2)", "Finding site (AG3)"); + String expectedData1 = String.join(exportSetting.getDelimiter(), concept1.getId(), handStructurePt.getTerm(), ""); + String expectedData2 = String.join(exportSetting.getDelimiter(), concept2.getId(), "", legStructurePt.getTerm()); + + var exporter = new SnomedSimpleTypeRefSetDSVExporter(null, conceptStreamFactory, ancestorCollector, exportSetting); + String exportContents = getContentsAndDelete(exporter); + assertEquals("Export file should have a header (with group numbers) and data rows with relationship destination term", + String.join(LS, expectedHeader, expectedData1, expectedData2) + LS, exportContents); + + // Second run: include the SCTID for destination concepts + exportSetting.setIncludeRelationshipTargetId(true); + + // As a result of the above setting, "Finding site" appears twice in the header (once for the ID and once for the description term) + expectedHeader = String.join(exportSetting.getDelimiter(), "Concept ID", "Finding site (AG2)", "Finding site (AG2)", "Finding site (AG3)", "Finding site (AG3)"); + String expectedDetails = String.join(exportSetting.getDelimiter(), "", "ID", "Destination", "ID", "Destination"); + expectedData1 = String.join(exportSetting.getDelimiter(), concept1.getId(), findingSiteAG2.getDestinationId(), handStructurePt.getTerm(), "", ""); + expectedData2 = String.join(exportSetting.getDelimiter(), concept2.getId(), "", "", findingSiteAG3.getDestinationId(), legStructurePt.getTerm()); + + exporter = new SnomedSimpleTypeRefSetDSVExporter(null, conceptStreamFactory, ancestorCollector, exportSetting); + exportContents = getContentsAndDelete(exporter); + assertEquals("Export file should have a header (with group numbers) and data rows with relationship destination SCTID and term", + String.join(LS, expectedHeader, expectedDetails, expectedData1, expectedData2) + LS, exportContents); + } + + private void exportSingleConceptMultipleRelationships(int relationshipGroup, String groupSuffix) throws IOException { + SnomedDescription handStructurePt = new SnomedDescription("d1"); + handStructurePt.setTerm("Hand structure"); + + SnomedConcept handStructure = new SnomedConcept("c1"); + handStructure.setPt(handStructurePt); + + SnomedDescription legStructurePt = new SnomedDescription("d2"); + legStructurePt.setTerm("Leg structure"); + + SnomedConcept legStructure = new SnomedConcept("c2"); + legStructure.setPt(legStructurePt); + + SnomedRelationship findingSite1 = new SnomedRelationship("r1"); + findingSite1.setRelationshipGroup(relationshipGroup); + findingSite1.setTypeId(Concepts.FINDING_SITE); + findingSite1.setDestination(handStructure); + findingSite1.setCharacteristicTypeId(Concepts.INFERRED_RELATIONSHIP); + + SnomedRelationship findingSite2 = new SnomedRelationship("r1"); + findingSite2.setRelationshipGroup(relationshipGroup); + findingSite2.setTypeId(Concepts.FINDING_SITE); + findingSite2.setDestination(legStructure); + findingSite2.setCharacteristicTypeId(Concepts.INFERRED_RELATIONSHIP); + + SnomedConcept concept = new SnomedConcept("c3"); + concept.setRelationships(new SnomedRelationships(List.of(findingSite1, findingSite2), null, 2, 2)); + + SnomedConcepts chunk = new SnomedConcepts(List.of(concept), null, 1, 1); + ConceptStreamFactory conceptStreamFactory = (expand, locales, context, includeInactiveMembers, refSetId) -> Stream.of(chunk); + AncestorCollector ancestorCollector = (locales, ancestorId, context) -> new SnomedConcepts(0, 0); + + SnomedRefSetDSVExportModel exportSetting = new SnomedRefSetDSVExportModel(); + exportSetting.setDelimiter("\t"); + exportSetting.addExportItem(new SimpleSnomedDsvExportItem(SnomedDsvExportItemType.CONCEPT_ID)); + exportSetting.addExportItem(new ComponentIdSnomedDsvExportItem(SnomedDsvExportItemType.RELATIONSHIP, Concepts.FINDING_SITE, "Finding site")); + + String expectedHeader = String.join(exportSetting.getDelimiter(), "Concept ID", "Finding site " + groupSuffix + "(1)", "Finding site " + groupSuffix + "(2)"); + String expectedData = String.join(exportSetting.getDelimiter(), concept.getId(), handStructurePt.getTerm(), legStructurePt.getTerm()); + + var exporter = new SnomedSimpleTypeRefSetDSVExporter(null, conceptStreamFactory, ancestorCollector, exportSetting); + String exportContents = getContentsAndDelete(exporter); + assertEquals("Export file should have a header (with group number if non-zero) and a data row with relationship destination terms", + String.join(LS, expectedHeader, expectedData) + LS, exportContents); + + // Second run: include the SCTID for destination concepts + exportSetting.setIncludeRelationshipTargetId(true); + + // As a result of the above setting, "Finding site" appears twice in the header (once for the ID and once for the description term) + expectedHeader = String.join(exportSetting.getDelimiter(), "Concept ID", + "Finding site " + groupSuffix + "(1)", "Finding site " + groupSuffix + "(1)", + "Finding site " + groupSuffix + "(2)", "Finding site " + groupSuffix + "(2)"); + + String expectedDetails = String.join(exportSetting.getDelimiter(), "", "ID", "Destination", "ID", "Destination"); + expectedData = String.join(exportSetting.getDelimiter(), concept.getId(), + findingSite1.getDestinationId(), handStructurePt.getTerm(), + findingSite2.getDestinationId(), legStructurePt.getTerm()); + + exporter = new SnomedSimpleTypeRefSetDSVExporter(null, conceptStreamFactory, ancestorCollector, exportSetting); + exportContents = getContentsAndDelete(exporter); + assertEquals("Export file should have a header (with group number if non-zero) and a data row with relationship destination SCTIDs and terms", + String.join(LS, expectedHeader, expectedDetails, expectedData) + LS, exportContents); + } + + @Test + public void exportSingleConceptMultipleZeroGroupRelationships() throws IOException { + exportSingleConceptMultipleRelationships(0, ""); + } + + @Test + public void exportSingleConceptMultipleGroupedRelationships() throws IOException { + exportSingleConceptMultipleRelationships(2, "(AG2) "); + } + + @Test + public void exportSingleConceptRelationshipValue() throws IOException { + + SnomedRelationship numberOfIngredients = new SnomedRelationship("r1"); + numberOfIngredients.setRelationshipGroup(0); + numberOfIngredients.setTypeId(Concepts.HAS_ACTIVE_INGREDIENT); // close enough + numberOfIngredients.setValue("#5"); + numberOfIngredients.setCharacteristicTypeId(Concepts.INFERRED_RELATIONSHIP); + + SnomedConcept concept = new SnomedConcept("c1"); + concept.setRelationships(new SnomedRelationships(List.of(numberOfIngredients), null, 1, 1)); + + SnomedConcepts chunk = new SnomedConcepts(List.of(concept), null, 1, 1); + ConceptStreamFactory conceptStreamFactory = (expand, locales, context, includeInactiveMembers, refSetId) -> Stream.of(chunk); + AncestorCollector ancestorCollector = (locales, ancestorId, context) -> new SnomedConcepts(0, 0); + + SnomedRefSetDSVExportModel exportSetting = new SnomedRefSetDSVExportModel(); + exportSetting.setDelimiter("\t"); + exportSetting.addExportItem(new SimpleSnomedDsvExportItem(SnomedDsvExportItemType.CONCEPT_ID)); + exportSetting.addExportItem(new ComponentIdSnomedDsvExportItem(SnomedDsvExportItemType.RELATIONSHIP, Concepts.HAS_ACTIVE_INGREDIENT, "Ingredient count")); + + String expectedHeader = exportSetting.getExportItems() + .stream() + .map(i -> i.getDisplayName()) + .collect(Collectors.joining(exportSetting.getDelimiter())); + + String expectedData = String.join(exportSetting.getDelimiter(), + concept.getId(), + numberOfIngredients.getValue()); + + var exporter = new SnomedSimpleTypeRefSetDSVExporter(null, conceptStreamFactory, ancestorCollector, exportSetting); + String exportContents = getContentsAndDelete(exporter); + assertEquals("Export file should have a header and a data row with relationship value", String.join(LS, expectedHeader, expectedData) + LS, exportContents); + + // Second run: try and include the destination SCTID for a relationship value (which it doesn't have) + exportSetting.setIncludeRelationshipTargetId(true); + + // As a result of the above setting, "Ingredient count" appears twice in the header (once for the ID and once for the description term) + expectedHeader = String.join(exportSetting.getDelimiter(), "Concept ID", "Ingredient count", "Ingredient count"); + String expectedDetails = String.join(exportSetting.getDelimiter(), "", "ID", "Destination"); + expectedData = String.join(exportSetting.getDelimiter(), concept.getId(), "", numberOfIngredients.getValue()); + + exporter = new SnomedSimpleTypeRefSetDSVExporter(null, conceptStreamFactory, ancestorCollector, exportSetting); + exportContents = getContentsAndDelete(exporter); + assertEquals("Export file should have a header and a data row with an empty relationship destination SCTID and populated value", String.join(LS, expectedHeader, expectedDetails, expectedData) + LS, exportContents); + } + + @Test + public void exportSingleConceptMember() throws IOException { + + SnomedReferenceSetMember numberOfIngredients = new SnomedReferenceSetMember(); + numberOfIngredients.setRefsetId(SnomedRefSetUtil.getRefSetId(DataType.INTEGER)); + numberOfIngredients.setReferencedComponent(new SnomedConcept("c1")); + numberOfIngredients.setProperties(Map.of( + SnomedRf2Headers.FIELD_RELATIONSHIP_GROUP, 0, + SnomedRf2Headers.FIELD_TYPE_ID, Concepts.HAS_ACTIVE_INGREDIENT, // close approximation again + SnomedRf2Headers.FIELD_CHARACTERISTIC_TYPE_ID, Concepts.INFERRED_RELATIONSHIP, + SnomedRf2Headers.FIELD_VALUE, "99" + )); + + // Grouped additional members are ignored + SnomedReferenceSetMember decoy1 = new SnomedReferenceSetMember(); + decoy1.setRefsetId(SnomedRefSetUtil.getRefSetId(DataType.INTEGER)); + decoy1.setReferencedComponent(new SnomedConcept("c1")); + decoy1.setProperties(Map.of( + SnomedRf2Headers.FIELD_RELATIONSHIP_GROUP, 1, + SnomedRf2Headers.FIELD_TYPE_ID, Concepts.HAS_ACTIVE_INGREDIENT, + SnomedRf2Headers.FIELD_CHARACTERISTIC_TYPE_ID, Concepts.ADDITIONAL_RELATIONSHIP, + SnomedRf2Headers.FIELD_VALUE, "10" + )); + + // Irrelevant relationship types are also ignored + SnomedReferenceSetMember decoy2 = new SnomedReferenceSetMember(); + decoy2.setRefsetId(SnomedRefSetUtil.getRefSetId(DataType.INTEGER)); + decoy2.setReferencedComponent(new SnomedConcept("c1")); + decoy2.setProperties(Map.of( + SnomedRf2Headers.FIELD_RELATIONSHIP_GROUP, 2, + SnomedRf2Headers.FIELD_TYPE_ID, Concepts.FINDING_SITE, + SnomedRf2Headers.FIELD_CHARACTERISTIC_TYPE_ID, Concepts.INFERRED_RELATIONSHIP, + SnomedRf2Headers.FIELD_VALUE, "5" + )); + + SnomedConcept concept = new SnomedConcept("c1"); + concept.setMembers(new SnomedReferenceSetMembers(List.of(numberOfIngredients, decoy1, decoy2), null, 3, 3)); + + SnomedConcepts chunk = new SnomedConcepts(List.of(concept), null, 1, 1); + ConceptStreamFactory conceptStreamFactory = (expand, locales, context, includeInactiveMembers, refSetId) -> Stream.of(chunk); + AncestorCollector ancestorCollector = (locales, ancestorId, context) -> new SnomedConcepts(0, 0); + + SnomedRefSetDSVExportModel exportSetting = new SnomedRefSetDSVExportModel(); + exportSetting.setDelimiter("\t"); + exportSetting.addExportItem(new SimpleSnomedDsvExportItem(SnomedDsvExportItemType.CONCEPT_ID)); + exportSetting.addExportItem(new DatatypeSnomedDsvExportItem(SnomedDsvExportItemType.DATAYPE, Concepts.HAS_ACTIVE_INGREDIENT, "Ingredient count", false)); + + String expectedHeader = exportSetting.getExportItems() + .stream() + .map(i -> i.getDisplayName()) + .collect(Collectors.joining(exportSetting.getDelimiter())); + + String expectedData = String.join(exportSetting.getDelimiter(), + concept.getId(), + (String) numberOfIngredients.getProperties().get(SnomedRf2Headers.FIELD_VALUE)); + + var exporter = new SnomedSimpleTypeRefSetDSVExporter(null, conceptStreamFactory, ancestorCollector, exportSetting); + String exportContents = getContentsAndDelete(exporter); + assertEquals("Export file should have a header and a data row with CD member value", String.join(LS, expectedHeader, expectedData) + LS, exportContents); + } + + @Test + public void exportSingleConceptBooleanMember() throws IOException { + + SnomedReferenceSetMember isVaccine = new SnomedReferenceSetMember(); + isVaccine.setRefsetId(SnomedRefSetUtil.getRefSetId(DataType.BOOLEAN)); + isVaccine.setReferencedComponent(new SnomedConcept("c1")); + isVaccine.setProperties(Map.of( + SnomedRf2Headers.FIELD_RELATIONSHIP_GROUP, 0, + SnomedRf2Headers.FIELD_TYPE_ID, "isVaccine", // close approximation once more + SnomedRf2Headers.FIELD_CHARACTERISTIC_TYPE_ID, Concepts.ADDITIONAL_RELATIONSHIP, + SnomedRf2Headers.FIELD_VALUE, "1" + )); + + SnomedConcept concept = new SnomedConcept("c1"); + concept.setMembers(new SnomedReferenceSetMembers(List.of(isVaccine), null, 1, 1)); + + SnomedConcepts chunk = new SnomedConcepts(List.of(concept), null, 1, 1); + ConceptStreamFactory conceptStreamFactory = (expand, locales, context, includeInactiveMembers, refSetId) -> Stream.of(chunk); + AncestorCollector ancestorCollector = (locales, ancestorId, context) -> new SnomedConcepts(0, 0); + + SnomedRefSetDSVExportModel exportSetting = new SnomedRefSetDSVExportModel(); + exportSetting.setDelimiter("\t"); + exportSetting.addExportItem(new SimpleSnomedDsvExportItem(SnomedDsvExportItemType.CONCEPT_ID)); + exportSetting.addExportItem(new DatatypeSnomedDsvExportItem(SnomedDsvExportItemType.DATAYPE, "isVaccine", "Vaccine", true)); + + String expectedHeader = exportSetting.getExportItems() + .stream() + .map(i -> i.getDisplayName()) + .collect(Collectors.joining(exportSetting.getDelimiter())); + + String expectedData = String.join(exportSetting.getDelimiter(), concept.getId(), "Yes"); + + var exporter = new SnomedSimpleTypeRefSetDSVExporter(null, conceptStreamFactory, ancestorCollector, exportSetting); + String exportContents = getContentsAndDelete(exporter); + assertEquals("Export file should have a header and a data row with boolean CD member value", String.join(LS, expectedHeader, expectedData) + LS, exportContents); + } + + @Test + public void exportSingleConceptGroupedMember() throws IOException { + + SnomedReferenceSetMember numberOfIngredients = new SnomedReferenceSetMember(); + numberOfIngredients.setRefsetId(SnomedRefSetUtil.getRefSetId(DataType.INTEGER)); + numberOfIngredients.setReferencedComponent(new SnomedConcept("c1")); + numberOfIngredients.setProperties(Map.of( + SnomedRf2Headers.FIELD_RELATIONSHIP_GROUP, 5, + SnomedRf2Headers.FIELD_TYPE_ID, Concepts.HAS_ACTIVE_INGREDIENT, + SnomedRf2Headers.FIELD_CHARACTERISTIC_TYPE_ID, Concepts.INFERRED_RELATIONSHIP, + SnomedRf2Headers.FIELD_VALUE, "99" + )); + + SnomedConcept concept = new SnomedConcept("c1"); + concept.setMembers(new SnomedReferenceSetMembers(List.of(numberOfIngredients), null, 1, 1)); + + SnomedConcepts chunk = new SnomedConcepts(List.of(concept), null, 1, 1); + ConceptStreamFactory conceptStreamFactory = (expand, locales, context, includeInactiveMembers, refSetId) -> Stream.of(chunk); + AncestorCollector ancestorCollector = (locales, ancestorId, context) -> new SnomedConcepts(0, 0); + + SnomedRefSetDSVExportModel exportSetting = new SnomedRefSetDSVExportModel(); + exportSetting.setDelimiter("\t"); + exportSetting.addExportItem(new SimpleSnomedDsvExportItem(SnomedDsvExportItemType.CONCEPT_ID)); + exportSetting.addExportItem(new DatatypeSnomedDsvExportItem(SnomedDsvExportItemType.DATAYPE, Concepts.HAS_ACTIVE_INGREDIENT, "Ingredient count", false)); + + String expectedHeader = String.join(exportSetting.getDelimiter(), + "Concept ID", + "Ingredient count (AG5)"); + + String expectedData = String.join(exportSetting.getDelimiter(), + concept.getId(), + (String) numberOfIngredients.getProperties().get(SnomedRf2Headers.FIELD_VALUE)); + + var exporter = new SnomedSimpleTypeRefSetDSVExporter(null, conceptStreamFactory, ancestorCollector, exportSetting); + String exportContents = getContentsAndDelete(exporter); + assertEquals("Export file should have a header with group number and a data row with CD member value", String.join(LS, expectedHeader, expectedData) + LS, exportContents); + } +} diff --git a/snomed/com.b2international.snowowl.snomed.datastore/src/com/b2international/snowowl/snomed/datastore/request/dsv/SnomedSimpleTypeRefSetDSVExporter.java b/snomed/com.b2international.snowowl.snomed.datastore/src/com/b2international/snowowl/snomed/datastore/request/dsv/SnomedSimpleTypeRefSetDSVExporter.java index def69072b96..f09f8c4f3ac 100644 --- a/snomed/com.b2international.snowowl.snomed.datastore/src/com/b2international/snowowl/snomed/datastore/request/dsv/SnomedSimpleTypeRefSetDSVExporter.java +++ b/snomed/com.b2international.snowowl.snomed.datastore/src/com/b2international/snowowl/snomed/datastore/request/dsv/SnomedSimpleTypeRefSetDSVExporter.java @@ -16,16 +16,19 @@ package com.b2international.snowowl.snomed.datastore.request.dsv; import static com.google.common.collect.Lists.newArrayList; -import static com.google.common.collect.Maps.newHashMap; -import static java.util.Optional.ofNullable; +import static com.google.common.collect.Maps.newTreeMap; import java.io.BufferedWriter; import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.util.*; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.SortedSet; import java.util.stream.Collectors; +import java.util.stream.Stream; import org.eclipse.core.runtime.IProgressMonitor; @@ -33,45 +36,85 @@ import com.b2international.snowowl.core.date.Dates; import com.b2international.snowowl.core.date.EffectiveTimes; import com.b2international.snowowl.core.domain.BranchContext; -import com.b2international.snowowl.core.request.SearchResourceRequest.Sort; -import com.b2international.snowowl.core.request.SearchResourceRequestIterator; import com.b2international.snowowl.snomed.common.SnomedConstants.Concepts; import com.b2international.snowowl.snomed.common.SnomedRf2Headers; import com.b2international.snowowl.snomed.core.domain.SnomedConcept; import com.b2international.snowowl.snomed.core.domain.SnomedConcepts; -import com.b2international.snowowl.snomed.core.domain.SnomedDescription; -import com.b2international.snowowl.snomed.core.domain.SnomedRelationship; -import com.b2international.snowowl.snomed.core.domain.refset.SnomedRefSetType; -import com.b2international.snowowl.snomed.datastore.index.entry.SnomedConceptDocument; import com.b2international.snowowl.snomed.datastore.internal.rf2.AbstractSnomedDsvExportItem; import com.b2international.snowowl.snomed.datastore.internal.rf2.ComponentIdSnomedDsvExportItem; import com.b2international.snowowl.snomed.datastore.internal.rf2.DatatypeSnomedDsvExportItem; import com.b2international.snowowl.snomed.datastore.internal.rf2.SnomedRefSetDSVExportModel; import com.b2international.snowowl.snomed.datastore.request.SnomedConceptSearchRequestBuilder; import com.b2international.snowowl.snomed.datastore.request.SnomedRequests; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Charsets; import com.google.common.base.Joiner; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSortedSet; -import com.google.common.collect.Ordering; +import com.google.common.collect.*; /** * Implements the export process of the DSV export for simple type reference sets. */ public class SnomedSimpleTypeRefSetDSVExporter implements IRefSetDSVExporter { - private static final String HEADER_EXPAND = "descriptions(active:true)," - + "relationships(active:true)," - + "members()"; + private static final String HEADER_EXPAND = "descriptions(active:true), " + + "relationships(active:true), " + + "members(active:true, refSetType:\"CONCRETE_DATA_TYPE\")"; - private static final String DATA_EXPAND = "pt()," - + "descriptions(active:true)," - + "relationships(active:true,expand(destination(expand(pt()))))," - + "members()"; + private static final String DATA_EXPAND = "pt(), " + + "descriptions(active:true), " + + "relationships(active:true, expand(destination(expand(pt())))), " + + "members(active:true, refSetType:\"CONCRETE_DATA_TYPE\")"; - private static final Map NO_OCCURRENCES = ImmutableMap.of(); + private static final Multiset NO_OCCURRENCES = ImmutableMultiset.of(); + + static interface ConceptStreamFactory { + + Stream getConceptStream( + String expand, + List locales, + BranchContext context, + boolean includeInactiveMembers, + String refSetId + ); + + ConceptStreamFactory DEFAULT = (expand, locales, context, includeInactiveMembers, refSetId) -> { + SnomedConceptSearchRequestBuilder builder = SnomedRequests.prepareSearchConcept() + .setLocales(locales) + .setExpand(expand) + .setLimit(context.getPageSize()); + + if (includeInactiveMembers) { + builder.isMemberOf(refSetId); + } else { + builder.isActiveMemberOf(refSetId); + } + + return builder.stream(context); + }; + } + + static interface AncestorCollector { + + SnomedConcepts getAncestors( + List locales, + String ancestorId, + BranchContext context + ); + + AncestorCollector DEFAULT = (locales, ancestorId, context) -> { + return SnomedRequests.prepareSearchConcept() + .all() + .setLocales(locales) + .filterByAncestor(ancestorId) + .setExpand("pt()") + .build() + .execute(context); + }; + } private final BranchContext context; + private final ConceptStreamFactory conceptStreamFactory; + private final AncestorCollector ancestorCollector; private String refSetId; private boolean includeDescriptionId; @@ -82,9 +125,8 @@ public class SnomedSimpleTypeRefSetDSVExporter implements IRefSetDSVExporter { private Joiner joiner; private String lineSeparator; - private Map descriptionCount; // maximum number of descriptions by type - private Map> propertyCountByGroup; // maximum number of properties by group and type - + private Multiset descriptionCount; // maximum number of descriptions by type + private Map> propertyCountByGroup; // maximum number of properties by group and type /** * Creates a new instance with the export parameters. @@ -92,13 +134,26 @@ public class SnomedSimpleTypeRefSetDSVExporter implements IRefSetDSVExporter { * @param exportSetting */ public SnomedSimpleTypeRefSetDSVExporter(final BranchContext context, final SnomedRefSetDSVExportModel exportSetting) { + this(context, ConceptStreamFactory.DEFAULT, AncestorCollector.DEFAULT, exportSetting); + } + + @VisibleForTesting + SnomedSimpleTypeRefSetDSVExporter( + final BranchContext context, + final ConceptStreamFactory conceptStreamFactory, + final AncestorCollector ancestorCollector, + final SnomedRefSetDSVExportModel exportSetting + ) { + this.context = context; + this.conceptStreamFactory = conceptStreamFactory; + this.ancestorCollector = ancestorCollector; + this.refSetId = exportSetting.getRefSetId(); this.includeDescriptionId = exportSetting.includeDescriptionId(); this.includeRelationshipId = exportSetting.includeRelationshipTargetId(); this.includeInactiveMembers = exportSetting.includeInactiveMembers(); this.exportItems = exportSetting.getExportItems(); this.locales = exportSetting.getLocales(); - this.context = context; this.joiner = Joiner.on(exportSetting.getDelimiter()); this.lineSeparator = System.lineSeparator(); } @@ -130,97 +185,119 @@ public File executeDSVExport(IProgressMonitor monitor) throws IOException { /* * Fetches members of the specified reference set */ - private SearchResourceRequestIterator getMemberConceptIterator(String expand) { - SnomedConceptSearchRequestBuilder builder = SnomedRequests.prepareSearchConcept() - .setLocales(locales) - .setExpand(expand) - .sortBy(Sort.fieldAsc(SnomedConceptDocument.Fields.ID)) - .setLimit(context.getPageSize()); - - if (includeInactiveMembers) { - builder.isMemberOf(refSetId); - } else { - builder.isActiveMemberOf(refSetId); - } - - return new SearchResourceRequestIterator<>(builder, b -> b.build().execute(context)); + private Stream getConceptStream(String expand) { + return conceptStreamFactory.getConceptStream(expand, locales, context, includeInactiveMembers, refSetId); } /* * Finds the maximum number of occurrences for each description, relationship and concrete data type; generates headers. */ private void computeHeader() { - descriptionCount = newHashMap(); - propertyCountByGroup = newHashMap(); - - SearchResourceRequestIterator conceptIterator = getMemberConceptIterator(HEADER_EXPAND); - while (conceptIterator.hasNext()) { - computeHeader(conceptIterator.next()); - } + descriptionCount = HashMultiset.create(); + propertyCountByGroup = newTreeMap(); + getConceptStream(HEADER_EXPAND).forEachOrdered(this::computeHeader); } private void computeHeader(SnomedConcepts chunk) { for (SnomedConcept concept : chunk) { + // Respect the exported item order that was sent in via the export request for (AbstractSnomedDsvExportItem exportItem : exportItems) { switch (exportItem.getType()) { - case DESCRIPTION: - ComponentIdSnomedDsvExportItem descriptionItem = (ComponentIdSnomedDsvExportItem) exportItem; - String descriptionTypeId = descriptionItem.getComponentId(); - Integer matchingDescriptions = concept.getDescriptions() - .stream() - .filter(d -> descriptionTypeId.equals(d.getTypeId())) - .collect(Collectors.reducing(0, description -> 1, Integer::sum)); + + case DESCRIPTION: { + final ComponentIdSnomedDsvExportItem descriptionItem = (ComponentIdSnomedDsvExportItem) exportItem; + final String typeId = descriptionItem.getComponentId(); + + // Find the number of active descriptions with this type ID (status is included in expand filter) + final int previousDescriptions = descriptionCount.count(typeId); + final int currentDescriptions = concept.getDescriptions() + .stream() + .filter(d -> typeId.equals(d.getTypeId())) + .collect(Collectors.summingInt(d -> 1)); - descriptionCount.merge(descriptionTypeId, matchingDescriptions, Math::max); + // Register in description type column counter -- bigger number wins + descriptionCount.setCount(typeId, Math.max(previousDescriptions, currentDescriptions)); break; - case RELATIONSHIP: - ComponentIdSnomedDsvExportItem relationshipItem = (ComponentIdSnomedDsvExportItem) exportItem; - String relationshipTypeId = relationshipItem.getComponentId(); + } - Map matchingRelationships = concept.getRelationships() - .stream() - .filter(r -> relationshipTypeId.equals(r.getTypeId()) - && (Concepts.INFERRED_RELATIONSHIP.equals(r.getCharacteristicTypeId()) - || Concepts.ADDITIONAL_RELATIONSHIP.equals(r.getCharacteristicTypeId()))) - .collect(Collectors.groupingBy( - SnomedRelationship::getRelationshipGroup, - Collectors.reducing(0, relationship -> 1, Integer::sum))); + case RELATIONSHIP: { + final ComponentIdSnomedDsvExportItem relationshipItem = (ComponentIdSnomedDsvExportItem) exportItem; + final String typeId = relationshipItem.getComponentId(); + + // Find the number of active relationships with this type ID (status is included in expand filter) + // ...but only count inferred relationships and additional relationships in group 0 + Map currentRelationshipsByGroup = concept.getRelationships() + .stream() + .filter(r -> { + final String relationshipTypeId = r.getTypeId(); + final String characteristicTypeId = r.getCharacteristicTypeId(); + final Integer relationshipGroup = r.getRelationshipGroup(); + + return typeId.equals(relationshipTypeId) + && (Concepts.INFERRED_RELATIONSHIP.equals(characteristicTypeId) + || (Concepts.ADDITIONAL_RELATIONSHIP.equals(characteristicTypeId) && relationshipGroup == 0)); + }) + .collect(Collectors.groupingBy( + // Results should be collected on a per-group basis + r -> r.getRelationshipGroup(), + Collectors.summingInt(r -> 1) + )); - matchingRelationships.entrySet() - .stream() - .forEach(entry -> { - propertyCountByGroup.compute(entry.getKey(), (key, oldValue) -> { - Map propertyCountForGroup = ofNullable(oldValue).orElseGet(HashMap::new); - propertyCountForGroup.merge(relationshipTypeId, entry.getValue(), Math::max); - return propertyCountForGroup; - }); - }); + currentRelationshipsByGroup.entrySet() + .stream() + .forEachOrdered(entry -> { + final int relationshipGroup = entry.getKey(); + // Number of properties encountered so far, for this type _and_ group number + final Multiset previousRelationshipsByType = propertyCountByGroup.computeIfAbsent(relationshipGroup, HashMultiset::create); + final int previousRelationships = previousRelationshipsByType.count(typeId); + final int currentRelationships = entry.getValue(); + + // Register in relationship type column counter for this group -- bigger number wins + previousRelationshipsByType.setCount(typeId, Math.max(previousRelationships, currentRelationships)); + }); + break; - case DATAYPE: - ComponentIdSnomedDsvExportItem dataTypeItem = (ComponentIdSnomedDsvExportItem) exportItem; - String dataTypeId = dataTypeItem.getComponentId(); + } + + case DATAYPE: { + final ComponentIdSnomedDsvExportItem dataTypeItem = (ComponentIdSnomedDsvExportItem) exportItem; + final String typeId = dataTypeItem.getComponentId(); - Map matchingMembers = concept.getMembers() - .stream() - .filter(m -> SnomedRefSetType.CONCRETE_DATA_TYPE.equals(m.type()) - && m.isActive() - && dataTypeId.equals(m.getProperties().get(SnomedRf2Headers.FIELD_TYPE_ID)) - && (Concepts.INFERRED_RELATIONSHIP.equals(m.getProperties().get(SnomedRf2Headers.FIELD_CHARACTERISTIC_TYPE_ID)) - || Concepts.ADDITIONAL_RELATIONSHIP.equals(m.getProperties().get(SnomedRf2Headers.FIELD_CHARACTERISTIC_TYPE_ID)))) - .collect(Collectors.groupingBy( - m -> (Integer) m.getProperties().get(SnomedRf2Headers.FIELD_RELATIONSHIP_GROUP), - Collectors.reducing(0, relationship -> 1, Integer::sum))); + // Find the number of active CD members with this type ID (refset type and status is included in expand filter) + // ...but only count inferred members and additional members in group 0 + Map currentMembersByGroup = concept.getMembers() + .stream() + .filter(m -> { + final String memberTypeId = (String) m.getProperties().get(SnomedRf2Headers.FIELD_TYPE_ID); + final String characteristicTypeId = (String) m.getProperties().get(SnomedRf2Headers.FIELD_CHARACTERISTIC_TYPE_ID); + final Integer memberGroup = (Integer) m.getProperties().get(SnomedRf2Headers.FIELD_RELATIONSHIP_GROUP); + + return typeId.equals(memberTypeId) + && (Concepts.INFERRED_RELATIONSHIP.equals(characteristicTypeId) + || (Concepts.ADDITIONAL_RELATIONSHIP.equals(characteristicTypeId) && memberGroup == 0)); + }) + .collect(Collectors.groupingBy( + // Results should be collected on a per-group basis + m -> (Integer) m.getProperties().get(SnomedRf2Headers.FIELD_RELATIONSHIP_GROUP), + Collectors.summingInt(m -> 1) + )); - matchingMembers.entrySet() - .stream() - .forEach(entry -> { - propertyCountByGroup.compute(entry.getKey(), (key, oldValue) -> { - Map propertyCountForGroup = ofNullable(oldValue).orElseGet(HashMap::new); - propertyCountForGroup.merge(dataTypeId, entry.getValue(), Math::max); - return propertyCountForGroup; - }); - }); + currentMembersByGroup.entrySet() + .stream() + .forEach(entry -> { + final int relationshipGroup = entry.getKey(); + // Number of properties encountered so far, for this type _and_ group number + final Multiset previousMembersByType = propertyCountByGroup.computeIfAbsent(relationshipGroup, HashMultiset::create); + final int previousMembers = previousMembersByType.count(typeId); + final int currentMembers = entry.getValue(); + + // Register in relationship type column counter for this group -- bigger number wins + previousMembersByType.setCount(typeId, Math.max(previousMembers, currentMembers)); + }); + break; + } + default: // Single-use fields don't need to be counted in advance break; @@ -237,29 +314,33 @@ private void writeHeader(BufferedWriter writer) throws IOException { for (AbstractSnomedDsvExportItem exportItem : exportItems) { switch (exportItem.getType()) { + case DESCRIPTION: { - ComponentIdSnomedDsvExportItem descriptionItem = (ComponentIdSnomedDsvExportItem) exportItem; - String typeId = descriptionItem.getComponentId(); - String displayName = descriptionTypeIdMap.getOrDefault(typeId, descriptionItem.getDisplayName()); - int occurrences = descriptionCount.get(typeId); + final ComponentIdSnomedDsvExportItem descriptionItem = (ComponentIdSnomedDsvExportItem) exportItem; + final String typeId = descriptionItem.getComponentId(); + final String displayName = descriptionTypeIdMap.getOrDefault(typeId, descriptionItem.getDisplayName()); + final int occurrences = descriptionCount.count(typeId); if (occurrences < 2) { + // No numbered suffix required if (includeDescriptionId) { - propertyHeader.add(displayName); propertyHeader.add(displayName); detailHeader.add("ID"); + propertyHeader.add(displayName); detailHeader.add("Term"); } else { propertyHeader.add(displayName); detailHeader.add(""); } } else { + // Numbered suffixes should start at index 1 for (int j = 1; j <= occurrences; j++) { - String numberedDisplayName = String.format("%s (%s)", displayName, j); + final String numberedDisplayName = String.format("%s (%s)", displayName, j); + if (includeDescriptionId) { - propertyHeader.add(numberedDisplayName); propertyHeader.add(numberedDisplayName); detailHeader.add("ID"); + propertyHeader.add(numberedDisplayName); detailHeader.add("Term"); } else { propertyHeader.add(numberedDisplayName); @@ -267,40 +348,50 @@ private void writeHeader(BufferedWriter writer) throws IOException { } } } + break; } case RELATIONSHIP: { - ComponentIdSnomedDsvExportItem relationshipItem = (ComponentIdSnomedDsvExportItem) exportItem; - String typeId = relationshipItem.getComponentId(); - String displayName = propertyTypeIdMap.getOrDefault(typeId, relationshipItem.getDisplayName()); + final ComponentIdSnomedDsvExportItem relationshipItem = (ComponentIdSnomedDsvExportItem) exportItem; + final String typeId = relationshipItem.getComponentId(); + final String displayName = propertyTypeIdMap.getOrDefault(typeId, relationshipItem.getDisplayName()); for (Integer group : propertyCountByGroup.keySet() ) { - Map occurrencesByType = propertyCountByGroup.getOrDefault(group, NO_OCCURRENCES); - int occurrences = occurrencesByType.getOrDefault(typeId, 0); + final Multiset occurrencesByType = propertyCountByGroup.getOrDefault(group, NO_OCCURRENCES); + final int occurrences = occurrencesByType.count(typeId); + + /* + * It is possible that a particular relationship type does not appear in a + * particular group at all, skip if this is the case. + */ if (occurrences < 1) { continue; } - String groupTag = group == 0 ? "" : String.format(" (AG%s)", group); + final String groupTag = (group == 0) ? "" : String.format(" (AG%s)", group); + final String groupedDisplayName = displayName + groupTag; + if (occurrences < 2) { + // No numbered suffix required if (includeRelationshipId) { - String groupedDisplayName = displayName + groupTag; - propertyHeader.add(groupedDisplayName); propertyHeader.add(groupedDisplayName); detailHeader.add("ID"); + propertyHeader.add(groupedDisplayName); detailHeader.add("Destination"); } else { - propertyHeader.add(displayName); + propertyHeader.add(groupedDisplayName); detailHeader.add(""); } } else { + // Numbered suffixes should start at index 1 for (int j = 1; j <= occurrences; j++) { - String numberedDisplayName = String.format("%s (%s)%s", displayName, j, groupTag); + final String numberedDisplayName = String.format("%s (%s)", groupedDisplayName, j); + if (includeRelationshipId) { - propertyHeader.add(numberedDisplayName); propertyHeader.add(numberedDisplayName); detailHeader.add("ID"); + propertyHeader.add(numberedDisplayName); detailHeader.add("Destination"); } else { propertyHeader.add(numberedDisplayName); @@ -314,49 +405,63 @@ private void writeHeader(BufferedWriter writer) throws IOException { } case DATAYPE: { - ComponentIdSnomedDsvExportItem dataTypeItem = (ComponentIdSnomedDsvExportItem) exportItem; - String typeId = dataTypeItem.getComponentId(); - String displayName = propertyTypeIdMap.getOrDefault(typeId, dataTypeItem.getDisplayName()); + final ComponentIdSnomedDsvExportItem dataTypeItem = (ComponentIdSnomedDsvExportItem) exportItem; + final String typeId = dataTypeItem.getComponentId(); + final String displayName = propertyTypeIdMap.getOrDefault(typeId, dataTypeItem.getDisplayName()); for (Integer groupId : propertyCountByGroup.keySet() ) { - Map occurrencesByType = propertyCountByGroup.getOrDefault(groupId, NO_OCCURRENCES); - int occurrences = occurrencesByType.getOrDefault(typeId, 0); + final Multiset occurrencesByType = propertyCountByGroup.getOrDefault(groupId, NO_OCCURRENCES); + final int occurrences = occurrencesByType.count(typeId); + + /* + * It is possible that a particular relationship type does not appear in a + * particular group at all, skip if this is the case. + */ if (occurrences < 1) { continue; } - String groupTag = groupId == 0 ? "" : String.format(" (AG%s)", groupId); + final String groupTag = (groupId == 0) ? "" : String.format(" (AG%s)", groupId); + final String groupedDisplayName = displayName + groupTag; + if (occurrences < 2) { - String groupedDisplayName = displayName + groupTag; + // No numbered suffix required propertyHeader.add(groupedDisplayName); detailHeader.add(""); } else { + // Numbered suffixes should start at index 1 for (int j = 1; j <= occurrences; j++) { - String numberedDisplayName = String.format("%s (%s)%s", displayName, j, groupTag); + final String numberedDisplayName = String.format("%s (%s)", groupedDisplayName, j); + propertyHeader.add(numberedDisplayName); detailHeader.add(""); } } } + break; } - case PREFERRED_TERM: + case PREFERRED_TERM: { if (includeDescriptionId) { - propertyHeader.add(exportItem.getDisplayName()); propertyHeader.add(exportItem.getDisplayName()); detailHeader.add("ID"); + propertyHeader.add(exportItem.getDisplayName()); detailHeader.add("Term"); } else { propertyHeader.add(exportItem.getDisplayName()); detailHeader.add(""); } + break; + } - default: + default: { propertyHeader.add(exportItem.getDisplayName()); detailHeader.add(""); + break; + } } } @@ -371,27 +476,19 @@ private void writeHeader(BufferedWriter writer) throws IOException { } private Map createTypeIdMap(String ancestorId) { - return createTypeIdMap(SnomedRequests.prepareSearchConcept() - .all() - .setLocales(locales) - .filterByAncestor(ancestorId) - .setExpand("pt()") - .build() - .execute(context)); + return createTypeIdMap(ancestorCollector.getAncestors(locales, ancestorId, context)); } private Map createTypeIdMap(SnomedConcepts concepts) { return concepts.stream() .collect(Collectors.toMap( - SnomedConcept::getId, + c -> c.getId(), c -> getPreferredTerm(c))); } private void writeValues(IProgressMonitor monitor, BufferedWriter writer) throws IOException { - SearchResourceRequestIterator conceptIterator = getMemberConceptIterator(DATA_EXPAND); - - while (conceptIterator.hasNext()) { - SnomedConcepts chunk = conceptIterator.next(); + final Iterable chunks = () -> getConceptStream(DATA_EXPAND).iterator(); + for (SnomedConcepts chunk : chunks) { writeValues(writer, chunk); monitor.worked(chunk.getItems().size()); } @@ -405,17 +502,21 @@ private void writeValues(BufferedWriter writer, SnomedConcepts chunk) throws IOE for (AbstractSnomedDsvExportItem exportItem : exportItems) { switch (exportItem.getType()) { + case DESCRIPTION: { final ComponentIdSnomedDsvExportItem descriptionItem = (ComponentIdSnomedDsvExportItem) exportItem; final String typeId = descriptionItem.getComponentId(); - int occurrences = descriptionCount.get(typeId); + final int occurrences = descriptionCount.count(typeId); - final Map termsById = concept.getDescriptions() - .stream() - .filter(d -> typeId.equals(d.getTypeId())) - .collect(Collectors.toMap( - SnomedDescription::getId, - SnomedDescription::getTerm)); + // Description ID keys, description term values (hopefully a 1:1 mapping, a Multimap is used only to satisfy other use cases) + final Multimap termsById = concept.getDescriptions() + .stream() + .filter(d -> typeId.equals(d.getTypeId())) + .collect(Multimaps.toMultimap( + d -> d.getId(), + d -> d.getTerm(), + ArrayListMultimap::create + )); addCells(dataRow, occurrences, includeDescriptionId, termsById); break; @@ -423,68 +524,87 @@ private void writeValues(BufferedWriter writer, SnomedConcepts chunk) throws IOE case RELATIONSHIP: { final ComponentIdSnomedDsvExportItem relationshipItem = (ComponentIdSnomedDsvExportItem) exportItem; + final String typeId = relationshipItem.getComponentId(); + for (Integer propertyGroup : propertyCountByGroup.keySet()) { - final String typeId = relationshipItem.getComponentId(); - final Map groupOccurrences = propertyCountByGroup.getOrDefault(propertyGroup, NO_OCCURRENCES); + final Multiset groupOccurrences = propertyCountByGroup.getOrDefault(propertyGroup, NO_OCCURRENCES); + final int occurrences = groupOccurrences.count(typeId); - final int occurrences = groupOccurrences.getOrDefault(typeId, 0); - concept.getRelationships() - .stream() - .filter(r -> typeId.equals(r.getTypeId()) - && Objects.equals(r.getRelationshipGroup(), propertyGroup) - && (Concepts.INFERRED_RELATIONSHIP.equals(r.getCharacteristicTypeId()) - || Concepts.ADDITIONAL_RELATIONSHIP.equals(r.getCharacteristicTypeId()))) - .forEach(relationship -> { - if (relationship.hasValue()) { - addCells(dataRow, occurrences, includeRelationshipId, ImmutableMap.of(relationship.getValue(), "")); - } else { - addCells(dataRow, occurrences, includeRelationshipId, ImmutableMap.of(relationship.getDestinationId(), getPreferredTerm(relationship.getDestination()))); - } - }); + if (occurrences < 1) { + // No header has been allocated for this attribute group-type ID pair, skip + break; + } + + // Destination ID keys, destination concept terms for values + // - OR - + // Empty string as key, relationships value literals for values + final Multimap destinationsById = concept.getRelationships() + .stream() + .filter(r -> { + final String relationshipTypeId = r.getTypeId(); + final String characteristicTypeId = r.getCharacteristicTypeId(); + final Integer relationshipGroup = r.getRelationshipGroup(); + + return typeId.equals(relationshipTypeId) + && Objects.equals(propertyGroup, relationshipGroup) + && (Concepts.INFERRED_RELATIONSHIP.equals(characteristicTypeId) + || (Concepts.ADDITIONAL_RELATIONSHIP.equals(characteristicTypeId) && relationshipGroup == 0)); + }) + .collect(Multimaps.toMultimap( + r -> r.hasValue() ? "" : r.getDestinationId(), + r -> r.hasValue() ? r.getValue() : getPreferredTerm(r.getDestination()), + ArrayListMultimap::create + )); + addCells(dataRow, occurrences, includeRelationshipId, destinationsById); } + break; } case DATAYPE: { final DatatypeSnomedDsvExportItem datatypeItem = (DatatypeSnomedDsvExportItem) exportItem; + final String typeId = datatypeItem.getComponentId(); + for (Integer propertyGroup : propertyCountByGroup.keySet()) { - Map groupedOccurrences = propertyCountByGroup.getOrDefault(propertyGroup, NO_OCCURRENCES); - final String typeId = datatypeItem.getComponentId(); - int occurrences = groupedOccurrences.getOrDefault(typeId, 0); + final Multiset groupOccurrences = propertyCountByGroup.getOrDefault(propertyGroup, NO_OCCURRENCES); + final int occurrences = groupOccurrences.count(typeId); if (occurrences < 1) { + // No header has been allocated for this attribute group-type ID pair, skip break; } - final List properties = concept.getMembers() - .stream() - .filter(m -> SnomedRefSetType.CONCRETE_DATA_TYPE.equals(m.type()) - && m.isActive() - && typeId.equals(m.getProperties().get(SnomedRf2Headers.FIELD_TYPE_ID)) - && Objects.equals(m.getProperties().get(SnomedRf2Headers.FIELD_RELATIONSHIP_GROUP), propertyGroup) - && (Concepts.INFERRED_RELATIONSHIP.equals(m.getProperties().get(SnomedRf2Headers.FIELD_CHARACTERISTIC_TYPE_ID)) - || Concepts.ADDITIONAL_RELATIONSHIP.equals(m.getProperties().get(SnomedRf2Headers.FIELD_CHARACTERISTIC_TYPE_ID)))) - .map(m -> m.getProperties().get(SnomedRf2Headers.FIELD_VALUE)) - .map(p -> { + // Empty string for keys, CD member values for values (only collected this way to conform to the method signature below) + final Multimap valuesById = concept.getMembers() + .stream() + .filter(m -> { + final String memberTypeId = (String) m.getProperties().get(SnomedRf2Headers.FIELD_TYPE_ID); + final String characteristicTypeId = (String) m.getProperties().get(SnomedRf2Headers.FIELD_CHARACTERISTIC_TYPE_ID); + final Integer memberGroup = (Integer) m.getProperties().get(SnomedRf2Headers.FIELD_RELATIONSHIP_GROUP); + + return typeId.equals(memberTypeId) + && Objects.equals(propertyGroup, memberGroup) + && (Concepts.INFERRED_RELATIONSHIP.equals(characteristicTypeId) + || (Concepts.ADDITIONAL_RELATIONSHIP.equals(characteristicTypeId) && memberGroup == 0)); + }) + .collect(Multimaps.toMultimap( + m -> "", + m -> { + final String serializedValue = (String) m.getProperties().get(SnomedRf2Headers.FIELD_VALUE); if (datatypeItem.isBooleanDatatype()) { - return "1".equals(p) ? "Yes" : "No"; + return "1".equals(serializedValue) ? "Yes" : "No"; } else { - return p.toString(); + return serializedValue; } - }) - .sorted() - .collect(Collectors.toList()); + }, + ArrayListMultimap::create + )); - for (String value : properties) { - dataRow.add(value); - occurrences--; - } - while (occurrences > 0) { - dataRow.add(""); - occurrences--; - } + // "Destination IDs" are never included for CD members + addCells(dataRow, occurrences, false, valuesById); } + break; } @@ -527,25 +647,34 @@ private void writeValues(BufferedWriter writer, SnomedConcepts chunk) throws IOE } } - private void addCells(List dataRow, int occurrences, boolean includeIds, Map idValuePairs) { + private void addCells(List dataRow, int occurrences, boolean includeIds, Multimap idValuePairs) { if (includeIds) { SortedSet sortedIds = ImmutableSortedSet.copyOf(idValuePairs.keySet()); + for (String id : sortedIds) { - dataRow.add(id); - dataRow.add(idValuePairs.get(id)); - occurrences--; + List sortedValues = Ordering.natural().sortedCopy(idValuePairs.get(id)); + for (String value : sortedValues) { + dataRow.add(id); + dataRow.add(value); + occurrences--; + } } + while (occurrences > 0) { dataRow.add(""); dataRow.add(""); occurrences--; } + } else { + List sortedValues = Ordering.natural().sortedCopy(idValuePairs.values()); + for (String value : sortedValues) { dataRow.add(value); occurrences--; } + while (occurrences > 0) { dataRow.add(""); occurrences--; @@ -553,12 +682,11 @@ private void addCells(List dataRow, int occurrences, boolean includeIds, } } - private String getPreferredTerm(SnomedConcept concept) { + private static String getPreferredTerm(SnomedConcept concept) { return (concept.getPt() == null) ? "" : concept.getPt().getTerm(); } - private String getPreferredTermId(SnomedConcept concept) { + private static String getPreferredTermId(SnomedConcept concept) { return (concept.getPt() == null) ? "" : concept.getPt().getId(); } - }