Skip to content

Commit

Permalink
issue #3448 - convert property-based concept hiearchies in getConcepts
Browse files Browse the repository at this point in the history
1. added `hasPropertyHierarchy` and `convertToSimpleCodeSystem`
utilities to CodeSystemSupport. the conversion utility uses JGraphT to
simplify the creation of an acyclic directed graph and then traverses
that structure to build an updated CodeSystem resource with nested
concepts. this allows the rest of our CodeSystemSupport to stay the
same.

2. call `convertToSimpleCodeSystem` from
RegistryTermServiceProvider.getConcepts to convert property-based
hierarchy into the more-common nested concept flavor so that we can use
our existing CodeSystemSupport filter approach for CodeSystems in the
registry

3. added hand-crafted CodeSystem resources that test parent-to-child and
child-to-parent edges with the conversion logic

4. added tests to verify our updated expansions match the expansions
from tx.fhir.org for select ValueSets that draw from polyhierarchical
CodeSystems like v3-ActCode and v3-RoleCode

Signed-off-by: Lee Surprenant <[email protected]>
  • Loading branch information
lmsurpre committed Jun 22, 2022
1 parent 2df79da commit deaff35
Show file tree
Hide file tree
Showing 20 changed files with 3,278 additions and 13 deletions.
11 changes: 11 additions & 0 deletions term/fhir-term/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,23 @@
<artifactId>fhir-registry</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.jgrapht</groupId>
<artifactId>jgrapht-core</artifactId>
<version>1.5.1</version>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>fhir-core-r4b</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>fhir-hl7-terminology</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* (C) Copyright IBM Corp. 2020, 2021
* (C) Copyright IBM Corp. 2020, 2022
*
* SPDX-License-Identifier: Apache-2.0
*/
Expand Down Expand Up @@ -65,7 +65,8 @@ public Set<Concept> getConcepts(CodeSystem codeSystem, List<Filter> filters) {
public <R> Set<R> getConcepts(CodeSystem codeSystem, List<Filter> filters, Function<Concept, ? extends R> function) {
checkArguments(codeSystem, filters, function);
try {
return CodeSystemSupport.getConcepts(codeSystem, filters, function);
CodeSystem simpleCodeSystem = CodeSystemSupport.convertToSimpleCodeSystem(codeSystem);
return CodeSystemSupport.getConcepts(simpleCodeSystem, filters, function);
} catch (FHIRTermException e) {
throw new FHIRTermServiceException(e.getMessage(), e, e.getIssues());
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* (C) Copyright IBM Corp. 2019, 2021
* (C) Copyright IBM Corp. 2019, 2022
*
* SPDX-License-Identifier: Apache-2.0
*/
Expand All @@ -23,6 +23,8 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
Expand All @@ -32,6 +34,14 @@
import java.util.regex.Pattern;
import java.util.stream.Collectors;

import org.jgrapht.Graph;
import org.jgrapht.event.TraversalListenerAdapter;
import org.jgrapht.event.VertexTraversalEvent;
import org.jgrapht.graph.DefaultDirectedGraph;
import org.jgrapht.graph.DefaultEdge;
import org.jgrapht.traverse.DepthFirstIterator;
import org.jgrapht.traverse.GraphIterator;

import com.ibm.fhir.cache.CacheKey;
import com.ibm.fhir.cache.CacheManager;
import com.ibm.fhir.cache.CacheManager.Configuration;
Expand All @@ -48,6 +58,7 @@
import com.ibm.fhir.model.type.Element;
import com.ibm.fhir.model.type.Integer;
import com.ibm.fhir.model.type.String;
import com.ibm.fhir.model.type.Uri;
import com.ibm.fhir.model.type.code.CodeSystemHierarchyMeaning;
import com.ibm.fhir.model.type.code.FilterOperator;
import com.ibm.fhir.model.type.code.IssueSeverity;
Expand All @@ -68,6 +79,10 @@ public final class CodeSystemSupport {
public static final Configuration ANCESTORS_AND_SELF_CACHE_CONFIG = Configuration.of(128);
public static final Configuration DESCENDANTS_AND_SELF_CACHE_CONFIG = Configuration.of(128);

private static final java.lang.String PARENT_PROP = "http://hl7.org/fhir/concept-properties#parent";
private static final java.lang.String CHILD_PROP = "http://hl7.org/fhir/concept-properties#child";
private static final Set<java.lang.String> PARENT_CHILD_PROPS = Set.of(PARENT_PROP, CHILD_PROP);

/**
* A function that maps a code system concept to its code value
*/
Expand Down Expand Up @@ -232,6 +247,124 @@ public static CodeSystem getCodeSystem(java.lang.String url) {
return FHIRRegistry.getInstance().getResource(url, CodeSystem.class);
}

/**
* @param codeSystem
* @return
* true if the passed codeSystem has a property-based concept hierarchy
*/
public static boolean hasPropertyHierarchy(CodeSystem codeSystem) {
for (CodeSystem.Property p : codeSystem.getProperty()) {
Uri uri = p.getUri();
if (uri != null && PARENT_CHILD_PROPS.contains(uri.getValue())) {
return true;
}
}
return false;
}

/**
* Convert the passed codeSystem into a "simple" CodeSystem...one where all concept hierarchy is expressed
* through nested concepts rather than properties.
*
* @param codeSystem
* @return
* the code system associated with the given input parameter, or null if no such code system exists
* @implSpec for CodeSystems with no parent or child properties, this will return the same CodeSystem that was passed
*/
public static CodeSystem convertToSimpleCodeSystem(CodeSystem codeSystem) {
if (!hasPropertyHierarchy(codeSystem)) {
return codeSystem;
}
// for (Concept c : codeSystem.getConcept()) {
// if (!c.getConcept().isEmpty()) {
// throw new UnsupportedOperationException("Mixing property-based hierarchy with nested concept hierarchy is not yet supported.");
// }
// }

CodeSystem.Builder codeSystemBuilder = codeSystem.toBuilder();

// the concepts are initially created as builders and then overwritten as we build them up from the leaf concepts
Map<java.lang.String, Object> concepts = new HashMap<>();
Graph<java.lang.String, DefaultEdge> g = new DefaultDirectedGraph<>(DefaultEdge.class);

Set<java.lang.String> parentProps = new HashSet<>();
Set<java.lang.String> childProps = new HashSet<>();
for (CodeSystem.Property p : codeSystem.getProperty()) {
Uri uri = p.getUri();
if (uri == null) {
continue;
}
if ("http://hl7.org/fhir/concept-properties#parent".equals(uri.getValue())) {
parentProps.add(p.getCode().getValue());
} else if ("http://hl7.org/fhir/concept-properties#child".equals(uri.getValue())) {
childProps.add(p.getCode().getValue());
}
}

// initialize the map of concepts and the directed graph for each code in the system
for (Concept c : codeSystem.getConcept()) {
java.lang.String codeValue = c.getCode().getValue();
Object prev = concepts.put(codeValue, c.toBuilder());
if (prev != null) {
java.lang.String msg = "Code '" + codeValue + "' is duplicated in the CodeSystem";
LOG.fine(() -> msg + ": " + codeSystem);
throw new UnsupportedOperationException(msg);
}
g.addVertex(codeValue);
}

// write all parent/child propties to the graph as directed edges (parent->child)
for (Concept c : codeSystem.getConcept()) {
java.lang.String codeValue = c.getCode().getValue();
for (Concept.Property p : c.getProperty()) {
if (parentProps.contains(p.getCode().getValue()) && p.getValue().is(Code.class)) {
java.lang.String parentCode = p.getValue().as(Code.class).getValue();
g.addEdge(parentCode, codeValue);
} else if (childProps.contains(p.getCode().getValue()) && p.getValue().is(Code.class)) {
java.lang.String childCode = p.getValue().as(Code.class).getValue();
g.addEdge(codeValue, childCode);
}
}
}

// Build the concepts depth-first so that a given concept's children will always be built before that concept is built
// and store the topLevelConcepts so that we can add them to the CodeSystem
Set<Concept> topLevelConcepts = new HashSet<>();
GraphIterator<java.lang.String, DefaultEdge> iterator = new DepthFirstIterator<>(g);
iterator.addTraversalListener(new TraversalListenerAdapter<java.lang.String, DefaultEdge>() {
@Override
public void vertexFinished(VertexTraversalEvent<java.lang.String> e) {
java.lang.String code = e.getVertex();

Object object = concepts.get(code);
if (!(object instanceof Concept.Builder)) {
throw new IllegalStateException("object should not have been built yet!");
}

Concept.Builder builder = (Concept.Builder) object;
for (DefaultEdge edge : g.outgoingEdgesOf(code)) {
object = concepts.get(g.getEdgeTarget(edge));
if (!(object instanceof Concept)) {
throw new IllegalStateException("object should have been built but wasn't; check CodeSystem for cycles!");
}
builder.concept((Concept) object);
}

Concept concept = builder.build();
concepts.put(code, concept);
if (g.inDegreeOf(code) == 0) {
topLevelConcepts.add(concept);
}
}
});
while (iterator.hasNext()) {
// JGraphT walks the graph top-down, but we do our work on the way back up via the TraversalListener above
iterator.next();
}

return codeSystemBuilder.concept(topLevelConcepts).build();
}

/**
* Get the code system filter with the given property code and filter operator.
*
Expand Down Expand Up @@ -742,17 +875,18 @@ private static List<ConceptFilter> buildConceptFilters(CodeSystem codeSystem, Li

private static FHIRTermException conceptFilterNotCreated(Class<? extends ConceptFilter> conceptFilterType, Filter filter) {
java.lang.String message = java.lang.String.format("%s not created (property: %s, op: %s, value: %s)",
conceptFilterType.getSimpleName(),
filter.getProperty().getValue(),
filter.getOp().getValue(),
filter.getValue().getValue());
conceptFilterType.getSimpleName(),
filter.getProperty().getValue(),
filter.getOp().getValue(),
filter.getValue().getValue());

throw new FHIRTermException(message, Collections.singletonList(Issue.builder()
.severity(IssueSeverity.ERROR)
.code(IssueType.NOT_SUPPORTED)
.details(CodeableConcept.builder()
.text(string(message))
.build())
.build()));
.severity(IssueSeverity.ERROR)
.code(IssueType.NOT_SUPPORTED)
.details(CodeableConcept.builder()
.text(string(message))
.build())
.build()));
}

private static Code code(String value) {
Expand Down Expand Up @@ -840,6 +974,7 @@ private static ConceptFilter createIsAFilter(CodeSystem codeSystem, Include.Filt
if ("concept".equals(filter.getProperty().getValue()) &&
(CodeSystemHierarchyMeaning.IS_A.equals(codeSystem.getHierarchyMeaning()) ||
codeSystem.getHierarchyMeaning() == null)) {

Concept concept = findConcept(codeSystem, code(filter.getValue()));
if (concept != null) {
return new IsAFilter(codeSystem, concept);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
/*
* (C) Copyright IBM Corp. 2022
*
* SPDX-License-Identifier: Apache-2.0
*/
package com.ibm.fhir.term.service.test;

import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertTrue;

import java.io.Reader;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.HashSet;
import java.util.Set;
import java.util.stream.Collectors;

import org.testng.annotations.Test;

import com.ibm.fhir.model.format.Format;
import com.ibm.fhir.model.parser.FHIRParser;
import com.ibm.fhir.model.resource.ValueSet;
import com.ibm.fhir.term.util.ValueSetSupport;

public class HL7TermValueSetExpansionTest {
FHIRParser parser = FHIRParser.parser(Format.JSON);

/**
* Compare FHIRTermService ValueSet expansion to expanded ValueSets obtained from https://tx.fhir.org
*
* Note: tx.fhir.org has seems to have some stale expansions. In all of the following cases, our expansion
* differed and I think our expansion is right for the version of the hl7.terminology package we're using (3.1.0).
*
* <ul>
* <li> ValueSet-parent-relationship-codes.json
* <li> ValueSet-v3-Confidentiality.json
* <li> ValueSet-v3-PurposeOfUse.json
* </ul>
*/
@Test
void testHl7TermValueSetExpansion() throws Exception {
Path expandedValueSetsDir = Path.of("src/test/resources/tx-expanded-valuesets");

Set<Path> valueSetFiles = Files.list(expandedValueSetsDir).collect(Collectors.toSet());
for (Path path : valueSetFiles) {
try (Reader reader = Files.newBufferedReader(path)) {
ValueSet txExpandedValueSet = parser.parse(reader);
String url = txExpandedValueSet.getUrl().getValue();

// the version might not match, but these ValueSets shouldn't change much and so thats probably ok
ValueSet ourValueSet = ValueSetSupport.getValueSet(url);
ValueSet ourExpandedValueSet = ValueSetSupport.expand(ourValueSet);

ValueSet.Expansion ourExpansion = ourExpandedValueSet.getExpansion();
assertNotNull(ourExpansion);

Set<String> ourConcepts = new HashSet<>();
for (ValueSet.Expansion.Contains concept : ourExpandedValueSet.getExpansion().getContains()) {
ourConcepts.add(concept.getSystem().getValue() + "|" + concept.getCode().getValue());
}

Set<String> theirConcepts = new HashSet<>();
for (ValueSet.Expansion.Contains concept : txExpandedValueSet.getExpansion().getContains()) {
theirConcepts.add(concept.getSystem().getValue() + "|" + concept.getCode().getValue());
}

Set<String> diffSet;

diffSet = new HashSet<>(theirConcepts);
diffSet.removeAll(ourConcepts);
assertTrue(diffSet.isEmpty(), "all their codes are in our set: " + diffSet.toString());

diffSet = new HashSet<>(ourConcepts);
diffSet.removeAll(theirConcepts);
assertTrue(diffSet.isEmpty(), "all our codes are in their set: " + diffSet.toString());
}
}
}
}
Loading

0 comments on commit deaff35

Please sign in to comment.