Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

SO-5596: snapshot import with unpublished references #1157

Merged
merged 4 commits into from
May 10, 2023
Merged

SO-5596: snapshot import with unpublished references #1157

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
6ca4969
refactor(test): rename method in SnomedImportApiTest
cmark May 10, 2023
838e5b8
chore(test): fix incorrect case in import api test method names
cmark May 10, 2023
8df4529
fix(snomed): properly process Snapshot RF2 files with mixed...
cmark May 10, 2023
824e44e
fix(test): give a better name to the newly introduced snapshot/delta...
cmark May 10, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Binary file added ...med/core/rest/io/SnomedCT_RF2Release_INT_20180223_content_w_and_wo_effective_time_mix.zip
View file
Open in desktop
Binary file not shown.
125 changes: 82 additions & 43 deletions ...e.rest.tests/src/com/b2international/snowowl/snomed/core/rest/io/SnomedImportApiTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@
import com.b2international.snowowl.snomed.core.rest.SnomedComponentType;
import com.b2international.snowowl.snomed.datastore.SnomedRefSetUtil;
import com.b2international.snowowl.snomed.datastore.request.rf2.SnomedRf2ImportRequestBuilder;
import com.google.common.collect.ImmutableMap;
import com.b2international.snowowl.test.commons.rest.RestExtensions;

import io.restassured.response.ValidatableResponse;

Expand Down Expand Up @@ -96,11 +96,11 @@ private void importArchive(IBranchPath path, boolean createVersion, Rf2ReleaseTy
}

private void importArchive(IBranchPath path, List<String> ignoreMissingReferencesIn, boolean createVersion, Rf2ReleaseType releaseType, final String fileName) {
final Map<String, ?> importConfiguration = ImmutableMap.<String, Object>builder()
.put("type", releaseType.name())
.put("createVersions", createVersion)
.put("ignoreMissingReferencesIn", ignoreMissingReferencesIn)
.build();
var importConfiguration = Map.of(
"type", releaseType.name(),
"createVersions", createVersion,
"ignoreMissingReferencesIn", ignoreMissingReferencesIn
);
importArchive(path, importConfiguration, fileName);
}

Expand All @@ -120,7 +120,7 @@ private void importArchive(final IBranchPath branchPath, Map<String, ?> importCo
.setToolingId(SnomedTerminologyComponentConstants.TOOLING_ID)
.setUrl(SnomedTerminologyComponentConstants.SNOMED_URI_SCT + "/" + codeSystemId)
.setTitle(codeSystemId)
.build("[email protected]", "Created new code system " + codeSystemId)
.build(RestExtensions.USER, "Created new code system " + codeSystemId)
.execute(getBus())
.getSync(1L, TimeUnit.MINUTES);
}
Expand All @@ -133,10 +133,10 @@ private void importArchive(final IBranchPath branchPath, Map<String, ?> importCo
@Test
public void import01InvalidBranchPath() throws Exception {
final IBranchPath branchPath = BranchPathUtils.createPath("MAIN/notfound");
final Map<String, ?> importConfiguration = ImmutableMap.<String, Object>builder()
.put("type", Rf2ReleaseType.DELTA.name())
.put("createVersions", false)
.build();
var importConfiguration = Map.of(
"type", Rf2ReleaseType.DELTA.name(),
"createVersions", false
);
final String importId = lastPathSegment(doImport(branchPath, importConfiguration, getClass(), "SnomedCT_Release_INT_20150131_new_concept.zip").statusCode(201)
.extract().header("Location"));
waitForImportJob(branchPath, importId).statusCode(200).body("status", equalTo(RemoteJobState.FAILED.name()));
Expand Down Expand Up @@ -250,10 +250,10 @@ public void import11ExtensionConceptWithVersion() throws Exception {
createCodeSystem(branchPath, "SNOMEDCT-NE").statusCode(201);
getComponent(branchPath, SnomedComponentType.CONCEPT, "555231000005107").statusCode(404);

final Map<String, ?> importConfiguration = ImmutableMap.<String, Object>builder()
.put("type", Rf2ReleaseType.DELTA.name())
.put("createVersions", true)
.build();
var importConfiguration = Map.of(
"type", Rf2ReleaseType.DELTA.name(),
"createVersions", true
);

importArchive(branchPath, importConfiguration, "SnomedCT_Release_INT_20150205_new_extension_concept.zip");
getComponent(branchPath, SnomedComponentType.CONCEPT, "555231000005107").statusCode(200);
Expand All @@ -262,73 +262,73 @@ public void import11ExtensionConceptWithVersion() throws Exception {

@Test
public void import12OnlyPubContentWithVersioning() throws Exception {
validateBranchHeadtimestampUpdate(branchPath,
importDeltaAndValidateBranchHeadTimestampUpdate(branchPath,
"SnomedCT_RF2Release_INT_20180223_content_with_effective_time.zip", true);
}

@Test
public void import13OnlyPubContentWithOutVersioning() throws Exception {
validateBranchHeadtimestampUpdate(branchPath,
public void import13OnlyPubContentWithoutVersioning() throws Exception {
importDeltaAndValidateBranchHeadTimestampUpdate(branchPath,
"SnomedCT_RF2Release_INT_20180223_content_with_effective_time.zip", false);
}

@Test
public void import14PubAndUnpubContentWithVersioning() throws Exception {
validateBranchHeadtimestampUpdate(branchPath,
importDeltaAndValidateBranchHeadTimestampUpdate(branchPath,
"SnomedCT_RF2Release_INT_20180223_content_w_and_wo_effective_time.zip", true);
}

@Test
public void import15PubAndUnpubContentWithOutVersioning() throws Exception {
validateBranchHeadtimestampUpdate(branchPath,
public void import15PubAndUnpubContentWithoutVersioning() throws Exception {
importDeltaAndValidateBranchHeadTimestampUpdate(branchPath,
"SnomedCT_RF2Release_INT_20180223_content_w_and_wo_effective_time.zip", false);
}

@Test
public void import16OnlyUnpubContentWithoutVersioning() throws Exception {
validateBranchHeadtimestampUpdate(branchPath,
importDeltaAndValidateBranchHeadTimestampUpdate(branchPath,
"SnomedCT_RF2Release_INT_20180223_content_without_effective_time.zip", false);
}

@Test
public void import17OnlyUnpubContentWithVersioning() throws Exception {
validateBranchHeadtimestampUpdate(branchPath,
importDeltaAndValidateBranchHeadTimestampUpdate(branchPath,
"SnomedCT_RF2Release_INT_20180223_content_without_effective_time.zip", true);
}

@Test
public void import18OnlyPubRefsetMembersWithVersioning() throws Exception {
validateBranchHeadtimestampUpdate(branchPath,
importDeltaAndValidateBranchHeadTimestampUpdate(branchPath,
"SnomedCT_RF2Release_INT_20180223_only_refset_w_effective_time.zip", true);
}

@Test
public void import19OnlyPubRefsetMembersWithoutVersioning() throws Exception {
validateBranchHeadtimestampUpdate(branchPath,
importDeltaAndValidateBranchHeadTimestampUpdate(branchPath,
"SnomedCT_RF2Release_INT_20180223_only_refset_w_effective_time.zip", false);
}

@Test
public void import20PubAndUnpubRefsetMembersWithVersioning() throws Exception {
validateBranchHeadtimestampUpdate(branchPath,
importDeltaAndValidateBranchHeadTimestampUpdate(branchPath,
"SnomedCT_RF2Release_INT_20180223_only_refset_w_and_wo_effective_time.zip", true);
}

@Test
public void import21PubAndUnpubRefsetMembersWithoutVersioning() throws Exception {
validateBranchHeadtimestampUpdate(branchPath,
importDeltaAndValidateBranchHeadTimestampUpdate(branchPath,
"SnomedCT_RF2Release_INT_20180223_only_refset_w_and_wo_effective_time.zip", false);
}

@Test
public void import22OnlyUnpubRefsetMembersWithoutVersioning() throws Exception {
validateBranchHeadtimestampUpdate(branchPath,
importDeltaAndValidateBranchHeadTimestampUpdate(branchPath,
"SnomedCT_RF2Release_INT_20180223_only_refset_wo_effective_time.zip", false);
}

@Test
public void import23OnlyUnpubRefsetMembersWithVersioning() throws Exception {
validateBranchHeadtimestampUpdate(branchPath,
importDeltaAndValidateBranchHeadTimestampUpdate(branchPath,
"SnomedCT_RF2Release_INT_20180223_only_refset_wo_effective_time.zip", true);
}

Expand Down Expand Up @@ -557,10 +557,10 @@ public void import32MissingComponentsWithoutSkip() throws Exception {
public void import33CreateVersionFromBranch() throws Exception {
try {
SnomedRf2ImportRequestBuilder.disableVersionsOnChildBranches();
final Map<String, ?> importConfiguration = ImmutableMap.<String, Object>builder()
.put("type", Rf2ReleaseType.DELTA.name())
.put("createVersions", true)
.build();
var importConfiguration = Map.of(
"type", Rf2ReleaseType.DELTA.name(),
"createVersions", true
);
final String importId = lastPathSegment(doImport(branchPath, importConfiguration, getClass(), "SnomedCT_Release_INT_20210502_concept_wo_eff_time.zip").statusCode(201)
.extract().header("Location"));
waitForImportJob(branchPath, importId).statusCode(200).body("status", equalTo(RemoteJobState.FAILED.name()));
Expand All @@ -573,7 +573,7 @@ public void import33CreateVersionFromBranch() throws Exception {
public void import34ImportWithAuthor() throws Exception {
createCodeSystem(branchPath, "SNOMEDCT-AUTR").statusCode(201);

final Map<String, ?> importConfiguration = Map.of(
var importConfiguration = Map.of(
"type", Rf2ReleaseType.DELTA.name(),
"createVersions", true
);
Expand Down Expand Up @@ -639,11 +639,11 @@ public void import37NewOwlParentInSecondImportBatch() {
// the batch size must be lowered to 1000, so then the import will try to consume changes in two sets
// the concept with id 10004011000154102 will get a new stated parent (2911000154100) via an OWL axiom which must be consumed in the same batch

Map<String, Object> configuration = ImmutableMap.<String, Object>builder()
.put("type", Rf2ReleaseType.DELTA)
.put("createVersions", false)
.put("batchSize", 1000)
.build();
var configuration = Map.of(
"type", Rf2ReleaseType.DELTA.name(),
"createVersions", false,
"batchSize", 1000
);

importArchive(branchPath, configuration, "SnomedCT_RF2Release_INT_20230421T120000Z_new_owl_parent_in_second_import_batch.zip");

Expand All @@ -653,10 +653,42 @@ public void import37NewOwlParentInSecondImportBatch() {
.body("statedAncestorIds", equalTo(List.of(IComponent.ROOT_ID, Concepts.ROOT_CONCEPT, "404684003")));

}

private void validateBranchHeadtimestampUpdate(IBranchPath branch, String importArchiveFileName,

@Test
public void import38SnapshotContentWithUnpublishedReferences() throws Exception {
importAndValidateBranchHeadTimestampUpdate(branchPath, "SnomedCT_RF2Release_INT_20180223_content_w_and_wo_effective_time_mix.zip", false, Rf2ReleaseType.SNAPSHOT);
}

@Test
public void import39SnapshotContentWithUnpublishedReferences() throws Exception {
cmark marked this conversation as resolved.
Show resolved Hide resolved
var codeSystemId = branchPath.lastSegment();
CodeSystemRequests.prepareNewCodeSystem()
.setBranchPath(branchPath.getPath())
.setId(codeSystemId)
.setToolingId(SnomedTerminologyComponentConstants.TOOLING_ID)
.setUrl(SnomedTerminologyComponentConstants.SNOMED_URI_SCT + "/" + codeSystemId)
.setTitle(codeSystemId)
.build(RestExtensions.USER, "Created new code system " + codeSystemId)
.execute(getBus())
.getSync(1L, TimeUnit.MINUTES);

var importConfiguration = Map.of(
"type", Rf2ReleaseType.DELTA.name(),
"createVersions", false
);

final String importId = lastPathSegment(doImport(branchPath, importConfiguration, getClass(), "SnomedCT_RF2Release_INT_20180223_content_w_and_wo_effective_time_mix.zip").statusCode(201)
.extract().header("Location"));
waitForImportJob(branchPath, importId).statusCode(200).body("status", equalTo(RemoteJobState.FINISHED.name()));
}

private void importDeltaAndValidateBranchHeadTimestampUpdate(IBranchPath branch, String importArchiveFileName,
boolean createVersions) {
importAndValidateBranchHeadTimestampUpdate(branchPath, importArchiveFileName, createVersions, Rf2ReleaseType.DELTA);
}


private void importAndValidateBranchHeadTimestampUpdate(IBranchPath branch, String importArchiveFileName, boolean createVersions, Rf2ReleaseType rf2ReleaseType) {
ValidatableResponse response = branching.getBranch(branch);

String baseTimestamp = response.extract().jsonPath().getString("baseTimestamp");
Expand All @@ -676,11 +708,18 @@ private void validateBranchHeadtimestampUpdate(IBranchPath branch, String import
.setToolingId(SnomedTerminologyComponentConstants.TOOLING_ID)
.setUrl(SnomedTerminologyComponentConstants.SNOMED_URI_SCT + "/" + codeSystemId)
.setTitle(codeSystemId)
.build("[email protected]", "Created new code system " + codeSystemId)
.build(RestExtensions.USER, "Created new code system " + codeSystemId)
.execute(getBus())
.getSync(1L, TimeUnit.MINUTES);

importArchive(branch, createVersions, Rf2ReleaseType.DELTA, importArchiveFileName);
var importConfiguration = Map.of(
"type", rf2ReleaseType.name(),
"createVersions", createVersions
);

final String importId = lastPathSegment(doImport(branchPath, importConfiguration, getClass(), importArchiveFileName).statusCode(201)
.extract().header("Location"));
waitForImportJob(branchPath, importId).statusCode(200).body("status", equalTo(RemoteJobState.FINISHED.name()));

ValidatableResponse response2 = branching.getBranch(branch);

Expand Down
12 changes: 6 additions & 6 deletions .../src/com/b2international/snowowl/snomed/datastore/request/rf2/SnomedRf2ImportRequest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -389,14 +389,14 @@ private void readFile(ZipEntry entry, final InputStream in, final ObjectReader o
}

private String getEffectiveTimeKey(final String effectiveTime) {
if (Strings.isNullOrEmpty(effectiveTime)) {
// Unset effective time rows are getting their own time slice in all import modes
return EffectiveTimes.UNSET_EFFECTIVE_TIME_LABEL;
} else if (Rf2ReleaseType.SNAPSHOT == releaseType) {
// All other rows are imported in a single run in snapshot mode
if (Rf2ReleaseType.SNAPSHOT == releaseType) {
// in case of Snapshot import treat all rows as a single slice
return Rf2EffectiveTimeSlice.SNAPSHOT_SLICE;
} if (Strings.isNullOrEmpty(effectiveTime)) {
// Otherwise if there is not effective time in the current row use the unset effective time label slice
return EffectiveTimes.UNSET_EFFECTIVE_TIME_LABEL;
} else {
// Delta and full modes, however, import each "chronological layer" in order
// if there is a valid effectiveTime, then use it to import each "chronological layer" in order
return effectiveTime;
}
}
Expand Down