Merge pull request #165 from IvanIsCoding/dcm4che-and-integration-tests

Update dcm4che to 5.31.2

dicom_util/build.gradle

@@ -38,8 +38,8 @@ dependencies {
     compile "com.google.api-client:google-api-client:1.30.5"
     compile "junit:junit:4.12"
     compile "org.junit.jupiter:junit-jupiter-engine:5.0.0"
-    compile "org.dcm4che:dcm4che-core:5.18.1"
-    compile "org.dcm4che:dcm4che-net:5.18.1"
+    compile "org.dcm4che:dcm4che-core:5.31.2"
+    compile "org.dcm4che:dcm4che-net:5.31.2"
 
     compile 'com.google.auth:google-auth-library-oauth2-http:0.17.2'
     compile group: 'org.eclipse.jetty.http2', name: 'http2-client', version: '9.4.20.v20190813'

export/build.gradle

@@ -57,8 +57,8 @@ dependencies {
     compile "com.beust:jcommander:1.72"
     compile 'com.google.auth:google-auth-library-oauth2-http:0.17.2'
     compile "com.google.api-client:google-api-client:1.30.5"
-    compile "org.dcm4che:dcm4che-core:5.18.1"
-    compile "org.dcm4che:dcm4che-net:5.18.1"
+    compile "org.dcm4che:dcm4che-core:5.31.2"
+    compile "org.dcm4che:dcm4che-net:5.31.2"
     compile project(":dicom_util")
     compile project(":util")
 

import/build.gradle

@@ -64,11 +64,11 @@ dependencies {
     compile "com.beust:jcommander:1.72"
     compile 'com.google.auth:google-auth-library-oauth2-http:0.17.2'
     compile "com.google.api-client:google-api-client:1.30.5"
-    compile "org.dcm4che:dcm4che-core:5.18.1"
-    compile "org.dcm4che:dcm4che-net:5.18.1"
+    compile "org.dcm4che:dcm4che-core:5.31.2"
+    compile "org.dcm4che:dcm4che-net:5.31.2"
 
-    compile "org.dcm4che:dcm4che-imageio:5.18.1"
-    runtime "org.dcm4che:dcm4che-imageio-rle:5.18.1"
+    compile "org.dcm4che:dcm4che-imageio:5.31.2"
+    runtime "org.dcm4che:dcm4che-imageio-rle:5.31.2"
     compile project(":jai-imageio-jpeg2000")
 
     compile project(":dicom_util")

import/src/main/java/com/google/cloud/healthcare/imaging/dicomadapter/CFindService.java

@@ -48,7 +48,7 @@ public class CFindService extends BasicCFindSCP {
   private final Flags cFINDFlags;
 
   CFindService(IDicomWebClient dicomWebClient, Flags flags) {
-    super(UID.StudyRootQueryRetrieveInformationModelFIND);
+    super(UID.StudyRootQueryRetrieveInformationModelFind);
     this.dicomWebClient = dicomWebClient;
     this.cFINDFlags = flags;
   }

import/src/main/java/com/google/cloud/healthcare/imaging/dicomadapter/CMoveService.java

@@ -51,7 +51,7 @@ public class CMoveService extends BasicCMoveSCP {
       IDicomWebClient dicomWebClient,
       AetDictionary aets,
       ISenderFactory senderFactory) {
-    super(UID.StudyRootQueryRetrieveInformationModelMOVE);
+    super(UID.StudyRootQueryRetrieveInformationModelMove);
     this.dicomWebClient = dicomWebClient;
     this.aets = aets;
     this.senderFactory = senderFactory;

import/src/main/java/com/google/cloud/healthcare/imaging/dicomadapter/StorageCommitmentService.java

@@ -54,7 +54,7 @@ public class StorageCommitmentService extends AbstractDicomService {
   private final String transferSyntax = UID.ImplicitVRLittleEndian;
 
   StorageCommitmentService(IDicomWebClient dicomWebClient, AetDictionary aets) {
-    super(UID.StorageCommitmentPushModelSOPClass);
+    super(UID.StorageCommitmentPushModel);
     this.dicomWebClient = dicomWebClient;
     this.aets = aets;
   }
@@ -66,11 +66,11 @@ protected void onDimseRQ(Association as, PresentationContext pc, Dimse dimse, At
       if (dimse != Dimse.N_ACTION_RQ) {
         throw new DicomServiceException(Status.UnrecognizedOperation);
       }
-      if (!cmd.getString(Tag.RequestedSOPClassUID).equals(UID.StorageCommitmentPushModelSOPClass)) {
+      if (!cmd.getString(Tag.RequestedSOPClassUID).equals(UID.StorageCommitmentPushModel)) {
         throw new DicomServiceException(Status.NoSuchSOPclass);
       }
       if (!cmd.getString(Tag.RequestedSOPInstanceUID)
-          .equals(UID.StorageCommitmentPushModelSOPInstance)) {
+          .equals(UID.StorageCommitmentPushModelInstance)) {
         throw new DicomServiceException(Status.NoSuchObjectInstance);
       }
       int actionTypeID = cmd.getInt(Tag.ActionTypeID, 0);
@@ -187,8 +187,8 @@ public void run() {
 
         int eventTypeId =
             absentInstances.size() > 0 ? EVENT_ID_FAILURES_PRESENT : EVENT_ID_ALL_SUCCESS;
-        association.neventReport(UID.StorageCommitmentPushModelSOPClass,
-            UID.StorageCommitmentPushModelSOPInstance,
+        association.neventReport(UID.StorageCommitmentPushModel,
+            UID.StorageCommitmentPushModelInstance,
             eventTypeId,
             makeDataset(presentInstances, absentInstances),
             transferSyntax,
@@ -219,10 +219,10 @@ private AAssociateRQ makeAAssociateRQ() {
       aarq.addPresentationContext(
           new PresentationContext(
               1,
-              UID.StorageCommitmentPushModelSOPClass,
+              UID.StorageCommitmentPushModel,
               transferSyntax));
       aarq.addRoleSelection(
-          new RoleSelection(UID.StorageCommitmentPushModelSOPClass, false, true));
+          new RoleSelection(UID.StorageCommitmentPushModel, false, true));
       return aarq;
     }
 

import/src/test/java/com/google/cloud/healthcare/imaging/dicomadapter/CFindServiceTest.java

@@ -159,15 +159,15 @@ public void basicCFindServiceTest(IDicomWebClient serverDicomWebClient,
     // Associate with peer AE.
     Association association =
         associate(serverHostname, serverPort,
-            UID.StudyRootQueryRetrieveInformationModelFIND, UID.ExplicitVRLittleEndian);
+            UID.StudyRootQueryRetrieveInformationModelFind, UID.ExplicitVRLittleEndian);
 
     Attributes findData = new Attributes();
     findData.setString(Tag.QueryRetrieveLevel, VR.CS, "IMAGE");
 
     // Issue CFIND
     DimseRSPAssert rspAssert = new DimseRSPAssert(association, expectedStatus);
     association.cfind(
-        UID.StudyRootQueryRetrieveInformationModelFIND,
+        UID.StudyRootQueryRetrieveInformationModelFind,
         1,
         findData, // I mock IDicomWebClient anyway, AttributesUtil will be tested separately
         UID.ExplicitVRLittleEndian,

import/src/test/java/com/google/cloud/healthcare/imaging/dicomadapter/CMoveServiceTest.java

@@ -251,7 +251,7 @@ public void basicCMoveServiceTest(IDicomWebClient serverDicomWebClient,
     // Associate with peer AE.
     Association association =
         associate(serverHostname, serverPort,
-            UID.StudyRootQueryRetrieveInformationModelMOVE, UID.ExplicitVRLittleEndian);
+            UID.StudyRootQueryRetrieveInformationModelMove, UID.ExplicitVRLittleEndian);
 
     Attributes moveDataset = new Attributes();
     moveDataset.setString(Tag.QueryRetrieveLevel, VR.CS, "STUDY");
@@ -260,7 +260,7 @@ public void basicCMoveServiceTest(IDicomWebClient serverDicomWebClient,
     // Issue CMOVE
     DimseRSPAssert rspAssert = new DimseRSPAssert(association, expectedStatus);
     association.cmove(
-        UID.StudyRootQueryRetrieveInformationModelMOVE,
+        UID.StudyRootQueryRetrieveInformationModelMove,
         1,
         moveDataset,
         UID.ExplicitVRLittleEndian,

import/src/test/java/com/google/cloud/healthcare/imaging/dicomadapter/StorageCommitmentServiceTest.java

@@ -145,13 +145,13 @@ public void basicCommitmentServiceTest(IDicomWebClient serverDicomWebClient,
     // Associate with peer AE.
     Association association =
         associate(serverHostname, serverPort,
-            UID.StorageCommitmentPushModelSOPClass, transferSyntax);
+            UID.StorageCommitmentPushModel, transferSyntax);
 
     // Issue N-ACTION
     DimseRSPAssert rspAssert = new DimseRSPAssert(association, expectedStatus);
     association.naction(
-        UID.StorageCommitmentPushModelSOPClass,
-        UID.StorageCommitmentPushModelSOPInstance,
+        UID.StorageCommitmentPushModel,
+        UID.StorageCommitmentPushModelInstance,
         1,
         requestData,
         transferSyntax,
@@ -205,7 +205,7 @@ private Device createSCUDevice(int listenerPort, Attributes expectReportAttrs,
 
     Device serverDevice = DeviceUtil.createServerDevice(clientAET, listenerPort, serviceRegistry,
         new TransferCapability(null,
-            UID.StorageCommitmentPushModelSOPClass,
+            UID.StorageCommitmentPushModel,
             TransferCapability.Role.SCU,
             transferSyntax));
     serverDevice.bindConnections();
@@ -219,7 +219,7 @@ private class StorageCommitmentSCUService extends AbstractDicomService {
 
     public StorageCommitmentSCUService(Attributes expectReportAttrs,
         CompletableFuture<Boolean> checkFuture) {
-      super(UID.StorageCommitmentPushModelSOPClass);
+      super(UID.StorageCommitmentPushModel);
       this.expectReportAttrs = expectReportAttrs;
       this.checkFuture = checkFuture;
     }
@@ -235,12 +235,12 @@ protected void onDimseRQ(Association as, PresentationContext pc, Dimse dimse, At
         throw new DicomServiceException(Status.UnrecognizedOperation);
       }
 
-      if (!cmd.getString(Tag.AffectedSOPClassUID).equals(UID.StorageCommitmentPushModelSOPClass)) {
+      if (!cmd.getString(Tag.AffectedSOPClassUID).equals(UID.StorageCommitmentPushModel)) {
         throw new DicomServiceException(Status.NoSuchSOPclass);
       }
 
       if (!cmd.getString(Tag.AffectedSOPInstanceUID)
-          .equals(UID.StorageCommitmentPushModelSOPInstance)) {
+          .equals(UID.StorageCommitmentPushModelInstance)) {
         throw new DicomServiceException(Status.NoSuchObjectInstance);
       }
 

integration_test/general.py

@@ -1,13 +1,14 @@
 import os
+import subprocess
 import random
 import string
 
 ERROR_CODE = 1
 
 def runCommand(command, message):
-    result = os.system(command)
-    print(message, result)
-    return result
+    result = subprocess.run(command, shell=True)
+    print(message, result.returncode)
+    return result.returncode
 
 def get_random_string(length):
     letters = string.ascii_lowercase
@@ -23,26 +24,10 @@ def get_short_sha():
 def get_imageproject():
    return os.environ.get("IMAGEPROJECT")
 
-# clear data
-def clear_data():
-    return runCommand("rm -R dcm4che", "remove dcm4che")
-
 # install environment
 def install_environment():
     return runCommand("./integration_test/scripts/install-env.sh", "install environment exit with")
 
-# clone-dcm4che
-def clone_dcm4che():
-    return runCommand("git clone https://github.com/dcm4che/dcm4che.git dcm4che", "clone-dcm4che exit with")
-
-# checkout-dcm4che-tag
-def checkout_dcm4che_tag():
-    return runCommand("cd dcm4che && git checkout \"tags/5.15.1\"", "checkout-dcm4che-tag exit with")
-
-# build-tools
-def build_tools():
-    return runCommand("./integration_test/scripts/mvn-install-tools.sh", "build-tools exit with")
-
 # store-scp
 def store_scp(store_csp_run_step, store_scp_port):
     return runCommand("./integration_test/scripts/run-store-scp-and-wait.sh " + store_csp_run_step + " "+store_scp_port, "store-scp exit with")
@@ -83,39 +68,33 @@ def wait_for_port(host, port):
 
 # run-store-scu
 def run_store_scu(host, adapter_port, file_path):
-    return runCommand("export PATH=/opt/apache-maven-3.6.3/bin:$PATH &&"
-           "cd dcm4che/dcm4che-tool/dcm4che-tool-storescu &&"
-           "mvn -ntp exec:java -Dexec.mainClass=org.dcm4che3.tool.storescu.StoreSCU -Dexec.args=-\"c IMPORTADAPTER@"+host+":"+adapter_port+" "+file_path+"\"", "run-store-scu exit with")
+    return runCommand("export PATH=/opt/dcm4che/bin:$PATH &&"
+           "storescu -c IMPORTADAPTER@"+host+":"+adapter_port+" "+file_path, "run-store-scu exit with")
 
 # run-find-scu-instance
 def run_find_scu_instance(host, adapter_port):
-    return runCommand("export PATH=/opt/apache-maven-3.6.3/bin:$PATH &&"
-           "cd /workspace/dcm4che/dcm4che-tool/dcm4che-tool-findscu &&"
-           "mvn -ntp  exec:java -Dexec.mainClass=org.dcm4che3.tool.findscu.FindSCU -Dexec.args=\"-c IMPORTADAPTER@"+host+":"+adapter_port+" -L IMAGE -X --out-cat --out-file findscu-instance-result.xml --out-dir ../../../integration_test/\"", "run-find-scu-instance exit with")
+    return runCommand("export PATH=/opt/dcm4che/bin:$PATH &&"
+           "findscu -c IMPORTADAPTER@"+host+":"+adapter_port+" -L IMAGE -X --out-cat --out-file findscu-instance-result.xml --out-dir /workspace/integration_test/", "run-find-scu-instance exit with")
 
 # run-find-scu-series
 def run_find_scu_series(host, adapter_port):
-    return runCommand("export PATH=/opt/apache-maven-3.6.3/bin:$PATH &&"
-           "cd /workspace/dcm4che/dcm4che-tool/dcm4che-tool-findscu &&"
-           "mvn -ntp  exec:java -Dexec.mainClass=org.dcm4che3.tool.findscu.FindSCU -Dexec.args=\"-c IMPORTADAPTER@"+host+":"+adapter_port+" -L SERIES -X --out-cat --out-file findscu-series-result.xml --out-dir ../../../integration_test/\"", "run-find-scu-series exit with")
+    return runCommand("export PATH=/opt/dcm4che/bin:$PATH &&"
+           "findscu -c IMPORTADAPTER@"+host+":"+adapter_port+" -L SERIES -X --out-cat --out-file findscu-series-result.xml --out-dir /workspace/integration_test/", "run-find-scu-series exit with")
 
 # run-find-scu-study
 def run_find_scu_study(host, adapter_port):
-    return runCommand("export PATH=/opt/apache-maven-3.6.3/bin:$PATH &&"
-           "cd /workspace/dcm4che/dcm4che-tool/dcm4che-tool-findscu &&"
-           "mvn -ntp  exec:java -Dexec.mainClass=org.dcm4che3.tool.findscu.FindSCU -Dexec.args=\"-c IMPORTADAPTER@"+host+":"+adapter_port+" -L STUDY -X --out-cat --out-file findscu-study-result.xml --out-dir ../../../integration_test/\"", "run-find-scu-study exit with")
+    return runCommand("export PATH=/opt/dcm4che/bin:$PATH &&"
+           "findscu -c IMPORTADAPTER@"+host+":"+adapter_port+" -L STUDY -X --out-cat --out-file findscu-study-result.xml --out-dir /workspace/integration_test/", "run-find-scu-study exit with")
 
 # run-move-scu
 def run_move_scu(host, adapter_port):
-    return runCommand("export PATH=/opt/apache-maven-3.6.3/bin:$PATH &&"
-           "cd /workspace/dcm4che/dcm4che-tool/dcm4che-tool-movescu &&"
-           "mvn -ntp  exec:java -Dexec.mainClass=org.dcm4che3.tool.movescu.MoveSCU -Dexec.args=\"-c IMPORTADAPTER@"+host+":"+adapter_port+" --dest STORESCP\"", "run-move-scu exit with")
+    return runCommand("export PATH=/opt/dcm4che/bin:$PATH &&"
+           "movescu -c IMPORTADAPTER@"+host+":"+adapter_port+" --dest STORESCP", "run-move-scu exit with")
 
 # run-commitment-scu
 def run_commitment_scu(host, adapter_port, com_scu_port, file_path):
-    return runCommand("export PATH=/opt/apache-maven-3.6.3/bin:$PATH &&"
-           "cd /workspace/dcm4che/dcm4che-tool/dcm4che-tool-stgcmtscu &&"
-           "mvn -ntp  exec:java -Dexec.mainClass=org.dcm4che3.tool.stgcmtscu.StgCmtSCU -Dexec.args=\"-c IMPORTADAPTER@"+host+":"+adapter_port+" -b STGCMTSCU:"+com_scu_port+" --explicit-vr --directory /workspace/integration_test/commitment_result "+file_path+"\"", "run-commitment-scu exit with")
+    return runCommand("export PATH=/opt/dcm4che/bin:$PATH &&"
+           "stgcmtscu -c IMPORTADAPTER@"+host+":"+adapter_port+" -b STGCMTSCU:"+com_scu_port+" --explicit-vr --directory /workspace/integration_test/commitment_result "+file_path, "run-commitment-scu exit with")
 
 # check-store-curl
 def check_store_curl(version, project, location, dataset, store_name, replaced_uid, file_path):
@@ -125,10 +104,14 @@ def check_store_curl(version, project, location, dataset, store_name, replaced_u
 def check_diff(file_path1, file_path2):
     return runCommand("diff "+file_path1+" "+file_path2, "check-diff exit with")
 
+#check-dcm-diff
+def check_diff_dcm(file_path1, file_path2):
+    return runCommand(
+           f"/workspace/integration_test/scripts/check-diff-dcm.sh {file_path1} {file_path2}", "check-diff-dcm exit with")
+
 # check-commitment-diff
 def check_commitment_diff():
-    return runCommand("chmod -R 777 dcm4che/dcm4che-tool/dcm4che-tool-dcm2xml &&"
-           "cd dcm4che/dcm4che-tool/dcm4che-tool-dcm2xml &&"
+    return runCommand(
            "/workspace/integration_test/scripts/diff-commitment-result.sh", "check-commitment-diff exit with")
 
 # delete-dicom-store

integration_test/scripts/build-adapter-image.sh

@@ -4,7 +4,7 @@
 set -e
 set -o pipefail
 
-export PATH=$PATH:/opt/gradle/gradle-6.7/bin
+export PATH=$PATH:/opt/gradle/gradle-6.9/bin
 base_name="gcr.io/${1}/healthcare-api-dicom-dicomweb-adapter-"
 
 build_adapter () {

integration_test/scripts/build-adapters.sh

@@ -3,7 +3,7 @@
 set -e
 set -o pipefail
 
-export PATH=$PATH:/opt/gradle/gradle-6.7/bin
+export PATH=$PATH:/opt/gradle/gradle-6.9/bin
 
 apt-get -qq update
 apt-get install -y gnupg2

integration_test/scripts/check-diff-dcm.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -e
+set -o pipefail
+
+export PATH=/opt/dcm4che/bin:$PATH
+tmp_dir=$(mktemp -d)
+
+dcm2json --with-bulkdata $PWD/$1 > $tmp_dir/got.json
+dcm2json --with-bulkdata $PWD/$2 > $tmp_dir/want.json
+
+# We ignore the Implementation Version Name when comparing files
+diff \
+    <(jq --sort-keys 'del(.["00020013"])' $tmp_dir/want.json) \
+    <(jq --sort-keys 'del(.["00020013"])' $tmp_dir/got.json)
+
+rm -rf $tmp_dir

integration_test/scripts/diff-commitment-result.sh

@@ -3,12 +3,16 @@
 set -e
 set -o pipefail
 
-export PATH=/opt/apache-maven-3.6.3/bin:$PATH
-
-mvn -q exec:java \
- -Dexec.mainClass=org.dcm4che3.tool.dcm2xml.Dcm2Xml \
- -Dexec.args=/workspace/integration_test/commitment_result/$(ls /workspace/integration_test/commitment_result/) \
- >/workspace/integration_test/tmp.xml
-cd /workspace/integration_test
-perl -pe 's|<DicomAttribute keyword="TransactionUID".*?<\/DicomAttribute>||' tmp.xml >commitment-clean.xml
-diff commitment-clean.xml data/commitment-expected.xml
+export PATH=/opt/dcm4che/bin:$PATH
+tmp_dir=$(mktemp -d)
+
+dcm2xml /workspace/integration_test/commitment_result/$(ls /workspace/integration_test/commitment_result/) \
+        > $tmp_dir/got.xml
+
+IGNORE_TRANSACTION_AND_IMPLEMENTATION='.NativeDicomModel.DicomAttribute[] | select(."+@keyword" != "ImplementationVersionName" and ."+@keyword" != "TransactionUID")'
+
+diff \
+	<(yq -oj "$IGNORE_TRANSACTION_AND_IMPLEMENTATION"  $tmp_dir/got.xml) \
+	<(yq -oj "$IGNORE_TRANSACTION_AND_IMPLEMENTATION"  /workspace/integration_test/data/commitment-expected.xml)
+
+rm -rf $tmp_dir