From ae77bb752b499c364b747870231470577f7343f8 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 13 Sep 2024 19:32:23 -0500 Subject: [PATCH 1/8] GODRIVER-3312 Use remaining test secrets from the vault --- .evergreen/config.yml | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 3056c9bcf1..75bbe5b1c7 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1817,10 +1817,6 @@ tasks: echo "Building build-kms-test ... end" source $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/secrets-export.sh echo "Copying files ... begin" - export GCPKMS_GCLOUD=${GCPKMS_GCLOUD} - export GCPKMS_PROJECT=${GCPKMS_PROJECT} - export GCPKMS_ZONE=${GCPKMS_ZONE} - export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME} tar czf testgcpkms.tgz ./testkms ./install/libmongocrypt/lib64/libmongocrypt.* GCPKMS_SRC=testgcpkms.tgz GCPKMS_DST=$GCPKMS_INSTANCENAME: $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/copy-file.sh echo "Copying files ... end" @@ -1837,10 +1833,6 @@ tasks: script: | ${PREPARE_SHELL} source $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/secrets-export.sh - export GCPKMS_GCLOUD=${GCPKMS_GCLOUD} - export GCPKMS_PROJECT=${GCPKMS_PROJECT} - export GCPKMS_ZONE=${GCPKMS_ZONE} - export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME} GCPKMS_CMD="LD_LIBRARY_PATH=./install/libmongocrypt/lib64 MONGODB_URI='mongodb://localhost:27017' PROVIDER='gcp' ./testkms" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh - name: "testgcpkms-fail-task" @@ -1906,9 +1898,7 @@ tasks: echo "Building build-kms-test ... end" echo "Copying files ... begin" - export AZUREKMS_RESOURCEGROUP=${AZUREKMS_RESOURCEGROUP} - export AZUREKMS_VMNAME=${AZUREKMS_VMNAME} - export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey + source $DRIVERS_TOOLS/.evergreen/csfle/azurekms/secrets-export.sh tar czf testazurekms.tgz ./testkms ./install/libmongocrypt/lib64/libmongocrypt.* AZUREKMS_SRC=testazurekms.tgz AZUREKMS_DST=/tmp $DRIVERS_TOOLS/.evergreen/csfle/azurekms/copy-file.sh echo "Copying files ... end" @@ -1924,9 +1914,6 @@ tasks: script: | ${PREPARE_SHELL} source ${DRIVERS_TOOLS}/.evergreen/csfle/azurekms/setup-secrets.sh - export AZUREKMS_RESOURCEGROUP=${AZUREKMS_RESOURCEGROUP} - export AZUREKMS_VMNAME=${AZUREKMS_VMNAME} - export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey AZUREKMS_CMD="LD_LIBRARY_PATH=./install/libmongocrypt/lib64 MONGODB_URI='mongodb://localhost:27017' PROVIDER='azure' AZUREKMS_KEY_NAME='${AZUREKMS_KEY_NAME}' AZUREKMS_KEY_VAULT_ENDPOINT='${AZUREKMS_KEY_VAULT_ENDPOINT}' ./testkms" $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh - name: "testazurekms-fail-task" @@ -1945,7 +1932,8 @@ tasks: PKG_CONFIG_PATH=$PKG_CONFIG_PATH \ make build-kms-test echo "Building build-kms-test ... end" - + source $DRIVERS_TOOLS/.evergreen/csfle/azurekms/secrets-export.sh + LD_LIBRARY_PATH=./install/libmongocrypt/lib64 \ MONGODB_URI='mongodb://localhost:27017' \ EXPECT_ERROR='unable to retrieve azure credentials' \ From a8e7d118b0ab1c16df3aab617e82fbe6fac5f018 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 13 Sep 2024 20:08:36 -0500 Subject: [PATCH 2/8] fix azure kms handling --- .evergreen/config.yml | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 75bbe5b1c7..2d0a2dc59c 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1932,12 +1932,11 @@ tasks: PKG_CONFIG_PATH=$PKG_CONFIG_PATH \ make build-kms-test echo "Building build-kms-test ... end" - source $DRIVERS_TOOLS/.evergreen/csfle/azurekms/secrets-export.sh LD_LIBRARY_PATH=./install/libmongocrypt/lib64 \ MONGODB_URI='mongodb://localhost:27017' \ EXPECT_ERROR='unable to retrieve azure credentials' \ - PROVIDER='azure' AZUREKMS_KEY_NAME='${AZUREKMS_KEY_NAME}' AZUREKMS_KEY_VAULT_ENDPOINT='${AZUREKMS_KEY_VAULT_ENDPOINT}' \ + PROVIDER='azure' AZUREKMS_KEY_NAME='' AZUREKMS_KEY_VAULT_ENDPOINT='' \ ./testkms - name: "test-fuzz" @@ -2249,18 +2248,14 @@ task_groups: shell: "bash" script: | ${PREPARE_SHELL} - $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/create-and-setup-instance.sh - # Load the GCPKMS_GCLOUD, GCPKMS_INSTANCE, GCPKMS_REGION, and GCPKMS_ZONE expansions. - - command: expansions.update - params: - file: testgcpkms-expansions.yml + $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/setup.sh teardown_group: - command: shell.exec params: shell: "bash" script: | ${PREPARE_SHELL} - $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/delete-instance.sh + $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/teardown.sh - func: handle-test-artifacts - func: cleanup tasks: @@ -2281,10 +2276,7 @@ task_groups: script: | ${PREPARE_SHELL} export AZUREKMS_VMNAME_PREFIX="GODRIVER" - export AZUREKMS_DRIVERS_TOOLS=$DRIVERS_TOOLS - # Get azurekms credentials from the vault. - . ${DRIVERS_TOOLS}/.evergreen/csfle/azurekms/setup-secrets.sh - ${DRIVERS_TOOLS}/.evergreen/csfle/azurekms/create-and-setup-vm.sh + . ${DRIVERS_TOOLS}/.evergreen/csfle/azurekms/setup.sh - command: expansions.update params: file: testazurekms-expansions.yml @@ -2298,9 +2290,7 @@ task_groups: shell: "bash" script: | ${PREPARE_SHELL} - export AZUREKMS_SCOPE=${AZUREKMS_SCOPE} - export AZUREKMS_RESOURCEGROUP=${AZUREKMS_RESOURCEGROUP} - $DRIVERS_TOOLS/.evergreen/csfle/azurekms/delete-vm.sh + $DRIVERS_TOOLS/.evergreen/csfle/azurekms/teardown.sh - func: handle-test-artifacts - func: cleanup tasks: From 59b85b7c783005d81afeafc034ffd2c3eaab2919 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 13 Sep 2024 20:22:53 -0500 Subject: [PATCH 3/8] try again --- .evergreen/config.yml | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 2d0a2dc59c..2e44607657 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -2276,21 +2276,17 @@ task_groups: script: | ${PREPARE_SHELL} export AZUREKMS_VMNAME_PREFIX="GODRIVER" - . ${DRIVERS_TOOLS}/.evergreen/csfle/azurekms/setup.sh - - command: expansions.update - params: - file: testazurekms-expansions.yml + export AZUREKMS_DRIVERS_TOOLS=$DRIVERS_TOOLS + # Get azurekms credentials from the vault. + . ${DRIVERS_TOOLS}/.evergreen/csfle/azurekms/setup-secrets.sh + ${DRIVERS_TOOLS}/.evergreen/csfle/azurekms/create-and-setup-vm.sh teardown_group: - # setup_group may have failed before updating expansions. Try to update again to clean up resources on setup failure. - - command: expansions.update - params: - file: testazurekms-expansions.yml - - command: shell.exec + - command: subprocess.exec params: - shell: "bash" - script: | - ${PREPARE_SHELL} - $DRIVERS_TOOLS/.evergreen/csfle/azurekms/teardown.sh + binary: "bash" + args: + - ${DRIVERS_TOOLS}/.evergreen/csfle/azurekms/teardown.sh + - func: handle-test-artifacts - func: cleanup tasks: From 685267f1fa06b7af6c53a4835999b06d1ac53dfc Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 13 Sep 2024 20:25:14 -0500 Subject: [PATCH 4/8] fix fail task --- .evergreen/config.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 2e44607657..54497be45b 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1932,11 +1932,13 @@ tasks: PKG_CONFIG_PATH=$PKG_CONFIG_PATH \ make build-kms-test echo "Building build-kms-test ... end" + + . ${DRIVERS_TOOLS}/.evergreen/csfle/azurekms/setup-secrets.sh LD_LIBRARY_PATH=./install/libmongocrypt/lib64 \ MONGODB_URI='mongodb://localhost:27017' \ EXPECT_ERROR='unable to retrieve azure credentials' \ - PROVIDER='azure' AZUREKMS_KEY_NAME='' AZUREKMS_KEY_VAULT_ENDPOINT='' \ + PROVIDER='azure' AZUREKMS_KEY_NAME='$AZUREKMS_KEY_NAME' AZUREKMS_KEY_VAULT_ENDPOINT='$AZUREKMS_KEY_VAULT_ENDPOINT' \ ./testkms - name: "test-fuzz" From daf5f78b20030555c6fdd315d217b3196bbcc70d Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 13 Sep 2024 20:35:04 -0500 Subject: [PATCH 5/8] try again --- .evergreen/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 54497be45b..ca61d36f20 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1914,7 +1914,7 @@ tasks: script: | ${PREPARE_SHELL} source ${DRIVERS_TOOLS}/.evergreen/csfle/azurekms/setup-secrets.sh - AZUREKMS_CMD="LD_LIBRARY_PATH=./install/libmongocrypt/lib64 MONGODB_URI='mongodb://localhost:27017' PROVIDER='azure' AZUREKMS_KEY_NAME='${AZUREKMS_KEY_NAME}' AZUREKMS_KEY_VAULT_ENDPOINT='${AZUREKMS_KEY_VAULT_ENDPOINT}' ./testkms" $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh + AZUREKMS_CMD="LD_LIBRARY_PATH=./install/libmongocrypt/lib64 MONGODB_URI='mongodb://localhost:27017' PROVIDER='azure' AZUREKMS_KEY_NAME=${AZUREKMS_KEY_NAME} AZUREKMS_KEY_VAULT_ENDPOINT=${AZUREKMS_KEY_VAULT_ENDPOINT} ./testkms" $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh - name: "testazurekms-fail-task" # testazurekms-fail-task runs without environment variables. @@ -1938,7 +1938,7 @@ tasks: LD_LIBRARY_PATH=./install/libmongocrypt/lib64 \ MONGODB_URI='mongodb://localhost:27017' \ EXPECT_ERROR='unable to retrieve azure credentials' \ - PROVIDER='azure' AZUREKMS_KEY_NAME='$AZUREKMS_KEY_NAME' AZUREKMS_KEY_VAULT_ENDPOINT='$AZUREKMS_KEY_VAULT_ENDPOINT' \ + PROVIDER='azure' AZUREKMS_KEY_NAME="$AZUREKMS_KEY_NAME" AZUREKMS_KEY_VAULT_ENDPOINT="$AZUREKMS_KEY_VAULT_ENDPOINT" \ ./testkms - name: "test-fuzz" From 74345f2877318b77fc49c4ecccb600009d622615 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 13 Sep 2024 20:47:02 -0500 Subject: [PATCH 6/8] try again --- .evergreen/config.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index ca61d36f20..4f7665af0f 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1913,7 +1913,7 @@ tasks: working_dir: src/go.mongodb.org/mongo-driver script: | ${PREPARE_SHELL} - source ${DRIVERS_TOOLS}/.evergreen/csfle/azurekms/setup-secrets.sh + source ${DRIVERS_TOOLS}/.evergreen/csfle/azurekms/secrets-export.sh AZUREKMS_CMD="LD_LIBRARY_PATH=./install/libmongocrypt/lib64 MONGODB_URI='mongodb://localhost:27017' PROVIDER='azure' AZUREKMS_KEY_NAME=${AZUREKMS_KEY_NAME} AZUREKMS_KEY_VAULT_ENDPOINT=${AZUREKMS_KEY_VAULT_ENDPOINT} ./testkms" $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh - name: "testazurekms-fail-task" @@ -1933,7 +1933,8 @@ tasks: make build-kms-test echo "Building build-kms-test ... end" - . ${DRIVERS_TOOLS}/.evergreen/csfle/azurekms/setup-secrets.sh + bash ${DRIVERS_TOOLS}/.evergreen/csfle/azurekms/setup-secrets.sh + source ${DRIVERS_TOOLS}/.evergreen/csfle/azurekms/secrets-export.sh LD_LIBRARY_PATH=./install/libmongocrypt/lib64 \ MONGODB_URI='mongodb://localhost:27017' \ From 2ad7e6758988c42497b7ab3b03c0900d5f33c601 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 13 Sep 2024 20:54:20 -0500 Subject: [PATCH 7/8] try again --- .evergreen/config.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 4f7665af0f..0149842f19 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1914,7 +1914,7 @@ tasks: script: | ${PREPARE_SHELL} source ${DRIVERS_TOOLS}/.evergreen/csfle/azurekms/secrets-export.sh - AZUREKMS_CMD="LD_LIBRARY_PATH=./install/libmongocrypt/lib64 MONGODB_URI='mongodb://localhost:27017' PROVIDER='azure' AZUREKMS_KEY_NAME=${AZUREKMS_KEY_NAME} AZUREKMS_KEY_VAULT_ENDPOINT=${AZUREKMS_KEY_VAULT_ENDPOINT} ./testkms" $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh + AZUREKMS_CMD="LD_LIBRARY_PATH=./install/libmongocrypt/lib64 MONGODB_URI='mongodb://localhost:27017' PROVIDER='azure' AZUREKMS_KEY_NAME=$AZUREKMS_KEYNAME AZUREKMS_KEY_VAULT_ENDPOINT=$AZUREKMS_KEYVAULTENDPOINT ./testkms" $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh - name: "testazurekms-fail-task" # testazurekms-fail-task runs without environment variables. @@ -1933,13 +1933,12 @@ tasks: make build-kms-test echo "Building build-kms-test ... end" - bash ${DRIVERS_TOOLS}/.evergreen/csfle/azurekms/setup-secrets.sh - source ${DRIVERS_TOOLS}/.evergreen/csfle/azurekms/secrets-export.sh + . ${DRIVERS_TOOLS}/.evergreen/csfle/azurekms/setup-secrets.sh LD_LIBRARY_PATH=./install/libmongocrypt/lib64 \ MONGODB_URI='mongodb://localhost:27017' \ EXPECT_ERROR='unable to retrieve azure credentials' \ - PROVIDER='azure' AZUREKMS_KEY_NAME="$AZUREKMS_KEY_NAME" AZUREKMS_KEY_VAULT_ENDPOINT="$AZUREKMS_KEY_VAULT_ENDPOINT" \ + PROVIDER='azure' AZUREKMS_KEY_NAME=$AZUREKMS_KEYNAME AZUREKMS_KEY_VAULT_ENDPOINT=AZUREKMS_KEYVAULTENDPOINT \ ./testkms - name: "test-fuzz" From cc7d5fff118ea494e31b23e0107f3f8b185d07fb Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 13 Sep 2024 21:03:13 -0500 Subject: [PATCH 8/8] fix var handling --- .evergreen/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 0149842f19..17e1e93221 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1938,7 +1938,7 @@ tasks: LD_LIBRARY_PATH=./install/libmongocrypt/lib64 \ MONGODB_URI='mongodb://localhost:27017' \ EXPECT_ERROR='unable to retrieve azure credentials' \ - PROVIDER='azure' AZUREKMS_KEY_NAME=$AZUREKMS_KEYNAME AZUREKMS_KEY_VAULT_ENDPOINT=AZUREKMS_KEYVAULTENDPOINT \ + PROVIDER='azure' AZUREKMS_KEY_NAME=$AZUREKMS_KEYNAME AZUREKMS_KEY_VAULT_ENDPOINT=$AZUREKMS_KEYVAULTENDPOINT \ ./testkms - name: "test-fuzz"