Azure blob read/write integration test (#7024)

Co-authored-by: Tom Wiseman <[email protected]>

.github/workflows/integration_tests.yml

@@ -83,6 +83,9 @@ jobs:
           - build_type: centaurSlurm
             build_mysql: 5.7
             friendly_name: "Centaur Slurm with MySQL 5.7"
+          - build_type: centaurBlob
+            build_mysql: 5.7
+            friendly_name: Centaur Blob
     name: ${{ matrix.friendly_name }}
     env:
       BUILD_NAME: ${{ matrix.build_type }}
@@ -92,6 +95,9 @@ jobs:
       BUILD_MARIADB: ${{ matrix.build_mariadb }}
       VAULT_ROLE_ID: ${{ secrets.VAULT_ROLE_ID_CI }}
       VAULT_SECRET_ID: ${{ secrets.VAULT_SECRET_ID_CI }}
+      AZURE_CLIENT_ID: ${{ secrets.VAULT_AZURE_CENTAUR_CLIENT_ID }}
+      AZURE_CLIENT_SECRET: ${{ secrets.VAULT_AZURE_CENTAUR_CLIENT_SECRET }}
+      AZURE_TENANT_ID: ${{ secrets.VAULT_AZURE_CENTAUR_TENANT_ID }}
     runs-on: ubuntu-latest
     timeout-minutes: 120
     steps:

centaur/src/main/resources/azureBlobTestCases/azure_blob_storage_read.test

@@ -0,0 +1,16 @@
+name: azure_blob_storage_read
+testFormat: workflowsuccess
+backends: [Local]
+tags: ["blob", "azure"]
+retryTestFailures: false
+
+files {
+  workflow: azure_blob_storage_read/azure_blob_storage_read.wdl
+  inputs: azure_blob_storage_read/azure_blob_storage_read.inputs
+  options: azure_blob_storage_read/azure_blob_storage_read.options
+}
+
+metadata {
+  status: Succeeded
+  "outputs.azure_blob_storage_read.s1": "This is my test file! Did it work??"
+}

centaur/src/main/resources/azureBlobTestCases/azure_blob_storage_read/azure_blob_storage_read.inputs

@@ -0,0 +1,3 @@
+{
+  "azure_blob_storage_read.file1": "https://centaurtesting.blob.core.windows.net/test-blob/testRead.txt"
+}

centaur/src/main/resources/azureBlobTestCases/azure_blob_storage_read/azure_blob_storage_read.options

@@ -0,0 +1,3 @@
+{
+  "final_workflow_log_dir": "https://centaurtesting.blob.core.windows.net/test-blob/test-cromwell-workflow-logs"
+}

centaur/src/main/resources/azureBlobTestCases/azure_blob_storage_read/azure_blob_storage_read.wdl

@@ -0,0 +1,12 @@
+version 1.0
+
+workflow azure_blob_storage_read {
+
+  input {
+    File file1
+  }
+
+  output {
+    String s1 = read_string(file1)
+  }
+}

filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobFileSystemManager.scala

@@ -230,7 +230,6 @@ case class NativeBlobSasTokenGenerator(container: BlobContainerName, endpoint: E
   private def authenticateWithSubscription(sub: SubscriptionId) = AzureResourceManager.authenticate(azureCredentialBuilder, azureProfile).withSubscription(sub.toString)
   private def authenticateWithDefaultSubscription = AzureResourceManager.authenticate(azureCredentialBuilder, azureProfile).withDefaultSubscription()
   private def azure = subscription.map(authenticateWithSubscription(_)).getOrElse(authenticateWithDefaultSubscription)
-
   private def findAzureStorageAccount(name: StorageAccountName) = azure.storageAccounts.list.asScala.find(_.name.equals(name.value))
       .map(Success(_)).getOrElse(Failure(new Exception("Azure Storage Account not found")))
   private def buildBlobContainerClient(credential: StorageSharedKeyCredential, endpoint: EndpointURL, container: BlobContainerName): BlobContainerClient = {

src/ci/bin/test.inc.sh

@@ -390,6 +390,9 @@ cromwell::private::create_build_variables() {
         centaurHoricromtalEngineUpgrade*)
             CROMWELL_BUILD_CROMWELL_CONFIG="${CROMWELL_BUILD_RESOURCES_DIRECTORY}/papi_v2alpha1_horicromtal_application.conf"
             ;;
+        centaurBlob*)
+            CROMWELL_BUILD_CROMWELL_CONFIG="${CROMWELL_BUILD_RESOURCES_DIRECTORY}/centaur_blob_test.conf"
+            ;;
         *)
             CROMWELL_BUILD_CROMWELL_CONFIG="${CROMWELL_BUILD_RESOURCES_DIRECTORY}/${CROMWELL_BUILD_BACKEND_TYPE}_application.conf"
             ;;
@@ -595,6 +598,7 @@ cromwell::private::create_centaur_variables() {
     CROMWELL_BUILD_CENTAUR_TYPE_PAPI_UPGRADE_NEW_WORKFLOWS="papiUpgradeNewWorkflows"
     CROMWELL_BUILD_CENTAUR_TYPE_HORICROMTAL_ENGINE_UPGRADE="horicromtalEngineUpgrade"
     CROMWELL_BUILD_CENTAUR_TYPE_HORICROMTAL="horicromtal"
+    CROMWELL_BUILD_CENTAUR_TYPE_AZURE_BLOB="azureBlob"
 
     case "${CROMWELL_BUILD_TYPE}" in
         centaurEngineUpgrade*)
@@ -612,6 +616,9 @@ cromwell::private::create_centaur_variables() {
         centaurHoricromtal*)
             CROMWELL_BUILD_CENTAUR_TYPE="${CROMWELL_BUILD_CENTAUR_TYPE_HORICROMTAL}"
             ;;
+        centaurBlob*)
+            CROMWELL_BUILD_CENTAUR_TYPE="${CROMWELL_BUILD_CENTAUR_TYPE_AZURE_BLOB}"
+            ;;
         *)
             # Only set the type if Jenkins, etc. has not already set the centaur type
             if [[ -z "${CROMWELL_BUILD_CENTAUR_TYPE-}" ]]; then

src/ci/bin/testCentaurBlob.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -o errexit -o nounset -o pipefail
+# import in shellcheck / CI / IntelliJ compatible ways
+# shellcheck source=/dev/null
+source "${BASH_SOURCE%/*}/test.inc.sh" || source test.inc.sh
+
+#NB: This function ensures that the correct .conf file is being used by cromwell (among other things).
+#Blob storage requires a configuration file tailored for Azure.
+cromwell::build::setup_common_environment
+
+cromwell::build::setup_centaur_environment
+
+cromwell::build::assemble_jars
+
+cromwell::build::run_centaur
+
+cromwell::build::generate_code_coverage

src/ci/resources/centaur_blob_test.conf

@@ -0,0 +1,66 @@
+include required(classpath("application.conf"))
+include "build_application.inc.conf"
+
+filesystems {
+  blob {
+    class = "cromwell.filesystems.blob.BlobPathBuilderFactory"
+    global {
+      # One BFSM is shared across all BlobPathBuilders
+      class = "cromwell.filesystems.blob.BlobFileSystemManager"
+      config {
+        container: "test-blob"
+        endpoint: "https://centaurtesting.blob.core.windows.net"
+        subscription: "62b22893-6bc1-46d9-8a90-806bb3cce3c9"
+      }
+    }
+  }
+}
+
+engine {
+  filesystems {
+    local {
+      enabled: false
+    }
+    http {
+      enabled: false
+    }
+    blob {
+      enabled: true
+    }
+  }
+}
+
+
+backend {
+  # Note that the backend is not selected purposefully here, 
+  # the existing tests at the time of writing do not submit tasks to a backend.
+  default = "TES"
+  providers {
+    TES {
+      actor-factory = "cromwell.backend.impl.tes.TesBackendLifecycleActorFactory"
+      config {
+        # Use for running on blob storage
+        #root = "https://<storage-account>.blob.core.windows.net/cromwell/cromwell-executions"
+        root = "cromwell-executions"
+        dockerRoot = "/cromwell-executions"
+        endpoint = "http://127.0.0.1:9000/v1/tasks"
+        concurrent-job-limit = 1000
+        transform-blob-to-local-path = false
+        filesystems {
+          blob {
+            enabled: true
+          }
+          local {
+            enabled: true
+          }
+          http {
+            enabled: true
+          }
+        }
+      }
+
+      # Have the engine (maybe) authenticate to docker.io. See BT-141 for more info.
+      include "dockerhub_provider_config_v1.inc.conf"
+    }
+  }
+}