add json dependency

WX-1210 Added JIRA ID for Cromwhelm auto commit message (#7180) make it scala lots of scaffolding WX-1179 DRS image build updates, remove self-hosted runners (#7179) WX-1210-action-fix Use PR title to find JIRA ID for cromwhelm commit message (#7184) WX-1110-query-fix Corrected Query to pull in attributes outside of executionStatus and backendStatus (#7185) WX-1179 GCP Batch new base (#7177) Co-authored-by: dspeck <[email protected]> Co-authored-by: jarroyo28 <[email protected]> Co-authored-by: Jacob Jennings <[email protected]> Co-authored-by: Janet Gainer-Dewar <[email protected]> Co-authored-by: Tom Wiseman <[email protected]> Co-authored-by: Chris Llanwarne <[email protected]> Co-authored-by: Christian Freitas <[email protected]> Co-authored-by: Tristan Garwood <[email protected]> WM-2100: Update terra-helmfile on publish (#7187) WM-2100: Correct commit and PR description of terra-helmfile auto update (#7188) forward progress stash more progress json WX-1230 Ignore `invalidate_bad_caches_use_good_local.test` (#7197) WX-1078 ACR support (#7192) WX-1179 Enable GCP Batch Integration Tests (#7199) Co-authored-by: Adam Nichols <[email protected]> Co-authored-by: Adam Nichols <[email protected]> WX-1179 GCP Batch Docs Update (#7196) Co-authored-by: Jacob Jennings <[email protected]> Co-authored-by: Adam Nichols <[email protected]> Co-authored-by: Adam Nichols <[email protected]> stash expired tokens ID-734 Increase Timeout for DRSHub Communication (#7198) [WX-1156] internal_path_prefix for TES 4.4 (#7190) think about stuff undo sins undo one more sin [WM-2184] Remove brackets from Jira ID (#7206) WX-1153 Azure blob read md5 from metadata for large files (#7204) Co-authored-by: Janet Gainer-Dewar <[email protected]> WX-1156 Fix internal_path_prefix (#7208) Co-authored-by: Janet Gainer-Dewar <[email protected]> WX-1256 Temporarily turn off engine hashing for blob files (#7209) Co-authored-by: Adam Nichols <[email protected]> WX-1173 Reopen filesystem for blob storage outside workspace (#7178) Co-authored-by: Janet Gainer-Dewar <[email protected]> Co-authored-by: Tom Wiseman <[email protected]> Co-authored-by: Adam Nichols <[email protected]> WX-1174 Adjust NIO Copy functionality (#7207) Co-authored-by: Adam Nichols <[email protected]> [WX-1168] TES Log Paths (#7210) deck chairs somethin that kinda works working test lots of good cleanup time for tests remove conf working on tests test progress much cleaner oops bunch of tests more better tests
broadinstitute · Sep 12, 2023 · 8e60203 · 8e60203
1 parent af9660e
commit 8e60203
Show file tree

Hide file tree

Showing 156 changed files with 11,370 additions and 767 deletions.
diff --git a/.github/workflows/chart_update_on_merge.yml b/.github/workflows/chart_update_on_merge.yml
@@ -9,8 +9,14 @@ jobs:
   chart-update:
     name: Cromwhelm Chart Auto Updater
     if: github.event.pull_request.merged == true
-    runs-on: self-hosted # Faster machines; see https://github.com/broadinstitute/cromwell/settings/actions/runners
+    runs-on: ubuntu-latest
     steps:
+    - name: Fetch Jira ID from the commit message
+      id: fetch-jira-id
+      run: |
+        JIRA_ID=$(echo '${{ github.event.pull_request.title }}' | grep -Eo '[A-Z][A-Z]+-[0-9]+' | xargs echo -n | tr '[:space:]' ',')
+        [[ -z "$JIRA_ID" ]] && { echo "No Jira ID found in $1" ; exit 1; }
+        echo "JIRA_ID=$JIRA_ID" >> $GITHUB_OUTPUT
     - name: Clone Cromwell
       uses: actions/checkout@v2
       with:
@@ -69,7 +75,7 @@ jobs:
         repository: broadinstitute/terra-helmfile
         event-type: update-service
         client-payload: '{"service": "cromiam", "version": "${{ env.CROMWELL_VERSION }}", "dev_only": false}'
-    - name: Edit & push chart
+    - name: Edit & push cromwhelm chart
       env:
         BROADBOT_GITHUB_TOKEN: ${{ secrets.BROADBOT_GITHUB_TOKEN }}
       run: |
@@ -82,5 +88,48 @@ jobs:
         git diff
         git config --global user.name "broadbot"
         git config --global user.email "[email protected]"
-        git commit -am "Auto update to Cromwell $CROMWELL_VERSION"
+        git commit -am "${{ steps.fetch-jira-id.outputs.JIRA_ID }}: Auto update to Cromwell $CROMWELL_VERSION"
         git push https://broadbot:[email protected]/broadinstitute/cromwhelm.git main
+        cd -
+
+    - name: Clone terra-helmfile
+      uses: actions/checkout@v3
+      with:
+        repository: broadinstitute/terra-helmfile
+        token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} # Has to be set at checkout AND later when pushing to work
+        path: terra-helmfile
+
+    - name: Update workflows-app in terra-helmfile
+      run: |
+        set -e
+        cd terra-helmfile
+        sed -i "s|image: broadinstitute/cromwell:.*|image: broadinstitute/cromwell:$CROMWELL_VERSION|" charts/workflows-app/values.yaml
+        cd -
+
+    - name: Update cromwell-runner-app in terra-helmfile
+      run: |
+        set -e
+        cd terra-helmfile
+        sed -i "s|image: broadinstitute/cromwell:.*|image: broadinstitute/cromwell:$CROMWELL_VERSION|" charts/cromwell-runner-app/values.yaml
+        cd -
+        
+
+    - name: Make PR in terra-helmfile
+      env:
+        BROADBOT_TOKEN: ${{ secrets.BROADBOT_GITHUB_TOKEN }}
+        GH_TOKEN: ${{ secrets.BROADBOT_GITHUB_TOKEN }}
+      run: |
+        set -e
+        JIRA_ID=${{ steps.fetch-jira-id.outputs.JIRA_ID }}
+        if [[ $JIRA_ID == "missing" ]]; then
+          echo "JIRA_ID missing, PR to terra-helmfile will not be created"
+          exit 0;
+        fi
+        cd terra-helmfile
+        git checkout -b ${JIRA_ID}-cromwell-update-$CROMWELL_VERSION
+        git config --global user.name "broadbot"
+        git config --global user.email "[email protected]"
+        git commit -am "${JIRA_ID}: Auto update Cromwell to $CROMWELL_VERSION in workflows-app and cromwell-runner-app"
+        git push -u origin ${JIRA_ID}-cromwell-update-$CROMWELL_VERSION
+        gh pr create --title "${JIRA_ID}: auto update Cromwell version to $CROMWELL_VERSION in workflows-app and cromwell-runner-app" --body "${JIRA_ID} helm chart update" --label "automerge"
+        cd -
diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml
@@ -28,6 +28,10 @@ jobs:
 
     #Invoke SBT to run all unit tests for Cromwell. 
     - name: Run tests
+      env:
+        AZURE_CLIENT_ID: ${{ secrets.VAULT_AZURE_CENTAUR_CLIENT_ID }}
+        AZURE_CLIENT_SECRET: ${{ secrets.VAULT_AZURE_CENTAUR_CLIENT_SECRET }}
+        AZURE_TENANT_ID: ${{ secrets.VAULT_AZURE_CENTAUR_TENANT_ID }}
       run: |
         set -e
         sbt "test"
diff --git a/.github/workflows/docker_build_test.yml b/.github/workflows/docker_build_test.yml
@@ -17,7 +17,7 @@ permissions:
 jobs:
   sbt-build:
     name: sbt docker build
-    runs-on: self-hosted
+    runs-on: ubuntu-latest
     steps:
       - name: Clone Cromwell
         uses: actions/checkout@v2

diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
@@ -24,7 +24,11 @@ jobs:
       #Each will be launched on its own runner so they can occur in parallel.
       #Friendly names are displayed on the Github UI and aren't used anywhere else.
       matrix:
+        # Batch test fixes to land later
         include:
+          - build_type: centaurGcpBatch
+            build_mysql: 5.7
+            friendly_name: Centaur GCP Batch with MySQL 5.7
           - build_type: centaurPapiV2beta
             build_mysql: 5.7
             friendly_name: Centaur Papi V2 Beta with MySQL 5.7

diff --git a/.github/workflows/make_publish_prs.yml b/.github/workflows/make_publish_prs.yml
@@ -16,7 +16,7 @@ on:
 jobs:
   make-firecloud-develop-pr:
     name: Create firecloud-develop PR
-    runs-on: self-hosted # Faster machines; see https://github.com/broadinstitute/cromwell/settings/actions/runners
+    runs-on: ubuntu-latest
     steps:
       - name: Clone firecloud-develop
         uses: actions/checkout@v2
@@ -70,4 +70,3 @@ jobs:
                 'It updates cromwell from version ${{ github.event.inputs.old_cromwell_version }} to ${{ github.event.inputs.new_cromwell_version }}.'
               ].join('\n')
             });
-
diff --git a/azure-blob-nio/src/main/java/com/azure/storage/blob/nio/AzureDirectoryStream.java b/azure-blob-nio/src/main/java/com/azure/storage/blob/nio/AzureDirectoryStream.java
@@ -3,12 +3,6 @@
 
 package com.azure.storage.blob.nio;
 
-import com.azure.core.util.logging.ClientLogger;
-import com.azure.storage.blob.BlobContainerClient;
-import com.azure.storage.blob.models.BlobItem;
-import com.azure.storage.blob.models.BlobListDetails;
-import com.azure.storage.blob.models.ListBlobsOptions;
-
 import java.io.IOException;
 import java.nio.file.DirectoryIteratorException;
 import java.nio.file.DirectoryStream;
@@ -18,6 +12,12 @@
 import java.util.NoSuchElementException;
 import java.util.Set;
 
+import com.azure.core.util.logging.ClientLogger;
+import com.azure.storage.blob.BlobContainerClient;
+import com.azure.storage.blob.models.BlobItem;
+import com.azure.storage.blob.models.BlobListDetails;
+import com.azure.storage.blob.models.ListBlobsOptions;
+
 /**
  * A type for iterating over the contents of a directory.
  *
@@ -88,7 +88,7 @@ private static class AzureDirectoryIterator implements Iterator<Path> {
             if (path.isRoot()) {
                 String containerName = path.toString().substring(0, path.toString().length() - 1);
                 AzureFileSystem afs = ((AzureFileSystem) path.getFileSystem());
-                containerClient = ((AzureFileStore) afs.getFileStore(containerName)).getContainerClient();
+                containerClient = ((AzureFileStore) afs.getFileStore()).getContainerClient();
             } else {
                 AzureResource azureResource = new AzureResource(path);
                 listOptions.setPrefix(azureResource.getBlobClient().getBlobName() + AzureFileSystem.PATH_SEPARATOR);

diff --git a/azure-blob-nio/src/main/java/com/azure/storage/blob/nio/AzureFileSystem.java b/azure-blob-nio/src/main/java/com/azure/storage/blob/nio/AzureFileSystem.java
@@ -3,19 +3,6 @@
 
 package com.azure.storage.blob.nio;
 
-import com.azure.core.credential.AzureSasCredential;
-import com.azure.core.http.HttpClient;
-import com.azure.core.http.policy.HttpLogDetailLevel;
-import com.azure.core.http.policy.HttpPipelinePolicy;
-import com.azure.core.util.CoreUtils;
-import com.azure.core.util.logging.ClientLogger;
-import com.azure.storage.blob.BlobServiceClient;
-import com.azure.storage.blob.BlobServiceClientBuilder;
-import com.azure.storage.blob.implementation.util.BlobUserAgentModificationPolicy;
-import com.azure.storage.common.StorageSharedKeyCredential;
-import com.azure.storage.common.policy.RequestRetryOptions;
-import com.azure.storage.common.policy.RetryPolicyType;
-
 import java.io.IOException;
 import java.nio.file.FileStore;
 import java.nio.file.FileSystem;
@@ -27,14 +14,31 @@
 import java.nio.file.attribute.FileAttributeView;
 import java.nio.file.attribute.UserPrincipalLookupService;
 import java.nio.file.spi.FileSystemProvider;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 import java.util.regex.PatternSyntaxException;
-import java.util.stream.Collectors;
+
+import com.azure.core.credential.AzureSasCredential;
+import com.azure.core.http.HttpClient;
+import com.azure.core.http.policy.HttpLogDetailLevel;
+import com.azure.core.http.policy.HttpPipelinePolicy;
+import com.azure.core.util.CoreUtils;
+import com.azure.core.util.logging.ClientLogger;
+import com.azure.storage.blob.BlobServiceClient;
+import com.azure.storage.blob.BlobServiceClientBuilder;
+import com.azure.storage.blob.implementation.util.BlobUserAgentModificationPolicy;
+import com.azure.storage.common.StorageSharedKeyCredential;
+import com.azure.storage.common.policy.RequestRetryOptions;
+import com.azure.storage.common.policy.RetryPolicyType;
 
 /**
  * Implement's Java's {@link FileSystem} interface for Azure Blob Storage.
@@ -67,6 +71,11 @@ public final class AzureFileSystem extends FileSystem {
      */
     public static final String AZURE_STORAGE_SAS_TOKEN_CREDENTIAL = "AzureStorageSasTokenCredential";
 
+    /**
+     * Expected type: String
+     */
+    public static final String AZURE_STORAGE_PUBLIC_ACCESS_CREDENTIAL = "AzureStoragePublicAccessCredential";
+
     /**
      * Expected type: com.azure.core.http.policy.HttpLogLevelDetail
      */
@@ -159,10 +168,12 @@ public final class AzureFileSystem extends FileSystem {
     private final Long putBlobThreshold;
     private final Integer maxConcurrencyPerRequest;
     private final Integer downloadResumeRetries;
-    private final Map<String, FileStore> fileStores;
     private FileStore defaultFileStore;
     private boolean closed;
 
+    private AzureSasCredential currentActiveSasCredential;
+    private Instant expiry;
+
     AzureFileSystem(AzureFileSystemProvider parentFileSystemProvider, String endpoint, Map<String, ?> config)
             throws IOException {
         // A FileSystem should only ever be instantiated by a provider.
@@ -179,9 +190,10 @@ public final class AzureFileSystem extends FileSystem {
             this.putBlobThreshold = (Long) config.get(AZURE_STORAGE_PUT_BLOB_THRESHOLD);
             this.maxConcurrencyPerRequest = (Integer) config.get(AZURE_STORAGE_MAX_CONCURRENCY_PER_REQUEST);
             this.downloadResumeRetries = (Integer) config.get(AZURE_STORAGE_DOWNLOAD_RESUME_RETRIES);
+            this.currentActiveSasCredential = (AzureSasCredential) config.get(AZURE_STORAGE_SAS_TOKEN_CREDENTIAL);
 
             // Initialize and ensure access to FileStores.
-            this.fileStores = this.initializeFileStores(config);
+            this.defaultFileStore = this.initializeFileStore(config);
         } catch (RuntimeException e) {
             throw LoggingUtility.logError(LOGGER, new IllegalArgumentException("There was an error parsing the "
                 + "configurations map. Please ensure all fields are set to a legal value of the correct type.", e));
@@ -221,7 +233,7 @@ public FileSystemProvider provider() {
     @Override
     public void close() throws IOException {
         this.closed = true;
-        this.parentFileSystemProvider.closeFileSystem(this.getFileSystemUrl());
+        this.parentFileSystemProvider.closeFileSystem(this.getFileSystemUrl() + "/" + defaultFileStore.name());
     }
 
     /**
@@ -282,9 +294,7 @@ public Iterable<Path> getRootDirectories() {
         If the file system was set to use all containers in the account, the account will be re-queried and the
         list may grow or shrink if containers were added or deleted.
          */
-        return fileStores.keySet().stream()
-            .map(name -> this.getPath(name + AzurePath.ROOT_DIR_SUFFIX))
-            .collect(Collectors.toList());
+        return Arrays.asList(this.getPath(defaultFileStore.name() + AzurePath.ROOT_DIR_SUFFIX));
     }
 
     /**
@@ -304,7 +314,7 @@ public Iterable<FileStore> getFileStores() {
         If the file system was set to use all containers in the account, the account will be re-queried and the
         list may grow or shrink if containers were added or deleted.
          */
-        return this.fileStores.values();
+        return Arrays.asList(defaultFileStore);
     }
 
     /**
@@ -397,6 +407,12 @@ private BlobServiceClient buildBlobServiceClient(String endpoint, Map<String, ?>
             builder.credential((StorageSharedKeyCredential) config.get(AZURE_STORAGE_SHARED_KEY_CREDENTIAL));
         } else if (config.containsKey(AZURE_STORAGE_SAS_TOKEN_CREDENTIAL)) {
             builder.credential((AzureSasCredential) config.get(AZURE_STORAGE_SAS_TOKEN_CREDENTIAL));
+            this.setExpiryFromSAS((AzureSasCredential) config.get(AZURE_STORAGE_SAS_TOKEN_CREDENTIAL));
+        } else if (config.containsKey(AZURE_STORAGE_PUBLIC_ACCESS_CREDENTIAL)) {
+            // The Blob Service Client Builder requires at least one kind of authentication to make requests
+            // For public files however, this is unnecessary. This key-value pair is to denote the case
+            // explicitly when we supply a placeholder SAS credential to bypass this requirement.
+            builder.credential((AzureSasCredential) config.get(AZURE_STORAGE_PUBLIC_ACCESS_CREDENTIAL));
         } else {
             throw LoggingUtility.logError(LOGGER, new IllegalArgumentException(String.format("No credentials were "
                     + "provided. Please specify one of the following when constructing an AzureFileSystem: %s, %s.",
@@ -430,23 +446,17 @@ private BlobServiceClient buildBlobServiceClient(String endpoint, Map<String, ?>
         return builder.buildClient();
     }
 
-    private Map<String, FileStore> initializeFileStores(Map<String, ?> config) throws IOException {
-        String fileStoreNames = (String) config.get(AZURE_STORAGE_FILE_STORES);
-        if (CoreUtils.isNullOrEmpty(fileStoreNames)) {
+    private FileStore initializeFileStore(Map<String, ?> config) throws IOException {
+        String fileStoreName = (String) config.get(AZURE_STORAGE_FILE_STORES);
+        if (CoreUtils.isNullOrEmpty(fileStoreName)) {
             throw LoggingUtility.logError(LOGGER, new IllegalArgumentException("The list of FileStores cannot be "
                 + "null."));
         }
 
         Boolean skipConnectionCheck = (Boolean) config.get(AZURE_STORAGE_SKIP_INITIAL_CONTAINER_CHECK);
         Map<String, FileStore> fileStores = new HashMap<>();
-        for (String fileStoreName : fileStoreNames.split(",")) {
-            FileStore fs = new AzureFileStore(this, fileStoreName, skipConnectionCheck);
-            if (this.defaultFileStore == null) {
-                this.defaultFileStore = fs;
-            }
-            fileStores.put(fileStoreName, fs);
-        }
-        return fileStores;
+        this.defaultFileStore = new AzureFileStore(this, fileStoreName, skipConnectionCheck);
+        return this.defaultFileStore;
     }
 
     @Override
@@ -470,12 +480,11 @@ Path getDefaultDirectory() {
         return this.getPath(this.defaultFileStore.name() + AzurePath.ROOT_DIR_SUFFIX);
     }
 
-    FileStore getFileStore(String name) throws IOException {
-        FileStore store = this.fileStores.get(name);
-        if (store == null) {
-            throw LoggingUtility.logError(LOGGER, new IOException("Invalid file store: " + name));
+    FileStore getFileStore() throws IOException {
+        if (this.defaultFileStore == null) {
+            throw LoggingUtility.logError(LOGGER, new IOException("FileStore not initialized"));
         }
-        return store;
+        return defaultFileStore;
     }
 
     Long getBlockSize() {
@@ -489,4 +498,32 @@ Long getPutBlobThreshold() {
     Integer getMaxConcurrencyPerRequest() {
         return this.maxConcurrencyPerRequest;
     }
+
+    public String createSASAppendedURL(String url) throws IllegalStateException {
+        if (Objects.isNull(currentActiveSasCredential)) {
+            throw new IllegalStateException("No current active SAS credential present");
+        }
+        return url + "?" + currentActiveSasCredential.getSignature();
+    }
+
+    public Optional<Instant> getExpiry() {
+        return Optional.ofNullable(expiry);
+    }
+
+    private void setExpiryFromSAS(AzureSasCredential token) {
+        List<String> strings = Arrays.asList(token.getSignature().split("&"));
+        Optional<String> expiryString = strings.stream()
+            .filter(s -> s.startsWith("se"))
+            .findFirst()
+            .map(s -> s.replaceFirst("se=",""))
+            .map(s -> s.replace("%3A", ":"));
+        this.expiry = expiryString.map(es -> Instant.parse(es)).orElse(null);
+    }
+
+    public boolean isExpired(Duration buffer) {
+        return Optional.ofNullable(this.expiry)
+            .map(e -> Instant.now().plus(buffer).isAfter(e))
+            .orElse(true);
+
+    }
 }