Merge branch 'master' into dp-sra_fetch-fix

broadinstitute · Sep 9, 2023 · c429d92 · c429d92
2 parents a8a6d8b + 7707958
commit c429d92
Show file tree

Hide file tree

Showing 16 changed files with 44 additions and 52 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -34,9 +34,9 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       # fetch git tags (tagged releases) because 
-      # actions/checkout@v2 does either a full checkout or a shallow checkout without tags
+      # actions/checkout@v3 does either a full checkout or a shallow checkout without tags
       - name: fetch tags
         run: git fetch --prune --unshallow --tags      
       - name: Programmatic environment setup
@@ -66,7 +66,7 @@ jobs:
         run: |
           env
       - name: install python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: '3.8'
       - name: install system dependencies
@@ -87,9 +87,9 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       # fetch git tags (tagged releases) because 
-      # actions/checkout@v2 does either a full checkout or a shallow checkout without tags
+      # actions/checkout@v3 does either a full checkout or a shallow checkout without tags
       - name: fetch tags
         run: git fetch --prune --unshallow --tags
       - name: Programmatic environment setup
@@ -115,7 +115,7 @@ jobs:
           echo "GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_BRANCH"
           echo "GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_BRANCH" >> $GITHUB_ENV
       - name: install java
-        uses: actions/setup-java@v2
+        uses: actions/setup-java@v3
         with:
           distribution: 'adopt'
           java-version: '11'
@@ -137,9 +137,9 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       # fetch git tags (tagged releases) because 
-      # actions/checkout@v2 does either a full checkout or a shallow checkout without tags
+      # actions/checkout@v3 does either a full checkout or a shallow checkout without tags
       - name: fetch tags
         run: git fetch --prune --unshallow --tags
       - name: Programmatic environment setup
@@ -165,7 +165,7 @@ jobs:
           echo "GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_BRANCH"
           echo "GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_BRANCH" >> $GITHUB_ENV
       - name: install python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with: 
           python-version: '3.8'
       - name: install docs dependencies
@@ -182,9 +182,9 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       # fetch git tags (tagged releases) because 
-      # actions/checkout@v2 does either a full checkout or a shallow checkout without tags
+      # actions/checkout@v3 does either a full checkout or a shallow checkout without tags
       - name: fetch tags
         run: git fetch --prune --unshallow --tags
       - name: Programmatic environment setup
@@ -210,7 +210,7 @@ jobs:
           echo "GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_BRANCH"
           echo "GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_BRANCH" >> $GITHUB_ENV
       - name: install java
-        uses: actions/setup-java@v2
+        uses: actions/setup-java@v3
         with:
           distribution: 'adopt'
           java-version: '11'
@@ -230,9 +230,9 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       # fetch git tags (tagged releases) because 
-      # actions/checkout@v2 does either a full checkout or a shallow checkout without tags
+      # actions/checkout@v3 does either a full checkout or a shallow checkout without tags
       - name: fetch tags
         run: git fetch --prune --unshallow --tags
       - name: Programmatic environment setup
@@ -258,7 +258,7 @@ jobs:
           echo "GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_BRANCH"
           echo "GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_BRANCH" >> $GITHUB_ENV
       - name: install python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: '3.8'
       - name: install system dependencies
@@ -277,7 +277,7 @@ jobs:
 
   deploy_dnanexus:
     #if: ${{ github.event.ref == 'dnanexus' }}
-    if: github.event_name == 'release' || github.ref == 'refs/heads/master'
+    if: github.event_name == 'release' || github.ref == 'refs/heads/master' || github.ref == 'refs/heads/dnanexus'
     needs:
       - validate_wdl_womtool
       - validate_wdl_miniwdl
@@ -289,9 +289,9 @@ jobs:
       DX_PROJECT: project-F8PQ6380xf5bK0Qk0YPjB17P
     steps:
       - name: checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       # fetch git tags (tagged releases) because 
-      # actions/checkout@v2 does either a full checkout or a shallow checkout without tags
+      # actions/checkout@v3 does either a full checkout or a shallow checkout without tags
       - name: fetch tags
         run: git fetch --prune --unshallow --tags
       - name: Programmatic environment setup
@@ -322,11 +322,11 @@ jobs:
           echo "${{ github.event.action }}"
           echo "${{ github.event.pull_request.merged }}"
       - name: install python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with: 
           python-version: '3.8'
       - name: install java
-        uses: actions/setup-java@v2
+        uses: actions/setup-java@v3
         with:
           distribution: 'adopt'
           java-version: '11'

diff --git a/pipes/WDL/tasks/tasks_interhost.wdl b/pipes/WDL/tasks/tasks_interhost.wdl
@@ -168,6 +168,9 @@ task beast {
   output {
     File        beast_log    = glob("*.log")[0]
     Array[File] trees        = glob("*.trees")
+    Array[File] ops          = glob("*.ops")
+    Array[File] rates        = glob("*.rates")
+    Array[File] root         = glob("*.root")
     File        beast_stdout = stdout()
   }
 

diff --git a/pipes/WDL/tasks/tasks_ncbi.wdl b/pipes/WDL/tasks/tasks_ncbi.wdl
@@ -60,7 +60,7 @@ task download_annotations {
   >>>
 
   output {
-    File        combined_fasta   = "~{combined_out_prefix}.fasta"
+    File        combined_fasta   = "combined/~{combined_out_prefix}.fasta"
     Array[File] genomes_fasta    = glob("*.fasta")
     Array[File] features_tbl     = glob("*.tbl")
     String      viralngs_version = read_string("VERSION")

diff --git a/pipes/WDL/tasks/tasks_sarscov2.wdl b/pipes/WDL/tasks/tasks_sarscov2.wdl
@@ -10,7 +10,7 @@ task pangolin_one_sample {
         Float?  max_ambig
         String? analysis_mode
         Boolean update_dbs_now=false
-        String  docker = "quay.io/staphb/pangolin:4.2-pdata-1.18.1.1"
+        String  docker = "quay.io/staphb/pangolin:4.3.1-pdata-1.22"
     }
     String basename = basename(genome_fasta, ".fasta")
     Int disk_size = 50
@@ -93,7 +93,7 @@ task pangolin_many_samples {
         String?      analysis_mode
         Boolean      update_dbs_now=false
         String       basename
-        String       docker = "quay.io/staphb/pangolin:4.2-pdata-1.18.1.1"
+        String       docker = "quay.io/staphb/pangolin:4.3.1-pdata-1.22"
     }
     Int disk_size = 100
     command <<<

diff --git a/pipes/WDL/workflows/bam_to_qiime.wdl b/pipes/WDL/workflows/bam_to_qiime.wdl
@@ -1,5 +1,7 @@
 version 1.0
 
+#DX_SKIP_WORKFLOW
+
 import "../tasks/tasks_16S_amplicon.wdl" as infile
 
 workflow qiime_import_bam {

diff --git a/pipes/WDL/workflows/beast_gpu.wdl b/pipes/WDL/workflows/beast_gpu.wdl
@@ -13,6 +13,9 @@ workflow beast_gpu {
     output {
         File        beast_log    = beast.beast_log
         Array[File] trees        = beast.trees
+        Array[File] ops          = beast.ops
+        Array[File] rates        = beast.rates
+        Array[File] root         = beast.root
         String      beast_stdout = beast.beast_stdout
     }
 }
diff --git a/pipes/WDL/workflows/classify_qiime2_multi.wdl b/pipes/WDL/workflows/classify_qiime2_multi.wdl
@@ -1,5 +1,7 @@
 version 1.0
 
+#DX_SKIP_WORKFLOW
+
 import "../tasks/tasks_16S_amplicon.wdl" as qiime 
 
 workflow amplicon16S_analysis {

diff --git a/pipes/WDL/workflows/demux_metag.wdl b/pipes/WDL/workflows/demux_metag.wdl
@@ -1,5 +1,7 @@
 version 1.0
 
+#DX_SKIP_WORKFLOW
+
 import "../tasks/tasks_demux.wdl" as demux
 import "../tasks/tasks_metagenomics.wdl" as metagenomics
 import "../tasks/tasks_read_utils.wdl" as read_utils

diff --git a/pipes/WDL/workflows/sarscov2_nextstrain_aligned_input.wdl b/pipes/WDL/workflows/sarscov2_nextstrain_aligned_input.wdl
@@ -13,7 +13,7 @@ workflow sarscov2_nextstrain_aligned_input {
     }
 
     input {
-        Array[File]+    aligned_sequences_fasta=["gs://nextstrain-data/files/ncov/open/aligned.fasta.xz"]
+        Array[File]+    aligned_sequences_fasta=["gs://nextstrain-data/files/ncov/open/aligned.fasta.zst"]
         Array[File]+    sample_metadata_tsvs=["gs://nextstrain-data/files/ncov/open/metadata.tsv.gz"]
 
         String          build_name
@@ -57,7 +57,7 @@ workflow sarscov2_nextstrain_aligned_input {
     call utils.zcat {
         input:
             infiles     = aligned_sequences_fasta,
-            output_name = "all_samples_combined_assembly.fasta"
+            output_name = "all_samples_combined_assembly.fasta.xz"
     }
 
     #### merge metadata, compute derived cols

diff --git a/pipes/WDL/workflows/terra_table_to_tsv.wdl b/pipes/WDL/workflows/terra_table_to_tsv.wdl
@@ -1,5 +1,7 @@
 version 1.0
 
+#DX_SKIP_WORKFLOW
+
 import "../tasks/tasks_terra.wdl" as terra
 
 workflow terra_table_to_tsv {

diff --git a/pipes/WDL/workflows/terra_update_assemblies.wdl b/pipes/WDL/workflows/terra_update_assemblies.wdl
@@ -1,5 +1,7 @@
 version 1.0
 
+#DX_SKIP_WORKFLOW
+
 import "../tasks/tasks_terra.wdl" as terra
 
 workflow update_data_tables {

diff --git a/pipes/dnax/dx-launcher/dx-yml-build b/pipes/dnax/dx-launcher/dx-yml-build
@@ -14,7 +14,7 @@ app_basename=os.path.splitext(os.path.basename(app_yaml_path))[0]
 output_json_filename=app_basename+'.json'
 
 with open(app_yaml_path) as infile:
-    data = yaml.load(infile)
+    data = yaml.safe_load(infile)
     data["00COMMENT"] = "This {}.json has been generated from {} automatically using dx-yml-build".format(app_basename,app_yaml_path)
     with open(output_json_filename, 'w') as outfile:
         json.dump(data, outfile, sort_keys=True, indent=2)

diff --git a/requirements-modules.txt b/requirements-modules.txt
@@ -7,5 +7,5 @@ broadinstitute/beast-beagle-cuda=1.10.5pre
 broadinstitute/ncbi-tools=2.10.7.10
 nextstrain/base=build-20211012T204409Z
 andersenlabapps/ivar=1.3.1
-quay.io/staphb/pangolin=4.2-pdata-1.18.1.1
+quay.io/staphb/pangolin=4.3.1-pdata-1.22
 nextstrain/nextclade=2.12.0
diff --git a/test/input/WDL/cromwell-local/test_inputs-genbank-local.json b/test/input/WDL/cromwell-local/test_inputs-genbank-local.json
diff --git a/test/input/WDL/miniwdl-local/test_inputs-sarscov2_lineages-local.json b/test/input/WDL/miniwdl-local/test_inputs-sarscov2_lineages-local.json
diff --git a/test/input/WDL/test_inputs-assemble_denovo-dnanexus.dx.json b/test/input/WDL/test_inputs-assemble_denovo-dnanexus.dx.json
@@ -6,7 +6,5 @@
 	],
 
 	"stage-common.filter_to_taxon_db": { "$dnanexus_link": { "project": "project-F8PQ6380xf5bK0Qk0YPjB17P", "id": "file-F1zXZ9Q0pvkp7FkZ646YJX2y" } },
-	"stage-common.reference_genome_fasta": [ { "$dnanexus_link": { "project": "project-F8PQ6380xf5bK0Qk0YPjB17P", "id": "file-F1zXZ9Q0pvkp7FkZ646YJX2y" } } ],
-
-	"stage-common.novocraft_license": { "$dnanexus_link": { "project": "project-F89kXY805X08BpXq7xjxbX9F", "id": "file-F1zkVGQ0jy1P68J88kyJ8zVF" } }
+	"stage-common.reference_genome_fasta": [ { "$dnanexus_link": { "project": "project-F8PQ6380xf5bK0Qk0YPjB17P", "id": "file-F1zXZ9Q0pvkp7FkZ646YJX2y" } } ]
 }