Merge branch 'indexg' of github.com:famosab/modules into indexg

nf-core · Nov 21, 2024 · 3eaedbe · 3eaedbe
2 parents 80bf749 + 8ad72fd
commit 3eaedbe
Show file tree

Hide file tree

Showing 110 changed files with 4,057 additions and 1,116 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -12,3 +12,4 @@ modules/nf-core/pints/caller/** @edmundmiller
 modules/nf-core/preseq/ccurve/** @edmundmiller
 modules/nf-core/preseq/lcextrap/** @edmundmiller
 subworkflows/nf-core/homer/** @edmundmiller
+modules/nf-core/**/Dockerfile @edmundmiller @maxulysse
diff --git a/.github/actions/nf-test-action/action.yml b/.github/actions/nf-test-action/action.yml
@@ -0,0 +1,83 @@
+name: "nf-test Action"
+description: "Runs nf-test with common setup steps"
+inputs:
+  path:
+    description: "Path to test"
+    required: true
+  profile:
+    description: "Profile to use"
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+
+    - uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4
+      with:
+        distribution: "temurin"
+        java-version: "17"
+
+    - name: Setup Nextflow
+      uses: nf-core/setup-nextflow@v2
+
+    - name: Set up Python
+      uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5
+      with:
+        python-version: "3.11"
+
+    - name: Install nf-test
+      uses: nf-core/setup-nf-test@v1
+      with:
+        version: "0.9.2"
+        install-pdiff: true
+
+    - name: Setup apptainer
+      if: contains(inputs.profile, 'singularity')
+      uses: eWaterCycle/setup-apptainer@main
+
+    - name: Set up Singularity
+      if: contains(inputs.profile, 'singularity')
+      shell: bash
+      run: |
+        mkdir -p $NXF_SINGULARITY_CACHEDIR
+        mkdir -p $NXF_SINGULARITY_LIBRARYDIR
+
+    - name: Conda setup
+      if: ${{inputs.profile == 'conda'}}
+      uses: conda-incubator/setup-miniconda@d2e6a045a86077fb6cad6f5adf368e9076ddaa8d # v3
+      with:
+        auto-update-conda: true
+        conda-solver: libmamba
+        conda-remove-defaults: true
+
+    # Set up secrets
+    - name: Set up nextflow secrets
+      if: env.SENTIEON_ENCRYPTION_KEY != '' && env.SENTIEON_LICENSE_MESSAGE != ''
+      shell: bash
+      run: |
+        nextflow secrets set SENTIEON_AUTH_DATA $(python3 tests/modules/nf-core/sentieon/license_message.py encrypt --key "$SENTIEON_ENCRYPTION_KEY" --message "$SENTIEON_LICENSE_MESSAGE")
+
+    - name: Run nf-test
+      shell: bash
+      env:
+        SENTIEON_LICSRVR_IP: ${{ env.SENTIEON_LICSRVR_IP }}
+        SENTIEON_AUTH_MECH: "GitHub Actions - token"
+      run: |
+        NFT_WORKDIR=~ \
+        nf-test test \
+          --profile=${{ inputs.profile }} \
+          --tap=test.tap \
+          --verbose \
+          ${{ inputs.path }}
+    - uses: pcolby/tap-summary@0959cbe1d4422e62afc65778cdaea6716c41d936 # v1
+      if: ${{ inputs.path != '' }}
+      with:
+        path: >-
+          test.tap
+
+    - name: Clean up
+      if: always()
+      shell: bash
+      run: |
+        sudo rm -rf /home/ubuntu/tests/
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -35,6 +35,8 @@ jobs:
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
       - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5
+        with:
+          python-version: "3.11"
       - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1
         # FIXME Flip this off once we get to less than a couple hundred. Adding
         # this so it will only run against changed files. It'll make it much
@@ -86,8 +88,7 @@ jobs:
         with:
           fetch-depth: 2 # To retrieve the preceding commit.
 
-      # TODO: change back to using dorny/paths-filter when https://github.com/dorny/paths-filter/pull/133 is implemented
-      - uses: mirpedrol/paths-filter@main
+      - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3
         id: filter
         with:
           filters: "tests/config/pytest_modules.yml"
@@ -135,6 +136,7 @@ jobs:
 
       - name: debug
         run: |
+          echo ${{ steps.filter.outputs.components }}
           echo ${{ steps.outputs.outputs.modules }}
           echo ${{ steps.outputs.outputs.subworkflows }}
 
@@ -344,14 +346,10 @@ jobs:
             tags: nanoplot
           - profile: conda
             tags: ncbitools/vecscreen
-          - profile: conda
-            tags: parabricks/applybqsr
           - profile: conda
             tags: parabricks/dbsnp
           - profile: conda
             tags: parabricks/deepvariant
-          - profile: conda
-            tags: parabricks/fq2bam
           - profile: conda
             tags: parabricks/genotypegvcf
           - profile: conda
@@ -445,6 +443,7 @@ jobs:
           miniconda-version: "latest"
           channels: conda-forge,bioconda
           python-version: ${{ matrix.python-version }}
+          conda-remove-defaults: true
 
       - name: Conda setup
         run: |
@@ -489,7 +488,7 @@ jobs:
             !${{ github.workspace }}/.singularity
 
   nf-test:
-    runs-on: ${{ github.event.inputs.runners  || 'ubuntu-latest' }}
+    runs-on: ${{ github.event.inputs.runners || 'self-hosted' }}
     name: nf-test
     needs: [nf-test-changes]
     if: ( needs.nf-test-changes.outputs.paths != '[]' )
@@ -648,18 +647,6 @@ jobs:
             path: modules/nf-core/islandpath
           - profile: conda
             path: modules/nf-core/scimap/mcmicro
-          - profile: conda
-            path: modules/nf-core/parabricks/fq2bammeth
-          - profile: docker_self_hosted
-            path: modules/nf-core/parabricks/fq2bammeth
-          - profile: singularity
-            path: modules/nf-core/parabricks/fq2bammeth
-          - profile: conda
-            path: modules/nf-core/parabricks/fq2bam
-          - profile: docker_self_hosted
-            path: modules/nf-core/parabricks/fq2bam
-          - profile: singularity
-            path: modules/nf-core/parabricks/fq2bam
           - profile: conda
             path: subworkflows/nf-core/vcf_annotate_ensemblvep
           - profile: conda
@@ -672,6 +659,10 @@ jobs:
             path: subworkflows/nf-core/fastq_align_bwa
           - profile: conda
             path: subworkflows/nf-core/fasta_newick_epang_gappa
+          - path: modules/nf-core/parabricks/fq2bammeth
+          - path: modules/nf-core/parabricks/fq2bam
+          - path: modules/nf-core/parabricks/applybqsr
+          - path: modules/nf-core/parabricks/indexgvcf
           - profile: conda
             path: modules/nf-core/xeniumranger/relabel
           - profile: conda
@@ -683,108 +674,49 @@ jobs:
 
     env:
       NXF_ANSI_LOG: false
-      NFTEST_VER: "0.9.0"
-      SENTIEON_LICENSE_MESSAGE: ${{ secrets.SENTIEON_LICENSE_MESSAGE }}
-      SENTIEON_ENCRYPTION_KEY: ${{ secrets.SENTIEON_ENCRYPTION_KEY }}
 
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
-
-      - uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4
-        with:
-          distribution: "temurin"
-          java-version: "17"
-      - name: Setup Nextflow
-        uses: nf-core/setup-nextflow@v2
-
-      - name: Install nf-test
-        uses: nf-core/setup-nf-test@v1
-        with:
-          version: ${{ env.NFTEST_VER }}
-
-      - name: Setup apptainer
-        if: matrix.profile == 'singularity'
-        uses: eWaterCycle/setup-apptainer@main
-
-      - name: Set up Singularity
-        if: matrix.profile == 'singularity'
-        run: |
-          mkdir -p $NXF_SINGULARITY_CACHEDIR
-          mkdir -p $NXF_SINGULARITY_LIBRARYDIR
-
-      - name: Set up Python
-        uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5
-        with:
-          python-version: "3.11"
-
-      - uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4
-        id: cache-pip-pdiff
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-pdiff
-          restore-keys: |
-            ${{ runner.os }}-pip-pdiff
-
-      - name: Install Python dependencies
-        run: python -m pip install --upgrade pip pdiff cryptography
-
-      - name: Set up miniconda
-        if: matrix.profile == 'conda'
-        uses: conda-incubator/setup-miniconda@d2e6a045a86077fb6cad6f5adf368e9076ddaa8d # v3
+      - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5
+      - name: Run nf-test Action
+        uses: ./.github/actions/nf-test-action
         with:
-          miniconda-version: "latest"
-          auto-update-conda: true
-          channels: conda-forge,bioconda
-
-      - name: Conda setup
-        if: matrix.profile == 'conda'
-        run: |
-          conda clean -a
-          conda install -n base conda-libmamba-solver
-          conda config --set solver libmamba
-          echo $(realpath $CONDA)/condabin >> $GITHUB_PATH
-          echo $(realpath python) >> $GITHUB_PATH
+          path: ${{ matrix.path }}
+          profile: ${{ matrix.profile }}
 
-      # Set up secrets
-      - name: Set up nextflow secrets
-        # TODO Only run if the tag includes `sentieon`
-        if: env.SENTIEON_ENCRYPTION_KEY != null && env.SENTIEON_LICENSE_MESSAGE != null
-        run: |
-          nextflow secrets set SENTIEON_AUTH_DATA $(python3 tests/modules/nf-core/sentieon/license_message.py encrypt --key "${{ secrets.SENTIEON_ENCRYPTION_KEY }}" --message "${{ secrets.SENTIEON_LICENSE_MESSAGE }}")
+  nf-test-gpu:
+    runs-on: "gpu"
+    name: nf-test-gpu
+    needs: [nf-test-changes]
+    if: ( needs.nf-test-changes.outputs.paths != '[]' && contains(needs.nf-test-changes.outputs.paths, 'modules/nf-core/parabricks') )
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - path: modules/nf-core/parabricks/applybqsr
+            profile: [docker_self_hosted, singularity]
+          - path: modules/nf-core/parabricks/fq2bam
+            profile: [docker_self_hosted, singularity]
+          - path: modules/nf-core/parabricks/fq2bammeth
+            profile: [docker_self_hosted, singularity]
+          - path: modules/nf-core/parabricks/indexgvcf
+            profile: [docker_self_hosted, singularity]
+    env:
+      NXF_ANSI_LOG: false
 
-      # Test the module
-      - name: Run nf-test
-        if: ${{ matrix.path != '' }}
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+      - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5
+      - name: Run nf-test Action
+        uses: ./.github/actions/nf-test-action
         env:
-          NFT_DIFF: "pdiff"
-          NFT_DIFF_ARGS: "--line-numbers --width 120 --expand-tabs=2"
+          SENTIEON_ENCRYPTION_KEY: ${{ secrets.SENTIEON_ENCRYPTION_KEY }}
+          SENTIEON_LICENSE_MESSAGE: ${{ secrets.SENTIEON_LICENSE_MESSAGE }}
           SENTIEON_LICSRVR_IP: ${{ secrets.SENTIEON_LICSRVR_IP }}
           SENTIEON_AUTH_MECH: "GitHub Actions - token"
-        run: |
-          # use "docker_self_hosted" if it runs on self-hosted runner and matrix.profile=docker
-          if [ "${{ matrix.profile }}" == "docker" ]; then
-            PROFILE="docker_self_hosted"
-          else
-            PROFILE=${{ matrix.profile }}
-          fi
-
-          NFT_WORKDIR=~ \
-          nf-test test \
-            --profile=${{ matrix.profile }} \
-            --tap=test.tap \
-            --verbose \
-            ${{ matrix.path }}
-
-      - uses: pcolby/tap-summary@0959cbe1d4422e62afc65778cdaea6716c41d936 # v1
-        if: ${{ matrix.path != '' }}
         with:
-          path: >-
-            test.tap
-
-      - name: Clean up
-        if: always()
-        run: |
-          sudo rm -rf /home/ubuntu/tests/
+          path: ${{ matrix.path }}
+          profile: ${{ matrix.profile }},gpu
 
   confirm-pass:
     runs-on: ubuntu-latest
@@ -798,6 +730,7 @@ jobs:
         pytest,
         nf-test-changes,
         nf-test,
+        nf-test-gpu,
       ]
     if: always()
     steps: