From 288cce44f375db658130a7403ba8dff33ff9d251 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Thu, 7 Mar 2024 21:30:09 +0100 Subject: [PATCH 1/2] bump workflows to 24.0 and sync with IUC --- .github/styler.R | 36 ++++ .github/workflows/ci.yaml | 86 ++++++-- .github/workflows/pr.yaml | 191 +++++++++++------- .github/workflows/pr_without_tool_change.yaml | 15 ++ .github/workflows/slash.yaml | 2 +- 5 files changed, 235 insertions(+), 95 deletions(-) create mode 100755 .github/styler.R create mode 100644 .github/workflows/pr_without_tool_change.yaml diff --git a/.github/styler.R b/.github/styler.R new file mode 100755 index 000000000..c5c0cb5d8 --- /dev/null +++ b/.github/styler.R @@ -0,0 +1,36 @@ +#!/usr/bin/env Rscript + +library("argparse") +library("styler") + +parser <- ArgumentParser(description = "Call styler") +parser$add_argument("dir", + metavar = "DIR", type = "character", + help = "File to parse" +) +parser$add_argument("--dry", + choices = c("off", "on"), default = "on" +) +args <- parser$parse_args() + +file_info <- file.info(args$dir) +is_directory <- file_info$isdir + +if (is_directory) { + captured_output <- capture.output({ + result <- style_dir(args$dir, indent_by = 4, dry = args$dry, recursive = TRUE) + }) +} else { + captured_output <- capture.output({ + result <- style_file(args$dir, indent_by = 4, dry = args$dry) + }) +} + +n <- nrow(subset(result, changed == TRUE)) +if (n > 0) { + if (args$dry == "off") { + print(paste("Changed", n, "files")) + } else { + stop(paste("Linting failed for", n, "files")) + } +} diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 32af3bf56..9f1086ec6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -7,7 +7,7 @@ on: types: [run-all-tool-tests-command] env: GALAXY_FORK: galaxyproject - GALAXY_BRANCH: release_22.01 + GALAXY_BRANCH: release_24.0 MAX_CHUNKS: 40 jobs: setup: @@ -23,11 +23,11 @@ jobs: chunk-list: ${{ steps.discover.outputs.chunk-list }} strategy: matrix: - python-version: ['3.7'] + python-version: ['3.11'] steps: - name: Add reaction if: ${{ github.event.client_payload.slash_command.command == 'run-all-tool-tests' }} - uses: peter-evans/create-or-update-comment@v1 + uses: peter-evans/create-or-update-comment@v4 with: token: ${{ secrets.PAT }} repository: ${{ github.event.client_payload.github.payload.repository.full_name }} @@ -37,17 +37,17 @@ jobs: id: get-fork-branch run: | TMP="${{ github.event.client_payload.slash_command.args.named.fork }}" - echo "::set-output name=fork::${TMP:-$GALAXY_FORK}" + echo "fork=${TMP:-$GALAXY_FORK}" >> $GITHUB_OUTPUT TMP="${{ github.event.client_payload.slash_command.args.named.branch }}" - echo "::set-output name=branch::${TMP:-$GALAXY_BRANCH}" + echo "branch=${TMP:-$GALAXY_BRANCH}" >> $GITHUB_OUTPUT - name: Determine latest commit in the Galaxy repo id: get-galaxy-sha - run: echo "::set-output name=galaxy-head-sha::$(git ls-remote https://github.com/${{ steps.get-fork-branch.outputs.fork }}/galaxy refs/heads/${{ steps.get-fork-branch.outputs.branch }} | cut -f1)" - - uses: actions/setup-python@v1 + run: echo "galaxy-head-sha=$(git ls-remote https://github.com/${{ steps.get-fork-branch.outputs.fork }}/galaxy refs/heads/${{ steps.get-fork-branch.outputs.branch }} | cut -f1)" >> $GITHUB_OUTPUT + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Cache .cache/pip - uses: actions/cache@v2 + uses: actions/cache@v4 id: cache-pip with: path: ~/.cache/pip @@ -56,7 +56,7 @@ jobs: # are not available as wheels, pip will build a wheel for them, which can be cached. - name: Install wheel run: pip install wheel - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: fetch-depth: 1 - name: Fake a Planemo run to update cache and determine commit range, repositories, and chunks @@ -74,6 +74,42 @@ jobs: run: | echo 'Using ${{ steps.discover.outputs.chunk-count }} chunks (${{ steps.discover.outputs.chunk-list }})' + lint: + name: Check for missing containers + needs: setup + if: ${{ needs.setup.outputs.repository-list != '' || needs.setup.outputs.tool-list != '' }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ['3.11'] + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 1 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Cache .cache/pip + uses: actions/cache@v4 + id: cache-pip + with: + path: ~/.cache/pip + key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy-head-sha }} + - name: Planemo lint + uses: galaxyproject/planemo-ci-action@v1 + id: lint + with: + mode: lint + repository-list: ${{ needs.setup.outputs.repository-list }} + tool-list: ${{ needs.setup.outputs.tool-list }} + additional-planemo-options: --biocontainers -s tests,output,inputs,help,general,command,citations,tool_xsd,xml_order,tool_urls,shed_metadata + - uses: actions/upload-artifact@v4 + if: ${{ failure() }} + with: + name: 'Tool linting output' + path: lint_report.txt + test: name: Test tools # This job runs on Linux @@ -84,7 +120,7 @@ jobs: fail-fast: false matrix: chunk: ${{ fromJson(needs.setup.outputs.chunk-list) }} - python-version: ['3.7'] + python-version: ['3.11'] services: postgres: image: postgres:11 @@ -97,18 +133,23 @@ jobs: steps: # checkout the repository # and use it as the current working directory - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: fetch-depth: 1 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Cache .cache/pip - uses: actions/cache@v2 + uses: actions/cache@v4 id: cache-pip with: path: ~/.cache/pip key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy-head-sha }} + - name: Get number of CPU cores + uses: SimenB/github-actions-cpu-cores@v2 + id: cpu-cores + - name: Clean dotnet folder for space + run: rm -Rf /usr/share/dotnet - name: Planemo test uses: galaxyproject/planemo-ci-action@v1 id: test @@ -119,7 +160,10 @@ jobs: galaxy-branch: ${{ needs.setup.outputs.branch }} chunk: ${{ matrix.chunk }} chunk-count: ${{ needs.setup.outputs.chunk-count }} - - uses: actions/upload-artifact@v2 + galaxy-slots: ${{ steps.cpu-cores.outputs.count }} + # Limit each test to 15 minutes + test_timeout: 900 + - uses: actions/upload-artifact@v4 with: name: 'Tool test output ${{ matrix.chunk }}' path: upload @@ -134,18 +178,18 @@ jobs: needs: [setup, test] strategy: matrix: - python-version: ['3.7'] + python-version: ['3.11'] # This job runs on Linux runs-on: ubuntu-latest steps: - - uses: actions/download-artifact@v2 + - uses: actions/download-artifact@v4 with: path: artifacts - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Cache .cache/pip - uses: actions/cache@v2 + uses: actions/cache@v4 id: cache-pip with: path: ~/.cache/pip @@ -156,18 +200,18 @@ jobs: with: mode: combine html-report: true - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v4 with: name: 'All tool test results' path: upload - name: Create URL to the run output if: ${{ github.event.client_payload.slash_command.command == 'run-all-tool-tests' }} id: vars - run: echo "::set-output name=run-url::https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" + run: echo "run-url=https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" >> $GITHUB_OUTPUT - name: Create comment if: ${{ github.event.client_payload.slash_command.command == 'run-all-tool-tests' }} - uses: peter-evans/create-or-update-comment@v1 + uses: peter-evans/create-or-update-comment@v4 with: token: ${{ secrets.PAT }} repository: ${{ github.event.client_payload.github.payload.repository.full_name }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index daf9a7234..944fb8bbf 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -1,8 +1,25 @@ name: Galaxy Tool Linting and Tests for push and PR -on: [push, pull_request] +on: + pull_request: + paths-ignore: + - '.github/**' + - 'datatypes/**' + - 'deprecated/**' + - 'docs/**' + - '*' + push: + branches: + - main + - master + paths-ignore: + - '.github/**' + - 'datatypes/**' + - 'deprecated/**' + - 'docs/**' + - '*' env: GALAXY_FORK: galaxyproject - GALAXY_BRANCH: release_22.01 + GALAXY_BRANCH: release_24.0 MAX_CHUNKS: 4 MAX_FILE_SIZE: 1M concurrency: @@ -30,7 +47,7 @@ jobs: commit-range: ${{ steps.discover.outputs.commit-range }} strategy: matrix: - python-version: ['3.7'] + python-version: ['3.11'] steps: - name: Print github context properties run: | @@ -43,18 +60,18 @@ jobs: echo 'event.after: ${{ github.event.after }}' - name: Determine latest commit in the Galaxy repo id: get-galaxy-sha - run: echo "::set-output name=galaxy-head-sha::$(git ls-remote https://github.com/${{ env.GALAXY_FORK }}/galaxy refs/heads/${{ env.GALAXY_BRANCH }} | cut -f1)" - - uses: actions/setup-python@v1 + run: echo "galaxy-head-sha=$(git ls-remote https://github.com/${{ env.GALAXY_FORK }}/galaxy refs/heads/${{ env.GALAXY_BRANCH }} | cut -f1)" >> $GITHUB_OUTPUT + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Cache .cache/pip - uses: actions/cache@v2 + uses: actions/cache@v4 id: cache-pip with: path: ~/.cache/pip key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ steps.get-galaxy-sha.outputs.galaxy-head-sha }} - name: Cache .planemo - uses: actions/cache@v2 + uses: actions/cache@v4 id: cache-planemo with: path: ~/.planemo @@ -65,7 +82,7 @@ jobs: run: pip install wheel - name: Install flake8 run: pip install flake8 flake8-import-order - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: fetch-depth: 0 - name: Fake a Planemo run to update cache and determine commit range, repositories, and chunks @@ -96,28 +113,38 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.7'] + python-version: ['3.11'] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: fetch-depth: 1 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Cache .cache/pip - uses: actions/cache@v2 + uses: actions/cache@v4 id: cache-pip with: path: ~/.cache/pip key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy-head-sha }} + + - name: Set fail level for pull request + if: ${{ github.event_name == 'pull_request' }} + run: + echo "FAIL_LEVEL=warn" >> "$GITHUB_ENV" + - name: Set fail level for merge + if: ${{ github.event_name != 'pull_request' }} + run: + echo "FAIL_LEVEL=error" >> "$GITHUB_ENV" - name: Planemo lint uses: galaxyproject/planemo-ci-action@v1 id: lint with: mode: lint + fail-level: ${{ env.FAIL_LEVEL }} repository-list: ${{ needs.setup.outputs.repository-list }} tool-list: ${{ needs.setup.outputs.tool-list }} - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v4 if: ${{ failure() }} with: name: 'Tool linting output' @@ -127,21 +154,21 @@ jobs: flake8: name: Lint Python scripts needs: setup - if: ${{ needs.setup.outputs.repository-list != '' }} + if: ${{ github.event_name == 'pull_request' && needs.setup.outputs.repository-list != '' }} runs-on: ubuntu-latest strategy: fail-fast: false matrix: - python-version: ['3.7'] + python-version: ['3.11'] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: fetch-depth: 1 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Cache .cache/pip - uses: actions/cache@v2 + uses: actions/cache@v4 id: cache-pip with: path: ~/.cache/pip @@ -150,7 +177,7 @@ jobs: run: pip install flake8 flake8-import-order - name: Flake8 run: echo '${{ needs.setup.outputs.repository-list }}' | xargs -d '\n' flake8 --output-file pylint_report.txt --tee - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v4 if: ${{ failure() }} with: name: 'Python linting output' @@ -164,52 +191,32 @@ jobs: strategy: matrix: os: [ubuntu-20.04] - r-version: ['4.0.1'] + r-version: ['release'] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: fetch-depth: 1 - - uses: r-lib/actions/setup-r@master + - uses: r-lib/actions/setup-r@v2 with: r-version: ${{ matrix.r-version }} - name: Cache R packages - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: ${{ env.R_LIBS_USER }} key: r_cache_${{ matrix.os }}_${{ matrix.r-version }} - - name: Install non-R lintr dependencies - run: sudo apt-get install libcurl4-openssl-dev - - name: Install lintr - run: | - install.packages('remotes') - remotes::install_cran("lintr") - shell: Rscript {0} - - name: Save repositories to file - run: echo '${{ needs.setup.outputs.repository-list }}' > repository_list.txt + - name: Install packages + uses: r-lib/actions/setup-r-dependencies@v2 + with: + packages: | + any::argparse + any::styler - name: lintr run: | - library(lintr) - linters <- with_defaults(line_length_linter = NULL, cyclocomp_linter = NULL, object_usage_linter = NULL) - con <- file("repository_list.txt", "r") - status <- 0 - while (TRUE) { - repo <- readLines(con, n = 1) - if (length(repo) == 0) { - break - } - lnt <- lint_dir(repo, relative_path=T, linters=linters) - if (length(lnt) > 0) { - status <- 1 - for (l in lnt) { - rel_path <- paste(repo, l$filename, sep="/") - write(paste(paste(rel_path, l$line_number, l$column_number, sep=":"), l$message), stderr()) - write(paste(paste(rel_path, l$line_number, l$column_number, sep=":"), l$message), "rlint_report.txt", append=TRUE) - } - } - } - quit(status = status) - shell: Rscript {0} - - uses: actions/upload-artifact@v2 + set -eo pipefail + echo '${{ needs.setup.outputs.repository-list }}' | xargs -d '\n' -n 1 ./.github/styler.R --dry off + git status + git diff --exit-code | tee rlint_report.txt + - uses: actions/upload-artifact@v4 if: ${{ failure() }} with: name: 'R linting output' @@ -221,7 +228,7 @@ jobs: if: ${{ github.event_name == 'pull_request' && needs.setup.outputs.repository-list != '' }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: fetch-depth: 0 - name: Check file sizes @@ -236,7 +243,7 @@ jobs: cat file_size_report.txt exit 1 fi - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v4 if: ${{ failure() }} with: name: 'File size report' @@ -253,7 +260,7 @@ jobs: fail-fast: false matrix: chunk: ${{ fromJson(needs.setup.outputs.chunk-list) }} - python-version: ['3.7'] + python-version: ['3.11'] services: postgres: image: postgres:11 @@ -264,24 +271,29 @@ jobs: ports: - 5432:5432 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: fetch-depth: 1 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Cache .cache/pip - uses: actions/cache@v2 + uses: actions/cache@v4 id: cache-pip with: path: ~/.cache/pip key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy-head-sha }} - name: Cache .planemo - uses: actions/cache@v2 + uses: actions/cache@v4 id: cache-planemo with: path: ~/.planemo key: planemo_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy-head-sha }} + - name: Get number of CPU cores + uses: SimenB/github-actions-cpu-cores@v2 + id: cpu-cores + - name: Clean dotnet folder for space + run: rm -Rf /usr/share/dotnet - name: Planemo test uses: galaxyproject/planemo-ci-action@v1 id: test @@ -292,7 +304,10 @@ jobs: galaxy-branch: ${{ env.GALAXY_BRANCH }} chunk: ${{ matrix.chunk }} chunk-count: ${{ needs.setup.outputs.chunk-count }} - - uses: actions/upload-artifact@v2 + galaxy-slots: ${{ steps.cpu-cores.outputs.count }} + # Limit each test to 15 minutes + test_timeout: 1800 + - uses: actions/upload-artifact@v4 with: name: 'Tool test output ${{ matrix.chunk }}' path: upload @@ -305,20 +320,20 @@ jobs: combine_outputs: name: Combine chunked test results needs: [setup, test] - if: ${{ needs.setup.outputs.repository-list != '' }} + if: ${{ always() && needs.setup.outputs.repository-list != '' }} runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.7'] + python-version: ['3.11'] steps: - - uses: actions/download-artifact@v2 + - uses: actions/download-artifact@v4 with: path: artifacts - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Cache .cache/pip - uses: actions/cache@v2 + uses: actions/cache@v4 id: cache-pip with: path: ~/.cache/pip @@ -329,34 +344,42 @@ jobs: with: mode: combine html-report: true - - uses: actions/upload-artifact@v2 + markdown-report: true + - uses: actions/upload-artifact@v4 with: name: 'All tool test results' path: upload + - run: cat upload/tool_test_output.md >> $GITHUB_STEP_SUMMARY - name: Check outputs uses: galaxyproject/planemo-ci-action@v1 id: check with: mode: check + - name: Check if all test chunks succeeded + run: | + NFILES=$(ls artifacts/ | grep "Tool test output" | wc -l) + if [[ "${{ needs.setup.outputs.chunk-count }}" != "$NFILES" ]]; then + exit 1 + fi # deploy the tools to the toolsheds (first TTS for testing) deploy: name: Deploy - needs: [setup, lint, flake8, lintr, combine_outputs] + needs: [setup, lint, combine_outputs] if: ${{ (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' ) && github.repository_owner == 'galaxyproject' }} runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.7'] + python-version: ['3.11'] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: fetch-depth: 1 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Cache .cache/pip - uses: actions/cache@v2 + uses: actions/cache@v4 id: cache-pip with: path: ~/.cache/pip @@ -377,6 +400,28 @@ jobs: shed-target: toolshed shed-key: ${{ secrets.TS_API_KEY }} + deploy-report: + name: Report deploy status + needs: [deploy] + if: ${{ always() && needs.deploy.result != 'success' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' ) && github.repository_owner == 'galaxyproject' }} + runs-on: ubuntu-latest + steps: + # report to the PR if deployment failed + - name: Get PR object + uses: 8BitJonny/gh-get-current-pr@2.2.0 + id: getpr + with: + sha: ${{ github.event.after }} + - name: Create comment + uses: peter-evans/create-or-update-comment@v4 + with: + token: ${{ secrets.PAT }} + issue-number: ${{ steps.getpr.outputs.number }} + body: | + Attention: deployment ${{ needs.deploy.result }}! + + https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} + determine-success: name: Check workflow success needs: [setup, lint, flake8, lintr, file_sizes, combine_outputs] diff --git a/.github/workflows/pr_without_tool_change.yaml b/.github/workflows/pr_without_tool_change.yaml new file mode 100644 index 000000000..edff41005 --- /dev/null +++ b/.github/workflows/pr_without_tool_change.yaml @@ -0,0 +1,15 @@ +name: Fallback +# Fallback workflow that provides a succeeding "Check workflow success" job +# as this is a requirement for being able to merge a PR +# see https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/defining-the-mergeability-of-pull-requests/troubleshooting-required-status-checks#handling-skipped-but-required-checks +on: + pull_request: +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true +jobs: + determine-success: + name: Check workflow success + runs-on: ubuntu-latest + steps: + - run: 'echo "No tool tests required for this PR"' diff --git a/.github/workflows/slash.yaml b/.github/workflows/slash.yaml index 664d53916..838257cbb 100644 --- a/.github/workflows/slash.yaml +++ b/.github/workflows/slash.yaml @@ -8,7 +8,7 @@ jobs: steps: - name: Slash Command Dispatch if: github.repository_owner == 'galaxyproject' - uses: peter-evans/slash-command-dispatch@v2 + uses: peter-evans/slash-command-dispatch@v3 with: token: ${{ secrets.PAT }} commands: | From 2f343e13780fe5f3783cbe4ce5b5557df1ed7e0e Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Fri, 8 Mar 2024 19:20:06 +0100 Subject: [PATCH 2/2] do not irgnore .github --- .github/workflows/pr.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 944fb8bbf..b612e38ef 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -2,7 +2,6 @@ name: Galaxy Tool Linting and Tests for push and PR on: pull_request: paths-ignore: - - '.github/**' - 'datatypes/**' - 'deprecated/**' - 'docs/**' @@ -12,7 +11,6 @@ on: - main - master paths-ignore: - - '.github/**' - 'datatypes/**' - 'deprecated/**' - 'docs/**'