Merge pull request #716 from ewels/sync-matrix

Sync matrix

.github/workflows/branch.yml

@@ -24,9 +24,7 @@ jobs:
           message: |
             Hi @${{ github.event.pull_request.user.login }},
 
-            It looks like this pull-request is has been made against the ${{github.event.pull_request.head.repo.full_name}} `master` branch.
-            The `master` branch on nf-core repositories should always contain code from the latest release.
-            Beacuse of this, PRs to `master` are only allowed if they come from the ${{github.event.pull_request.head.repo.full_name}} `dev` branch.
+            It looks like this pull-request is has been made against the ${{github.event.pull_request.head.repo.full_name}} `master` branch. The `master` branch on nf-core repositories should always contain code from the latest release. Beacuse of this, PRs to `master` are only allowed if they come from the ${{github.event.pull_request.head.repo.full_name}} `dev` branch.
 
             You do not need to close this PR, you can change the target branch to `dev` by clicking the _"Edit"_ button at the top of this page.
 

.github/workflows/sync.yml

@@ -4,13 +4,34 @@ on:
     types: [published]
 
 jobs:
-  sync-all:
-    name: Sync all pipelines
+  get-pipelines:
     runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - id: set-matrix
+        run: |
+          curl -O https://nf-co.re/pipeline_names.json
+          echo "::set-output name=matrix::$(cat pipeline_names.json)"
 
+  sync:
+    needs: get-pipelines
+    runs-on: ubuntu-latest
+    strategy:
+      matrix: ${{fromJson(needs.get-pipelines.outputs.matrix)}}
+      fail-fast: false
     steps:
+
       - uses: actions/checkout@v2
-        name: Check out source-code repository
+        name: Check out nf-core/tools
+
+      - uses: actions/checkout@v2
+        name: Check out nf-core/${{ matrix.pipeline }}
+        with:
+          repository: nf-core/${{ matrix.pipeline }}
+          ref: dev
+          token: ${{ secrets.nf_core_bot_auth_token }}
+          path: nf-core/${{ matrix.pipeline }}
 
       - name: Set up Python 3.8
         uses: actions/setup-python@v1
@@ -32,14 +53,20 @@ jobs:
       - name: Run synchronisation
         if: github.repository == 'nf-core/tools'
         env:
-          AUTH_TOKEN: ${{ secrets.nf_core_bot_auth_token }}
+          GITHUB_AUTH_TOKEN: ${{ secrets.nf_core_bot_auth_token }}
         run: |
           git config --global user.email "[email protected]"
           git config --global user.name "nf-core-bot"
-          nf-core --log-file sync_log.txt sync --all --username nf-core-bot --auth-token $AUTH_TOKEN
+          nf-core --log-file sync_log_${{ matrix.pipeline }}.txt sync nf-core/${{ matrix.pipeline }} \
+            --from-branch dev \
+            --pull-request \
+            --username nf-core-bot \
+            --repository nf-core/${{ matrix.pipeline }}
+
 
       - name: Upload sync log file artifact
+        if: ${{ always() }}
         uses: actions/upload-artifact@v2
         with:
-          name: sync-log-file
-          path: sync_log.txt
+          name: sync_log_${{ matrix.pipeline }}
+          path: sync_log_${{ matrix.pipeline }}.txt

CHANGELOG.md

@@ -8,15 +8,25 @@ Apologies for the inconvenience.
 * Fix syntax error in `/push_dockerhub.yml` GitHub Action workflow
 * Change `params.readPaths` -> `params.input_paths` in `test_full.config`
 * Check results when posting the lint results as a GitHub comment
+  * This feature is unfortunately not possible when making PRs from forks outside of the nf-core organisation for now.
+* More major refactoring of the automated pipeline sync
+  * New GitHub Actions matrix parallelisation of sync jobs across pipelines [[#673](https://github.com/nf-core/tools/issues/673)]
+  * Removed the `--all` behaviour from `nf-core sync` as we no longer need it
+  * Sync now uses a new list of pipelines on the website which does not include archived pipelines [[#712](https://github.com/nf-core/tools/issues/712)]
+  * When making a PR it checks if a PR already exists - if so it updates it [[#710](https://github.com/nf-core/tools/issues/710)]
+  * More tests and code refactoring for more stable code. Hopefully fixes 404 error [[#711](https://github.com/nf-core/tools/issues/711)]
 
 ## [v1.10.1 - Copper Camel _(patch)_](https://github.com/nf-core/tools/releases/tag/1.10.1) - [2020-07-30]
 
 Patch release to fix the automatic template synchronisation, which failed in the v1.10 release.
 
 * Improved logging: `nf-core --log-file log.txt` now saves a verbose log to disk.
-* GitHub actions sync now uploads verbose log as an artifact.
-* Sync - fixed several minor bugs, improved logging.
+* nf-core/tools GitHub Actions pipeline sync now uploads verbose log as an artifact.
+* Sync - fixed several minor bugs, made logging less verbose.
 * Python Rich library updated to `>=4.2.1`
+* Hopefully fix git config for pipeline sync so that commit comes from @nf-core-bot
+* Fix sync auto-PR text indentation so that it doesn't all show as code
+* Added explicit flag `--show-passed` for `nf-core lint` instead of taking logging verbosity
 
 ## [v1.10 - Copper Camel](https://github.com/nf-core/tools/releases/tag/1.10) - [2020-07-30]
 

nf_core/__main__.py

@@ -550,14 +550,12 @@ def bump_version(pipeline_dir, new_version, nextflow):
 
 
 @nf_core_cli.command("sync", help_priority=10)
-@click.argument("pipeline_dir", type=click.Path(exists=True), nargs=-1, metavar="<pipeline directory>")
+@click.argument("pipeline_dir", required=True, type=click.Path(exists=True), metavar="<pipeline directory>")
 @click.option("-b", "--from-branch", type=str, help="The git branch to use to fetch workflow vars.")
 @click.option("-p", "--pull-request", is_flag=True, default=False, help="Make a GitHub pull-request with the changes.")
-@click.option("-u", "--username", type=str, help="GitHub username for the PR.")
-@click.option("-r", "--repository", type=str, help="GitHub repository name for the PR.")
-@click.option("-a", "--auth-token", type=str, help="GitHub API personal access token.")
-@click.option("--all", is_flag=True, default=False, help="Sync template for all nf-core pipelines.")
-def sync(pipeline_dir, from_branch, pull_request, username, repository, auth_token, all):
+@click.option("-r", "--repository", type=str, help="GitHub PR: target repository.")
+@click.option("-u", "--username", type=str, help="GitHub PR: auth username.")
+def sync(pipeline_dir, from_branch, pull_request, repository, username):
     """
     Sync a pipeline TEMPLATE branch with the nf-core template.
 
@@ -571,24 +569,13 @@ def sync(pipeline_dir, from_branch, pull_request, username, repository, auth_tok
     new release of nf-core/tools (and the included template) is made.
     """
 
-    # Pull and sync all nf-core pipelines
-    if all:
-        nf_core.sync.sync_all_pipelines(username, auth_token)
-    else:
-        # Manually check for the required parameter
-        if not pipeline_dir or len(pipeline_dir) != 1:
-            log.error("Either use --all or specify one <pipeline directory>")
-            sys.exit(1)
-        else:
-            pipeline_dir = pipeline_dir[0]
-
-        # Sync the given pipeline dir
-        sync_obj = nf_core.sync.PipelineSync(pipeline_dir, from_branch, pull_request)
-        try:
-            sync_obj.sync()
-        except (nf_core.sync.SyncException, nf_core.sync.PullRequestException) as e:
-            log.error(e)
-            sys.exit(1)
+    # Sync the given pipeline dir
+    sync_obj = nf_core.sync.PipelineSync(pipeline_dir, from_branch, pull_request, repository, username)
+    try:
+        sync_obj.sync()
+    except (nf_core.sync.SyncException, nf_core.sync.PullRequestException) as e:
+        log.error(e)
+        sys.exit(1)
 
 
 if __name__ == "__main__":

nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md

@@ -46,7 +46,7 @@ These tests are run both with the latest available version of `Nextflow` and als
 
 ## Patch
 
-: warning: Only in the unlikely and regretful event of a release happening with a bug.
+:warning: Only in the unlikely and regretful event of a release happening with a bug.
 
 * On your own fork, make a new branch `patch` based on `upstream/master`.
 * Fix the bug, and bump version (X.Y.Z+1).

nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/branch.yml

@@ -17,6 +17,7 @@ jobs:
 
 {% raw %}
       # If the above check failed, post a comment on the PR explaining the failure
+      # NOTE - this doesn't currently work if the PR is coming from a fork, due to limitations in GitHub actions secrets
       - name: Post PR comment
         if: failure()
         uses: mshick/add-pr-comment@v1