balance chunks in CI workflow

fixes: galaxyproject#2959
bernt-matthias · Oct 9, 2020 · 3bf87ca · 3bf87ca
1 parent 3e9f9dd
commit 3bf87ca
Showing 1 changed file with 13 additions and 15 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -1,10 +1,11 @@
 name: Weekly global Tool Linting and Tests
-on:
-  schedule:
-    # Run at midnight every monday
-    - cron: '0 0 * * 1'
-  repository_dispatch:
-    types: [run-all-tool-tests-command]
+on: [push, pull_request]
+# on:
+#   schedule:
+#     # Run at midnight every monday
+#     - cron: '0 0 * * 1'
+#   repository_dispatch:
+#     types: [run-all-tool-tests-command]
 env:
   GALAXY_RELEASE: release_20.09
 jobs:
@@ -80,27 +81,24 @@ jobs:
     - name: Install Planemo
       run: pip install planemo
     - name: Planemo ci_find_tools
-      run: planemo ci_find_repos --chunk_count 40 --chunk ${{ matrix.chunk  }} --exclude test_repositories --exclude packages --exclude deprecated --exclude_from .tt_skip --exclude_from .tt_biocontainer_skip --output tool.list
+      run: planemo ci_find_tools --chunk_count 40 --chunk ${{ matrix.chunk  }} --exclude test_repositories --exclude packages --exclude deprecated --exclude_from .tt_skip --exclude_from .tt_biocontainer_skip --group_tools --output tool.list
     - name: Show repo list
       run: cat tool.list
     - name: Planemo test tools
       run: |
-           while read -r DIR; do
-               if [[ "$DIR" =~ ^data_managers.* ]]; then
-                   TESTPATH=$(planemo ci_find_tools "$DIR")
-               else
-                   TESTPATH="$DIR"
-               fi
+           mkdir json_output/
+           while read -r TOOLS; do
                if grep -lqf .tt_biocontainer_skip <(echo $DIR); then
                    PLANEMO_OPTIONS=""
                else
                    PLANEMO_OPTIONS="--biocontainers --no_dependency_resolution --no_conda_auto_init"
                fi
-               PIP_QUIET=1 planemo test --database_connection postgresql://postgres:postgres@localhost:5432/galaxy --no_conda_auto_init --galaxy_branch $GALAXY_RELEASE --biocontainers --no_dependency_resolution --galaxy_python_version ${{ matrix.python-version }} --test_output_json "$DIR"/tool_test_output.json "$TESTPATH" || true
+               json=$(mktemp -p json_out --suff .json)
+               PIP_QUIET=1 planemo test --database_connection postgresql://postgres:postgres@localhost:5432/galaxy --no_conda_auto_init --galaxy_branch $GALAXY_RELEASE --biocontainers --no_dependency_resolution --galaxy_python_version ${{ matrix.python-version }} --test_output_json $json "$TOOLS" || true
                docker system prune --all --force --volumes || true
            done < tool.list
     - name: Merge tool_test_output.json files
-      run: find . -name tool_test_output.json -exec sh -c 'planemo merge_test_reports "$@" tool_test_output.json' sh {} +
+      run: planemo merge_test_reports json_output/*json tool_test_output.json
     - name: Create tool_test_output.html
       run: planemo test_reports tool_test_output.json --test_output tool_test_output.html
     - name: Copy artifacts into place