From aa73b23a4b41bbd9e5a5f1fde8225f47be3efc0e Mon Sep 17 00:00:00 2001 From: "Olivier Wilkinson (reivilibre)" Date: Wed, 28 Dec 2022 17:39:54 +0000 Subject: [PATCH 1/5] Reduce linting, unit test and integration test runs to when the files they check have been changed --- .github/workflows/tests.yml | 85 +++++++++++++++++++++++++++++++------ 1 file changed, 73 insertions(+), 12 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5a0c0a0d65c9..56fbc27fe1bf 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -17,6 +17,9 @@ jobs: runs-on: ubuntu-latest outputs: rust: ${{ !startsWith(github.ref, 'refs/pull/') || steps.filter.outputs.rust }} + trial: ${{ !startsWith(github.ref, 'refs/pull/') || steps.filter.outputs.trial }} + integration: ${{ !startsWith(github.ref, 'refs/pull/') || steps.filter.outputs.integration }} + linting: ${{ !startsWith(github.ref, 'refs/pull/') || steps.filter.outputs.linting }} steps: - uses: dorny/paths-filter@v2 id: filter @@ -29,8 +32,41 @@ jobs: - 'Cargo.toml' - 'Cargo.lock' + trial: + - 'synapse/**' + - 'tests/**' + - 'rust/**' + - 'Cargo.toml' + - 'Cargo.lock' + - 'pyproject.toml' + - 'poetry.lock' + + integration: + - 'synapse/**' + - 'rust/**' + - 'Cargo.toml' + - 'Cargo.lock' + - 'pyproject.toml' + - 'poetry.lock' + - 'docker/**' + + linting: + - 'synapse/**' + - 'docker/**' + - 'tests/**' + - 'scripts-dev/**' + - 'contrib/**' + - 'synmark/**' + - 'stubs/**' + - '.ci/**' + - 'pyproject.toml' + - 'poetry.lock' + check-sampleconfig: runs-on: ubuntu-latest + needs: changes + if: ${{ needs.changes.outputs.linting == 'true' }} + steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 @@ -44,6 +80,9 @@ jobs: check-schema-delta: runs-on: ubuntu-latest + needs: changes + if: ${{ needs.changes.outputs.linting == 'true' }} + steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 @@ -54,6 +93,8 @@ jobs: lint: uses: "matrix-org/backend-meta/.github/workflows/python-poetry-ci.yml@v2" + needs: changes + if: ${{ needs.changes.outputs.linting == 'true' }} with: typechecking-extras: "all" @@ -82,6 +123,9 @@ jobs: lint-pydantic: runs-on: ubuntu-latest + needs: changes + if: ${{ needs.changes.outputs.linting == 'true' }} + steps: - uses: actions/checkout@v3 with: @@ -185,8 +229,10 @@ jobs: sytest_test_matrix: ${{ steps.get-matrix.outputs.sytest_test_matrix }} trial: - if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail - needs: calculate-test-jobs + if: ${{ !cancelled() && !failure() && needs.changes.outputs.trial == 'true' }} # Allow previous steps to be skipped, but not fail + needs: + - calculate-test-jobs + - changes runs-on: ubuntu-latest strategy: matrix: @@ -245,8 +291,10 @@ jobs: trial-olddeps: # Note: sqlite only; no postgres - if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail - needs: linting-done + if: ${{ !cancelled() && !failure() && needs.changes.outputs.trial == 'true' }} # Allow previous steps to be skipped, but not fail + needs: + - linting-done + - changes runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v3 @@ -313,8 +361,10 @@ jobs: trial-pypy: # Very slow; only run if the branch name includes 'pypy' # Note: sqlite only; no postgres. Completely untested since poetry move. - if: ${{ contains(github.ref, 'pypy') && !failure() && !cancelled() }} - needs: linting-done + if: ${{ contains(github.ref, 'pypy') && !failure() && !cancelled() && needs.changes.outputs.trial == 'true' }} + needs: + - linting-done + - changes runs-on: ubuntu-latest strategy: matrix: @@ -344,8 +394,10 @@ jobs: || true sytest: - if: ${{ !failure() && !cancelled() }} - needs: calculate-test-jobs + if: ${{ !failure() && !cancelled() && needs.changes.outputs.integration == 'true' }} + needs: + - calculate-test-jobs + - changes runs-on: ubuntu-latest container: image: matrixdotorg/sytest-synapse:${{ matrix.job.sytest-tag }} @@ -429,8 +481,10 @@ jobs: portdb: - if: ${{ !failure() && !cancelled() }} # Allow previous steps to be skipped, but not fail - needs: linting-done + if: ${{ !failure() && !cancelled() && needs.changes.outputs.linting == 'true' }} # Allow previous steps to be skipped, but not fail + needs: + - linting-done + - changes runs-on: ubuntu-latest strategy: matrix: @@ -489,8 +543,10 @@ jobs: schema_diff complement: - if: "${{ !failure() && !cancelled() }}" - needs: linting-done + if: "${{ !failure() && !cancelled() && needs.changes.outputs.integration == 'true' }}" + needs: + - linting-done + - changes runs-on: ubuntu-latest strategy: @@ -571,5 +627,10 @@ jobs: # The newsfile lint may be skipped on non PR builds # Cargo test is skipped if there is no changes on Rust code skippable: | + trial + trial-olddeps + sytest + portdb + complement lint-newsfile cargo-test From b39e17527a5488fcbfbc75fb62620e8d257782e3 Mon Sep 17 00:00:00 2001 From: "Olivier Wilkinson (reivilibre)" Date: Wed, 28 Dec 2022 17:40:08 +0000 Subject: [PATCH 2/5] Newsfile Signed-off-by: Olivier Wilkinson (reivilibre) --- changelog.d/14745.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/14745.misc diff --git a/changelog.d/14745.misc b/changelog.d/14745.misc new file mode 100644 index 000000000000..eae0501d6b5d --- /dev/null +++ b/changelog.d/14745.misc @@ -0,0 +1 @@ +Avoid running CI steps when the files they check have not been changed. \ No newline at end of file From 25a5bbfbd71d55641872de4246c2c622ed17cf4a Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Wed, 6 Sep 2023 14:03:06 -0400 Subject: [PATCH 3/5] Update lists of files. --- .github/workflows/tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 56fbc27fe1bf..b71139053a53 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -31,6 +31,7 @@ jobs: - 'rust/**' - 'Cargo.toml' - 'Cargo.lock' + - '.rustfmt.toml' trial: - 'synapse/**' @@ -59,6 +60,7 @@ jobs: - 'synmark/**' - 'stubs/**' - '.ci/**' + - 'mypy.ini' - 'pyproject.toml' - 'poetry.lock' From b1a00998c1083cf9d6e719955495bd2951d2c902 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Wed, 6 Sep 2023 14:06:11 -0400 Subject: [PATCH 4/5] Clarify comment --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b71139053a53..bd3e748abf00 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -626,8 +626,8 @@ jobs: with: needs: ${{ toJSON(needs) }} - # The newsfile lint may be skipped on non PR builds - # Cargo test is skipped if there is no changes on Rust code + # Various bits are skipped if there was no applicable changes. + # The newsfile lint may be skipped on non PR builds. skippable: | trial trial-olddeps From 65178b813f5b1e7955568d75ce4039e6bdb0a83b Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Wed, 6 Sep 2023 14:14:05 -0400 Subject: [PATCH 5/5] Include docker for integration tests. --- .github/workflows/tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0f599d21c939..916bed4bb9db 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -46,6 +46,7 @@ jobs: integration: - 'synapse/**' - 'rust/**' + - 'docker/**' - 'Cargo.toml' - 'Cargo.lock' - 'pyproject.toml'