Tests

Implement MSC4210: Remove legacy mentions #3521

Workflow file for this run

	name: Tests

	on:
	push:
	branches: ["develop", "release-*"]
	pull_request:
	merge_group:
	workflow_dispatch:

	concurrency:
	group: ${{ github.workflow }}-${{ github.ref }}
	cancel-in-progress: true

	jobs:
	# Job to detect what has changed so we don't run e.g. Rust checks on PRs that
	# don't modify Rust code.
	changes:
	runs-on: ubuntu-latest
	outputs:
	rust: ${{ !startsWith(github.ref, 'refs/pull/') \|\| steps.filter.outputs.rust }}
	trial: ${{ !startsWith(github.ref, 'refs/pull/') \|\| steps.filter.outputs.trial }}
	integration: ${{ !startsWith(github.ref, 'refs/pull/') \|\| steps.filter.outputs.integration }}
	linting: ${{ !startsWith(github.ref, 'refs/pull/') \|\| steps.filter.outputs.linting }}
	linting_readme: ${{ !startsWith(github.ref, 'refs/pull/') \|\| steps.filter.outputs.linting_readme }}
	steps:
	- uses: dorny/paths-filter@v3
	id: filter
	# We only check on PRs
	if: startsWith(github.ref, 'refs/pull/')
	with:
	filters: \|
	rust:
	- 'rust/**'
	- 'Cargo.toml'
	- 'Cargo.lock'
	- '.rustfmt.toml'
	- '.github/workflows/tests.yml'

	trial:
	- 'synapse/**'
	- 'tests/**'
	- 'rust/**'
	- '.ci/scripts/calculate_jobs.py'
	- 'Cargo.toml'
	- 'Cargo.lock'
	- 'pyproject.toml'
	- 'poetry.lock'
	- '.github/workflows/tests.yml'

	integration:
	- 'synapse/**'
	- 'rust/**'
	- 'docker/**'
	- 'Cargo.toml'
	- 'Cargo.lock'
	- 'pyproject.toml'
	- 'poetry.lock'
	- 'docker/**'
	- '.ci/**'
	- 'scripts-dev/complement.sh'
	- '.github/workflows/tests.yml'

	linting:
	- 'synapse/**'
	- 'docker/**'
	- 'tests/**'
	- 'scripts-dev/**'
	- 'contrib/**'
	- 'synmark/**'
	- 'stubs/**'
	- '.ci/**'
	- 'mypy.ini'
	- 'pyproject.toml'
	- 'poetry.lock'
	- '.github/workflows/tests.yml'

	linting_readme:
	- 'README.rst'

	check-sampleconfig:
	runs-on: ubuntu-latest
	needs: changes
	if: ${{ needs.changes.outputs.linting == 'true' }}

	steps:
	- uses: actions/checkout@v4
	- name: Install Rust
	uses: dtolnay/[email protected]
	- uses: Swatinem/rust-cache@v2
	- uses: matrix-org/setup-python-poetry@v1
	with:
	python-version: "3.x"
	poetry-version: "1.3.2"
	extras: "all"
	- run: poetry run scripts-dev/generate_sample_config.sh --check
	- run: poetry run scripts-dev/config-lint.sh

	check-schema-delta:
	runs-on: ubuntu-latest
	needs: changes
	if: ${{ needs.changes.outputs.linting == 'true' }}

	steps:
	- uses: actions/checkout@v4
	- uses: actions/setup-python@v5
	with:
	python-version: "3.x"
	- run: "pip install 'click==8.1.1' 'GitPython>=3.1.20'"
	- run: scripts-dev/check_schema_delta.py --force-colors

	check-lockfile:
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- uses: actions/setup-python@v5
	with:
	python-version: "3.x"
	- run: .ci/scripts/check_lockfile.py

	lint:
	runs-on: ubuntu-latest
	needs: changes
	if: ${{ needs.changes.outputs.linting == 'true' }}

	steps:
	- name: Checkout repository
	uses: actions/checkout@v4

	- name: Setup Poetry
	uses: matrix-org/setup-python-poetry@v1
	with:
	install-project: "false"

	- name: Run ruff check
	run: poetry run ruff check --output-format=github .

	- name: Run ruff format
	run: poetry run ruff format --check .

	lint-mypy:
	runs-on: ubuntu-latest
	name: Typechecking
	needs: changes
	if: ${{ needs.changes.outputs.linting == 'true' }}

	steps:
	- name: Checkout repository
	uses: actions/checkout@v4

	- name: Install Rust
	uses: dtolnay/[email protected]
	- uses: Swatinem/rust-cache@v2

	- name: Setup Poetry
	uses: matrix-org/setup-python-poetry@v1
	with:
	# We want to make use of type hints in optional dependencies too.
	extras: all
	# We have seen odd mypy failures that were resolved when we started
	# installing the project again:
	# https://github.com/matrix-org/synapse/pull/15376#issuecomment-1498983775
	# To make CI green, err towards caution and install the project.
	install-project: "true"

	# Cribbed from
	# https://github.com/AustinScola/mypy-cache-github-action/blob/85ea4f2972abed39b33bd02c36e341b28ca59213/src/restore.ts#L10-L17
	- name: Restore/persist mypy's cache
	uses: actions/cache@v4
	with:
	path: \|
	.mypy_cache
	key: mypy-cache-${{ github.context.sha }}
	restore-keys: mypy-cache-

	- name: Run mypy
	run: poetry run mypy

	lint-crlf:
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- name: Check line endings
	run: scripts-dev/check_line_terminators.sh

	lint-newsfile:
	if: ${{ (github.base_ref == 'develop' \|\| contains(github.base_ref, 'release-')) && github.actor != 'dependabot[bot]' }}
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	with:
	ref: ${{ github.event.pull_request.head.sha }}
	fetch-depth: 0
	- uses: actions/setup-python@v5
	with:
	python-version: "3.x"
	- run: "pip install 'towncrier>=18.6.0rc1'"
	- run: scripts-dev/check-newsfragment.sh
	env:
	PULL_REQUEST_NUMBER: ${{ github.event.number }}

	lint-pydantic:
	runs-on: ubuntu-latest
	needs: changes
	if: ${{ needs.changes.outputs.linting == 'true' }}

	steps:
	- uses: actions/checkout@v4
	with:
	ref: ${{ github.event.pull_request.head.sha }}
	- name: Install Rust
	uses: dtolnay/[email protected]
	- uses: Swatinem/rust-cache@v2
	- uses: matrix-org/setup-python-poetry@v1
	with:
	poetry-version: "1.3.2"
	extras: "all"
	- run: poetry run scripts-dev/check_pydantic_models.py

	lint-clippy:
	runs-on: ubuntu-latest
	needs: changes
	if: ${{ needs.changes.outputs.rust == 'true' }}

	steps:
	- uses: actions/checkout@v4

	- name: Install Rust
	uses: dtolnay/[email protected]
	with:
	components: clippy
	- uses: Swatinem/rust-cache@v2

	- run: cargo clippy -- -D warnings

	# We also lint against a nightly rustc so that we can lint the benchmark
	# suite, which requires a nightly compiler.
	lint-clippy-nightly:
	runs-on: ubuntu-latest
	needs: changes
	if: ${{ needs.changes.outputs.rust == 'true' }}

	steps:
	- uses: actions/checkout@v4

	- name: Install Rust
	uses: dtolnay/rust-toolchain@master
	with:
	toolchain: nightly-2022-12-01
	components: clippy
	- uses: Swatinem/rust-cache@v2

	- run: cargo clippy --all-features -- -D warnings

	lint-rustfmt:
	runs-on: ubuntu-latest
	needs: changes
	if: ${{ needs.changes.outputs.rust == 'true' }}

	steps:
	- uses: actions/checkout@v4

	- name: Install Rust
	uses: dtolnay/rust-toolchain@master
	with:
	# We use nightly so that it correctly groups together imports
	toolchain: nightly-2022-12-01
	components: rustfmt
	- uses: Swatinem/rust-cache@v2

	- run: cargo fmt --check

	# This is to detect issues with the rst file, which can otherwise cause issues
	# when uploading packages to PyPi.
	lint-readme:
	runs-on: ubuntu-latest
	needs: changes
	if: ${{ needs.changes.outputs.linting_readme == 'true' }}
	steps:
	- uses: actions/checkout@v4
	- uses: actions/setup-python@v5
	with:
	python-version: "3.x"
	- run: "pip install rstcheck"
	- run: "rstcheck --report-level=WARNING README.rst"

	# Dummy step to gate other tests on without repeating the whole list
	linting-done:
	if: ${{ !cancelled() }} # Run this even if prior jobs were skipped
	needs:
	- lint
	- lint-mypy
	- lint-crlf
	- lint-newsfile
	- lint-pydantic
	- check-sampleconfig
	- check-schema-delta
	- check-lockfile
	- lint-clippy
	- lint-clippy-nightly
	- lint-rustfmt
	- lint-readme
	runs-on: ubuntu-latest
	steps:
	- uses: matrix-org/done-action@v3
	with:
	needs: ${{ toJSON(needs) }}

	# Various bits are skipped if there was no applicable changes.
	skippable: \|
	check-sampleconfig
	check-schema-delta
	lint
	lint-mypy
	lint-newsfile
	lint-pydantic
	lint-clippy
	lint-clippy-nightly
	lint-rustfmt
	lint-readme


	calculate-test-jobs:
	if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
	needs: linting-done
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- uses: actions/setup-python@v5
	with:
	python-version: "3.x"
	- id: get-matrix
	run: .ci/scripts/calculate_jobs.py
	outputs:
	trial_test_matrix: ${{ steps.get-matrix.outputs.trial_test_matrix }}
	sytest_test_matrix: ${{ steps.get-matrix.outputs.sytest_test_matrix }}

	trial:
	if: ${{ !cancelled() && !failure() && needs.changes.outputs.trial == 'true' }} # Allow previous steps to be skipped, but not fail
	needs:
	- calculate-test-jobs
	- changes
	runs-on: ubuntu-latest
	strategy:
	matrix:
	job: ${{ fromJson(needs.calculate-test-jobs.outputs.trial_test_matrix) }}

	steps:
	- uses: actions/checkout@v4
	- run: sudo apt-get -qq install xmlsec1
	- name: Set up PostgreSQL ${{ matrix.job.postgres-version }}
	if: ${{ matrix.job.postgres-version }}
	# 1. Mount postgres data files onto a tmpfs in-memory filesystem to reduce overhead of docker's overlayfs layer.
	# 2. Expose the unix socket for postgres. This removes latency of using docker-proxy for connections.
	run: \|
	docker run -d -p 5432:5432 \
	--tmpfs /var/lib/postgres:rw,size=6144m \
	--mount 'type=bind,src=/var/run/postgresql,dst=/var/run/postgresql' \
	-e POSTGRES_PASSWORD=postgres \
	-e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \
	postgres:${{ matrix.job.postgres-version }}

	- name: Install Rust
	uses: dtolnay/[email protected]
	- uses: Swatinem/rust-cache@v2

	- uses: matrix-org/setup-python-poetry@v1
	with:
	python-version: ${{ matrix.job.python-version }}
	poetry-version: "1.3.2"
	extras: ${{ matrix.job.extras }}
	- name: Await PostgreSQL
	if: ${{ matrix.job.postgres-version }}
	timeout-minutes: 2
	run: until pg_isready -h localhost; do sleep 1; done
	- run: poetry run trial --jobs=6 tests
	env:
	SYNAPSE_POSTGRES: ${{ matrix.job.database == 'postgres' \|\| '' }}
	SYNAPSE_POSTGRES_HOST: /var/run/postgresql
	SYNAPSE_POSTGRES_USER: postgres
	SYNAPSE_POSTGRES_PASSWORD: postgres
	- name: Dump logs
	# Logs are most useful when the command fails, always include them.
	if: ${{ always() }}
	# Note: Dumps to workflow logs instead of using actions/upload-artifact
	# This keeps logs colocated with failing jobs
	# It also ignores find's exit code; this is a best effort affair
	run: >-
	find _trial_temp -name '*.log'
	-exec echo "::group::{}" \;
	-exec cat {} \;
	-exec echo "::endgroup::" \;
	\|\| true

	trial-olddeps:
	# Note: sqlite only; no postgres
	if: ${{ !cancelled() && !failure() && needs.changes.outputs.trial == 'true' }} # Allow previous steps to be skipped, but not fail
	needs:
	- linting-done
	- changes
	runs-on: ubuntu-20.04
	steps:
	- uses: actions/checkout@v4

	- name: Install Rust
	uses: dtolnay/[email protected]
	- uses: Swatinem/rust-cache@v2

	# There aren't wheels for some of the older deps, so we need to install
	# their build dependencies
	- run: \|
	sudo apt-get -qq update
	sudo apt-get -qq install build-essential libffi-dev python-dev \
	libxml2-dev libxslt-dev xmlsec1 zlib1g-dev libjpeg-dev libwebp-dev

	- uses: actions/setup-python@v5
	with:
	python-version: '3.8'

	- name: Prepare old deps
	if: steps.cache-poetry-old-deps.outputs.cache-hit != 'true'
	run: .ci/scripts/prepare_old_deps.sh

	# Note: we install using `pip` here, not poetry. `poetry install` ignores the
	# build-system section (https://github.com/python-poetry/poetry/issues/6154), but
	# we explicitly want to test that you can `pip install` using the oldest version
	# of poetry-core and setuptools-rust.
	- run: pip install .[all,test]

	# We nuke the local copy, as we've installed synapse into the virtualenv
	# (rather than use an editable install, which we no longer support). If we
	# don't do this then python can't find the native lib.
	- run: rm -rf synapse/

	# Sanity check we can import/run Synapse
	- run: python -m synapse.app.homeserver --help

	- run: python -m twisted.trial -j6 tests
	- name: Dump logs
	# Logs are most useful when the command fails, always include them.
	if: ${{ always() }}
	# Note: Dumps to workflow logs instead of using actions/upload-artifact
	# This keeps logs colocated with failing jobs
	# It also ignores find's exit code; this is a best effort affair
	run: >-
	find _trial_temp -name '*.log'
	-exec echo "::group::{}" \;
	-exec cat {} \;
	-exec echo "::endgroup::" \;
	\|\| true

	trial-pypy:
	# Very slow; only run if the branch name includes 'pypy'
	# Note: sqlite only; no postgres. Completely untested since poetry move.
	if: ${{ contains(github.ref, 'pypy') && !failure() && !cancelled() && needs.changes.outputs.trial == 'true' }}
	needs:
	- linting-done
	- changes
	runs-on: ubuntu-latest
	strategy:
	matrix:
	python-version: ["pypy-3.8"]
	extras: ["all"]

	steps:
	- uses: actions/checkout@v4
	# Install libs necessary for PyPy to build binary wheels for dependencies
	- run: sudo apt-get -qq install xmlsec1 libxml2-dev libxslt-dev
	- uses: matrix-org/setup-python-poetry@v1
	with:
	python-version: ${{ matrix.python-version }}
	poetry-version: "1.3.2"
	extras: ${{ matrix.extras }}
	- run: poetry run trial --jobs=2 tests
	- name: Dump logs
	# Logs are most useful when the command fails, always include them.
	if: ${{ always() }}
	# Note: Dumps to workflow logs instead of using actions/upload-artifact
	# This keeps logs colocated with failing jobs
	# It also ignores find's exit code; this is a best effort affair
	run: >-
	find _trial_temp -name '*.log'
	-exec echo "::group::{}" \;
	-exec cat {} \;
	-exec echo "::endgroup::" \;
	\|\| true

	sytest:
	if: ${{ !failure() && !cancelled() && needs.changes.outputs.integration == 'true' }}
	needs:
	- calculate-test-jobs
	- changes
	runs-on: ubuntu-latest
	container:
	image: matrixdotorg/sytest-synapse:${{ matrix.job.sytest-tag }}
	volumes:
	- ${{ github.workspace }}:/src
	env:
	# If this is a pull request to a release branch, use that branch as default branch for sytest, else use develop
	# This works because the release script always create a branch on the sytest repo with the same name as the release branch
	SYTEST_DEFAULT_BRANCH: ${{ startsWith(github.base_ref, 'release-') && github.base_ref \|\| 'develop' }}
	SYTEST_BRANCH: ${{ github.head_ref }}
	POSTGRES: ${{ matrix.job.postgres && 1}}
	MULTI_POSTGRES: ${{ (matrix.job.postgres == 'multi-postgres') \|\| '' }}
	ASYNCIO_REACTOR: ${{ (matrix.job.reactor == 'asyncio') \|\| '' }}
	WORKERS: ${{ matrix.job.workers && 1 }}
	BLACKLIST: ${{ matrix.job.workers && 'synapse-blacklist-with-workers' }}
	TOP: ${{ github.workspace }}

	strategy:
	fail-fast: false
	matrix:
	job: ${{ fromJson(needs.calculate-test-jobs.outputs.sytest_test_matrix) }}

	steps:
	- uses: actions/checkout@v4
	- name: Prepare test blacklist
	run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers

	- name: Install Rust
	uses: dtolnay/[email protected]
	- uses: Swatinem/rust-cache@v2

	- name: Run SyTest
	run: /bootstrap.sh synapse
	working-directory: /src
	- name: Summarise results.tap
	if: ${{ always() }}
	run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
	- name: Upload SyTest logs
	uses: actions/upload-artifact@v4
	if: ${{ always() }}
	with:
	name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.job.*, ', ') }})
	path: \|
	/logs/results.tap
	/logs/*/.log*

	export-data:
	if: ${{ !failure() && !cancelled() && needs.changes.outputs.integration == 'true'}} # Allow previous steps to be skipped, but not fail
	needs: [linting-done, portdb, changes]
	runs-on: ubuntu-latest
	env:
	TOP: ${{ github.workspace }}

	services:
	postgres:
	image: postgres
	ports:
	- 5432:5432
	env:
	POSTGRES_PASSWORD: "postgres"
	POSTGRES_INITDB_ARGS: "--lc-collate C --lc-ctype C --encoding UTF8"
	options: >-
	--health-cmd pg_isready
	--health-interval 10s
	--health-timeout 5s
	--health-retries 5

	steps:
	- uses: actions/checkout@v4
	- run: sudo apt-get -qq install xmlsec1 postgresql-client
	- uses: matrix-org/setup-python-poetry@v1
	with:
	poetry-version: "1.3.2"
	extras: "postgres"
	- run: .ci/scripts/test_export_data_command.sh
	env:
	PGHOST: localhost
	PGUSER: postgres
	PGPASSWORD: postgres
	PGDATABASE: postgres


	portdb:
	if: ${{ !failure() && !cancelled() && needs.changes.outputs.integration == 'true'}} # Allow previous steps to be skipped, but not fail
	needs:
	- linting-done
	- changes
	runs-on: ubuntu-latest
	strategy:
	matrix:
	include:
	- python-version: "3.8"
	postgres-version: "11"

	- python-version: "3.11"
	postgres-version: "15"

	services:
	postgres:
	image: postgres:${{ matrix.postgres-version }}
	ports:
	- 5432:5432
	env:
	POSTGRES_PASSWORD: "postgres"
	POSTGRES_INITDB_ARGS: "--lc-collate C --lc-ctype C --encoding UTF8"
	options: >-
	--health-cmd pg_isready
	--health-interval 10s
	--health-timeout 5s
	--health-retries 5

	steps:
	- uses: actions/checkout@v4
	- name: Add PostgreSQL apt repository
	# We need a version of pg_dump that can handle the version of
	# PostgreSQL being tested against. The Ubuntu package repository lags
	# behind new releases, so we have to use the PostreSQL apt repository.
	# Steps taken from https://www.postgresql.org/download/linux/ubuntu/
	run: \|
	sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
	wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc \| sudo apt-key add -
	sudo apt-get update
	- run: sudo apt-get -qq install xmlsec1 postgresql-client
	- uses: matrix-org/setup-python-poetry@v1
	with:
	python-version: ${{ matrix.python-version }}
	poetry-version: "1.3.2"
	extras: "postgres"
	- run: .ci/scripts/test_synapse_port_db.sh
	id: run_tester_script
	env:
	PGHOST: localhost
	PGUSER: postgres
	PGPASSWORD: postgres
	PGDATABASE: postgres
	- name: "Upload schema differences"
	uses: actions/upload-artifact@v4
	if: ${{ failure() && !cancelled() && steps.run_tester_script.outcome == 'failure' }}
	with:
	name: Schema dumps
	path: \|
	unported.sql
	ported.sql
	schema_diff

	complement:
	if: "${{ !failure() && !cancelled() && needs.changes.outputs.integration == 'true' }}"
	needs:
	- linting-done
	- changes
	runs-on: ubuntu-latest

	strategy:
	fail-fast: false
	matrix:
	include:
	- arrangement: monolith
	database: SQLite

	- arrangement: monolith
	database: Postgres

	- arrangement: workers
	database: Postgres

	steps:
	- name: Run actions/checkout@v4 for synapse
	uses: actions/checkout@v4
	with:
	path: synapse

	- name: Install Rust
	uses: dtolnay/[email protected]
	- uses: Swatinem/rust-cache@v2

	- name: Prepare Complement's Prerequisites
	run: synapse/.ci/scripts/setup_complement_prerequisites.sh

	- uses: actions/setup-go@v5
	with:
	cache-dependency-path: complement/go.sum
	go-version-file: complement/go.mod

	# use p=1 concurrency as GHA boxes are underpowered and don't like running tons of synapses at once.
	- run: \|
	set -o pipefail
	COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -p 1 -json 2>&1 \| synapse/.ci/scripts/gotestfmt
	shell: bash
	env:
	POSTGRES: ${{ (matrix.database == 'Postgres') && 1 \|\| '' }}
	WORKERS: ${{ (matrix.arrangement == 'workers') && 1 \|\| '' }}
	name: Run Complement Tests

	cargo-test:
	if: ${{ needs.changes.outputs.rust == 'true' }}
	runs-on: ubuntu-latest
	needs:
	- linting-done
	- changes

	steps:
	- uses: actions/checkout@v4

	- name: Install Rust
	uses: dtolnay/[email protected]
	- uses: Swatinem/rust-cache@v2

	- run: cargo test

	# We want to ensure that the cargo benchmarks still compile, which requires a
	# nightly compiler.
	cargo-bench:
	if: ${{ needs.changes.outputs.rust == 'true' }}
	runs-on: ubuntu-latest
	needs:
	- linting-done
	- changes

	steps:
	- uses: actions/checkout@v4

	- name: Install Rust
	uses: dtolnay/rust-toolchain@master
	with:
	toolchain: nightly-2022-12-01
	- uses: Swatinem/rust-cache@v2

	- run: cargo bench --no-run

	# a job which marks all the other jobs as complete, thus allowing PRs to be merged.
	tests-done:
	if: ${{ always() }}
	needs:
	- trial
	- trial-olddeps
	- sytest
	- export-data
	- portdb
	- complement
	- cargo-test
	- cargo-bench
	- linting-done
	runs-on: ubuntu-latest
	steps:
	- uses: matrix-org/done-action@v3
	with:
	needs: ${{ toJSON(needs) }}

	# Various bits are skipped if there was no applicable changes.
	# The newsfile lint may be skipped on non PR builds.
	skippable: \|
	trial
	trial-olddeps
	sytest
	portdb
	export-data
	complement
	lint-newsfile
	cargo-test
	cargo-bench

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Implement MSC4210: Remove legacy mentions #3521

Workflow file

Implement MSC4210: Remove legacy mentions #3521

Jobs

Run details

Workflow file for this run