From 1917f7490cbc0a41aa22e1ef25fa0be0000897c8 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Tue, 7 Aug 2018 20:41:49 +1000 Subject: [PATCH 01/10] some initial dockerfile stuff --- .dockerignore | 3 +++ Dockerfile-sytest | 26 +++++++++++++++++++ Dockerfile-sytestpy2 | 3 +++ Dockerfile-sytestpy3 | 3 +++ docker_sytest.sh | 59 ++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 94 insertions(+) create mode 100644 Dockerfile-sytest create mode 100644 Dockerfile-sytestpy2 create mode 100644 Dockerfile-sytestpy3 create mode 100755 docker_sytest.sh diff --git a/.dockerignore b/.dockerignore index f36f86fbb722..6cdb8532d332 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,3 +3,6 @@ Dockerfile .gitignore demo/etc tox.ini +synctl +.git/* +.tox/* diff --git a/Dockerfile-sytest b/Dockerfile-sytest new file mode 100644 index 000000000000..b87676ac51f5 --- /dev/null +++ b/Dockerfile-sytest @@ -0,0 +1,26 @@ +FROM debian:stretch + +ENV DEBIAN_FRONTEND noninteractive + +RUN apt-get -qq update +RUN apt-get -qq install -y python python-dev python-virtualenv python3-virtualenv python3-dev build-essential perl wget postgresql postgresql-client libpq-dev libssl-dev libz-dev libffi-dev sqlite3 libjpeg-dev libxslt1-dev git locales + +RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ + dpkg-reconfigure --frontend=noninteractive locales && \ + update-locale LANG=en_US.UTF-8 + +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 + +RUN wget -q https://github.com/matrix-org/sytest/raw/develop/install-deps.pl +RUN wget -q https://github.com/matrix-org/sytest/raw/develop/cpanfile +RUN perl install-deps.pl + +RUN mkdir /test +RUN mkdir /src +RUN mkdir /logs +WORKDIR /test + +ADD docker_sytest.sh /docker_sytest.sh +ENTRYPOINT /docker_sytest.sh diff --git a/Dockerfile-sytestpy2 b/Dockerfile-sytestpy2 new file mode 100644 index 000000000000..7b01f529ee2c --- /dev/null +++ b/Dockerfile-sytestpy2 @@ -0,0 +1,3 @@ +FROM hawkowl/sytest:latest + +ENV PYTHON=python2 diff --git a/Dockerfile-sytestpy3 b/Dockerfile-sytestpy3 new file mode 100644 index 000000000000..b294caa4d79b --- /dev/null +++ b/Dockerfile-sytestpy3 @@ -0,0 +1,3 @@ +FROM hawkowl/sytest:latest + +ENV PYTHON=python3.5 diff --git a/docker_sytest.sh b/docker_sytest.sh new file mode 100755 index 000000000000..dd136c038942 --- /dev/null +++ b/docker_sytest.sh @@ -0,0 +1,59 @@ +#! /usr/bin/env bash + +set -x + +if [ -e "/test/run-tests.pl" ] +then + echo "Using local sytests..." +else + branch_name="$(git --git-dir=/src/.git symbolic-ref HEAD 2>/dev/null)" || branch_name="(unnamed branch)" + + echo "Trying to get same-named sytest..." + wget -q https://github.com/matrix-org/sytest/archive/$branch_name.tar.gz -O sytest.tar.gz +4 + if [ $? -eq 0 ] + then + echo "Using $branch_name!" + else + echo "Using develop instead..." + wget -q https://github.com/matrix-org/sytest/archive/develop.tar.gz -O sytest.tar.gz + fi + + tar --strip-components=1 -xf sytest.tar.gz + +fi + +# PostgreSQL setup +if [ -n "$POSTGRES" ] +then + + export PGDATA=/var/lib/postgresql/data + export PGUSER=postgres + export POSTGRES_DB_1=pg1 + export POSTGRES_DB_2=pg2 + + su -c '/usr/lib/postgresql/9.6/bin/initdb -E "UTF-8" --lc-collate="en_US.UTF-8" --lc-ctype="en_US.UTF-8" --username=postgres' postgres + su -c '/usr/lib/postgresql/9.6/bin/pg_ctl -w -D /var/lib/postgresql/data start' postgres + + jenkins/prep_sytest_for_postgres.sh + + su -c 'psql -c "CREATE DATABASE pg1;"' postgres + su -c 'psql -c "CREATE DATABASE pg2;"' postgres + +fi + +# Build the virtualenv +$PYTHON -m virtualenv -p $PYTHON /venv/ +/venv/bin/pip install -q --no-cache-dir -e /src/ +/venv/bin/pip install -q --no-cache-dir lxml psycopg2 + +# Make sure all deps are installed -- this is done in the docker build so it shouldn't be too many +./install-deps.pl + +# Run the tests +./run-tests.pl -I Synapse --python=/venv/bin/python -O tap --all > results.tap + +# Copy out the logs +cp results.tap /logs/results.tap +cp server-0/homeserver.log /logs/homeserver-0.log +cp server-1/homeserver.log /logs/homeserver-1.log From 03306b7cea1322d782f749a925afd85f82f152be Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Tue, 7 Aug 2018 20:41:56 +1000 Subject: [PATCH 02/10] circleci --- .circleci/config.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 000000000000..c0e7105f6bbc --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,11 @@ +version: 2 +jobs: + sytestpy2: + machine: true + steps: + - checkout + - run: docker pull hawkowl/sytestpy2 + - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs -e POSTGRES=1 hawkowl/sytestpy2 + - store_artifacts: + path: ~/logs + destination: logs From d9e8a82ca0b452b77121369633dc72085f418834 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Tue, 7 Aug 2018 20:44:06 +1000 Subject: [PATCH 03/10] py3 as well --- .circleci/config.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index c0e7105f6bbc..b51b28f9fd3d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,3 +9,12 @@ jobs: - store_artifacts: path: ~/logs destination: logs + sytestpy3: + machine: true + steps: + - checkout + - run: docker pull hawkowl/sytestpy3 + - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs -e POSTGRES=1 hawkowl/sytestpy3 + - store_artifacts: + path: ~/logs + destination: logs From b48392cf291516ded63f3a1da86f6159069aa985 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Tue, 7 Aug 2018 20:46:01 +1000 Subject: [PATCH 04/10] run on workflows --- .circleci/config.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index b51b28f9fd3d..0c6dd685591b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -18,3 +18,10 @@ jobs: - store_artifacts: path: ~/logs destination: logs + +workflows: + version: 2 + build: + jobs: + - sytestpy2 + - sytestpy3 From c58fd2fbff8e3193af5bd4290bc927aef3c890ab Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Tue, 7 Aug 2018 20:53:12 +1000 Subject: [PATCH 05/10] sqlite and postgres and proper logs --- .circleci/config.yml | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0c6dd685591b..04d8a4ead75b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,22 +1,31 @@ version: 2 jobs: sytestpy2: + machine: true + steps: + - checkout + - run: docker pull hawkowl/sytestpy2 + - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs hawkowl/sytestpy2 + - store_artifacts: + path: ~/project/logs + destination: logs + sytestpy2postgres: machine: true steps: - checkout - run: docker pull hawkowl/sytestpy2 - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs -e POSTGRES=1 hawkowl/sytestpy2 - store_artifacts: - path: ~/logs + path: ~/project/logs destination: logs sytestpy3: machine: true steps: - checkout - run: docker pull hawkowl/sytestpy3 - - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs -e POSTGRES=1 hawkowl/sytestpy3 + - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs hawkowl/sytestpy3 - store_artifacts: - path: ~/logs + path: ~/project/logs destination: logs workflows: @@ -24,4 +33,5 @@ workflows: build: jobs: - sytestpy2 + - sytestpy2postgres - sytestpy3 From 8a28ba2f1c5bc18ce896685997fcddfc7b512ef2 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Tue, 7 Aug 2018 21:23:22 +1000 Subject: [PATCH 06/10] fix tests --- MANIFEST.in | 3 +++ changelog.d/3660.misc | 1 + 2 files changed, 4 insertions(+) create mode 100644 changelog.d/3660.misc diff --git a/MANIFEST.in b/MANIFEST.in index 1ff98d95dff0..d635ec8e9e37 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -27,6 +27,8 @@ exclude jenkins.sh exclude jenkins*.sh exclude jenkins* exclude Dockerfile +exclude Dockerfile* +exclude docker_sytest.sh exclude .dockerignore recursive-exclude jenkins *.sh @@ -36,3 +38,4 @@ recursive-include changelog.d * prune .github prune demo/etc prune docker +prune .circleci diff --git a/changelog.d/3660.misc b/changelog.d/3660.misc new file mode 100644 index 000000000000..acd814c27391 --- /dev/null +++ b/changelog.d/3660.misc @@ -0,0 +1 @@ +Sytests can now be run inside a Docker container. From 84822386e4b03a8bf99405a26880f0e3b1f59e9e Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 8 Aug 2018 17:05:45 +1000 Subject: [PATCH 07/10] make it go green --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 04d8a4ead75b..8f91308b0624 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -34,4 +34,5 @@ workflows: jobs: - sytestpy2 - sytestpy2postgres - - sytestpy3 +# Currently broken! +# - sytestpy3 From 6c6f628e9481f7de20856f221d566f431d5c0256 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 8 Aug 2018 20:03:03 +1000 Subject: [PATCH 08/10] move out the dockerfiles, update in CI --- .circleci/config.yml | 10 ++++---- Dockerfile-sytest | 26 ------------------- Dockerfile-sytestpy2 | 3 --- Dockerfile-sytestpy3 | 3 --- docker_sytest.sh | 59 -------------------------------------------- 5 files changed, 5 insertions(+), 96 deletions(-) delete mode 100644 Dockerfile-sytest delete mode 100644 Dockerfile-sytestpy2 delete mode 100644 Dockerfile-sytestpy3 delete mode 100755 docker_sytest.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 8f91308b0624..f3c0c875f95f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -4,8 +4,8 @@ jobs: machine: true steps: - checkout - - run: docker pull hawkowl/sytestpy2 - - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs hawkowl/sytestpy2 + - run: docker pull matrixdotorg/sytest-synapsepy2 + - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs matrixdotorg/sytest-synapsepy2 - store_artifacts: path: ~/project/logs destination: logs @@ -13,8 +13,8 @@ jobs: machine: true steps: - checkout - - run: docker pull hawkowl/sytestpy2 - - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs -e POSTGRES=1 hawkowl/sytestpy2 + - run: docker pull matrixdotorg/sytest-synapsepy2 + - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs -e POSTGRES=1 matrixdotorg/sytest-synapsepy2 - store_artifacts: path: ~/project/logs destination: logs @@ -22,7 +22,7 @@ jobs: machine: true steps: - checkout - - run: docker pull hawkowl/sytestpy3 + - run: docker pull matrixdotorg/sytest-synapsepy3 - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs hawkowl/sytestpy3 - store_artifacts: path: ~/project/logs diff --git a/Dockerfile-sytest b/Dockerfile-sytest deleted file mode 100644 index b87676ac51f5..000000000000 --- a/Dockerfile-sytest +++ /dev/null @@ -1,26 +0,0 @@ -FROM debian:stretch - -ENV DEBIAN_FRONTEND noninteractive - -RUN apt-get -qq update -RUN apt-get -qq install -y python python-dev python-virtualenv python3-virtualenv python3-dev build-essential perl wget postgresql postgresql-client libpq-dev libssl-dev libz-dev libffi-dev sqlite3 libjpeg-dev libxslt1-dev git locales - -RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ - dpkg-reconfigure --frontend=noninteractive locales && \ - update-locale LANG=en_US.UTF-8 - -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US:en -ENV LC_ALL en_US.UTF-8 - -RUN wget -q https://github.com/matrix-org/sytest/raw/develop/install-deps.pl -RUN wget -q https://github.com/matrix-org/sytest/raw/develop/cpanfile -RUN perl install-deps.pl - -RUN mkdir /test -RUN mkdir /src -RUN mkdir /logs -WORKDIR /test - -ADD docker_sytest.sh /docker_sytest.sh -ENTRYPOINT /docker_sytest.sh diff --git a/Dockerfile-sytestpy2 b/Dockerfile-sytestpy2 deleted file mode 100644 index 7b01f529ee2c..000000000000 --- a/Dockerfile-sytestpy2 +++ /dev/null @@ -1,3 +0,0 @@ -FROM hawkowl/sytest:latest - -ENV PYTHON=python2 diff --git a/Dockerfile-sytestpy3 b/Dockerfile-sytestpy3 deleted file mode 100644 index b294caa4d79b..000000000000 --- a/Dockerfile-sytestpy3 +++ /dev/null @@ -1,3 +0,0 @@ -FROM hawkowl/sytest:latest - -ENV PYTHON=python3.5 diff --git a/docker_sytest.sh b/docker_sytest.sh deleted file mode 100755 index dd136c038942..000000000000 --- a/docker_sytest.sh +++ /dev/null @@ -1,59 +0,0 @@ -#! /usr/bin/env bash - -set -x - -if [ -e "/test/run-tests.pl" ] -then - echo "Using local sytests..." -else - branch_name="$(git --git-dir=/src/.git symbolic-ref HEAD 2>/dev/null)" || branch_name="(unnamed branch)" - - echo "Trying to get same-named sytest..." - wget -q https://github.com/matrix-org/sytest/archive/$branch_name.tar.gz -O sytest.tar.gz -4 - if [ $? -eq 0 ] - then - echo "Using $branch_name!" - else - echo "Using develop instead..." - wget -q https://github.com/matrix-org/sytest/archive/develop.tar.gz -O sytest.tar.gz - fi - - tar --strip-components=1 -xf sytest.tar.gz - -fi - -# PostgreSQL setup -if [ -n "$POSTGRES" ] -then - - export PGDATA=/var/lib/postgresql/data - export PGUSER=postgres - export POSTGRES_DB_1=pg1 - export POSTGRES_DB_2=pg2 - - su -c '/usr/lib/postgresql/9.6/bin/initdb -E "UTF-8" --lc-collate="en_US.UTF-8" --lc-ctype="en_US.UTF-8" --username=postgres' postgres - su -c '/usr/lib/postgresql/9.6/bin/pg_ctl -w -D /var/lib/postgresql/data start' postgres - - jenkins/prep_sytest_for_postgres.sh - - su -c 'psql -c "CREATE DATABASE pg1;"' postgres - su -c 'psql -c "CREATE DATABASE pg2;"' postgres - -fi - -# Build the virtualenv -$PYTHON -m virtualenv -p $PYTHON /venv/ -/venv/bin/pip install -q --no-cache-dir -e /src/ -/venv/bin/pip install -q --no-cache-dir lxml psycopg2 - -# Make sure all deps are installed -- this is done in the docker build so it shouldn't be too many -./install-deps.pl - -# Run the tests -./run-tests.pl -I Synapse --python=/venv/bin/python -O tap --all > results.tap - -# Copy out the logs -cp results.tap /logs/results.tap -cp server-0/homeserver.log /logs/homeserver-0.log -cp server-1/homeserver.log /logs/homeserver-1.log From bcb23c0edaa2f417c045ce0bcb58918f62780d85 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 8 Aug 2018 20:30:35 +1000 Subject: [PATCH 09/10] Update MANIFEST.in --- MANIFEST.in | 2 -- 1 file changed, 2 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index d635ec8e9e37..e0826ba544c8 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -27,8 +27,6 @@ exclude jenkins.sh exclude jenkins*.sh exclude jenkins* exclude Dockerfile -exclude Dockerfile* -exclude docker_sytest.sh exclude .dockerignore recursive-exclude jenkins *.sh From 44b85174cab632ba5143dc4af765fb388ccac1ec Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 8 Aug 2018 20:59:45 +1000 Subject: [PATCH 10/10] Update config.yml --- .circleci/config.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f3c0c875f95f..e03f01b83703 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -27,6 +27,15 @@ jobs: - store_artifacts: path: ~/project/logs destination: logs + sytestpy3postgres: + machine: true + steps: + - checkout + - run: docker pull matrixdotorg/sytest-synapsepy3 + - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs -e POSTGRES=1 matrixdotorg/sytest-synapsepy3 + - store_artifacts: + path: ~/project/logs + destination: logs workflows: version: 2 @@ -34,5 +43,6 @@ workflows: jobs: - sytestpy2 - sytestpy2postgres -# Currently broken! +# Currently broken while the Python 3 port is incomplete # - sytestpy3 +# - sytestpy3postgres