Don't modify source volumes when running in docker (#748)

I've fallen into this trap several times now: I don't expect the docker image
to modify my local files.

docker/README.md

@@ -19,20 +19,20 @@ Once pulled from Docker Hub, a container can be run on a homeserver checkout:
 ### Synapse
 
 ```
-docker run --rm -it -v /path/to/synapse\:/src -v /path/to/where/you/want/logs\:/logs matrixdotorg/sytest-synapse:py35
+docker run --rm -it -v /path/to/synapse\:/src:ro -v /path/to/where/you/want/logs\:/logs matrixdotorg/sytest-synapse:py35
 ```
 
 ### Dendrite
 
 ```
-docker run --rm -it -v /path/to/dendrite\:/src -v /path/to/where/you/want/logs\:/logs matrixdotorg/sytest-dendrite
+docker run --rm -it -v /path/to/dendrite\:/src:ro -v /path/to/where/you/want/logs\:/logs matrixdotorg/sytest-dendrite
 ```
 
 This will run on the same branch in SyTest as the checkout, if possible, but
 will fall back to using either Synapse or Dendrite's `develop` branch.
 
 If you want to use an existing checkout of SyTest, mount it to `/sytest` inside
-the container by adding `-v /path/to/sytest\:/sytest` to the docker command.
+the container by adding `-v /path/to/sytest\:/sytest:ro` to the docker command.
 
 You can pass arguments to sytest by adding them at the end of the docker
 command. For example, you can use

docker/bootstrap.sh

@@ -35,9 +35,5 @@ fi
 export SYTEST_LIB="/sytest/lib"
 SYTEST_SCRIPT="/sytest/docker/$1_sytest.sh"
 
-# dos2unix files that need to be UNIX line ending
-dos2unix $SYTEST_SCRIPT
-dos2unix /sytest/*.pl
-
 # Run the sytest script
 $SYTEST_SCRIPT "${@:2}"

docker/dendrite_sytest.sh

@@ -4,6 +4,8 @@ set -ex
 
 cd /sytest
 
+mkdir /work
+
 # Make sure all Perl deps are installed -- this is done in the docker build so will only install packages added since the last Docker build
 ./install-deps.pl
 
@@ -15,19 +17,23 @@ su -c "psql -c \"CREATE USER dendrite PASSWORD 'itsasecret'\" postgres"
 su -c 'for i in account device mediaapi syncapi roomserver serverkey federationsender publicroomsapi appservice naffka sytest_template; do psql -c "CREATE DATABASE $i OWNER dendrite;"; done' postgres
 
 # Write dendrite configuration
-mkdir -p "server-0"
-cat > "server-0/database.yaml" << EOF
+mkdir -p "/work/server-0"
+cat > "/work/server-0/database.yaml" << EOF
 args:
     user: $PGUSER
     database: $PGUSER
     host: $PGHOST
 type: pg
 EOF
 
+
 # Run the tests
-dos2unix ./run-tests.pl
+mkdir -p /logs
+
 TEST_STATUS=0
-./run-tests.pl -I Dendrite::Monolith -d /src/bin -W /src/testfile -O tap --all "$@" > results.tap || TEST_STATUS=$?
+./run-tests.pl -I Dendrite::Monolith -d /src/bin -W /src/testfile -O tap --all \
+    --work-directory="/work" \
+    "$@" > /logs/results.tap || TEST_STATUS=$?
 
 if [ $TEST_STATUS -ne 0 ]; then
     echo >&2 -e "run-tests \e[31mFAILED\e[0m: exit code $TEST_STATUS"
@@ -41,8 +47,6 @@ fi
 echo >&2 "--- Copying assets"
 
 # Copy out the logs
-mkdir -p /logs
-cp results.tap /logs/results.tap
 rsync --ignore-missing-args --min-size=1B -av server-0 server-1 /logs --include "*/" --include="*.log.*" --include="*.log" --exclude="*"
 
 # Write out JUnit

docker/prep_sytest_for_postgres.sh

@@ -6,7 +6,7 @@
 
 set -e
 
-cd "`dirname $0`/.."
+cd "/work"
 
 if [ -z "$POSTGRES_DB_1" ]; then
     echo >&2 "Variable POSTGRES_DB_1 not set"

docker/synapse_sytest.sh

@@ -6,6 +6,8 @@ set -ex
 
 cd /sytest
 
+mkdir /work
+
 # PostgreSQL setup
 if [ -n "$POSTGRES" ]; then
     export PGUSER=postgres
@@ -16,7 +18,6 @@ if [ -n "$POSTGRES" ]; then
     su -c 'eatmydata /usr/lib/postgresql/*/bin/pg_ctl -w -D $PGDATA start' postgres
 
     # Write out the configuration for a PostgreSQL using Synapse
-    dos2unix docker/prep_sytest_for_postgres.sh
     docker/prep_sytest_for_postgres.sh
 
     # Make the test databases for the two Synapse servers that will be spun up
@@ -36,7 +37,7 @@ if [ -n "$OFFLINE" ]; then
 else
     # We've already created the virtualenv, but lets double check we have all
     # deps.
-    /venv/bin/pip install -q --upgrade --no-cache-dir -e /src
+    /venv/bin/pip install -q --upgrade --no-cache-dir /src
     /venv/bin/pip install -q --upgrade --no-cache-dir \
         lxml psycopg2 coverage codecov tap.py coverage_enable_subprocess
 
@@ -55,18 +56,20 @@ echo >&2 "+++ Running tests"
 export COVERAGE_PROCESS_START="/src/.coveragerc"
 
 RUN_TESTS=(
-    perl -I "$SYTEST_LIB" ./run-tests.pl --python=/venv/bin/python --synapse-directory=/src -B "/src/$BLACKLIST" --coverage -O tap --all
+    perl -I "$SYTEST_LIB" /sytest/run-tests.pl --python=/venv/bin/python --synapse-directory=/src -B "/src/$BLACKLIST" --coverage -O tap --all
+    --work-directory="/work"
 )
 
-TEST_STATUS=0
-
 if [ -n "$WORKERS" ]; then
     RUN_TESTS+=(-I Synapse::ViaHaproxy --dendron-binary=/pydron.py)
 else
     RUN_TESTS+=(-I Synapse)
 fi
 
-"${RUN_TESTS[@]}" "$@" >results.tap || TEST_STATUS=$?
+mkdir -p /logs
+
+TEST_STATUS=0
+"${RUN_TESTS[@]}" "$@" >/logs/results.tap || TEST_STATUS=$?
 
 if [ $TEST_STATUS -ne 0 ]; then
     echo >&2 -e "run-tests \e[31mFAILED\e[0m: exit code $TEST_STATUS"
@@ -77,14 +80,12 @@ fi
 echo >&2 "--- Copying assets"
 
 # Copy out the logs
-mkdir -p /logs
-cp results.tap /logs/results.tap
-rsync --ignore-missing-args --min-size=1B -av server-0 server-1 /logs --include "*/" --include="*.log.*" --include="*.log" --exclude="*"
-cp /.coverage.* /src || true
+rsync --ignore-missing-args --min-size=1B -av /work/server-0 /work/server-1 /logs --include "*/" --include="*.log.*" --include="*.log" --exclude="*"
+#cp /.coverage.* /src || true
 
-cd /src
-export TOP=/src
-/venv/bin/coverage combine
+#cd /src
+#export TOP=/src
+#/venv/bin/coverage combine
 
 if [ $TEST_STATUS -ne 0 ]; then
     # Build the annotation

run-tests.pl

@@ -66,6 +66,9 @@
 # the server under test'.
 our $TEST_ROOM_VERSION;
 
+# where we put working files (server configs, mostly)
+our $WORK_DIR = ".";
+
 Getopt::Long::Configure('pass_through');
 GetOptions(
    'I|server-implementation=s' => \$SERVER_IMPL,
@@ -87,6 +90,8 @@
 
    'n|no-tls' => sub { $WANT_TLS = 0 },
 
+   'work-directory=s' => \$WORK_DIR,
+
    'room-version=s' => \$TEST_ROOM_VERSION,
 
    # these two are superceded by -I, but kept for backwards compatibility
@@ -178,6 +183,9 @@ sub usage
 
    -p, --port-range START:MAX   - pool of TCP ports to allocate from
 
+   --work-directory DIR         - where we put working files (server configs,
+                                  mostly). Defaults to '.'.
+
    --room-version VERSION       - use the given room version for the majority of
                                   tests
 .

tests/01http-server.pl

@@ -30,7 +30,7 @@ =head2 start_test_server_ssl
 sub start_test_server_ssl {
    my ( $server ) = @_;
 
-   my $test_server_dir = abs_path( "test-server" );
+   my $test_server_dir = abs_path( $main::WORK_DIR."/test-server" );
    -d $test_server_dir or make_path( $test_server_dir );
 
    my $ssl_cert = "$test_server_dir/server.crt";

tests/05homeserver.pl

@@ -33,7 +33,7 @@
          $OUTPUT->diag( "Starting Homeserver using $HS_FACTORY" );
 
          my $server = eval { $HS_FACTORY->create_server(
-            hs_dir              => abs_path( "server-$idx" ),
+            hs_dir              => abs_path( $main::WORK_DIR . "/server-$idx" ),
             hs_index            => $idx,
             bind_host           => $BIND_HOST,
             output              => $OUTPUT,