Respect poetry lockfile in Synapse container, if present

docker/synapse.Dockerfile

@@ -4,12 +4,53 @@ FROM matrixdotorg/sytest:${SYTEST_IMAGE_TAG}
 
 ARG PYTHON_VERSION=python3
 RUN apt-get -qq update && apt-get -qq install -y \
-    ${PYTHON_VERSION} ${PYTHON_VERSION}-dev ${PYTHON_VERSION}-venv eatmydata \
-    redis-server
+    ${PYTHON_VERSION} ${PYTHON_VERSION}-dev ${PYTHON_VERSION}-venv \
+    ${PYTHON_VERSION}-pip eatmydata redis-server
 
-# /src is where we expect Synapse to be
+RUN ${PYTHON_VERSION} -m pip install -q --no-cache-dir poetry==1.1.12
+
+# As part of the Docker build, we attempt to pre-install Synapse's dependencies
+# in the hope that it speeds up the real install of Synapse. To make this work,
+# we have to reuse the same virtual env both times. There are three ways to do
+# this with poetry:
+#  1. Ensure that the Synapse source directory lives in the same path both
+#     times. Poetry creates and reuses a virtual env based off the package name
+#     ("matrix-synapse") and directory path.
+#  2. Configure `virtualenvs.in-project` to `true`. This makes poetry create or
+#     use a virtual env at `./.venv` in the source directory.
+#  3. Run poetry with a virtual env already active. Poetry will use the active
+#     virtual env, if there is one.
+# We use the second option and make `.venv` a symlink.
+RUN poetry config virtualenvs.in-project true
+
+# /src is where we expect the Synapse source directory to be mounted
 RUN mkdir /src
 
+# Download a cache of build dependencies to support offline mode.
+# `setuptools` and `wheel` are only required for pre-poetry Synapse versions.
+# These version numbers are arbitrary and were the latest at the time.
+RUN ${PYTHON_VERSION} -m pip download --dest /pypi-offline-cache \
+        poetry-core==1.0.8 setuptools==60.10.0 wheel==0.37.1
+
+# TODO: Once poetry lands in the develop branch of Synapse, uncomment these
+#       lines and delete the pip version of the virtual env preparation.
+# Create the virtual env upfront so we don't need to keep reinstalling
+# dependencies.
+# RUN wget -q https://github.com/matrix-org/synapse/archive/develop.tar.gz \
+#         -O /synapse.tar.gz && \
+#     mkdir /synapse && \
+#     tar -C /synapse --strip-components=1 -xf synapse.tar.gz && \
+#     ln -s -T /venv /synapse/.venv && \
+#     cd /synapse && \
+#     poetry install -q --no-root --extras all && \
+#     # Finally clean up the poetry cache and the copy of Synapse.
+#     # This must be done in the same RUN command, otherwise intermediate layers
+#     # of the Docker image will contain all the unwanted files we think we've
+#     # deleted.
+#     rm -rf `poetry config cache-dir` && \
+#     rm -rf /synapse && \
+#     rm /synapse.tar.gz
+
 # Create the virutal env upfront so we don't need to keep reinstall dependencies
 # Manually upgrade pip to ensure it can locate Cryptography's binary wheels
 RUN ${PYTHON_VERSION} -m venv /venv && /venv/bin/pip install -U pip
@@ -20,4 +61,8 @@ RUN /venv/bin/pip install -q --no-cache-dir lxml psycopg2 coverage codecov
 # and test
 RUN /venv/bin/pip uninstall -q --no-cache-dir -y matrix-synapse
 
+# Pre-install test dependencies installed by `scripts/synapse_sytest.sh`.
+RUN /venv/bin/pip install -q --no-cache-dir \
+        lxml psycopg2 coverage codecov tap.py coverage_enable_subprocess
+
 ENTRYPOINT [ "/bin/bash", "/bootstrap.sh", "synapse" ]

scripts/synapse_sytest.sh

@@ -114,34 +114,54 @@ fi
 # default value for SYNAPSE_SOURCE
 : ${SYNAPSE_SOURCE:=/src}
 
-# if we're running against a source directory, turn it into a tarball.  pip
-# will then unpack it to a temporary location, and build it.  (As of pip 20.1,
-# it will otherwise try to build it in-tree, which means writing changes to the
-# source volume outside the container.)
-#
-if [ -d "$SYNAPSE_SOURCE" ]; then
-    echo "Creating tarball from synapse source"
-    tar -C "$SYNAPSE_SOURCE" -czf /tmp/synapse.tar.gz \
-        synapse setup.py README.rst MANIFEST.in
-    SYNAPSE_SOURCE="/tmp/synapse.tar.gz"
-elif [ ! -r "$SYNAPSE_SOURCE" ]; then
+if [ ! -r "$SYNAPSE_SOURCE" ]; then
     echo "Unable to read synapse source at $SYNAPSE_SOURCE" >&2
     exit 1
 fi
 
+if [ ! -d "$SYNAPSE_SOURCE" ]; then
+    echo "$SYNAPSE_SOURCE must be a source directory" >&2
+    exit 1
+fi
+
+# Make a copy of the source directory to avoid writing changes to the source
+# volume outside the container.
+cp -r "$SYNAPSE_SOURCE" /synapse
+
 if [ -n "$OFFLINE" ]; then
     # if we're in offline mode, just put synapse into the virtualenv, and
     # hope that the deps are up-to-date.
     #
-    # --no-use-pep517 works around what appears to be a pip issue
-    # (https://github.com/pypa/pip/issues/5402 possibly) where pip wants
-    # to reinstall any requirements for the build system, even if they are
-    # already installed.
-    /venv/bin/pip install --no-index --no-use-pep517 "$SYNAPSE_SOURCE"
+    # pip will want to install any requirements for the build system
+    # (https://github.com/pypa/pip/issues/5402), so we have to provide a
+    # directory of pre-downloaded build requirements.
+    echo "Installing Synapse using pip in offline mode..."
+    /venv/bin/pip install --no-index --find-links /pypi-offline-cache /synapse
 else
-    # We've already created the virtualenv, but lets double check we have all
-    # deps.
-    /venv/bin/pip install -q --upgrade --no-cache-dir "$SYNAPSE_SOURCE"[redis]
+    if [ -f "/synapse/poetry.lock" ]; then
+        # Install Synapse and dependencies using poetry, respecting the lockfile.
+        # The virtual env will already be populated with dependencies from the
+        # Docker build.
+        echo "Installing Synapse using poetry..."
+        if [ -d /synapse/.venv ]; then
+            # There was a virtual env in the source directory for some reason.
+            # We want to use our own, so remove it.
+            rm -rf /synapse/.venv
+        fi
+        ln -s -T /venv /synapse/.venv # reuse the existing virtual env
+        pushd /synapse
+        poetry install --extras redis
+        popd
+    else
+        # Install Synapse and dependencies using pip. As of pip 20.1, this will
+        # try to build Synapse in-tree, which means writing changes to the source
+        # directory.
+        # The virtual env will already be populated with dependencies from the
+        # Docker build.
+        echo "Installing Synapse using pip..."
+        /venv/bin/pip install -q --upgrade --no-cache-dir /synapse[redis]
+    fi
+
     /venv/bin/pip install -q --upgrade --no-cache-dir \
         lxml psycopg2 coverage codecov tap.py coverage_enable_subprocess