From ea322a8ddd132890cbb6b78e17604ff6adc3d3cb Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 14 Jul 2022 22:28:55 +0100 Subject: [PATCH 1/2] Docker: copy postgres from base image When building the docker images for complement testing, copy a preinstalled complement over from a base image, rather than apt installing it. This avoids network traffic and is much faster. --- changelog.d/13279.misc | 1 + docker/complement/Dockerfile | 39 +++++++++++++------ .../complement/conf/postgres.supervisord.conf | 2 +- 3 files changed, 29 insertions(+), 13 deletions(-) create mode 100644 changelog.d/13279.misc diff --git a/changelog.d/13279.misc b/changelog.d/13279.misc new file mode 100644 index 000000000000..a083d2af2a19 --- /dev/null +++ b/changelog.d/13279.misc @@ -0,0 +1 @@ +Reduce the rebuild time for the complement-synapse docker image. diff --git a/docker/complement/Dockerfile b/docker/complement/Dockerfile index 8bec0f611649..99d7dfae1509 100644 --- a/docker/complement/Dockerfile +++ b/docker/complement/Dockerfile @@ -4,21 +4,36 @@ # # Instructions for building this image from those it depends on is detailed in this guide: # https://github.com/matrix-org/synapse/blob/develop/docker/README-testing.md#testing-with-postgresql-and-single-or-multi-process-synapse + ARG SYNAPSE_VERSION=latest + +# first of all, we create a base image with a postgres server and database, +# which we can copy into the target image. For repeated rebuilds, this is +# much faster than apt installing postgres each time. +# +# This trick only works because (a) the Synapse image happens to have all the +# shared libraries that postgres wants, (b) we use a postgres image based on +# the same debian version as Synapse's docker image (so the versions of the +# shared libraries match). + +FROM postgres:13-bullseye AS postgres_base + # initialise the database cluster in /var/lib/postgresql + RUN gosu postgres initdb --locale=C --encoding=UTF-8 --auth-host password + + # Configure a password and create a database for Synapse + RUN echo "ALTER USER postgres PASSWORD 'somesecret'" | gosu postgres postgres --single + RUN echo "CREATE DATABASE synapse" | gosu postgres postgres --single + FROM matrixdotorg/synapse-workers:$SYNAPSE_VERSION -# Install postgresql -RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -yqq postgresql-13 - -# Configure a user and create a database for Synapse -RUN pg_ctlcluster 13 main start && su postgres -c "echo \ - \"ALTER USER postgres PASSWORD 'somesecret'; \ - CREATE DATABASE synapse \ - ENCODING 'UTF8' \ - LC_COLLATE='C' \ - LC_CTYPE='C' \ - template=template0;\" | psql" && pg_ctlcluster 13 main stop +# copy the postgres installation over from the image we built above +RUN adduser --system --uid 999 postgres --home /var/lib/postgresql +COPY --from=postgres_base /var/lib/postgresql /var/lib/postgresql +COPY --from=postgres_base /usr/lib/postgresql /usr/lib/postgresql +COPY --from=postgres_base /usr/share/postgresql /usr/share/postgresql +RUN mkdir /var/run/postgresql && chown postgres /var/run/postgresql +ENV PATH="${PATH}:/usr/lib/postgresql/13/bin" +ENV PGDATA=/var/lib/postgresql/data # Extend the shared homeserver config to disable rate-limiting, # set Complement's static shared secret, enable registration, amongst other diff --git a/docker/complement/conf/postgres.supervisord.conf b/docker/complement/conf/postgres.supervisord.conf index 5dae3e63309a..b88bfc772e40 100644 --- a/docker/complement/conf/postgres.supervisord.conf +++ b/docker/complement/conf/postgres.supervisord.conf @@ -1,5 +1,5 @@ [program:postgres] -command=/usr/local/bin/prefix-log /usr/bin/pg_ctlcluster 13 main start --foreground +command=/usr/local/bin/prefix-log gosu postgres postgres # Only start if START_POSTGRES=1 autostart=%(ENV_START_POSTGRES)s From 77864ffaae36844a333a812d3a87d147676eeeb1 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 15 Jul 2022 10:14:37 +0100 Subject: [PATCH 2/2] more indent --- docker/complement/Dockerfile | 67 ++++++++++++++++++------------------ 1 file changed, 34 insertions(+), 33 deletions(-) diff --git a/docker/complement/Dockerfile b/docker/complement/Dockerfile index 99d7dfae1509..c5e7984a2835 100644 --- a/docker/complement/Dockerfile +++ b/docker/complement/Dockerfile @@ -24,37 +24,38 @@ FROM postgres:13-bullseye AS postgres_base RUN echo "ALTER USER postgres PASSWORD 'somesecret'" | gosu postgres postgres --single RUN echo "CREATE DATABASE synapse" | gosu postgres postgres --single -FROM matrixdotorg/synapse-workers:$SYNAPSE_VERSION - -# copy the postgres installation over from the image we built above -RUN adduser --system --uid 999 postgres --home /var/lib/postgresql -COPY --from=postgres_base /var/lib/postgresql /var/lib/postgresql -COPY --from=postgres_base /usr/lib/postgresql /usr/lib/postgresql -COPY --from=postgres_base /usr/share/postgresql /usr/share/postgresql -RUN mkdir /var/run/postgresql && chown postgres /var/run/postgresql -ENV PATH="${PATH}:/usr/lib/postgresql/13/bin" -ENV PGDATA=/var/lib/postgresql/data - -# Extend the shared homeserver config to disable rate-limiting, -# set Complement's static shared secret, enable registration, amongst other -# tweaks to get Synapse ready for testing. -# To do this, we copy the old template out of the way and then include it -# with Jinja2. -RUN mv /conf/shared.yaml.j2 /conf/shared-orig.yaml.j2 -COPY conf/workers-shared-extra.yaml.j2 /conf/shared.yaml.j2 - -WORKDIR /data - -COPY conf/postgres.supervisord.conf /etc/supervisor/conf.d/postgres.conf +# now build the final image, based on the Synapse image. -# Copy the entrypoint -COPY conf/start_for_complement.sh / - -# Expose nginx's listener ports -EXPOSE 8008 8448 - -ENTRYPOINT ["/start_for_complement.sh"] - -# Update the healthcheck to have a shorter check interval -HEALTHCHECK --start-period=5s --interval=1s --timeout=1s \ - CMD /bin/sh /healthcheck.sh +FROM matrixdotorg/synapse-workers:$SYNAPSE_VERSION + # copy the postgres installation over from the image we built above + RUN adduser --system --uid 999 postgres --home /var/lib/postgresql + COPY --from=postgres_base /var/lib/postgresql /var/lib/postgresql + COPY --from=postgres_base /usr/lib/postgresql /usr/lib/postgresql + COPY --from=postgres_base /usr/share/postgresql /usr/share/postgresql + RUN mkdir /var/run/postgresql && chown postgres /var/run/postgresql + ENV PATH="${PATH}:/usr/lib/postgresql/13/bin" + ENV PGDATA=/var/lib/postgresql/data + + # Extend the shared homeserver config to disable rate-limiting, + # set Complement's static shared secret, enable registration, amongst other + # tweaks to get Synapse ready for testing. + # To do this, we copy the old template out of the way and then include it + # with Jinja2. + RUN mv /conf/shared.yaml.j2 /conf/shared-orig.yaml.j2 + COPY conf/workers-shared-extra.yaml.j2 /conf/shared.yaml.j2 + + WORKDIR /data + + COPY conf/postgres.supervisord.conf /etc/supervisor/conf.d/postgres.conf + + # Copy the entrypoint + COPY conf/start_for_complement.sh / + + # Expose nginx's listener ports + EXPOSE 8008 8448 + + ENTRYPOINT ["/start_for_complement.sh"] + + # Update the healthcheck to have a shorter check interval + HEALTHCHECK --start-period=5s --interval=1s --timeout=1s \ + CMD /bin/sh /healthcheck.sh