Wagtail 4.0 upgrade

.devcontainer/devcontainer.json

@@ -0,0 +1,9 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
+// https://github.com/microsoft/vscode-dev-containers/tree/v0.245.2/containers/docker-existing-docker-compose
+// If you want to run as a non-root user in the container, see .devcontainer/docker-compose.yml.
+{
+  "name": "wagtail.org devcontainer",
+  "dockerComposeFile": ["../docker-compose.yml"],
+  "service": "web",
+  "workspaceFolder": "/app"
+}

.github/workflows/ci.yml

@@ -29,6 +29,10 @@ jobs:
           key: ${{ runner.os }}-node-${{ hashFiles('**/.nvmrc') }}-${{ hashFiles('**/package-lock.json') }}
       - if: steps.node-cache.outputs.cache-hit != 'true'
         run: npm ci --no-audit
+      - uses: actions/cache@v3
+        with:
+          path: ${{ env.HOME }}/.cache/pre-commit
+          key: ${{ runner.os }}-pre-commit-${{ hashFiles('.pre-commit-config.yaml') }}
       - uses: pre-commit/[email protected]
   test_js:
     runs-on: ubuntu-latest
@@ -69,18 +73,18 @@ jobs:
         options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
     steps:
       - uses: actions/checkout@v3
+      - name: Install Poetry
+        run: pipx install poetry==1.2.1 # Make sure this matches POETRY_VERSION in the Dockerfile
       - name: Set up Python
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v4
         with:
           python-version: 3.8
-          cache: 'pip'
-      - name: Install python dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements.txt
+          cache: 'poetry'
+      - name: Install Python dependencies
+        run: poetry install
       - name: System checks
-        run: python manage.py check
-      - name: Missing migrations
-        run: python manage.py makemigrations --check --noinput
+        run: poetry run ./manage.py check
+      - name: Check for missing migrations
+        run: poetry run ./manage.py makemigrations --check --noinput
       - name: Test
-        run: python manage.py test
+        run: poetry run ./manage.py test

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 default_language_version:
   node: 18.7.0
-  python: python3
+  python: python3.8
 repos:
   - repo: https://github.com/psf/black
     rev: 22.6.0
@@ -20,8 +20,9 @@ repos:
     hooks:
       - id: flake8
         additional_dependencies:
-          - flake8-comprehensions
-          - flake8-assertive
+          - flake8-assertive==2.1.0
+          - flake8-blind-except==0.2.1
+          - flake8-comprehensions==3.10.0
   - repo: https://github.com/pre-commit/mirrors-prettier
     rev: v2.7.1
     hooks:

Dockerfile

@@ -9,15 +9,19 @@ RUN npm ci --no-optional --no-audit --progress=false
 COPY ./wagtailio/static/ ./wagtailio/static/
 RUN npm run build:prod
 
-# Build Python app.
+
+# Build Python app - this stage is a common base for the prod and dev stages
 FROM python:3.8-bullseye AS backend
 
-ENV PYTHONUNBUFFERED=1 \
-    PYTHONPATH=/app \
-    DJANGO_SETTINGS_MODULE=wagtailio.settings.production \
+ARG POETRY_VERSION=1.2.1
+
+ENV DJANGO_SETTINGS_MODULE=wagtailio.settings.production \
+    GUNICORN_CMD_ARGS="--max-requests 1200 --max-requests-jitter 50 --access-logfile -" \
+    PATH=/home/wagtailio/.local/bin:/venv/bin:$PATH \
     PORT=8000 \
-    WEB_CONCURRENCY=3 \
-    GUNICORN_CMD_ARGS="--max-requests 1200 --access-logfile -"
+    PYTHONUNBUFFERED=1 \
+    VIRTUAL_ENV=/venv \
+    WEB_CONCURRENCY=3
 
 # Install operating system dependencies.
 RUN apt-get update -y && \
@@ -27,45 +31,44 @@ RUN apt-get update -y && \
 WORKDIR /app
 EXPOSE 8000
 
+# Create a virtual environment and install Poetry
+RUN python -m venv /venv \
+    && /venv/bin/pip install --upgrade pip wheel \
+    && /usr/local/bin/python -m pip install poetry==$POETRY_VERSION
+
 # Create a non-root application user.
-ARG UID=1000
-RUN useradd wagtailio -u $UID -m
-RUN chown -R wagtailio /app
+ARG UID=1000 GID=1000
+RUN groupadd --gid $GID --force wagtailio \
+    && useradd --create-home --uid $UID -g wagtailio wagtailio
+RUN chown --recursive $UID:$GID /app /venv
 
 
+# This stage builds the image that will run in production
 FROM backend AS prod
 
-# Switch to application user.
+# Switch to application user
 USER wagtailio
 
-# Create a virtual environment
-RUN python3 -m venv /home/wagtailio/venv
-ENV PATH="/home/wagtailio/venv/bin:$PATH"
-RUN pip install --upgrade pip wheel
-
-# Install Gunicorn.
-RUN pip install "gunicorn>=20.1,<20.2"
-
-# Install production Python requirements.
-COPY --chown=wagtailio requirements.txt /
-RUN pip install -r /requirements.txt
+# Install production dependencies
+COPY --chown=wagtailio pyproject.toml poetry.lock ./
+RUN poetry install --only main --no-root
 
-COPY --chown=wagtailio --from=frontend ./wagtailio/static_compiled ./wagtailio/static_compiled
-
-# Install application code.
+# Copy in application code and install the root package
 COPY --chown=wagtailio . .
+RUN poetry install --only-root
 
-# Install assets
+# Collect static files
+COPY --chown=wagtailio --from=frontend ./wagtailio/static_compiled ./wagtailio/static_compiled
 RUN SECRET_KEY=none django-admin collectstatic --noinput --clear
 
 # Run application
 CMD gunicorn wagtailio.wsgi:application
 
 
+# This stage builds the image that we use for development
 FROM backend AS dev
 
 # Install Node.js because newer versions of Heroku CLI have a node binary dependency
-
 RUN curl -fsSL https://deb.nodesource.com/setup_18.x | bash -
 RUN apt-get install -y nodejs
 
@@ -78,13 +81,16 @@ RUN curl "https://s3.amazonaws.com/aws-cli/awscli-bundle.zip" -o "/tmp/awscli-bu
     /tmp/awscli-bundle/install -i /usr/local/aws -b /usr/local/bin/aws && \
     rm /tmp/awscli-bundle.zip && rm -r /tmp/awscli-bundle
 
-# Switch to application user.
+# Switch to the application user
 USER wagtailio
 
+# Install development dependencies
+COPY --chown=wagtailio pyproject.toml poetry.lock ./
+RUN poetry install --no-root
+
 # Pull in the node modules for the frontend
 COPY --chown=wagtailio --from=frontend ./node_modules ./node_modules
 
-# Install development Python requirements.
-ENV PATH="/home/wagtailio/.local/bin:$PATH"
-COPY --chown=wagtailio requirements.txt requirements-dev.txt /
-RUN pip install -r /requirements-dev.txt
+# Make sure the working directory is on PYTHONPATH (so django-admin etc. can
+# import wagtailio)
+ENV PYTHONPATH=/app

Makefile

@@ -1,36 +1,44 @@
-.PHONY: setup rebuild start superuser migrations migrate pull-production-data pull-production-media pull-staging-data pull-staging-media
-
-setup:
-	make rebuild
+.PHONY: setup
+setup: rebuild
 	docker-compose run web django-admin migrate
 	docker-compose run web django-admin createcachetable
 
+.PHONY: rebuild
 rebuild:
-	bash -c "docker-compose build --build-arg UID=$$(id -u) --build-arg DOCKER_GID=$$(ls -ln /var/run/docker.sock | awk '{print $$4}')"
+	bash -c "docker-compose build --build-arg UID=$$(id -u) --build-arg GID=$$(id -g)"
 
+.PHONY: start
 start:
 	docker-compose up
 
+.PHONY: runserver
 runserver:
 	docker-compose exec web django-admin runserver 0.0.0.0:8000
 
+.PHONY: superuser
 superuser:
 	docker-compose run web django-admin createsuperuser
 
+.PHONY: migrations
 migrations:
-	docker-compose run web django-admin migrations
+	docker-compose run web django-admin makemigrations
 
+.PHONY: migrate
 migrate:
 	docker-compose run web django-admin migrate
 
+.PHONY: pull-production-data
 pull-production-data:
 	docker-compose run web fab pull_production_data
 
+.PHONY: pull-production-media
 pull-production-media:
 	docker-compose run web fab pull_production_media
 
+.PHONY: pull-staging-data
 pull-staging-data:
 	docker-compose run web fab pull_staging_data
 
+.PHONY: rebuild
 pull-staging-media:
 	docker-compose run web fab pull_staging_media

docker-compose.yml

@@ -12,7 +12,7 @@ services:
       SECRET_KEY: fake
       DJANGO_SETTINGS_MODULE: wagtailio.settings.dev
       DATABASE_URL: postgres://wagtailorg:wagtailorg@db:5432/wagtailorg
-
+      PRIMARY_HOST: http://localhost:8000
       # The env vars get picked up by postgresql client tools (psql, pg_dump, etc)
       PGHOST: db
       PGUSER: wagtailorg

fabfile.py

@@ -3,10 +3,12 @@
 from fabric.context_managers import warn_only
 
 PRODUCTION_APP_INSTANCE = "wagtail-org-production"
-STAGING_APP_INSTANCE = "wagtailio-staging"
+
+# FIXME - there's no staging instance at present
+# STAGING_APP_INSTANCE = "wagtailio-staging"
 
 LOCAL_MEDIA_FOLDER = "./media"
-LOCAL_DATABASE_NAME = "wagtailio"
+LOCAL_DATABASE_NAME = "wagtailorg"
 
 
 ############
@@ -51,36 +53,36 @@ def production_shell():
 #########
 
 
-@task
-def pull_staging_media():
-    pull_media_from_s3_heroku(STAGING_APP_INSTANCE)
+# @task
+# def pull_staging_media():
+#     pull_media_from_s3_heroku(STAGING_APP_INSTANCE)
 
 
-@task
-def push_staging_media():
-    push_media_to_s3_heroku(STAGING_APP_INSTANCE)
+# @task
+# def push_staging_media():
+#     push_media_to_s3_heroku(STAGING_APP_INSTANCE)
 
 
-@task
-def pull_staging_data():
-    pull_database_from_heroku(STAGING_APP_INSTANCE)
+# @task
+# def pull_staging_data():
+#     pull_database_from_heroku(STAGING_APP_INSTANCE)
 
 
-@task
-def push_staging_data():
-    push_database_to_heroku(STAGING_APP_INSTANCE)
+# @task
+# def push_staging_data():
+#     push_database_to_heroku(STAGING_APP_INSTANCE)
 
 
-@task
-def deploy_staging():
-    deploy_to_heroku(
-        STAGING_APP_INSTANCE, local_branch="staging", remote_branch="master"
-    )
+# @task
+# def deploy_staging():
+#     deploy_to_heroku(
+#         STAGING_APP_INSTANCE, local_branch="staging", remote_branch="master"
+#     )
 
 
-@task
-def staging_shell():
-    open_heroku_shell(STAGING_APP_INSTANCE)
+# @task
+# def staging_shell():
+#     open_heroku_shell(STAGING_APP_INSTANCE)
 
 
 #######