STuff

.github/actions/build-docker/action.yml

@@ -17,6 +17,10 @@ inputs:
     required: false
     description: Push the image?
     default: 'false'
+  deps_artifact:
+    required: false
+    description: The artifact containing the dependencies
+    default: 'deps'
 
 outputs:
   tag:
@@ -28,6 +32,9 @@ outputs:
   digest:
     description: The docker build digest of the built image
     value: ${{ steps.build_meta.outputs.digest }}
+  deps_artifact:
+    description: The artifact containing the dependencies
+    value: ${{ steps.context.outputs.deps_artifact }}
 
 runs:
   using: 'composite'
@@ -40,6 +47,7 @@ runs:
 
         echo "git_build_url=$git_repo_url/actions/runs/${{ github.run_id }}" >> $GITHUB_OUTPUT
         echo "git_sha=${{ github.sha }}" >> $GITHUB_OUTPUT
+        echo "deps_artifact=${{ inputs.deps_artifact }}" >> $GITHUB_OUTPUT
 
         cat $GITHUB_OUTPUT
 
@@ -65,7 +73,7 @@ runs:
         echo "ANNOTATIONS_FILE=${{ steps.meta.outputs.bake-file-annotations }}" >> $GITHUB_ENV
         echo "DOCKER_METADATA_FILE=buildx-bake-metadata.json" >> $GITHUB_ENV
 
-        make setup
+        make setup_files
 
     - name: Build Image
       id: build
@@ -75,6 +83,20 @@ runs:
           ARGS="--file ${{ env.TAGS_FILE }} --file ${{ env.ANNOTATIONS_FILE }}" \
           DOCKER_PUSH=${{ inputs.push }}
 
+    - name: Update deps
+      shell: bash
+      run: |
+        make setup_deps
+
+    - name: Upload deps
+      id: upload_deps
+      uses: actions/upload-artifact@v4
+      with:
+        name: ${{ steps.context.outputs.deps_artifact }}
+        path: deps
+        overwrite: true
+        include-hidden-files: true
+
     - name: Get image digest
       id: build_meta
       shell: bash

Dockerfile

@@ -18,7 +18,7 @@ ENV HOME=/data/olympia
 WORKDIR ${HOME}
 RUN chown -R olympia:olympia ${HOME}
 
-FROM olympia AS development
+FROM olympia AS base
 # Add keys and repos for node and mysql
 # TODO: replace this with a bind mount on the RUN command
 COPY docker/*.gpg.asc /etc/apt/trusted.gpg.d/
@@ -107,9 +107,10 @@ COPY docker/etc/mime.types /etc/mime.types
 # Define production dependencies as a single layer
 # let's the rest of the stages inherit prod dependencies
 # and makes copying the /deps dir to the final layer easy.
-FROM development AS pip_production
+FROM base AS pip_production
 
 RUN \
+    --mount=type=bind,source=Makefile-docker,target=${HOME}/Makefile-docker \
     # Files required to install pip dependencies
     --mount=type=bind,source=./requirements/prod.txt,target=${HOME}/requirements/prod.txt \
     # Files required to install npm dependencies
@@ -119,11 +120,10 @@ RUN \
     --mount=type=cache,target=${PIP_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_UID} \
     --mount=type=cache,target=${NPM_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_UID} \
 <<EOF
-${PIP_COMMAND} install --progress-bar=off --no-deps --exists-action=w -r requirements/prod.txt
-npm ci ${NPM_ARGS} --include=prod
+make -f Makefile-docker update_deps_production
 EOF
 
-FROM development AS locales
+FROM base AS locales
 ARG LOCALE_DIR=${HOME}/locale
 # Compile locales
 # Copy the locale files from the host so it is writable by the olympia user
@@ -158,7 +158,9 @@ echo "from olympia.lib.settings_base import *" > settings_local.py
 DJANGO_SETTINGS_MODULE="settings_local" make -f Makefile-docker update_assets
 EOF
 
-FROM development AS production
+FROM base AS development
+
+FROM base AS production
 
 ARG DOCKER_BUILD DOCKER_COMMIT DOCKER_VERSION
 

Makefile-docker

@@ -78,11 +78,18 @@ update_assets:
 	# Collect static files: This MUST be run last or files will be missing
 	$(PYTHON_COMMAND) manage.py collectstatic --noinput
 
-.PHONY: update_deps
-update_deps: ## install the dependencies required to run this project
+.PHONY: update_deps_base
+update_deps_base:
+	cp /data/olympia/package*.json /deps || echo true
+
+.PHONY: update_deps_prod
+update_deps_production: update_deps_base ## install the production dependencies
 	$(PIP_COMMAND) install --progress-bar=off --no-deps --exists-action=w -r requirements/prod.txt
+	npm ci $(NPM_ARGS) --include=prod
+
+.PHONY: update_deps_dev
+update_deps_development: update_deps_base update_deps_production ## install the development dependencies
 	$(PIP_COMMAND) install --progress-bar=off --no-deps --exists-action=w -r requirements/dev.txt
-	cp /data/olympia/package*.json /deps || echo true
 	npm install $(NPM_ARGS) --no-save
 
 # TOOD: remove this after we migrate addons-frontned to not depend on it.

Makefile-os

@@ -13,6 +13,18 @@ export DOCKER_VERSION ?=
 export DATA_BACKUP_SKIP ?=
 override DOCKER_MYSQLD_VOLUME = addons-server_data_mysqld
 
+define DOCKER_TAG
+$(shell docker compose config web --format json | jq -r '.services.web.image')
+endef
+
+define IS_LOCAL
+$(if $(findstring :local, $(DOCKER_TAG)),true,false)
+endef
+
+define READ_ENV
+$(shell cat .env | grep $(1) | cut -d '=' -f2)
+endef
+
 INITIALIZE_ARGS ?=
 INIT_CLEAN ?=
 INIT_LOAD ?=
@@ -25,13 +37,12 @@ ifneq ($(INIT_LOAD),)
 	INITIALIZE_ARGS += --load $(INIT_LOAD)
 endif
 
-
-
 DOCKER_BAKE_ARGS := \
 	--file docker-bake.hcl \
 	--file .env \
 	--progress $(DOCKER_PROGRESS) \
 	--metadata-file $(DOCKER_METADATA_FILE) \
+	--set "*.args.DOCKER_TARGET=$(call READ_ENV,DOCKER_TARGET)"
 
 ifeq ($(DOCKER_PUSH), true)
 	DOCKER_BAKE_ARGS += --push
@@ -69,11 +80,43 @@ help_submake:
 test_setup:
 	npm exec jest -- ./tests/make --runInBand
 
-.PHONY: setup
-setup: ## create configuration files version.json and .env required to run this project
-	for path in $(CLEAN_PATHS); do rm -rf "$(PWD)/$$path" && echo "$$path removed"; done
+.PHONY: setup_files
+setup_files: ## Setup the host files
+	$(foreach path,$(CLEAN_PATHS),rm -rf $(PWD)/$(path) && echo "$(path) removed";)
 	./scripts/setup.py
 
+.PHONY: setup_image ## Pull or build the docker image based on the image version
+setup_image: setup_files
+ifeq ($(IS_LOCAL), true)
+	$(MAKE) docker_build_web
+else
+	$(MAKE) docker_pull_web
+endif
+
+.PHONY: setup_deps
+setup_deps: setup_image ## Extract or install dependencies to the host ./deps directory
+ifeq ($(IS_LOCAL), false)
+	@echo "Non-local image detected, cleaning deps"
+	$(MAKE) clean_deps
+endif
+ifeq ($(call READ_ENV,DOCKER_TARGET), production)
+	@echo "Production image detected, cleaning deps"
+	$(MAKE) clean_deps
+endif
+	docker compose run \
+		--no-deps \
+		--rm \
+		--volume $(PWD)/deps:/deps \
+		--volume $(PWD):/data/olympia \
+		web \
+		make -f Makefile-docker update_deps_$(shell echo $(call READ_ENV,DOCKER_TARGET))
+
+.PHONY: setup_compose
+setup_compose: addons-server_data_mysqld ## Setup the docker compose project
+
+.PHONY: setup
+setup: setup_files setup_image setup_deps setup_compose ## Setup the project to run
+
 .PHONY: push_locales
 push_locales: ## extracts and merges translation strings
 	bash ./scripts/push_l10n_extraction.sh $(ARGS)
@@ -136,22 +179,9 @@ clean_deps:
 	rm -rf deps/**
 
 .PHONY: docker_compose_up
-docker_compose_up: docker_mysqld_volume_create ## Start the docker containers
+docker_compose_up: ## Start the docker containers
 	docker compose up $(DOCKER_COMPOSE_ARGS) $(ARGS)
 
-.PHONY: docker_run
-docker_run: ## Run a command in a specified docker image
-	docker compose run \
-		--no-deps \
-		--rm \
-		--entrypoint bash \
-		$(shell echo "$(DOCKER_RUN_ARGS)") \
-		web \
-		$(shell if [ -n "$(DOCKER_RUN_COMMAND)" ]; \
-			then echo '-c "$(DOCKER_RUN_COMMAND)"'; \
-			else echo "bash"; \
-		fi)
-
 .PHONY: up
 up: setup docker_compose_up docker_clean_images docker_clean_volumes ## Create and start docker compose
 	# Explicitly run initialize via the web container as make can get confused

docker-compose.ci.yml

@@ -1,10 +1,7 @@
 services:
   worker:
-    environment:
-      - HOST_UID=9500
     volumes:
       - /data/olympia
-      - /deps
 
   web:
     extends:

docker-compose.yml

@@ -23,6 +23,7 @@ x-env-mapping: &env
     - HOST_UID
     - DEBUG
     - DATA_BACKUP_SKIP
+    - DOCKER_TARGET
 
 x-olympia: &olympia
   <<: *env
@@ -58,8 +59,6 @@ services:
       - /data/olympia/static-build
       - /data/olympia/site-static
       - ./deps:/deps
-      - ./package.json:/deps/package.json
-      - ./package-lock.json:/deps/package-lock.json
     extra_hosts:
      - "olympia.test:127.0.0.1"
     restart: on-failure:5

scripts/setup.py

@@ -1,9 +1,6 @@
 #!/usr/bin/env python3
 
 import os
-import subprocess
-import sys
-
 
 root_dir = os.path.join(os.path.dirname(__file__), '..')
 
@@ -43,17 +40,6 @@ def get_value(key, default_value):
     return default_value
 
 
-def make_run(*args, makefile='Makefile-os'):
-    print('Running:', ' '.join(['make', '-f', makefile, *args]))
-    subprocess.run(['make', '-f', makefile, *args])
-
-def docker_run(args='', command=''):
-    make_run(
-        'docker_run',
-        f"DOCKER_RUN_ARGS={args}",
-        f"DOCKER_RUN_COMMAND='{command}'",
-    )
-
 def get_docker_tag():
     image = 'mozilla/addons-server'
     version = 'local'
@@ -141,25 +127,6 @@ def main():
         }
     )
 
-    # Pull or build the docker image based on the image version
-    if is_local:
-        make_run('docker_build_web')
-    else:
-        make_run('docker_pull_web')
-
-    # Extract or install dependencies to the host ./deps directory
-    if not is_local or is_production:
-        make_run('clean_deps')
-        docker_run(
-            args=f'--user root --volume {path("deps")}:/_deps',
-            command='mv /deps/* /_deps',
-        )
-    else:
-        docker_run(
-            args=f'--user olympia --volume {path("deps")}:/deps --volume {path("")}:/data/olympia',
-            command='make -f Makefile-docker update_deps',
-        )
-
 
 if __name__ == '__main__':
     main()