Use docker compose for our build tests

.github/workflows/test-build.yml

@@ -241,18 +241,6 @@ jobs:
       matrix:
         platform: ["linux/amd64", "linux/arm64"]
         image: ["scripts", "webapp"]
-    env:
-      POSTGRES_USER: palace_user
-      POSTGRES_PASSWORD: test
-      POSTGRES_DB: palace_circulation
-
-    services:
-      postgres:
-        image: postgres:12
-        env:
-          POSTGRES_USER: ${{ env.POSTGRES_USER }}
-          POSTGRES_PASSWORD: ${{ env.POSTGRES_PASSWORD }}
-          POSTGRES_DB: ${{ env.POSTGRES_DB }}
 
     steps:
       - uses: actions/checkout@v4
@@ -269,36 +257,23 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
-      - name: Build image
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          file: ./docker/Dockerfile
-          tags: test_image
-          load: true
-          target: ${{ matrix.image }}
-          cache-from: type=gha,scope=buildkit-${{ github.run_id }}
-          platforms: ${{ matrix.platform }}
-          build-args: |
-            BASE_IMAGE=${{ needs.docker-image-build.outputs.baseimage }}
-
-      - name: Start container
-        run: >
-          docker run --rm --name test_container -d --platform ${{ matrix.platform }}
-          --network ${{job.services.postgres.network}}
-          -e SIMPLIFIED_PRODUCTION_DATABASE="postgresql://${{ env.POSTGRES_USER }}:${{ env.POSTGRES_PASSWORD }}@postgres:5432/${{ env.POSTGRES_DB }}"
-          test_image
+      - name: Build & Start container
+        run: docker compose up -d --build ${{ matrix.image }}
+        env:
+          BUILD_PLATFORM: ${{ matrix.platform }}
+          BUILD_CACHE_FROM: type=gha,scope=buildkit-${{ github.run_id }}
+          BUILD_BASE_IMAGE: ${{ needs.docker-image-build.outputs.baseimage }}
 
       - name: Run tests
-        run: ./docker/ci/test_${{ matrix.image }}.sh test_container
+        run: ./docker/ci/test_${{ matrix.image }}.sh ${{ matrix.image }}
 
       - name: Output logs
         if: failure()
-        run: docker logs test_container
+        run: docker logs circulation-${{ matrix.image }}-1
 
       - name: Stop container
         if: always()
-        run: docker stop test_container
+        run: docker compose down
 
   docker-image-push:
     name: Push circ-${{ matrix.image }}

docker-compose.yml

@@ -1,42 +1,63 @@
 version: "3.9"
 
-# Common set of CM environment variables
+# Common CM setup
 # see: https://github.com/compose-spec/compose-spec/blob/master/spec.md#extension
-x-cm-env-variables: &cm-env-variables
-  SIMPLIFIED_PRODUCTION_DATABASE: "postgresql://palace:test@pg:5432/circ"
-  PALACE_STORAGE_ACCESS_KEY: "palace"
-  PALACE_STORAGE_SECRET_KEY: "test123456789"
-  PALACE_STORAGE_ENDPOINT_URL: "http://minio:9000"
-  PALACE_STORAGE_PUBLIC_ACCESS_BUCKET: "public"
-  PALACE_STORAGE_ANALYTICS_BUCKET: "analytics"
-  PALACE_STORAGE_URL_TEMPLATE: "http://localhost:9000/{bucket}/{key}"
-  PALACE_REPORTING_NAME: "TEST CM"
+x-cm-variables: &cm
+  platform: "${BUILD_PLATFORM-}"
+  environment:
+    SIMPLIFIED_PRODUCTION_DATABASE: "postgresql://palace:test@pg:5432/circ"
+    PALACE_SEARCH_URL: "http://os:9200"
+    PALACE_STORAGE_ACCESS_KEY: "palace"
+    PALACE_STORAGE_SECRET_KEY: "test123456789"
+    PALACE_STORAGE_ENDPOINT_URL: "http://minio:9000"
+    PALACE_STORAGE_PUBLIC_ACCESS_BUCKET: "public"
+    PALACE_STORAGE_ANALYTICS_BUCKET: "analytics"
+    PALACE_STORAGE_URL_TEMPLATE: "http://localhost:9000/{bucket}/{key}"
+    PALACE_REPORTING_NAME: "TEST CM"
+  depends_on:
+    pg:
+      condition: service_healthy
+    minio:
+      condition: service_healthy
+    os:
+      condition: service_healthy
+
+x-cm-build: &cm-build
+  context: .
+  dockerfile: docker/Dockerfile
+  args:
+    - BASE_IMAGE=${BUILD_BASE_IMAGE-ghcr.io/thepalaceproject/circ-baseimage:latest}
+  cache_from:
+    - ${BUILD_CACHE_FROM-ghcr.io/thepalaceproject/circ-webapp:main}
 
 services:
   # example docker compose configuration for testing and development
 
   webapp:
+    <<: *cm
     build:
-      context: .
-      dockerfile: docker/Dockerfile
+      <<: *cm-build
       target: webapp
     ports:
       - "6500:80"
-    environment: *cm-env-variables
 
   scripts:
+    <<: *cm
     build:
-      context: .
-      dockerfile: docker/Dockerfile
+      <<: *cm-build
       target: scripts
-    environment: *cm-env-variables
 
   pg:
     image: "postgres:12"
     environment:
       POSTGRES_USER: palace
       POSTGRES_PASSWORD: test
       POSTGRES_DB: circ
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 30s
+      timeout: 30s
+      retries: 3
 
   minio:
     image: "bitnami/minio:2023.2.27"
@@ -48,11 +69,22 @@ services:
       MINIO_ROOT_PASSWORD: "test123456789"
       MINIO_SCHEME: "http"
       MINIO_DEFAULT_BUCKETS: "public:download,analytics"
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+      interval: 30s
+      timeout: 20s
+      retries: 3
 
   os:
     build:
       dockerfile: docker/Dockerfile.ci
       target: opensearch
+      context: .
     environment:
-      discovery.type: single-node
-      DISABLE_SECURITY_PLUGIN: true
+      discovery.type: "single-node"
+      DISABLE_SECURITY_PLUGIN: "true"
+    healthcheck:
+      test: curl --silent http://localhost:9200 >/dev/null; if [[ $$? == 52 ]]; then echo 0; else echo 1; fi
+      interval: 30s
+      timeout: 10s
+      retries: 5

docker/ci/check_service_status.sh

@@ -4,7 +4,7 @@ function wait_for_runit()
   # The container to run the command in
   container="$1"
 
-  timeout 120s grep -q 'Runit started' <(docker logs "$container" -f 2>&1)
+  timeout 120s grep -q 'Runit started' <(docker compose logs "$container" -f 2>&1)
 }
 
 # A method to check that runit services are running inside the container
@@ -17,7 +17,7 @@ function check_service_status()
   service="$2"
 
   # Check the status of the service.
-  service_status=$(docker exec "$container" /bin/bash -c "sv check $service")
+  service_status=$(docker compose exec "$container" /bin/bash -c "sv check $service")
 
   # Get the exit code for the sv call.
   sv_status=$?
@@ -34,7 +34,7 @@ function check_crontab() {
   container="$1"
 
   # Installing the crontab will reveal any errors and exit with an error code
-  $(docker exec "$container" /bin/bash -c "crontab /etc/cron.d/circulation")
+  $(docker compose exec "$container" /bin/bash -c "crontab /etc/cron.d/circulation")
   validate_status=$?
   if [[ "$validate_status" != 0 ]]; then
     echo "  FAIL: crontab is incorrect"
@@ -48,7 +48,7 @@ function run_script() {
   container="$1"
   script="$2"
 
-  output=$(docker exec "$container" /bin/bash -c "$script")
+  output=$(docker compose exec "$container" /bin/bash -c "$script")
   script_status=$?
   if [[ "$script_status" != 0 ]]; then
     echo "  FAIL: script run failed"

docker/ci/test_scripts.sh

@@ -12,6 +12,9 @@ source "${dir}/check_service_status.sh"
 # Wait for container to start
 wait_for_runit "$container"
 
+# Make sure database initialization completed successfully
+timeout 240s grep -q 'Initialization complete' <(docker compose logs "$container" -f 2>&1)
+
 # Make sure that cron is running in the scripts container
 check_service_status "$container" /etc/service/cron
 

docker/ci/test_webapp.sh

@@ -17,18 +17,18 @@ check_service_status "$container" /etc/service/nginx
 check_service_status "$container" /etc/service/uwsgi
 
 # Wait for UWSGI to be ready to accept connections.
-timeout 240s grep -q 'WSGI app .* ready in [0-9]* seconds' <(docker logs "$container" -f 2>&1)
+timeout 240s grep -q 'WSGI app .* ready in [0-9]* seconds' <(docker compose logs "$container" -f 2>&1)
 
 # Make sure the web server is running.
-healthcheck=$(docker exec "$container" curl --write-out "%{http_code}" --silent --output /dev/null http://localhost/healthcheck.html)
+healthcheck=$(docker compose exec "$container" curl --write-out "%{http_code}" --silent --output /dev/null http://localhost/healthcheck.html)
 if ! [[ ${healthcheck} == "200" ]]; then
   exit 1
 else
   echo "  OK"
 fi
 
 # Also make sure the app server is running.
-feed_type=$(docker exec "$container" curl --write-out "%{content_type}" --silent --output /dev/null http://localhost/version.json)
+feed_type=$(docker compose exec "$container" curl --write-out "%{content_type}" --silent --output /dev/null http://localhost/version.json)
 if ! [[ ${feed_type} == "application/json" ]]; then
   exit 1
 else