From 0b78e58d3eefc6d78a7673c29fa8ba62d94637cf Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Mon, 11 Dec 2023 17:32:37 -0400 Subject: [PATCH] Use docker compose for our build tests --- .github/workflows/test-build.yml | 43 ++++--------------- docker-compose.yml | 68 +++++++++++++++++++++++-------- docker/ci/check_service_status.sh | 8 ++-- docker/ci/test_scripts.sh | 3 ++ docker/ci/test_webapp.sh | 6 +-- 5 files changed, 69 insertions(+), 59 deletions(-) diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index 18fc21a9d9..29f193db3e 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -241,18 +241,6 @@ jobs: matrix: platform: ["linux/amd64", "linux/arm64"] image: ["scripts", "webapp"] - env: - POSTGRES_USER: palace_user - POSTGRES_PASSWORD: test - POSTGRES_DB: palace_circulation - - services: - postgres: - image: postgres:12 - env: - POSTGRES_USER: ${{ env.POSTGRES_USER }} - POSTGRES_PASSWORD: ${{ env.POSTGRES_PASSWORD }} - POSTGRES_DB: ${{ env.POSTGRES_DB }} steps: - uses: actions/checkout@v4 @@ -269,36 +257,23 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Build image - uses: docker/build-push-action@v5 - with: - context: . - file: ./docker/Dockerfile - tags: test_image - load: true - target: ${{ matrix.image }} - cache-from: type=gha,scope=buildkit-${{ github.run_id }} - platforms: ${{ matrix.platform }} - build-args: | - BASE_IMAGE=${{ needs.docker-image-build.outputs.baseimage }} - - - name: Start container - run: > - docker run --rm --name test_container -d --platform ${{ matrix.platform }} - --network ${{job.services.postgres.network}} - -e SIMPLIFIED_PRODUCTION_DATABASE="postgresql://${{ env.POSTGRES_USER }}:${{ env.POSTGRES_PASSWORD }}@postgres:5432/${{ env.POSTGRES_DB }}" - test_image + - name: Build & Start container + run: docker compose up -d --build ${{ matrix.image }} + env: + BUILD_PLATFORM: ${{ matrix.platform }} + BUILD_CACHE_FROM: type=gha,scope=buildkit-${{ github.run_id }} + BUILD_BASE_IMAGE: ${{ needs.docker-image-build.outputs.baseimage }} - name: Run tests - run: ./docker/ci/test_${{ matrix.image }}.sh test_container + run: ./docker/ci/test_${{ matrix.image }}.sh ${{ matrix.image }} - name: Output logs if: failure() - run: docker logs test_container + run: docker logs circulation-${{ matrix.image }}-1 - name: Stop container if: always() - run: docker stop test_container + run: docker compose down docker-image-push: name: Push circ-${{ matrix.image }} diff --git a/docker-compose.yml b/docker-compose.yml index 2588473739..4e500f6b0c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,35 +1,51 @@ version: "3.9" -# Common set of CM environment variables +# Common CM setup # see: https://github.com/compose-spec/compose-spec/blob/master/spec.md#extension -x-cm-env-variables: &cm-env-variables - SIMPLIFIED_PRODUCTION_DATABASE: "postgresql://palace:test@pg:5432/circ" - PALACE_STORAGE_ACCESS_KEY: "palace" - PALACE_STORAGE_SECRET_KEY: "test123456789" - PALACE_STORAGE_ENDPOINT_URL: "http://minio:9000" - PALACE_STORAGE_PUBLIC_ACCESS_BUCKET: "public" - PALACE_STORAGE_ANALYTICS_BUCKET: "analytics" - PALACE_STORAGE_URL_TEMPLATE: "http://localhost:9000/{bucket}/{key}" - PALACE_REPORTING_NAME: "TEST CM" +x-cm-variables: &cm + platform: "${BUILD_PLATFORM-}" + environment: + SIMPLIFIED_PRODUCTION_DATABASE: "postgresql://palace:test@pg:5432/circ" + PALACE_SEARCH_URL: "http://os:9200" + PALACE_STORAGE_ACCESS_KEY: "palace" + PALACE_STORAGE_SECRET_KEY: "test123456789" + PALACE_STORAGE_ENDPOINT_URL: "http://minio:9000" + PALACE_STORAGE_PUBLIC_ACCESS_BUCKET: "public" + PALACE_STORAGE_ANALYTICS_BUCKET: "analytics" + PALACE_STORAGE_URL_TEMPLATE: "http://localhost:9000/{bucket}/{key}" + PALACE_REPORTING_NAME: "TEST CM" + depends_on: + pg: + condition: service_healthy + minio: + condition: service_healthy + os: + condition: service_healthy + +x-cm-build: &cm-build + context: . + dockerfile: docker/Dockerfile + args: + - BASE_IMAGE=${BUILD_BASE_IMAGE-ghcr.io/thepalaceproject/circ-baseimage:latest} + cache_from: + - ${BUILD_CACHE_FROM-ghcr.io/thepalaceproject/circ-webapp:main} services: # example docker compose configuration for testing and development webapp: + <<: *cm build: - context: . - dockerfile: docker/Dockerfile + <<: *cm-build target: webapp ports: - "6500:80" - environment: *cm-env-variables scripts: + <<: *cm build: - context: . - dockerfile: docker/Dockerfile + <<: *cm-build target: scripts - environment: *cm-env-variables pg: image: "postgres:12" @@ -37,6 +53,11 @@ services: POSTGRES_USER: palace POSTGRES_PASSWORD: test POSTGRES_DB: circ + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 30s + timeout: 30s + retries: 3 minio: image: "bitnami/minio:2023.2.27" @@ -48,11 +69,22 @@ services: MINIO_ROOT_PASSWORD: "test123456789" MINIO_SCHEME: "http" MINIO_DEFAULT_BUCKETS: "public:download,analytics" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 30s + timeout: 20s + retries: 3 os: build: dockerfile: docker/Dockerfile.ci target: opensearch + context: . environment: - discovery.type: single-node - DISABLE_SECURITY_PLUGIN: true + discovery.type: "single-node" + DISABLE_SECURITY_PLUGIN: "true" + healthcheck: + test: curl --silent http://localhost:9200 >/dev/null; if [[ $$? == 52 ]]; then echo 0; else echo 1; fi + interval: 30s + timeout: 10s + retries: 5 diff --git a/docker/ci/check_service_status.sh b/docker/ci/check_service_status.sh index a76ff6d02d..9adbe35f81 100644 --- a/docker/ci/check_service_status.sh +++ b/docker/ci/check_service_status.sh @@ -4,7 +4,7 @@ function wait_for_runit() # The container to run the command in container="$1" - timeout 120s grep -q 'Runit started' <(docker logs "$container" -f 2>&1) + timeout 120s grep -q 'Runit started' <(docker compose logs "$container" -f 2>&1) } # A method to check that runit services are running inside the container @@ -17,7 +17,7 @@ function check_service_status() service="$2" # Check the status of the service. - service_status=$(docker exec "$container" /bin/bash -c "sv check $service") + service_status=$(docker compose exec "$container" /bin/bash -c "sv check $service") # Get the exit code for the sv call. sv_status=$? @@ -34,7 +34,7 @@ function check_crontab() { container="$1" # Installing the crontab will reveal any errors and exit with an error code - $(docker exec "$container" /bin/bash -c "crontab /etc/cron.d/circulation") + $(docker compose exec "$container" /bin/bash -c "crontab /etc/cron.d/circulation") validate_status=$? if [[ "$validate_status" != 0 ]]; then echo " FAIL: crontab is incorrect" @@ -48,7 +48,7 @@ function run_script() { container="$1" script="$2" - output=$(docker exec "$container" /bin/bash -c "$script") + output=$(docker compose exec "$container" /bin/bash -c "$script") script_status=$? if [[ "$script_status" != 0 ]]; then echo " FAIL: script run failed" diff --git a/docker/ci/test_scripts.sh b/docker/ci/test_scripts.sh index d283e87093..55dc74de15 100755 --- a/docker/ci/test_scripts.sh +++ b/docker/ci/test_scripts.sh @@ -12,6 +12,9 @@ source "${dir}/check_service_status.sh" # Wait for container to start wait_for_runit "$container" +# Make sure database initialization completed successfully +timeout 240s grep -q 'Initialization complete' <(docker compose logs "$container" -f 2>&1) + # Make sure that cron is running in the scripts container check_service_status "$container" /etc/service/cron diff --git a/docker/ci/test_webapp.sh b/docker/ci/test_webapp.sh index aa5680b1ce..51241f9198 100755 --- a/docker/ci/test_webapp.sh +++ b/docker/ci/test_webapp.sh @@ -17,10 +17,10 @@ check_service_status "$container" /etc/service/nginx check_service_status "$container" /etc/service/uwsgi # Wait for UWSGI to be ready to accept connections. -timeout 240s grep -q 'WSGI app .* ready in [0-9]* seconds' <(docker logs "$container" -f 2>&1) +timeout 240s grep -q 'WSGI app .* ready in [0-9]* seconds' <(docker compose logs "$container" -f 2>&1) # Make sure the web server is running. -healthcheck=$(docker exec "$container" curl --write-out "%{http_code}" --silent --output /dev/null http://localhost/healthcheck.html) +healthcheck=$(docker compose exec "$container" curl --write-out "%{http_code}" --silent --output /dev/null http://localhost/healthcheck.html) if ! [[ ${healthcheck} == "200" ]]; then exit 1 else @@ -28,7 +28,7 @@ else fi # Also make sure the app server is running. -feed_type=$(docker exec "$container" curl --write-out "%{content_type}" --silent --output /dev/null http://localhost/version.json) +feed_type=$(docker compose exec "$container" curl --write-out "%{content_type}" --silent --output /dev/null http://localhost/version.json) if ! [[ ${feed_type} == "application/json" ]]; then exit 1 else