[QOLDEV-1005] update for CKAN 2.11

- update test container to support CKAN 2.11 - update to Selenium 4 - drop Python 2 support - add Docker Compose 2 support - replace Nosetests with pytest - sync test config with other repos
qld-gov-au · Nov 12, 2024 · be95c94 · be95c94
1 parent 85649cd
commit be95c94
Show file tree

Hide file tree

Showing 26 changed files with 278 additions and 268 deletions.
diff --git a/.ahoy.yml b/.ahoy.yml
@@ -20,6 +20,7 @@ commands:
   build-network:
     usage: Ensure that the amazeeio network exists.
     cmd: |
+      ahoy title "Creating amazeeio Docker network"
       docker network prune -f > /dev/null
       docker network inspect amazeeio-network > /dev/null || docker network create amazeeio-network
 
@@ -28,63 +29,74 @@ commands:
     cmd: |
       ahoy line "Project                  : " ${PROJECT}
       ahoy line "Site local URL           : " ${LAGOON_LOCALDEV_URL}
-      ahoy line "DB port on host          : " $(docker port $(docker-compose ps -q postgres) 5432 | cut -d : -f 2)
-      ahoy line "Solr port on host        : " $(docker port $(docker-compose ps -q solr) 8983 | cut -d : -f 2)
+      ahoy line "DB port on host          : " $(docker port $(sh bin/docker-compose.sh ps -q postgres) 5432 | cut -d : -f 2)
+      ahoy line "Solr port on host        : " $(docker port $(sh bin/docker-compose.sh ps -q solr) 8983 | cut -d : -f 2)
       ahoy line "Mailhog URL              : " http://mailhog.docker.amazee.io/
 
   up:
     usage: Build and start Docker containers.
     cmd: |
-      docker-compose up -d "$@"
+      ahoy title "Building and starting Docker containers"
+      sh bin/docker-compose.sh up -d "$@"
       sleep 10
-      docker-compose logs
       ahoy cli "dockerize -wait tcp://ckan:5000 -timeout 1m"
-      if docker-compose logs | grep -q "\[Error\]"; then docker-compose logs; exit 1; fi
-      if docker-compose logs | grep -q "Exception"; then docker-compose logs; exit 1; fi
+      if sh bin/docker-compose.sh logs | grep -q "\[Error\]"; then exit 1; fi
+      if sh bin/docker-compose.sh logs | grep -q "Exception"; then exit 1; fi
       docker ps -a --filter name=^/${COMPOSE_PROJECT_NAME}_
       export DOCTOR_CHECK_CLI=0
 
   down:
     usage: Stop Docker containers and remove container, images, volumes and networks.
-    cmd: 'if [ -f "docker-compose.yml" ]; then docker-compose down --volumes; fi'
+    cmd: |
+      ahoy title 'Stopping and removing old containers, images, volumes, networks'
+      if [ -f "docker-compose.yml" ]; then sh bin/docker-compose.sh down --volumes; fi
 
   start:
     usage: Start existing Docker containers.
-    cmd: docker-compose start "$@"
+    cmd: sh bin/docker-compose.sh start "$@"
 
   stop:
     usage: Stop running Docker containers.
-    cmd: docker-compose stop "$@"
+    cmd: sh bin/docker-compose.sh stop "$@"
 
   restart:
     usage: Restart all stopped and running Docker containers.
-    cmd: docker-compose restart "$@"
+    cmd: sh bin/docker-compose.sh restart "$@"
 
   logs:
     usage: Show Docker logs.
-    cmd: docker-compose logs "$@"
+    cmd: |
+        ahoy title "Output logs"
+        sh bin/docker-compose.sh logs "$@"
 
   pull:
     usage: Pull latest docker images.
-    cmd: if [ ! -z "$(docker image ls -q)" ]; then docker image ls --format \"{{.Repository}}:{{.Tag}}\" | grep amazeeio/ | grep -v none | xargs -n1 docker pull | cat; fi
+    cmd: if [ ! -z "$(docker image ls -q)" ]; then docker image ls --format \"{{.Repository}}:{{.Tag}}\" | grep ckan/ckan- | grep -v none | xargs -n1 docker pull | cat; fi
 
   cli:
     usage: Start a shell inside CLI container or run a command.
-    cmd: if \[ "${#}" -ne 0 \]; then docker exec $(docker-compose ps -q ckan) sh -c '. ${APP_DIR}/bin/activate; cd $APP_DIR;'" $*"; else docker exec $(docker-compose ps -q ckan) sh -c '. ${APP_DIR}/bin/activate && cd $APP_DIR && sh'; fi
+    cmd: |
+      CKAN_CONTAINER=$(sh bin/docker-compose.sh ps -q ckan)
+      if [ "${#}" -ne 0 \]; then
+        docker exec $CKAN_CONTAINER sh -c '. ${APP_DIR}/bin/activate; cd $APP_DIR;'" $*"
+      else
+        docker exec $CKAN_CONTAINER sh -c '. ${APP_DIR}/bin/activate && cd $APP_DIR && sh'
+      fi
 
   doctor:
     usage: Find problems with current project setup.
     cmd: bin/doctor.sh "$@"
 
   install-site:
-    usage: Install a site.
+    usage: Install test site data.
     cmd: |
       ahoy title "Installing a fresh site"
       ahoy cli '$APP_DIR/bin/init.sh'
 
   clean:
     usage: Remove containers and all build files.
     cmd: |
+      ahoy title "Cleaning up old builds"
       ahoy down
       # Remove other directories.
       # @todo: Add destinations below.
@@ -101,33 +113,37 @@ commands:
 
   flush-redis:
     usage: Flush Redis cache.
-    cmd: docker exec -i $(docker-compose ps -q redis) redis-cli flushall > /dev/null
+    cmd: docker exec -i $(sh bin/docker-compose.sh ps -q redis) redis-cli flushall > /dev/null
 
   lint:
     usage: Lint code.
     cmd: |
+      ahoy title 'Check for lint'
       ahoy cli "flake8 ${@:-ckanext}" || \
       [ "${ALLOW_LINT_FAIL:-0}" -eq 1 ]
 
   copy-local-files:
     usage: Update files from local repo.
     cmd: |
-      docker cp . $(docker-compose ps -q ckan):/srv/app/
-      docker cp bin/ckan_cli $(docker-compose ps -q ckan):/usr/bin/
-      ahoy cli 'chmod -v u+x /usr/bin/ckan_cli; cp -v .docker/test.ini $CKAN_INI'
+      docker cp . $(sh bin/docker-compose.sh ps -q ckan):/srv/app/
+      docker cp bin/ckan_cli $(sh bin/docker-compose.sh ps -q ckan):/usr/bin/
+      ahoy cli 'chmod -v u+x /usr/bin/ckan_cli $APP_DIR/bin/*; cp -v .docker/test.ini $CKAN_INI'
 
   test-unit:
     usage: Run unit tests.
     cmd: |
+      ahoy title 'Run unit tests'
       ahoy cli 'pytest --ckan-ini=${CKAN_INI} $APP_DIR/ckanext' || \
       [ "${ALLOW_UNIT_FAIL:-0}" -eq 1 ]
 
   test-bdd:
     usage: Run BDD tests.
     cmd: |
+      ahoy title 'Run scenario tests'
+      ahoy cli "rm -f test/screenshots/*"
       ahoy start-ckan-job-worker &
       ahoy start-mailmock &
-      sleep 5 &&
+      sleep 5
       ahoy cli "behave -k ${*:-test/features} --tags=-format_autocomplete" || \
       [ "${ALLOW_BDD_FAIL:-0}" -eq 1 ]
       ahoy stop-mailmock
@@ -146,9 +162,9 @@ commands:
       ahoy cli "killall -2 mailmock"
 
   start-ckan-job-worker:
-    usage: Starts CKAN background job worker
+    usage: Starts default CKAN background job worker
     cmd: |
-      ahoy title 'Starting CKAN background job worker'
+      ahoy title 'Starting default CKAN background job worker'
       ahoy cli "ckan_cli jobs clear && \
                 ckan_cli jobs worker"
 

diff --git a/.docker/Dockerfile-template.ckan b/.docker/Dockerfile-template.ckan
@@ -1,18 +1,18 @@
-FROM openknowledge/ckan-dev:{CKAN_VERSION}
+FROM ckan/ckan-dev:{CKAN_VERSION}
 
 ARG SITE_URL=http://ckan:5000/
 ENV PYTHON_VERSION={PYTHON_VERSION}
+ENV CKAN_VERSION={CKAN_VERSION}
 ENV CKAN_SITE_URL="${SITE_URL}"
 ENV PYTHON={PYTHON}
 
 WORKDIR "${APP_DIR}"
 
 ENV DOCKERIZE_VERSION v0.6.1
-RUN apk add --no-cache build-base \
-    && curl -sL https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \
+RUN wget -O - https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \
         | tar -C /usr/local/bin -xzvf -
 
-# Install CKAN.
+RUN which ps || apt-get install -y procps
 
 COPY .docker/test.ini $CKAN_INI
 

diff --git a/.docker/test.ini b/.docker/test.ini
@@ -20,6 +20,7 @@ beaker.session.key = ckan
 # This is the secret token that the beaker library uses to hash the cookie sent
 # to the client. `paster make-config` generates a unique value for this each
 # time it generates a config file.
+SECRET_KEY = bSmgPpaxg2M+ZRes3u1TXwIcE
 beaker.session.secret = bSmgPpaxg2M+ZRes3u1TXwIcE
 
 # `paster make-config` generates a unique value for this each time it generates
@@ -67,7 +68,7 @@ ckan.plugins = report tagless_report
 smtp.test_server = localhost:8025
 smtp.mail_from = [email protected]
 
-# Logging configuration
+## Logging configuration
 [loggers]
 keys = root, ckan, ckanext
 

diff --git a/.env b/.env
@@ -13,7 +13,7 @@
 PROJECT="ckanext-report"
 
 # Docker Compose project name. All containers will have this name.
-COMPOSE_PROJECT_NAME="$PROJECT"
+COMPOSE_PROJECT_NAME="ckanext-report"
 
 # Flag to allow code linting failures. 0=enforce, 1=ignore
 ALLOW_LINT_FAIL=0

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -7,20 +7,39 @@ on:
       - master
 
 jobs:
+  # Quick check so we don't waste minutes if there's a Flake8 error
+  lint:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.x'
+      - name: Install requirements
+        run: pip install flake8 pycodestyle
+      - name: Check syntax
+        run: flake8 . --count --show-source --statistics
+
   test:
+    needs: lint
     strategy:
       fail-fast: false
       matrix:
-        ckan-version: ["2.10", 2.9, 2.9-py2, 2.8, 2.7]
+        ckan-version: ["2.11", "2.10", 2.9]
 
-    name: Continuous Integration build on CKAN ${{ matrix.ckan-version }}
+    name: Test on CKAN ${{ matrix.ckan-version }}
     runs-on: ubuntu-latest
-    container: integratedexperts/ci-builder
+    container: drevops/ci-runner:23.12.0
     env:
       CKAN_VERSION: ${{ matrix.ckan-version }}
 
     steps:
-      - uses: actions/checkout@v2
+      # Patch https://github.com/actions/runner/issues/863
+      - name: Preserve $HOME set in the container
+        run: echo HOME=/root >> "$GITHUB_ENV"
+
+      - uses: actions/checkout@v4
         timeout-minutes: 2
 
       - name: Build
@@ -29,16 +48,21 @@ jobs:
 
       - name: Test
         run: bin/test.sh
-        timeout-minutes: 30
+        timeout-minutes: 15
+
+      - name: Retrieve logs
+        if: always()
+        run: ahoy logs
+        timeout-minutes: 1
 
       - name: Retrieve screenshots
-        if: failure()
+        if: always()
         run: bin/process-artifacts.sh
         timeout-minutes: 1
 
       - name: Upload screenshots
         if: failure()
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: CKAN ${{ matrix.ckan-version }} screenshots
           path: /tmp/artifacts/behave/screenshots

diff --git a/README.md b/README.md
@@ -25,10 +25,8 @@ TODO:
 
 | CKAN version    | Compatibility       |
 | --------------- | ------------------- |
-| 2.6 and earlier | yes                 |
-| 2.7             | yes                 |
-| 2.8             | yes                 |
-| 2.9-py2         | yes                 |
+| 2.8 and earlier | no                  |
+| 2.9-py2         | no                  |
 | 2.9  (py3)      | yes                 |
 | 2.10 (py3)      | yes                 |
 

diff --git a/bin/build.sh b/bin/build.sh
@@ -2,7 +2,7 @@
 ##
 # Build site in CI.
 #
-set -e
+set -ex
 
 # Process Docker Compose configuration. This is used to avoid multiple
 # docker-compose.yml files.
@@ -14,16 +14,12 @@ sed -i -e "s/##//" docker-compose.yml
 # Pull the latest images.
 ahoy pull
 
-PYTHON=python
-if [ "$CKAN_VERSION" = "2.7" ] || [ "$CKAN_VERSION" = "2.8" ] || [ "$CKAN_VERSION" = "2.9-py2" ]; then
-    PYTHON_VERSION=py2
-else
-    PYTHON_VERSION=py3
-    PYTHON="${PYTHON}3"
-fi
+PYTHON_VERSION=py3
+PYTHON="python3"
 
 sed "s|{CKAN_VERSION}|$CKAN_VERSION|g" .docker/Dockerfile-template.ckan \
+    | sed "s|{PYTHON_VERSION}|$PYTHON_VERSION|g" \
     | sed "s|{PYTHON}|$PYTHON|g" \
-    | sed "s|{PYTHON_VERSION}|$PYTHON_VERSION|g" > .docker/Dockerfile.ckan
+    > .docker/Dockerfile.ckan
 
-ahoy build || (ahoy logs; exit 1)
+ahoy build
diff --git a/bin/ckan_cli b/bin/ckan_cli
@@ -59,14 +59,14 @@ fi
 if [ "$COMMAND" = "ckan" ]; then
     # adjust args to match ckan expectations
     COMMAND=$(echo "$1" | sed -e 's/create-test-data/seed/')
-    echo "Using 'ckan' command from $ENV_DIR with config ${CKAN_INI} to run $COMMAND..." >&2
     shift
+    echo "Using 'ckan' command from $ENV_DIR with config ${CKAN_INI} to run $COMMAND $1..." >&2
     exec $ENV_DIR/ckan -c ${CKAN_INI} $COMMAND "$@" $CLICK_ARGS
 elif [ "$COMMAND" = "paster" ]; then
     # adjust args to match paster expectations
     COMMAND=$1
-    echo "Using 'paster' command from $ENV_DIR with config ${CKAN_INI} to run $COMMAND..." >&2
     shift
+    echo "Using 'paster' command from $ENV_DIR with config ${CKAN_INI} to run $COMMAND $1..." >&2
     if [ "$1" = "show" ]; then shift; fi
     exec $ENV_DIR/paster --plugin=$PASTER_PLUGIN $COMMAND "$@" -c ${CKAN_INI}
 else