From 5e71dd2dbc45d2f7fd9aa2efc4b9daf6a02f4d2f Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Wed, 12 Jul 2023 14:09:33 +1000 Subject: [PATCH 1/6] [QOLSVC-2077] add scenario tests --- .ahoy.yml | 199 ++++++ .docker/Dockerfile-template.ckan | 39 ++ .docker/test.ini | 153 +++++ .env | 29 + .flake8 | 2 + .github/workflows/test.yml | 86 +-- .gitignore | 27 + bin/activate | 3 + bin/build.sh | 52 ++ bin/ckan_cli | 75 ++ bin/create-test-data.sh | 73 ++ bin/deactivate | 3 + bin/doctor.sh | 202 ++++++ bin/extract-id.py | 5 + bin/get-logs.sh | 7 + bin/init-ext.sh | 49 ++ bin/init.sh | 10 + bin/process-artifacts.sh | 13 + bin/process-config.sh | 0 bin/serve.sh | 23 + bin/test-all.sh | 13 + bin/test-bdd.sh | 9 + bin/test-lint.sh | 9 + bin/test.sh | 8 + dev-requirements-2.9-py2.txt | 14 + dev-requirements-2.9.txt | 12 + dev-requirements.txt | 12 +- docker-compose.yml | 95 +++ test/features/environment.py | 136 ++++ test/features/resource_availability.feature | 92 +++ test/features/steps/steps.py | 721 ++++++++++++++++++++ 31 files changed, 2117 insertions(+), 54 deletions(-) create mode 100644 .ahoy.yml create mode 100644 .docker/Dockerfile-template.ckan create mode 100644 .docker/test.ini create mode 100644 .env create mode 100644 bin/activate create mode 100755 bin/build.sh create mode 100644 bin/ckan_cli create mode 100644 bin/create-test-data.sh create mode 100644 bin/deactivate create mode 100755 bin/doctor.sh create mode 100644 bin/extract-id.py create mode 100755 bin/get-logs.sh create mode 100755 bin/init-ext.sh create mode 100755 bin/init.sh create mode 100755 bin/process-artifacts.sh create mode 100644 bin/process-config.sh create mode 100755 bin/serve.sh create mode 100755 bin/test-all.sh create mode 100755 bin/test-bdd.sh create mode 100755 bin/test-lint.sh create mode 100755 bin/test.sh create mode 100644 dev-requirements-2.9-py2.txt create mode 100644 dev-requirements-2.9.txt create mode 100644 docker-compose.yml create mode 100644 test/features/environment.py create mode 100644 test/features/resource_availability.feature create mode 100644 test/features/steps/steps.py diff --git a/.ahoy.yml b/.ahoy.yml new file mode 100644 index 0000000..b265775 --- /dev/null +++ b/.ahoy.yml @@ -0,0 +1,199 @@ +--- +ahoyapi: v2 + +commands: + + # Docker commands. + build: + usage: Build or rebuild project. + cmd: | + ahoy title "Building project" + ahoy pre-flight + ahoy clean + ahoy build-network + ahoy up -- --build --force-recreate + ahoy title "Build complete" + ahoy doctor + ahoy info 1 + + build-network: + usage: Ensure that the amazeeio network exists. + cmd: | + docker network prune -f > /dev/null + docker network inspect amazeeio-network > /dev/null || docker network create amazeeio-network + + info: + usage: Print information about this project. + cmd: | + ahoy line "Project : " ${PROJECT} + ahoy line "Site local URL : " ${LAGOON_LOCALDEV_URL} + ahoy line "DB port on host : " $(docker port $(docker-compose ps -q postgres) 5432 | cut -d : -f 2) + ahoy line "Solr port on host : " $(docker port $(docker-compose ps -q solr) 8983 | cut -d : -f 2) + ahoy line "Mailhog URL : " http://mailhog.docker.amazee.io/ + + up: + usage: Build and start Docker containers. + cmd: | + docker-compose up -d "$@" + sleep 10 + docker-compose logs + ahoy cli "dockerize -wait tcp://ckan:5000 -timeout 1m" + if docker-compose logs | grep -q "\[Error\]"; then docker-compose logs; exit 1; fi + if docker-compose logs | grep -q "Exception"; then docker-compose logs; exit 1; fi + docker ps -a --filter name=^/${COMPOSE_PROJECT_NAME}_ + ahoy cli '$APP_DIR/bin/init.sh' + export DOCTOR_CHECK_CLI=0 + + down: + usage: Stop Docker containers and remove container, images, volumes and networks. + cmd: 'if [ -f "docker-compose.yml" ]; then docker-compose down --volumes; fi' + + start: + usage: Start existing Docker containers. + cmd: docker-compose start "$@" + + stop: + usage: Stop running Docker containers. + cmd: docker-compose stop "$@" + + restart: + usage: Restart all stopped and running Docker containers. + cmd: docker-compose restart "$@" + + logs: + usage: Show Docker logs. + cmd: docker-compose logs "$@" + + pull: + usage: Pull latest docker images. + cmd: if [ ! -z "$(docker image ls -q)" ]; then docker image ls --format \"{{.Repository}}:{{.Tag}}\" | grep ckan/ckan- | grep -v none | xargs -n1 docker pull | cat; fi + + cli: + usage: Start a shell inside CLI container or run a command. + cmd: if \[ "${#}" -ne 0 \]; then docker exec $(docker-compose ps -q ckan) sh -c '. ${APP_DIR}/bin/activate; cd $APP_DIR;'" $*"; else docker exec $(docker-compose ps -q ckan) sh -c '. ${APP_DIR}/bin/activate && cd $APP_DIR && sh'; fi + + doctor: + usage: Find problems with current project setup. + cmd: bin/doctor.sh "$@" + + install-site: + usage: Install test site data. + cmd: | + ahoy title "Installing a fresh site" + ahoy cli '$APP_DIR/bin/init.sh && $APP_DIR/bin/create-test-data.sh' + + clean: + usage: Remove containers and all build files. + cmd: | + ahoy down + # Remove other directories. + # @todo: Add destinations below. + rm -rf \ + ./ckan + + reset: + usage: "Reset environment: remove containers, all build, manually created and Drupal-Dev files." + cmd: | + ahoy clean + git ls-files --others -i --exclude-from=.git/info/exclude | xargs chmod 777 + git ls-files --others -i --exclude-from=.git/info/exclude | xargs rm -Rf + find . -type d -not -path "./.git/*" -empty -delete + + flush-redis: + usage: Flush Redis cache. + cmd: docker exec -i $(docker-compose ps -q redis) redis-cli flushall > /dev/null + + lint: + usage: Lint code. + cmd: | + ahoy cli "flake8 ${@:-ckanext}" || \ + [ "${ALLOW_LINT_FAIL:-0}" -eq 1 ] + + copy-local-files: + usage: Update files from local repo. + cmd: | + docker cp . $(docker-compose ps -q ckan):/srv/app/ + docker cp bin/ckan_cli $(docker-compose ps -q ckan):/usr/bin/ + ahoy cli 'chmod -v u+x /usr/bin/ckan_cli $APP_DIR/bin/*; cp -v .docker/test.ini $CKAN_INI; $APP_DIR/bin/process-config.sh' + + pip-list: + usage: List pip install version details + cmd: | + ahoy cli 'pip list' + + test-unit: + usage: Run unit tests. + cmd: | + ahoy cli 'pytest --ckan-ini=${CKAN_INI} --cov=ckanext $APP_DIR/ckanext' || \ + [ "${ALLOW_UNIT_FAIL:-0}" -eq 1 ] + + test-bdd: + usage: Run BDD tests. + cmd: | + ahoy cli "rm -f test/screenshots/*" + ahoy start-mailmock & + sleep 5 && + if [ "$BEHAVE_TAG" = "" ]; then + # no tag specified, probably running locally + (ahoy cli "behave -k ${*:-test/features} --tags=smoke" \ + && ahoy cli "behave -k ${*:-test/features} --tags=-smoke" \ + ) || [ "${ALLOW_BDD_FAIL:-0}" -eq 1 ] + elif [ "$BEHAVE_TAG" = "authenticated" ]; then + # run any tests that don't have a specific tag + ahoy cli "behave -k ${*:-test/features} --tags=-unauthenticated --tags=-smoke" \ + || [ "${ALLOW_BDD_FAIL:-0}" -eq 1 ] + else + # run tests with the specified tag + ahoy cli "behave -k ${*:-test/features} --tags=$BEHAVE_TAG" \ + || [ "${ALLOW_BDD_FAIL:-0}" -eq 1 ] + fi + ahoy stop-mailmock + + start-mailmock: + usage: Starts email mock server used for email BDD tests + cmd: | + ahoy title 'Starting mailmock' + ahoy cli 'mailmock -p 8025 -o ${APP_DIR}/test/emails' # for debugging mailmock email output remove --no-stdout + + stop-mailmock: + usage: Stops email mock server used for email BDD tests + cmd: | + ahoy title 'Stopping mailmock' + ahoy cli "killall -2 mailmock" + + # Utilities. + title: + cmd: printf "$(tput -Txterm setaf 4)==> ${1}$(tput -Txterm sgr0)\n" + hide: true + + line: + cmd: printf "$(tput -Txterm setaf 2)${1}$(tput -Txterm sgr0)${2}\n" + hide: true + + getvar: + cmd: eval echo "${@}" + hide: true + + pre-flight: + cmd: | + export DOCTOR_CHECK_DB=${DOCTOR_CHECK_DB:-1} + export DOCTOR_CHECK_TOOLS=${DOCTOR_CHECK_TOOLS:-1} + export DOCTOR_CHECK_PORT=${DOCTOR_CHECK_PORT:-0} + export DOCTOR_CHECK_PYGMY=${DOCTOR_CHECK_PYGMY:-1} + export DOCTOR_CHECK_CLI=${DOCTOR_CHECK_CLI:-0} + export DOCTOR_CHECK_SSH=${DOCTOR_CHECK_SSH:-0} + export DOCTOR_CHECK_WEBSERVER=${DOCTOR_CHECK_WEBSERVER:-0} + export DOCTOR_CHECK_BOOTSTRAP=${DOCTOR_CHECK_BOOTSTRAP:-0} + ahoy doctor + hide: true + +entrypoint: + - bash + - "-c" + - "-e" + - | + export LAGOON_LOCALDEV_URL=http://$PROJECT.docker.amazee.io + [ -f .env ] && [ -s .env ] && export $(grep -v '^#' .env | xargs) && if [ -f .env.local ] && [ -s .env.local ]; then export $(grep -v '^#' .env.local | xargs); fi + bash -e -c "$0" "$@" + - "{{cmd}}" + - "{{name}}" diff --git a/.docker/Dockerfile-template.ckan b/.docker/Dockerfile-template.ckan new file mode 100644 index 0000000..b0222e0 --- /dev/null +++ b/.docker/Dockerfile-template.ckan @@ -0,0 +1,39 @@ +FROM openknowledge/ckan-dev:{CKAN_VERSION} + +ARG SITE_URL=http://ckan:5000/ +ENV PYTHON_VERSION={PYTHON_VERSION} +ENV CKAN_VERSION={CKAN_VERSION} +ENV CKAN_SITE_URL="${SITE_URL}" +ENV PYTHON={PYTHON} + +WORKDIR "${APP_DIR}" + +ENV DOCKERIZE_VERSION v0.6.1 +RUN apk add --no-cache build-base \ + && curl -sL https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \ + | tar -C /usr/local/bin -xzvf - + +# Update urllib3 to fix urlopen bug +RUN pip install -U urllib3 + +# Install CKAN. + +RUN cd $SRC_DIR/ckan \ + && git config --global --add safe.directory "$SRC_DIR/ckan" \ + && git remote set-url origin 'https://github.com/{CKAN_GIT_ORG}/ckan.git' \ + && git fetch \ + && git reset --hard && git clean -f \ + && git checkout '{CKAN_GIT_VERSION}' + +COPY .docker/test.ini $CKAN_INI + +COPY . ${APP_DIR}/ + +COPY bin/ckan_cli /usr/bin/ + +RUN chmod +x ${APP_DIR}/bin/*.sh /usr/bin/ckan_cli + +# Init current extension. +RUN ${APP_DIR}/bin/init-ext.sh + +CMD ["/srv/app/bin/serve.sh"] diff --git a/.docker/test.ini b/.docker/test.ini new file mode 100644 index 0000000..cea0d05 --- /dev/null +++ b/.docker/test.ini @@ -0,0 +1,153 @@ +#UNIT TEST CONFIG +# +# CKAN - Pylons configuration +# +# These are some of the configuration options available for your CKAN +# instance. Check the documentation in 'doc/configuration.rst' or at the +# following URL for a description of what they do and the full list of +# available options: +# +# http://docs.ckan.org/en/latest/maintaining/configuration.html +# +# The %(here)s variable will be replaced with the parent directory of this file +# + +[DEFAULT] +debug = false +smtp_server = localhost:8025 +error_email_from = paste@localhost + +[server:main] +use = egg:Paste#http +host = 0.0.0.0 +port = 5000 + +[app:main] +ckan.devserver.host = 0.0.0.0 +ckan.devserver.port = 5000 + +use = egg:ckan +full_stack = true +cache_dir = /tmp/%(ckan.site_id)s/ +beaker.session.key = ckan + +# This is the secret token that the beaker library uses to hash the cookie sent +# to the client. `paster make-config` generates a unique value for this each +# time it generates a config file. +beaker.session.secret = bSmgPpaxg2M+ZRes3u1TXwIcE + +# `paster make-config` generates a unique value for this each time it generates +# a config file. +app_instance_uuid = 6e3daf8e-1c6b-443b-911f-c7ab4c5f9605 + +# repoze.who config +who.config_file = %(here)s/who.ini +who.log_level = warning +who.log_file = %(cache_dir)s/who_log.ini + +## Database Settings +sqlalchemy.url = postgresql://ckan_default:pass@postgres/ckan_test + +# PostgreSQL' full-text search parameters +ckan.datastore.default_fts_lang = english +ckan.datastore.default_fts_index_method = gist + +## Site Settings. +ckan.site_url = http://ckan:5000/ + +## Search Settings + +ckan.site_id = default +solr_url = http://solr:8983/solr/ckan + +## Redis Settings + +# URL to your Redis instance, including the database to be used. +ckan.redis.url = redis://redis:6379 + +## Authorization Settings +ckan.auth.anon_create_dataset = false +ckan.auth.create_unowned_dataset = false +ckan.auth.create_dataset_if_not_in_organization = false +ckan.auth.user_create_groups = false +ckan.auth.user_create_organizations = false +ckan.auth.user_delete_groups = true +ckan.auth.user_delete_organizations = true +ckan.auth.create_user_via_api = false +ckan.auth.create_user_via_web = true +ckan.auth.roles_that_cascade_to_sub_groups = admin +ckan.auth.public_user_details = False + +## Plugins Settings +ckan.plugins = + resource_visibility + + +# Define which views should be created by default +# (plugins must be loaded in ckan.plugins) +ckan.views.default_views = image_view text_view recline_view + +# Customize which text formats the text_view plugin will show +#ckan.preview.json_formats = json +#ckan.preview.xml_formats = xml rdf rdf+xml owl+xml atom rss +#ckan.preview.text_formats = text plain text/plain + +# Customize which image formats the image_view plugin will show +#ckan.preview.image_formats = png jpeg jpg gif + +## Internationalisation Settings +ckan.locale_default = en_AU +ckan.locale_order = en pt_BR ja it cs_CZ ca es fr el sv sr sr@latin no sk fi ru de pl nl bg ko_KR hu sa sl lv +ckan.locales_offered = +ckan.locales_filtered_out = en_AU +ckan.display_timezone = Australia/Queensland + +## Storage Settings + +ckan.storage_path = /app/filestore + +## Activity Streams Settings + +ckan.hide_activity_from_users = %(ckan.site_id)s + + +## Email settings + +smtp.server = localhost:8025 +smtp.test_server = localhost:8025 +smtp.mail_from = info@test.ckan.net + +## Logging configuration +[loggers] +keys = root, ckan, ckanext + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARNING +handlers = console + +[logger_ckan] +level = INFO +handlers = console +qualname = ckan +propagate = 0 + +[logger_ckanext] +level = DEBUG +handlers = console +qualname = ckanext +propagate = 0 + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s diff --git a/.env b/.env new file mode 100644 index 0000000..b435428 --- /dev/null +++ b/.env @@ -0,0 +1,29 @@ +## +# Project environment variables. +# +# It is used by Ahoy and other scripts to read default values. +# +# The values must be scalar (cannot be another variable). +# +# You may also create .env.local file to override any values locally (it is +# excluded from git). +# + +# Project name. +PROJECT="ckanext-resource-visibility" + +# Docker Compose project name. All containers will have this name. +COMPOSE_PROJECT_NAME="$PROJECT" + +# Flag to allow code linting failures. 0=enforce, 1=ignore +ALLOW_LINT_FAIL=0 + +# Flag to allow unit tests failures. 0=enforce, 1=ignore +ALLOW_UNIT_FAIL=0 + +# Flag to allow BDD tests failures. 0=enforce, 1=ignore +ALLOW_BDD_FAIL=0 + +# Disable amazeeio based health checks +DOCTOR_CHECK_WEBSERVER=0 +DOCTOR_CHECK_BOOTSTRAP=0 diff --git a/.flake8 b/.flake8 index 8287ed1..55f2a01 100644 --- a/.flake8 +++ b/.flake8 @@ -16,4 +16,6 @@ max-line-length = 127 # List ignore rules one per line. ignore = + C901 + E501 W503 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3879b8a..0a7aa46 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,80 +9,60 @@ on: jobs: lint: runs-on: ubuntu-latest + timeout-minutes: 5 steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: python-version: '3.x' - - name: Cache pip - uses: actions/cache@v2 - with: - # This path is specific to Ubuntu - path: ~/.cache/pip - # Look to see if there is a cache hit for the corresponding requirements file - key: ${{ runner.os }}-pip-flake8-${{ hashFiles('requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip-flake8- - ${{ runner.os }}- - name: Install requirements run: pip install flake8 pycodestyle - name: Check syntax - run: flake8 . + run: flake8 test: needs: lint strategy: + fail-fast: false matrix: ckan-version: ["2.10", 2.9, 2.9-py2] - fail-fast: false name: CKAN ${{ matrix.ckan-version }} runs-on: ubuntu-latest - container: - image: openknowledge/ckan-dev:${{ matrix.ckan-version }} - services: - solr: - image: ckan/ckan-solr:${{ matrix.ckan-version }} - postgres: - image: ckan/ckan-postgres-dev:${{ matrix.ckan-version }} - env: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: postgres - options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 - redis: - image: redis:3 + container: drevops/ci-builder env: - CKAN_SQLALCHEMY_URL: postgresql://ckan_default:pass@postgres/ckan_test - CKAN_DATASTORE_WRITE_URL: postgresql://datastore_write:pass@postgres/datastore_test - CKAN_DATASTORE_READ_URL: postgresql://datastore_read:pass@postgres/datastore_test - CKAN_SOLR_URL: http://solr:8983/solr/ckan - CKAN_REDIS_URL: redis://redis:6379/1 + CKAN_VERSION: ${{ matrix.ckan-version }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 + timeout-minutes: 2 - - name: Cache pip - uses: actions/cache@v2 - with: - # This path is specific to Ubuntu - path: ~/.cache/pip - # Look to see if there is a cache hit for the corresponding requirements file - key: ${{ runner.os }}-pip-${{ hashFiles('*requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip- - ${{ runner.os }}- + - name: Build + run: bin/build.sh + timeout-minutes: 15 - - name: Install requirements - run: | - pip install -r requirements.txt - pip install -r dev-requirements.txt - pip install -e . - # Replace default path to CKAN core config file with the one on the container - sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini + - name: Unit test + run: bin/test.sh + timeout-minutes: 10 + + - name: Scenario test + run: bin/test-bdd.sh + timeout-minutes: 30 - - name: Setup extension - run: ckan -c test.ini db init + - name: Retrieve logs + if: failure() + run: ahoy logs + timeout-minutes: 5 - - name: Run tests - run: pytest --ckan-ini=test.ini --cov=ckanext.resource_visibility --disable-warnings ckanext/resource_visibility/tests + - name: Retrieve screenshots + if: failure() + run: bin/process-artifacts.sh + timeout-minutes: 1 + + - name: Upload screenshots + if: failure() + uses: actions/upload-artifact@v2 + with: + name: CKAN ${{ matrix.ckan-version }} screenshots + path: /tmp/artifacts/behave/screenshots + timeout-minutes: 1 diff --git a/.gitignore b/.gitignore index 8570dc5..6552abd 100644 --- a/.gitignore +++ b/.gitignore @@ -15,10 +15,17 @@ env/ build/ develop-eggs/ dist/ +downloads/ +eggs/ +lib/ +lib64/ +parts/ sdist/ +var/ *.egg-info/ .installed.cfg *.egg +*.eggs # PyInstaller # Usually these files are written by a python script from a template @@ -37,6 +44,26 @@ htmlcov/ .cache nosetests.xml coverage.xml +test/screenshots + +# Translations +*.mo +*.pot + +# Django stuff: +*.log # Sphinx documentation docs/_build/ +.env.local +screenshots +!/test/screenshots/.gitkeep + +# PyBuilder +target/ + +#Intellij +.idea + +# generated from template +Dockerfile.ckan diff --git a/bin/activate b/bin/activate new file mode 100644 index 0000000..982827c --- /dev/null +++ b/bin/activate @@ -0,0 +1,3 @@ +if [ "$VENV_DIR" != "" ]; then + . ${VENV_DIR}/bin/activate +fi diff --git a/bin/build.sh b/bin/build.sh new file mode 100755 index 0000000..4854247 --- /dev/null +++ b/bin/build.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +## +# Build site in CI. +# +set -ex + +# Process Docker Compose configuration. This is used to avoid multiple +# docker-compose.yml files. +# Remove lines containing '###'. +sed -i -e "/###/d" docker-compose.yml +# Uncomment lines containing '##'. +sed -i -e "s/##//" docker-compose.yml + +# Pull the latest images. +ahoy pull + +PYTHON=python + +CKAN_GIT_VERSION=$CKAN_VERSION +CKAN_GIT_ORG=ckan + +if [ "$CKAN_VERSION" = "2.10" ]; then + if [ "$CKAN_TYPE" = "custom" ]; then + CKAN_GIT_VERSION=ckan-2.10.0-qgov.1 + CKAN_GIT_ORG=qld-gov-au + fi + + PYTHON_VERSION=py3 + PYTHON="${PYTHON}3" +else + if [ "$CKAN_TYPE" = "custom" ]; then + CKAN_GIT_VERSION=ckan-2.9.5-qgov.9 + CKAN_GIT_ORG=qld-gov-au + fi + + if [ "$CKAN_VERSION" = "2.9-py2" ]; then + PYTHON_VERSION=py2 + CKAN_GIT_VERSION=2.9 + else + PYTHON_VERSION=py3 + PYTHON="${PYTHON}3" + fi +fi + +sed "s|{CKAN_VERSION}|$CKAN_VERSION|g" .docker/Dockerfile-template.ckan \ + | sed "s|{CKAN_GIT_VERSION}|$CKAN_GIT_VERSION|g" \ + | sed "s|{CKAN_GIT_ORG}|$CKAN_GIT_ORG|g" \ + | sed "s|{PYTHON_VERSION}|$PYTHON_VERSION|g" \ + | sed "s|{PYTHON}|$PYTHON|g" \ + > .docker/Dockerfile.ckan + +ahoy build diff --git a/bin/ckan_cli b/bin/ckan_cli new file mode 100644 index 0000000..2349646 --- /dev/null +++ b/bin/ckan_cli @@ -0,0 +1,75 @@ +#!/bin/sh + +# Call either 'ckan' (from CKAN >= 2.9) or 'paster' (from CKAN <= 2.8) +# with appropriate syntax, depending on what is present on the system. +# This is intended to smooth the upgrade process from 2.8 to 2.9. +# Eg: +# ckan_cli jobs list +# could become either: +# paster --plugin=ckan jobs list -c /etc/ckan/default/production.ini +# or: +# ckan -c /etc/ckan/default/production.ini jobs list + +# This script is aware of the VIRTUAL_ENV environment variable, and will +# attempt to respect it with similar behaviour to commands like 'pip'. +# Eg placing this script in a virtualenv 'bin' directory will cause it +# to call the 'ckan' or 'paster' command in that directory, while +# placing this script elsewhere will cause it to rely on the VIRTUAL_ENV +# variable, or if that is not set, the system PATH. + +# Since the positioning of the CKAN configuration file is central to the +# differences between 'paster' and 'ckan', this script needs to be aware +# of the config file location. It will use the CKAN_INI environment +# variable if it exists, or default to /etc/ckan/default/production.ini. + +# If 'paster' is being used, the default plugin is 'ckan'. A different +# plugin can be specified by setting the PASTER_PLUGIN environment +# variable. This variable is irrelevant if using the 'ckan' command. + +CKAN_INI="${CKAN_INI:-/etc/ckan/default/production.ini}" +PASTER_PLUGIN="${PASTER_PLUGIN:-ckan}" +# First, look for a command alongside this file +ENV_DIR=$(dirname "$0") +if [ -f "$ENV_DIR/ckan" ]; then + COMMAND=ckan +elif [ -f "$ENV_DIR/paster" ]; then + COMMAND=paster +elif [ "$VIRTUAL_ENV" != "" ]; then + # If command not found alongside this file, check the virtualenv + ENV_DIR="$VIRTUAL_ENV/bin" + if [ -f "$ENV_DIR/ckan" ]; then + COMMAND=ckan + elif [ -f "$ENV_DIR/paster" ]; then + COMMAND=paster + fi +else + # if no virtualenv is active, try the system path + if (which ckan > /dev/null 2>&1); then + ENV_DIR=$(dirname $(which ckan)) + COMMAND=ckan + elif (which paster > /dev/null 2>&1); then + ENV_DIR=$(dirname $(which paster)) + COMMAND=paster + else + echo "Unable to locate 'ckan' or 'paster' command" >&2 + exit 1 + fi +fi + +if [ "$COMMAND" = "ckan" ]; then + # adjust args to match ckan expectations + COMMAND=$(echo "$1" | sed -e 's/create-test-data/seed/') + shift + echo "Using 'ckan' command from $ENV_DIR with config ${CKAN_INI} to run $COMMAND $1..." >&2 + exec $ENV_DIR/ckan -c ${CKAN_INI} $COMMAND "$@" $CLICK_ARGS +elif [ "$COMMAND" = "paster" ]; then + # adjust args to match paster expectations + COMMAND=$1 + shift + echo "Using 'paster' command from $ENV_DIR with config ${CKAN_INI} to run $COMMAND $1..." >&2 + if [ "$1" = "show" ]; then shift; fi + exec $ENV_DIR/paster --plugin=$PASTER_PLUGIN $COMMAND "$@" -c ${CKAN_INI} +else + echo "Unable to locate 'ckan' or 'paster' command in $ENV_DIR" >&2 + exit 1 +fi diff --git a/bin/create-test-data.sh b/bin/create-test-data.sh new file mode 100644 index 0000000..ff88b69 --- /dev/null +++ b/bin/create-test-data.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env sh +## +# Create some example content for extension BDD tests. +# +set -e +set -x + +CKAN_ACTION_URL=${CKAN_SITE_URL}api/action +CKAN_USER_NAME="${CKAN_USER_NAME:-admin}" +CKAN_DISPLAY_NAME="${CKAN_DISPLAY_NAME:-Administrator}" +CKAN_USER_EMAIL="${CKAN_USER_EMAIL:-admin@localhost}" + +. ${APP_DIR}/bin/activate + +add_user_if_needed () { + echo "Adding user '$2' ($1) with email address [$3]" + ckan_cli user show "$1" | grep "$1" || ckan_cli user add "$1"\ + fullname="$2"\ + email="$3"\ + password="${4:-Password123!}" +} + +add_user_if_needed "$CKAN_USER_NAME" "$CKAN_DISPLAY_NAME" "$CKAN_USER_EMAIL" +ckan_cli sysadmin add "${CKAN_USER_NAME}" + +API_KEY=$(ckan_cli user show "${CKAN_USER_NAME}" | tr -d '\n' | sed -r 's/^(.*)apikey=(\S*)(.*)/\2/') +if [ "$API_KEY" = "None" ]; then + echo "No API Key found on ${CKAN_USER_NAME}, generating API Token..." + API_KEY=$(ckan_cli user token add "${CKAN_USER_NAME}" test_setup |tail -1 | tr -d '[:space:]') +fi + +## +# BEGIN: Create a test organisation with test users for admin, editor and member +# +TEST_ORG_NAME=test-organisation +TEST_ORG_TITLE="Test Organisation" + +echo "Creating test users for ${TEST_ORG_TITLE} Organisation:" + +add_user_if_needed ckan_user "CKAN User" ckan_user@localhost +add_user_if_needed test_org_admin "Test Admin" test_org_admin@localhost +add_user_if_needed test_org_editor "Test Editor" test_org_editor@localhost +add_user_if_needed test_org_member "Test Member" test_org_member@localhost + +echo "Creating ${TEST_ORG_TITLE} organisation:" + +TEST_ORG=$( \ + curl -LsH "Authorization: ${API_KEY}" \ + --data '{"name": "'"${TEST_ORG_NAME}"'", "title": "'"${TEST_ORG_TITLE}"'", + "description": "Organisation for testing issues"}' \ + ${CKAN_ACTION_URL}/organization_create +) + +TEST_ORG_ID=$(echo $TEST_ORG | $PYTHON ${APP_DIR}/bin/extract-id.py) + +echo "Assigning test users to '${TEST_ORG_TITLE}' organisation (${TEST_ORG_ID}):" + +curl -LsH "Authorization: ${API_KEY}" \ + --data '{"id": "'"${TEST_ORG_ID}"'", "object": "test_org_admin", "object_type": "user", "capacity": "admin"}' \ + ${CKAN_ACTION_URL}/member_create + +curl -LsH "Authorization: ${API_KEY}" \ + --data '{"id": "'"${TEST_ORG_ID}"'", "object": "test_org_editor", "object_type": "user", "capacity": "editor"}' \ + ${CKAN_ACTION_URL}/member_create + +curl -LsH "Authorization: ${API_KEY}" \ + --data '{"id": "'"${TEST_ORG_ID}"'", "object": "test_org_member", "object_type": "user", "capacity": "member"}' \ + ${CKAN_ACTION_URL}/member_create +## +# END. +# + +. ${APP_DIR}/bin/deactivate diff --git a/bin/deactivate b/bin/deactivate new file mode 100644 index 0000000..7cd77b9 --- /dev/null +++ b/bin/deactivate @@ -0,0 +1,3 @@ +if [ "$VENV_DIR" != "" ]; then + deactivate +fi diff --git a/bin/doctor.sh b/bin/doctor.sh new file mode 100755 index 0000000..9c7f455 --- /dev/null +++ b/bin/doctor.sh @@ -0,0 +1,202 @@ +#!/usr/bin/env bash +# +# Check Drupal-Dev project requirements. +# +set -e + +DOCTOR_CHECK_TOOLS="${DOCTOR_CHECK_TOOLS:-1}" +DOCTOR_CHECK_PORT="${DOCTOR_CHECK_PORT:-0}" +DOCTOR_CHECK_PYGMY="${DOCTOR_CHECK_PYGMY:-1}" +DOCTOR_CHECK_CLI="${DOCTOR_CHECK_CLI:-1}" +DOCTOR_CHECK_SSH="${DOCTOR_CHECK_SSH:-0}" +DOCTOR_CHECK_WEBSERVER="${DOCTOR_CHECK_WEBSERVER:-1}" +DOCTOR_CHECK_BOOTSTRAP="${DOCTOR_CHECK_BOOTSTRAP:-1}" + +APP_PORT="${APP_PORT:-5000}" +CLI="${CLI:-cli}" +LAGOON_LOCALDEV_URL="${LAGOON_LOCALDEV_URL:-http://your-site.docker.amazee.io/}" +SSH_KEY_FILE="${SSH_KEY_FILE:-$HOME/.ssh/id_rsa}" +DATAROOT="${DATAROOT:-.data}" + +#------------------------------------------------------------------------------- +# DO NOT CHANGE ANYTHING BELOW THIS LINE +#------------------------------------------------------------------------------- + + +# +# Main entry point. +# +main() { + status "Checking project requirements" + + if [ "${DOCTOR_CHECK_TOOLS}" == "1" ]; then + [ "$(command_exists docker)" == "1" ] && error "Please install Docker (https://www.docker.com/get-started)" && exit 1 + [ "$(command_exists docker-compose)" == "1" ] && error "Please install docker-compose (https://docs.docker.com/compose/install/)" && exit 1 + [ "$(command_exists composer)" == "1" ] && error "Please install Composer (https://getcomposer.org/)" && exit 1 + [ "$(command_exists pygmy)" == "1" ] && error "Please install Pygmy (https://pygmy.readthedocs.io/)" && exit 1 + [ "$(command_exists ahoy)" == "1" ] && error "Please install Ahoy (https://ahoy-cli.readthedocs.io/)" && exit 1 + success "All required tools are present" + fi + + if [ "${DOCTOR_CHECK_PORT}" == "1" ]; then + if ! lsof -i :$APP_PORT | grep LISTEN | grep -q om.docke; then + error "Port $APP_PORT is occupied by a service other than Docker. Stop this service and run 'pygmy up'" + fi + success "Port $APP_PORT is available" + fi + + if [ "${DOCTOR_CHECK_PYGMY}" == "1" ]; then + if ! pygmy status > /dev/null 2>&1; then + error "pygmy is not running. Run 'pygmy up' to start pygmy." + exit 1 + fi + success "Pygmy is running" + fi + + # Check that the stack is running. + if [ "${DOCTOR_CHECK_CLI}" == "1" ]; then + if ! docker ps -q --no-trunc | grep "$(docker-compose ps -q ckan)" > /dev/null 2>&1; then + error "CLI container is not running. Run 'ahoy up'." + exit 1 + fi + success "CLI container is running" + fi + + if [ "${DOCTOR_CHECK_SSH}" == "1" ]; then + # SSH key injection is required to access Lagoon services from within + # containers. For example, to connect to production environment to run + # drush script. + # Pygmy makes this possible in the following way: + # 1. Pygmy starts `amazeeio/ssh-agent` container with a volume `/tmp/amazeeio_ssh-agent` + # 2. Pygmy adds a default SSH key from the host into this volume. + # 3. `docker-compose.yml` should have volume inclusion specified for CLI container: + # ``` + # volumes_from: + # - container:amazeeio-ssh-agent + # ``` + # 4. When CLI container starts, the volume is mounted and an entrypoint script + # adds SSH key into agent. + # @see https://github.com/amazeeio/lagoon/blob/master/images/php/cli/10-ssh-agent.sh + # + # Running `ssh-add -L` within CLI container should show that the SSH key + # is correctly loaded. + # + # As rule of a thumb, one must restart the CLI container after restarting + # Pygmy ONLY if SSH key was not loaded in pygmy before the stack starts. + # No need to restart CLI container if key was added, but pygmy was + # restarted - the volume mount will retain and the key will still be + # available in CLI container. + + # Check that the key is injected into pygmy ssh-agent container. + if ! pygmy status 2>&1 | grep -q "${SSH_KEY_FILE}"; then + error "SSH key is not added to pygmy. Run 'pygmy stop && pygmy start' and then 'ahoy up -- --build'." + exit 1 + fi + + # Check that the volume is mounted into CLI container. + if ! docker exec -i "$(docker-compose ps -q ckan)" sh -c "grep \"^/dev\" /etc/mtab|grep -q /tmp/amazeeio_ssh-agent"; then + error "SSH key is added to Pygmy, but the volume is not mounted into container. Make sure that your your \"docker-compose.yml\" has the following lines:" + error "volumes_from:" + error " - container:amazeeio-ssh-agent" + error "After adding these lines, run 'ahoy up -- --build'" + exit 1 + fi + + # Check that ssh key is available in the container. + if ! docker exec -i "$(docker-compose ps -q ckan)" bash -c "ssh-add -L | grep -q 'ssh-rsa'" ; then + error "SSH key was not added into container. Run 'ahoy up -- --build'." + exit 1 + fi + + success "SSH key is available within CLI container" + fi + + + if [ "${DOCTOR_CHECK_WEBSERVER}" == "1" ]; then + host_app_port="$(docker port $(docker-compose ps -q ckan) $APP_PORT | cut -d : -f 2)" + if ! curl -L -s -o /dev/null -w "%{http_code}" "${LAGOON_LOCALDEV_URL}:${host_app_port}" | grep -q 200; then + error "Web server is not accessible at ${LAGOON_LOCALDEV_URL}:${host_app_port}" + exit 1 + fi + success "Web server is running and accessible at ${LAGOON_LOCALDEV_URL}:${host_app_port}" + fi + + if [ "${DOCTOR_CHECK_BOOTSTRAP}" == "1" ]; then + host_app_port="$(docker port $(docker-compose ps -q ckan) $APP_PORT | cut -d : -f 2)" + if ! curl -L -s -N "${LAGOON_LOCALDEV_URL}:${host_app_port}" | grep -q -i "meta name=\"generator\" content=\"ckan"; then + error "Website is running, but cannot be bootstrapped. Try pulling latest container images with 'ahoy pull'" + exit 1 + fi + success "Successfully bootstrapped website at ${LAGOON_LOCALDEV_URL}:${host_app_port}" + fi + + status "All required checks have passed" +} + +# +# Check that command exists. +# +command_exists() { + local cmd=$1 + command -v "${cmd}" | grep -ohq "${cmd}" + local res=$? + + # Try homebrew lookup, if brew is available. + if command -v "brew" | grep -ohq "brew" && [ "$res" == "1" ] ; then + brew --prefix "${cmd}" > /dev/null + res=$? + fi + + echo ${res} +} + +# +# Status echo. +# +status() { + cecho blue "✚ $1"; +} + +# +# Success echo. +# +success() { + cecho green " ✓ $1"; +} + +# +# Error echo. +# +error() { + cecho red " ✘ $1"; + exit 1 +} + +# +# Colored echo. +# +cecho() { + local prefix="\033[" + local input_color=$1 + local message="$2" + + local color="" + case "$input_color" in + black | bk) color="${prefix}0;30m";; + red | r) color="${prefix}1;31m";; + green | g) color="${prefix}1;32m";; + yellow | y) color="${prefix}1;33m";; + blue | b) color="${prefix}1;34m";; + purple | p) color="${prefix}1;35m";; + cyan | c) color="${prefix}1;36m";; + gray | gr) color="${prefix}0;37m";; + *) message="$1" + esac + + # Format message with color codes, but only if a correct color was provided. + [ -n "$color" ] && message="${color}${message}${prefix}0m" + + echo -e "$message" +} + +main "$@" diff --git a/bin/extract-id.py b/bin/extract-id.py new file mode 100644 index 0000000..ae9cf79 --- /dev/null +++ b/bin/extract-id.py @@ -0,0 +1,5 @@ +# encoding: utf-8 +import json +import sys + +print(json.loads(sys.stdin.read())['result']['id']) diff --git a/bin/get-logs.sh b/bin/get-logs.sh new file mode 100755 index 0000000..ce828cb --- /dev/null +++ b/bin/get-logs.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +## +# Run tests in CI. +# +set -e +echo "output logs" +ahoy logs diff --git a/bin/init-ext.sh b/bin/init-ext.sh new file mode 100755 index 0000000..de4e638 --- /dev/null +++ b/bin/init-ext.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env sh +## +# Install current extension. +# +set -e +set -x + +install_requirements () { + PROJECT_DIR=$1 + shift + # Identify the best match requirements file, ignore the others. + # If there is one specific to our CKAN or Python version, use that. + for filename_pattern in "$@"; do + filename="$PROJECT_DIR/${filename_pattern}-$CKAN_VERSION.txt" + if [ -f "$filename" ]; then + pip install -r "$filename" + return 0 + fi + done + for filename_pattern in "$@"; do + filename="$PROJECT_DIR/${filename_pattern}-$PYTHON_VERSION.txt" + if [ -f "$filename" ]; then + pip install -r "$filename" + return 0 + fi + done + for filename_pattern in "$@"; do + filename="$PROJECT_DIR/$filename_pattern.txt" + if [ -f "$filename" ]; then + pip install -r "$filename" + return 0 + fi + done +} + +. ${APP_DIR}/bin/activate + +install_requirements . dev-requirements requirements-dev +for extension in . `ls -d $SRC_DIR/ckanext-*`; do + install_requirements $extension requirements pip-requirements +done +pip install -e . +installed_name=$(grep '^\s*name=' setup.py |sed "s|[^']*'\([-a-zA-Z0-9]*\)'.*|\1|") + +# Validate that the extension was installed correctly. +if ! pip list | grep "$installed_name" > /dev/null; then echo "Unable to find the extension in the list"; exit 1; fi + +. $APP_DIR/bin/process-config.sh +. ${APP_DIR}/bin/deactivate diff --git a/bin/init.sh b/bin/init.sh new file mode 100755 index 0000000..6423709 --- /dev/null +++ b/bin/init.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env sh +## +# Initialise CKAN data for testing. +# +set -e + +. ${APP_DIR}/bin/activate +CLICK_ARGS="--yes" ckan_cli db clean +ckan_cli db init +ckan_cli db upgrade diff --git a/bin/process-artifacts.sh b/bin/process-artifacts.sh new file mode 100755 index 0000000..6324f60 --- /dev/null +++ b/bin/process-artifacts.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +## +# Process test artifacts. +# +set -e + +# Create screenshots directory in case it was not created before. This is to +# avoid this script to fail when copying artifacts. +ahoy cli "mkdir -p test/screenshots" + +# Copy from the app container to the build host for storage. +mkdir -p /tmp/artifacts/behave +docker cp "$(docker-compose ps -q ckan)":/srv/app/test/screenshots /tmp/artifacts/behave/ diff --git a/bin/process-config.sh b/bin/process-config.sh new file mode 100644 index 0000000..e69de29 diff --git a/bin/serve.sh b/bin/serve.sh new file mode 100755 index 0000000..ed1a20a --- /dev/null +++ b/bin/serve.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env sh +set -e + +dockerize -wait tcp://postgres:5432 -timeout 1m +dockerize -wait tcp://solr:8983 -timeout 1m +dockerize -wait tcp://redis:6379 -timeout 1m + +for i in `seq 1 60`; do + if (PGPASSWORD=pass psql -h postgres -U ckan_default -d ckan_test -c "\q"); then + echo "Database became ready on attempt $i" + break + else + echo "Database not yet ready, retrying (attempt $i)..." + sleep 1 + fi +done + +. ${APP_DIR}/bin/activate +if (which ckan > /dev/null); then + ckan -c ${CKAN_INI} run --disable-reloader +else + paster serve ${CKAN_INI} +fi diff --git a/bin/test-all.sh b/bin/test-all.sh new file mode 100755 index 0000000..3362476 --- /dev/null +++ b/bin/test-all.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +## +# Run tests in CI. +# +set -e + +SCRIPT_DIR=`dirname $0` + +$SCRIPT_DIR/test-lint.sh + +$SCRIPT_DIR/test.sh + +$SCRIPT_DIR/test-bdd.sh diff --git a/bin/test-bdd.sh b/bin/test-bdd.sh new file mode 100755 index 0000000..85f5b4e --- /dev/null +++ b/bin/test-bdd.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +## +# Run tests in CI. +# +set -ex + +ahoy install-site +echo "==> Run BDD tests" +ahoy test-bdd diff --git a/bin/test-lint.sh b/bin/test-lint.sh new file mode 100755 index 0000000..c15afd3 --- /dev/null +++ b/bin/test-lint.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +## +# Run tests in CI. +# +set -e + +echo "==> Lint code" +ahoy lint + diff --git a/bin/test.sh b/bin/test.sh new file mode 100755 index 0000000..e96a6ff --- /dev/null +++ b/bin/test.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +## +# Run tests in CI. +# +set -e + +echo "==> Run Unit tests" +ahoy test-unit diff --git a/dev-requirements-2.9-py2.txt b/dev-requirements-2.9-py2.txt new file mode 100644 index 0000000..f70e5d7 --- /dev/null +++ b/dev-requirements-2.9-py2.txt @@ -0,0 +1,14 @@ +behave==1.2.6 +behaving==2.0.0 +Appium-Python-Client<=0.52 +ckanapi==4.3 +ckantoolkit>=0.0.4 +factory-boy +faker==3.0.1 +pytest-factoryboy +flake8==3.8.3 +mock +pytest-ckan +six>=1.13.0 +splinter>=0.13.0,<0.17 + diff --git a/dev-requirements-2.9.txt b/dev-requirements-2.9.txt new file mode 100644 index 0000000..4d9f8be --- /dev/null +++ b/dev-requirements-2.9.txt @@ -0,0 +1,12 @@ +behaving==2.0.0 +Appium-Python-Client==1.3.0 +ckanapi==4.3 +ckantoolkit>=0.0.4 +factory-boy +Faker +flake8==3.8.3 +mock +pytest-ckan +six>=1.13.0 +splinter>=0.13.0,<0.17 + diff --git a/dev-requirements.txt b/dev-requirements.txt index 778004a..ba86fb2 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,2 +1,12 @@ +behaving==3.1.5 +Appium-Python-Client==2.10.1 +ckanapi==4.3 +factory-boy +Faker +flake8==6.0.0 +mock pytest-ckan -pytest-cov \ No newline at end of file +pytest-cov +selenium<4.10 +six>=1.13.0 + diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..e0be555 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,95 @@ +version: '2.3' + +x-project: + &project "${PROJECT}" + +x-volumes: + &default-volumes + volumes: + - /app/ckan ### Local overrides to mount host filesystem. Automatically removed in CI and PROD. + - ./ckanext:/app/ckanext:${VOLUME_FLAGS:-delegated} ### Local overrides to mount host filesystem. Automatically removed in CI and PROD. + - ./test:/app/test:${VOLUME_FLAGS:-delegated} ### Local overrides to mount host filesystem. Automatically removed in CI and PROD. + ##- /app/filestore # Override for environment without host mounts. Automatically uncommented in CI. + +x-environment: + &default-environment + AMAZEEIO: AMAZEEIO + no_proxy: "ckan,postgres,postgres-datastore,redis,solr,chrome,mailhog.docker.amazee.io" + +services: + + ckan: + build: + context: . + dockerfile: .docker/Dockerfile.ckan + depends_on: + postgres: + condition: service_healthy + solr: + condition: service_started + networks: + - amazeeio-network + - default + ports: + - "5000" + image: *project + <<: *default-volumes + environment: + <<: *default-environment + AMAZEEIO_HTTP_PORT: 5000 + LAGOON_LOCALDEV_URL: "http://${PROJECT}.docker.amazee.io" + AMAZEEIO_URL: "${PROJECT}.docker.amazee.io" + + postgres: + image: ckan/ckan-postgres-dev:${CKAN_VERSION} + ports: + - "5432" + networks: + - amazeeio-network + - default + environment: + <<: *default-environment + healthcheck: + test: ["CMD", "pg_isready", "-U", "postgres"] + + redis: + image: redis:6-alpine + environment: + <<: *default-environment + networks: + - amazeeio-network + - default + + solr: + image: ckan/ckan-solr:${CKAN_VERSION} + ports: + - "8983" + environment: + <<: *default-environment + networks: + - amazeeio-network + - default + + chrome: + image: selenium/standalone-chrome:3.141.59-oxygen + shm_size: '1gb' + depends_on: + - ckan + <<: *default-volumes + environment: + <<: *default-environment + networks: + - amazeeio-network + - default + extra_hosts: + # point some unnecessary domains at localhost + - "www.googletagmanager.com:127.0.0.1" + - "fonts.gstatic.com:127.0.0.1" + - "gravatar.com:127.0.0.1" + +volumes: + solr-data: {} + +networks: + amazeeio-network: + external: true diff --git a/test/features/environment.py b/test/features/environment.py new file mode 100644 index 0000000..7b78d0a --- /dev/null +++ b/test/features/environment.py @@ -0,0 +1,136 @@ +# encoding: utf-8 + +import os + +from behaving import environment as benv +from splinter.browser import Browser + +# Path to the root of the project. +ROOT_PATH = os.path.realpath(os.path.join( + os.path.dirname(os.path.realpath(__file__)), + '../../')) + +# Base URL for relative paths resolution. +BASE_URL = 'http://ckan:5000/' + +# URL of remote Chrome instance. +REMOTE_CHROME_URL = 'http://chrome:4444/wd/hub' + +# @see bin/init.sh for credentials. +PERSONAS = { + 'SysAdmin': { + 'name': u'admin', + 'email': u'admin@localhost', + 'password': u'Password123!' + }, + 'Unauthenticated': { + 'name': u'', + 'email': u'', + 'password': u'' + }, + 'Group Admin': { + 'name': u'group_admin', + 'email': u'group_admin@localhost', + 'password': u'Password123!' + }, + 'Walker': { + 'name': u'walker', + 'email': u'walker@localhost', + 'password': u'Password123!' + }, + 'Foodie': { + 'name': u'foodie', + 'email': u'foodie@localhost', + 'password': u'Password123!' + }, + # This user will not be assigned to any organisations + 'CKANUser': { + 'name': u'ckan_user', + 'email': u'ckan_user@localhost', + 'password': u'Password123!' + }, + 'TestOrgAdmin': { + 'name': u'test_org_admin', + 'email': u'test_org_admin@localhost', + 'password': u'Password123!' + }, + 'TestOrgEditor': { + 'name': u'test_org_editor', + 'email': u'test_org_editor@localhost', + 'password': u'Password123!' + }, + 'TestOrgMember': { + 'name': u'test_org_member', + 'email': u'test_org_member@localhost', + 'password': u'Password123!' + }, + 'DataRequestOrgAdmin': { + 'name': u'dr_admin', + 'email': u'dr_admin@localhost', + 'password': u'Password123!' + }, + 'DataRequestOrgEditor': { + 'name': u'dr_editor', + 'email': u'dr_editor@localhost', + 'password': u'Password123!' + }, + 'DataRequestOrgMember': { + 'name': u'dr_member', + 'email': u'dr_member@localhost', + 'password': u'Password123!' + }, + 'ReportingOrgAdmin': { + 'name': u'report_admin', + 'email': u'report_admin@localhost', + 'password': u'Password123!' + }, + 'ReportingOrgEditor': { + 'name': u'report_editor', + 'email': u'report_editor@localhost', + 'password': u'Password123!' + } +} + + +def before_all(context): + # The path where screenshots will be saved. + context.screenshots_dir = os.path.join(ROOT_PATH, 'test/screenshots') + # The path where file attachments can be found. + context.attachment_dir = os.path.join(ROOT_PATH, 'test/fixtures') + # The path where emails can be found. + context.mail_path = os.path.join(ROOT_PATH, 'test/emails') + + # Set base url for all relative links. + context.base_url = BASE_URL + + # Set the rest of the settings to default Behaving's settings. + benv.before_all(context) + + +def after_all(context): + benv.after_all(context) + + +def before_feature(context, feature): + benv.before_feature(context, feature) + + +def after_feature(context, feature): + benv.after_feature(context, feature) + + +def before_scenario(context, scenario): + benv.before_scenario(context, scenario) + # Always use remote browser. + remote_browser = Browser( + driver_name="remote", browser="chrome", + command_executor=REMOTE_CHROME_URL + ) + for persona_name in PERSONAS.keys(): + context.browsers[persona_name] = remote_browser + # Set personas. + context.personas = PERSONAS + + +def after_scenario(context, scenario): + benv.after_scenario(context, scenario) diff --git a/test/features/resource_availability.feature b/test/features/resource_availability.feature new file mode 100644 index 0000000..7909d16 --- /dev/null +++ b/test/features/resource_availability.feature @@ -0,0 +1,92 @@ +@resource_visibility +@OpenData +Feature: Re-identification risk governance acknowledgement or Resource visibility + + Scenario: As a publisher, I can view hidden resources + Given "TestOrgEditor" as the persona + When I log in + And I create a dataset and resource with key-value parameters "name=package-with-invisible-resource::notes=Package with invisible resource::de_identified_data=NO::private=False" and "name=invisible-resource::resource_visible=FALSE" + Then I should see "invisible-resource" + + Given "CKANUser" as the persona + When I log out + And I log in + And I go to dataset "package-with-invisible-resource" + Then I should see "Package with invisible resource" + And I should not see "invisible-resource" + + Scenario: As an unprivileged user, I cannot see resources with privacy assessment requested and risk governance completed + Given "TestOrgEditor" as the persona + When I log in + And I create a dataset and resource with key-value parameters "name=package-with-assessed-resource::notes=Package with assessed resource::de_identified_data=NO::private=False" and "name=resource-for-assessment::request_privacy_assessment=YES::governance_acknowledgement=YES::resource_visible=TRUE" + Then I should see "resource-for-assessment" + + Given "CKANUser" as the persona + When I log out + And I log in + And I go to dataset "package-with-assessed-resource" + Then I should not see "resource-for-assessment" + + Scenario: As an unprivileged user, I can see de-identified resources marked as visible without a privacy assessment + Given "TestOrgEditor" as the persona + When I log in + And I create a dataset and resource with key-value parameters "name=de-identified-package-with-unassessed-resource::de_identified_data=YES::private=False" and "name=visible-resource::request_privacy_assessment=NO::governance_acknowledgement=YES::resource_visible=TRUE" + + Given "CKANUser" as the persona + When I log out + And I log in + And I go to dataset "de-identified-package-with-unassessed-resource" + Then I should see "visible-resource" + + When I log out + And I go to dataset "de-identified-package-with-unassessed-resource" + Then I should see "visible-resource" + + Scenario: As an unprivileged user, I cannot see de-identified resources with an incomplete privacy assessment + Given "TestOrgEditor" as the persona + When I log in + And I create a dataset and resource with key-value parameters "name=de-identified-package-with-partially-assessed-resource::de_identified_data=YES" and "name=invisible-resource::request_privacy_assessment=YES::governance_acknowledgement=NO::resource_visible=TRUE" + Then I should see "invisible-resource" + + Given "CKANUser" as the persona + When I log out + And I log in + And I go to dataset "de-identified-package-with-partially-assessed-resource" + Then I should not see "invisible-resource" + + When I log out + And I go to dataset "de-identified-package-with-partially-assessed-resource" + Then I should not see "invisible-resource" + + Scenario: As an unprivileged user, I cannot see de-identified resources + Given "TestOrgEditor" as the persona + When I log in + And I create a dataset and resource with key-value parameters "name=de-identified-package-with-assessed-resource::de_identified_data=YES" and "name=invisible-resource::request_privacy_assessment=YES::governance_acknowledgement=YES::resource_visible=TRUE" + Then I should see "invisible-resource" + + Given "CKANUser" as the persona + When I log out + And I log in + And I go to dataset "random_package" + Then I should not see "invisible-resource" + + Scenario: As a publisher, when I edit a resource, I can set its visibility + Given "TestOrgEditor" as the persona + When I log in + And I create a dataset and resource with key-value parameters "de_identified_data=NO" and "name=invisible-resource::resource_visible=FALSE" + And I press "invisible-resource" + And I press "Manage" + Then I should not see an element with xpath "//label[@for="field-request_privacy_assessment"]//*[@class="control-required"]" + And I should see an element with xpath "//select[@id="field-request_privacy_assessment"]//option[@value="" or @value="YES" or @value="NO"]" + + When I press the element with xpath "//button[string()='Update Resource']" + Then I should see an element with xpath "//th[string()='Request privacy assessment']/following-sibling::td[not(string())]" + + Scenario: As an anonymous user, I can see resources without de-identified data + Given "TestOrgEditor" as the persona + When I log in + And I create a dataset and resource with key-value parameters "name=package-without-de-identified-data::de_identified_data=NO::private=False" and "name=visible-resource::governance_acknowledgement=NO::resource_visible=TRUE" + + When I log out + And I go to dataset "package-without-de-identified-data" + Then I should see "visible-resource" diff --git a/test/features/steps/steps.py b/test/features/steps/steps.py new file mode 100644 index 0000000..5ae2e9b --- /dev/null +++ b/test/features/steps/steps.py @@ -0,0 +1,721 @@ +import datetime +import email +import quopri +import re +import requests +import six +from six.moves.urllib.parse import urlparse +import uuid + +from behave import when, then +from behaving.personas.steps import * # noqa: F401, F403 +from behaving.mail.steps import * # noqa: F401, F403 +from behaving.web.steps import * # noqa: F401, F403 + +# Monkey-patch Selenium 3 to handle Python 3.9 +import base64 +if not hasattr(base64, 'encodestring'): + base64.encodestring = base64.encodebytes + +# Monkey-patch Behaving to handle function rename +from behaving.web.steps import forms +if not hasattr(forms, 'fill_in_elem_by_name'): + forms.fill_in_elem_by_name = forms.i_fill_in_field + +URL_RE = re.compile(r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|\ + (?:%[0-9a-fA-F][0-9a-fA-F]))+', re.I | re.S | re.U) + + +@when(u'I take a debugging screenshot') +def debug_screenshot(context): + """ Take a screenshot only if debugging is enabled in the persona. + """ + if context.persona and context.persona.get('debug') == 'True': + context.execute_steps(u""" + Then I take a screenshot + """) + + +@when(u'I go to homepage') +def go_to_home(context): + context.execute_steps(u""" + When I visit "/" + """) + + +@when(u'I go to register page') +def go_to_register_page(context): + context.execute_steps(u""" + When I go to homepage + And I press "Register" + """) + + +@when(u'I log in') +def log_in(context): + context.execute_steps(u""" + When I go to homepage + And I expand the browser height + And I press "Log in" + And I log in directly + """) + + +@when(u'I expand the browser height') +def expand_height(context): + # Work around x=null bug in Selenium set_window_size + context.browser.driver.set_window_rect(x=0, y=0, width=1024, height=4096) + + +@when(u'I log in directly') +def log_in_directly(context): + """ + This differs to the `log_in` function above by logging in directly to a page where the user login form is presented + :param context: + :return: + """ + + assert context.persona, "A persona is required to log in, found [{}] in context. Have you configured the personas in before_scenario?".format(context.persona) + context.execute_steps(u""" + When I attempt to log in with password "$password" + Then I should see an element with xpath "//a[@title='Log out']" + """) + + +@when(u'I attempt to log in with password "{password}"') +def attempt_login(context, password): + assert context.persona + context.execute_steps(u""" + When I fill in "login" with "$name" + And I fill in "password" with "{}" + And I press the element with xpath "//button[contains(string(), 'Login')]" + """.format(password)) + + +@then(u'I should see the login form') +def login_link_visible(context): + context.execute_steps(u""" + Then I should see an element with xpath "//h1[contains(string(), 'Login')]" + """) + + +@when(u'I request a password reset') +def request_reset(context): + assert context.persona + context.execute_steps(u""" + When I visit "/user/reset" + And I fill in "user" with "$name" + And I press the element with xpath "//button[contains(string(), 'Request Reset')]" + """) + + +@when(u'I fill in "{name}" with "{value}" if present') +def fill_in_field_if_present(context, name, value): + context.execute_steps(u""" + When I execute the script "field = $('#field-{0}'); if (!field.length) field = $('#{0}'); if (!field.length) field = $('[name={0}]'); field.val('{1}'); field.keyup();" + """.format(name, value)) + + +@when(u'I clear the URL field') +def clear_url(context): + context.execute_steps(u""" + When I execute the script "$('a.btn-remove-url:contains(Clear)').click();" + """) + + +@when(u'I confirm the dialog containing "{text}" if present') +def confirm_dialog_if_present(context, text): + if context.browser.is_text_present(text): + context.execute_steps(u""" + When I press the element with xpath "//*[contains(@class, 'modal-dialog')]//button[contains(@class, 'btn-primary')]" + """) + + +@when(u'I confirm dataset deletion') +def confirm_dataset_deletion_dialog_if_present(context): + dialog_text = "Briefly describe the reason for deleting this dataset" + if context.browser.is_text_present(dialog_text): + context.execute_steps(u""" + Then I should see an element with xpath "//div[@class='modal-footer']//button[@class='btn btn-primary' and @disabled='disabled']" + When I fill in "deletion-reason" with "it should be longer than 10 characters" if present + Then I should not see an element with xpath "//div[@class='modal-footer']//button[@class='btn btn-primary' and @disabled='disabled']" + """) + # Press the Confirm button whether it is in a dialog or a page + context.execute_steps(u""" + When I press the element with xpath "//button[contains(@class, 'btn-primary') and contains(string(), 'Confirm') ]" + Then I should see "Dataset has been deleted" + """) + + +@when(u'I open the new resource form for dataset "{name}"') +def go_to_new_resource_form(context, name): + context.execute_steps(u""" + When I edit the "{0}" dataset + """.format(name)) + if context.browser.is_element_present_by_xpath("//*[contains(@class, 'btn-primary') and contains(string(), 'Next:')]"): + # Draft dataset, proceed directly to resource form + context.execute_steps(u""" + When I press "Next:" + """) + else: + # Existing dataset, browse to the resource form + context.execute_steps(u""" + When I press "Resources" + And I press "Add new resource" + """) + + +@when(u'I fill in title with random text') +def title_random_text(context): + assert context.persona + context.execute_steps(u""" + When I fill in "title" with "Test Title {0}" + And I fill in "name" with "test-title-{0}" if present + And I set "last_generated_name" to "test-title-{0}" + """.format(uuid.uuid4())) + + +@when(u'I go to dataset page') +def go_to_dataset_page(context): + context.execute_steps(u""" + When I visit "/dataset" + """) + + +@when(u'I go to dataset "{name}"') +def go_to_dataset(context, name): + context.execute_steps(u""" + When I visit "/dataset/{0}" + """.format(name)) + + +@when(u'I go to the first resource in the dataset') +def go_to_first_resource(context): + context.execute_steps(u""" + When I press the element with xpath "//li[@class="resource-item"]/a" + """) + + +@when(u'I edit the "{name}" dataset') +def edit_dataset(context, name): + context.execute_steps(u""" + When I visit "/dataset/edit/{0}" + """.format(name)) + + +@when(u'I select the "{licence_id}" licence') +def select_licence(context, licence_id): + # Licence requires special interaction due to fancy JavaScript + context.execute_steps(u""" + When I execute the script "$('#field-license_id').val('{0}').trigger('change')" + """.format(licence_id)) + + +@when(u'I enter the resource URL "{url}"') +def enter_resource_url(context, url): + context.execute_steps(u""" + When I execute the script "$('#resource-edit [name=url]').val('{0}')" + """.format(url)) + + +@when(u'I fill in default dataset fields') +def fill_in_default_dataset_fields(context): + context.execute_steps(u""" + When I fill in title with random text + And I fill in "notes" with "Description" + And I fill in "version" with "1.0" + And I fill in "author_email" with "test@me.com" + And I select the "other-open" licence + And I fill in "de_identified_data" with "NO" if present + """) + + +@when(u'I fill in default resource fields') +def fill_in_default_resource_fields(context): + context.execute_steps(u""" + When I fill in "name" with "Test Resource" + And I fill in "description" with "Test Resource Description" + And I fill in "size" with "1024" if present + """) + + +@when(u'I fill in link resource fields') +def fill_in_default_link_resource_fields(context): + context.execute_steps(u""" + When I enter the resource URL "https://example.com" + And I execute the script "document.getElementById('field-format').value='HTML'" + And I fill in "size" with "1024" if present + """) + + +@when(u'I upload "{file_name}" of type "{file_format}" to resource') +def upload_file_to_resource(context, file_name, file_format): + context.execute_steps(u""" + When I execute the script "button = document.getElementById('resource-upload-button'); if (button) button.click();" + And I attach the file "{file_name}" to "upload" + # Don't quote the injected string since it can have trailing spaces + And I execute the script "document.getElementById('field-format').value='{file_format}'" + And I fill in "size" with "1024" if present + """.format(file_name=file_name, file_format=file_format)) + + +@when(u'I go to group page') +def go_to_group_page(context): + context.execute_steps(u""" + When I visit "/group" + """) + + +@when(u'I go to organisation page') +def go_to_organisation_page(context): + context.execute_steps(u""" + When I visit "/organization" + """) + + +@when(u'I search the autocomplete API for user "{username}"') +def go_to_user_autocomplete(context, username): + context.execute_steps(u""" + When I visit "/api/2/util/user/autocomplete?q={0}" + """.format(username)) + + +@when(u'I go to the user list API') +def go_to_user_list(context): + context.execute_steps(u""" + When I visit "/api/3/action/user_list" + """) + + +@when(u'I go to the "{user_id}" profile page') +def go_to_user_profile(context, user_id): + context.execute_steps(u""" + When I visit "/user/{0}" + """.format(user_id)) + + +@when(u'I go to the dashboard') +def go_to_dashboard(context): + context.execute_steps(u""" + When I visit "/dashboard/datasets" + """) + + +@then(u'I should see my datasets') +def dashboard_datasets(context): + context.execute_steps(u""" + Then I should see an element with xpath "//li[contains(@class, 'active') and contains(string(), 'My Datasets')]" + """) + + +@when(u'I go to the "{user_id}" user API') +def go_to_user_show(context, user_id): + context.execute_steps(u""" + When I visit "/api/3/action/user_show?id={0}" + """.format(user_id)) + + +@when(u'I view the "{group_id}" {group_type} API "{including}" users') +def go_to_group_including_users(context, group_id, group_type, including): + if group_type == "organisation": + group_type = "organization" + context.execute_steps(u""" + When I visit "/api/3/action/{1}_show?id={0}&include_users={2}" + """.format(group_id, group_type, including in ['with', 'including'])) + + +@then(u'I should be able to download via the element with xpath "{expression}"') +def test_download_element(context, expression): + url = context.browser.find_by_xpath(expression).first['href'] + assert requests.get(url, cookies=context.browser.cookies.all()).status_code == 200 + + +@then(u'I should be able to patch dataset "{package_id}" via the API') +def test_package_patch(context, package_id): + url = context.base_url + 'api/action/package_patch' + response = requests.post(url, json={'id': package_id}, cookies=context.browser.cookies.all()) + print("Response from endpoint {} is: {}, {}".format(url, response, response.text)) + assert response.status_code == 200 + assert '"success": true' in response.text + + +# Parse a "key=value::key2=value2" parameter string and return an iterator of (key, value) pairs. +def _parse_params(param_string): + params = {} + for param in param_string.split("::"): + entry = param.split("=", 1) + params[entry[0]] = entry[1] if len(entry) > 1 else "" + return six.iteritems(params) + + +# Enter a JSON schema value +# This can require JavaScript interaction, and doesn't fit well into +# a step invocation due to all the double quotes. +def _enter_manual_schema(context, schema_json): + # Click the button to select manual JSON input if it exists + context.execute_steps(u""" + When I execute the script "$('a.btn[title*=JSON]:contains(JSON)').click();" + """) + # Call function directly so we can properly quote our parameter + forms.fill_in_elem_by_name(context, "schema_json", schema_json) + + +def _create_dataset_from_params(context, params): + context.execute_steps(u""" + When I visit "/dataset/new" + And I fill in default dataset fields + """) + for key, value in _parse_params(params): + if key == "name": + context.execute_steps(u""" + When I set "last_generated_name" to "{0}" + """.format(value)) + if key == "owner_org": + # Owner org uses UUIDs as its values, so we need to rely on displayed text + context.execute_steps(u""" + When I select by text "{1}" from "{0}" + """.format(key, value)) + elif key in ["update_frequency", "request_privacy_assessment", "private"]: + context.execute_steps(u""" + When I select "{1}" from "{0}" + """.format(key, value)) + elif key == "license_id": + context.execute_steps(u""" + When I select the "{0}" licence + """.format(value)) + elif key == "schema_json": + if value == "default": + value = """ + {"fields": [ + {"format": "default", "name": "Game Number", "type": "integer"}, + {"format": "default", "name": "Game Length", "type": "integer"} + ], + "missingValues": ["Default schema"] + } + """ + _enter_manual_schema(context, value) + else: + context.execute_steps(u""" + When I fill in "{0}" with "{1}" if present + """.format(key, value)) + context.execute_steps(u""" + When I take a debugging screenshot + And I press "Add Data" + Then I should see "Add New Resource" + """) + + +@when(u'I create a dataset with key-value parameters "{params}"') +def create_dataset_from_params(context, params): + _create_dataset_from_params(context, params) + context.execute_steps(u""" + When I go to dataset "$last_generated_name" + """) + + +@when(u'I create a dataset and resource with key-value parameters "{params}" and "{resource_params}"') +def create_dataset_and_resource_from_params(context, params, resource_params): + _create_dataset_from_params(context, params) + context.execute_steps(u""" + When I create a resource with key-value parameters "{0}" + Then I should see "Data and Resources" + """.format(resource_params)) + + +def _is_truthy(text): + return text and text.lower() in ["true", "t", "yes", "y"] + + +# Creates a resource using default values apart from the ones specified. +# The browser should already be on the create/edit resource page. +@when(u'I create a resource with key-value parameters "{resource_params}"') +def create_resource_from_params(context, resource_params): + context.execute_steps(u""" + When I fill in default resource fields + And I fill in link resource fields + """) + for key, value in _parse_params(resource_params): + if key == "url": + if value != "default": + context.execute_steps(u""" + When I clear the URL field + And I execute the script "$('#resource-edit [name=url]').val('{0}')" + """.format(value)) + elif key == "upload": + if value == "default": + value = "test_game_data.csv" + context.execute_steps(u""" + When I clear the URL field + And I execute the script "$('#resource-upload-button').click();" + And I attach the file "{0}" to "upload" + """.format(value)) + elif key == "format": + context.execute_steps(u""" + When I execute the script "document.getElementById('field-format').value='{0}'" + """.format(value)) + elif key in ["align_default_schema"]: + action = "check" if _is_truthy(value) else "uncheck" + context.execute_steps(u""" + When I {0} "{1}" + """.format(action, key)) + elif key == "resource_visible": + option = "TRUE" if _is_truthy(value) else "FALSE" + context.execute_steps(u""" + When I select "{1}" from "{0}" + """.format(key, option)) + elif key in ["governance_acknowledgement", "request_privacy_assessment"]: + option = "YES" if _is_truthy(value) else "NO" + context.execute_steps(u""" + When I select "{1}" from "{0}" + """.format(key, option)) + elif key == "schema": + if value == "default": + value = """{ + "fields": [{ + "format": "default", + "name": "Game Number", + "type": "integer" + }, { + "format": "default", + "name": "Game Length", + "type": "integer" + }], + "missingValues": ["Resource schema"] + }""" + _enter_manual_schema(context, value) + else: + context.execute_steps(u""" + When I fill in "{0}" with "{1}" if present + """.format(key, value)) + context.execute_steps(u""" + When I take a debugging screenshot + And I press the element with xpath "//form[contains(@class, 'resource-form')]//button[contains(@class, 'btn-primary')]" + And I take a debugging screenshot + """) + + +@then(u'I should receive a base64 email at "{address}" containing "{text}"') +def should_receive_base64_email_containing_text(context, address, text): + should_receive_base64_email_containing_texts(context, address, text, None) + + +@then(u'I should receive a base64 email at "{address}" containing both "{text}" and "{text2}"') +def should_receive_base64_email_containing_texts(context, address, text, text2): + # The default behaving step does not convert base64 emails + # Modified the default step to decode the payload from base64 + def filter_contents(mail): + mail = email.message_from_string(mail) + payload = mail.get_payload() + payload += "=" * ((4 - len(payload) % 4) % 4) # do fix the padding error issue + payload_bytes = quopri.decodestring(payload) + if len(payload_bytes) > 0: + payload_bytes += b'=' # do fix the padding error issue + if six.PY2: + decoded_payload = payload_bytes.decode('base64') + else: + import base64 + decoded_payload = six.ensure_text(base64.b64decode(six.ensure_binary(payload_bytes))) + print('decoded_payload: ', decoded_payload) + return text in decoded_payload and (not text2 or text2 in decoded_payload) + + assert context.mail.user_messages(address, filter_contents) + + +@when(u'I go to admin config page') +def go_to_admin_config(context): + context.execute_steps(u""" + When I visit "/ckan-admin/config" + """) + + +@when(u'I log out') +def log_out(context): + context.execute_steps(u""" + When I visit "/user/_logout" + Then I should see "Log in" + """) + + +# ckanext-data-qld + + +@when(u'I visit resource schema generation page') +def resource_schema_generation(context): + path = urlparse(context.browser.url).path + context.execute_steps(u""" + When I visit "{0}/generate_schema" + """.format(path)) + + +@when(u'I reload page every {seconds:d} seconds until I see an element with xpath "{xpath}" but not more than {reload_times:d} times') +def reload_page_every_n_until_find(context, xpath, seconds=5, reload_times=5): + for _ in range(reload_times): + element = context.browser.is_element_present_by_xpath( + xpath, wait_time=seconds + ) + if element: + assert element, 'Element with xpath "{}" was found'.format(xpath) + return + else: + print("Element with xpath '{}' was not found, reloading at {}...".format(xpath, datetime.datetime.now())) + context.browser.reload() + + assert False, 'Element with xpath "{}" was not found'.format(xpath) + + +@when(u'I trigger notification about updated privacy assessment results') +def i_trigger_notification_assessment_results(context): + context.execute_steps(u""" + When I visit "api/action/qld_test_trigger_notify_privacy_assessment_result" + """) + + +@when(u'I click the resource link in the email I received at "{address}"') +def click_link_in_email(context, address): + mails = context.mail.user_messages(address) + assert mails, u"message not found" + + mail = email.message_from_string(mails[-1]) + links = [] + + payload = mail.get_payload(decode=True).decode("utf-8") + links = URL_RE.findall(payload.replace("=\n", "")) + + assert links, u"link not found" + url = links[0].rstrip(':') + + context.browser.visit(url) + + +# ckanext-ytp-comments + + +@when(u'I go to dataset "{name}" comments') +def go_to_dataset_comments(context, name): + context.execute_steps(u""" + When I go to dataset "%s" + And I press "Comments" + """ % (name)) + + +@then(u'I should see the add comment form') +def comment_form_visible(context): + context.execute_steps(u""" + Then I should see an element with xpath "//textarea[@name='comment']" + """) + + +@then(u'I should not see the add comment form') +def comment_form_not_visible(context): + context.execute_steps(u""" + Then I should not see an element with xpath "//input[@name='subject']" + And I should not see an element with xpath "//textarea[@name='comment']" + """) + + +@when(u'I submit a comment with subject "{subject}" and comment "{comment}"') +def submit_comment_with_subject_and_comment(context, subject, comment): + """ + There can be multiple comment forms per page (add, edit, reply) each with fields named "subject" and "comment" + This step overcomes a limitation of the fill() method which only fills a form field by name + :param context: + :param subject: + :param comment: + :return: + """ + context.browser.execute_script(""" + document.querySelector('form#comment_form input[name="subject"]').value = '%s'; + """ % subject) + context.browser.execute_script(""" + document.querySelector('form#comment_form textarea[name="comment"]').value = '%s'; + """ % comment) + context.browser.execute_script(""" + document.querySelector('form#comment_form .form-actions input[type="submit"]').click(); + """) + + +@when(u'I submit a reply with comment "{comment}"') +def submit_reply_with_comment(context, comment): + """ + There can be multiple comment forms per page (add, edit, reply) each with fields named "subject" and "comment" + This step overcomes a limitation of the fill() method which only fills a form field by name + :param context: + :param comment: + :return: + """ + context.browser.execute_script(""" + document.querySelector('.comment-wrapper form textarea[name="comment"]').value = '%s'; + """ % comment) + context.browser.execute_script(""" + document.querySelector('.comment-wrapper form .form-actions input[type="submit"]').click(); + """) + + +# ckanext-qgov + + +@when(u'I lock my account') +def lock_account(context): + context.execute_steps(u""" + When I visit "/user/login" + """) + for x in range(11): + context.execute_steps(u""" + When I attempt to log in with password "incorrect password" + """) + + +# ckanext-datarequests + + +@when(u'I go to the data requests page containing "{keyword}"') +def go_to_datarequest_page_search(context, keyword): + context.execute_steps(u""" + When I visit "/datarequest?q={0}" + """.format(keyword)) + + +@when(u'I go to the data requests page') +def go_to_datarequest_page(context): + context.execute_steps(u""" + When I visit "/datarequest" + """) + + +@when(u'I go to data request "{subject}"') +def go_to_data_request(context, subject): + context.execute_steps(u""" + When I go to the data requests page containing "{0}" + And I click the link with text "{0}" + Then I should see "{0}" within 5 seconds + """.format(subject)) + + +@when(u'I create a datarequest') +def create_datarequest(context): + assert context.persona + context.execute_steps(u""" + When I go to the data requests page + And I press "Add data request" + And I fill in title with random text + And I fill in "description" with "Test description" + And I press the element with xpath "//button[contains(@class, 'btn-primary')]" + """) + + +@when(u'I go to data request "{subject}" comments') +def go_to_data_request_comments(context, subject): + context.execute_steps(u""" + When I go to data request "%s" + And I press "Comments" + """ % (subject)) + + +# ckanext-report + + +@when(u'I go to my reports page') +def go_to_reporting_page(context): + context.execute_steps(u""" + When I visit "/dashboard/reporting" + """) From 7fda5e47426a2eb93269e2076c9f377f78f23ce2 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Wed, 12 Jul 2023 14:27:47 +1000 Subject: [PATCH 2/6] [QOLSVC-2077] add scheming extension to scenario tests - Resource visibility relies on scheming fields --- .docker/test.ini | 8 ++++++++ .github/workflows/test.yml | 2 +- dev-requirements-2.9-py2.txt | 1 + dev-requirements-2.9.txt | 1 + dev-requirements.txt | 1 + 5 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.docker/test.ini b/.docker/test.ini index cea0d05..bfb1e03 100644 --- a/.docker/test.ini +++ b/.docker/test.ini @@ -80,6 +80,7 @@ ckan.auth.public_user_details = False ## Plugins Settings ckan.plugins = + scheming_datasets resource_visibility @@ -117,6 +118,13 @@ smtp.server = localhost:8025 smtp.test_server = localhost:8025 smtp.mail_from = info@test.ckan.net +## ckanext-scheming settings +# see https://github.com/ckan/ckanext-scheming#configuration +scheming.presets = + ckanext.scheming:presets.json + ckanext.resource_visibility:schema/presets.json +scheming.dataset_fallback = false + ## Logging configuration [loggers] keys = root, ckan, ckanext diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0a7aa46..bee6580 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -61,7 +61,7 @@ jobs: - name: Upload screenshots if: failure() - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: CKAN ${{ matrix.ckan-version }} screenshots path: /tmp/artifacts/behave/screenshots diff --git a/dev-requirements-2.9-py2.txt b/dev-requirements-2.9-py2.txt index f70e5d7..dcd2993 100644 --- a/dev-requirements-2.9-py2.txt +++ b/dev-requirements-2.9-py2.txt @@ -12,3 +12,4 @@ pytest-ckan six>=1.13.0 splinter>=0.13.0,<0.17 +-e git+https://github.com/ckan/ckanext-scheming.git@release-3.0.0#egg=ckanext-scheming diff --git a/dev-requirements-2.9.txt b/dev-requirements-2.9.txt index 4d9f8be..5f2cfa0 100644 --- a/dev-requirements-2.9.txt +++ b/dev-requirements-2.9.txt @@ -10,3 +10,4 @@ pytest-ckan six>=1.13.0 splinter>=0.13.0,<0.17 +-e git+https://github.com/ckan/ckanext-scheming.git@release-3.0.0#egg=ckanext-scheming diff --git a/dev-requirements.txt b/dev-requirements.txt index ba86fb2..2cb82ab 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -10,3 +10,4 @@ pytest-cov selenium<4.10 six>=1.13.0 +-e git+https://github.com/ckan/ckanext-scheming.git@release-3.0.0#egg=ckanext-scheming From 6474d037b1c17678a5e7184d26132c2d0c76e190 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Wed, 12 Jul 2023 14:57:33 +1000 Subject: [PATCH 3/6] [QOLSVC-2077] add schema file that just adds our fields to the CKAN default --- .docker/Dockerfile-template.ckan | 3 - .docker/test.ini | 1 + ckanext/resource_visibility/ckan_dataset.yaml | 102 ++++++++++++++++++ 3 files changed, 103 insertions(+), 3 deletions(-) create mode 100644 ckanext/resource_visibility/ckan_dataset.yaml diff --git a/.docker/Dockerfile-template.ckan b/.docker/Dockerfile-template.ckan index b0222e0..53acf91 100644 --- a/.docker/Dockerfile-template.ckan +++ b/.docker/Dockerfile-template.ckan @@ -13,9 +13,6 @@ RUN apk add --no-cache build-base \ && curl -sL https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \ | tar -C /usr/local/bin -xzvf - -# Update urllib3 to fix urlopen bug -RUN pip install -U urllib3 - # Install CKAN. RUN cd $SRC_DIR/ckan \ diff --git a/.docker/test.ini b/.docker/test.ini index bfb1e03..ee04a5f 100644 --- a/.docker/test.ini +++ b/.docker/test.ini @@ -120,6 +120,7 @@ smtp.mail_from = info@test.ckan.net ## ckanext-scheming settings # see https://github.com/ckan/ckanext-scheming#configuration +scheming.dataset_schemas = ckanext.resource_visibility:ckan_dataset.yaml scheming.presets = ckanext.scheming:presets.json ckanext.resource_visibility:schema/presets.json diff --git a/ckanext/resource_visibility/ckan_dataset.yaml b/ckanext/resource_visibility/ckan_dataset.yaml new file mode 100644 index 0000000..06335a7 --- /dev/null +++ b/ckanext/resource_visibility/ckan_dataset.yaml @@ -0,0 +1,102 @@ +scheming_version: 2 +dataset_type: dataset +about: A reimplementation of the default CKAN dataset schema +about_url: http://github.com/ckan/ckanext-scheming + + +dataset_fields: + +- field_name: title + label: Title + preset: title + form_placeholder: eg. A descriptive title + +- field_name: name + label: URL + preset: dataset_slug + form_placeholder: eg. my-dataset + +- field_name: notes + label: Description + form_snippet: markdown.html + form_placeholder: eg. Some useful notes about the data + +- field_name: tag_string + label: Tags + preset: tag_string_autocomplete + form_placeholder: eg. economy, mental health, government + +- field_name: license_id + label: License + form_snippet: license.html + help_text: License definitions and additional information can be found at http://opendefinition.org/ + +- field_name: owner_org + label: Organization + preset: dataset_organization + +- field_name: url + label: Source + form_placeholder: http://example.com/dataset.json + display_property: foaf:homepage + display_snippet: link.html + +- field_name: version + label: Version + validators: ignore_missing unicode_safe package_version_validator + form_placeholder: '1.0' + +- field_name: author + label: Author + form_placeholder: Joe Bloggs + display_property: dc:creator + +- field_name: author_email + label: Author Email + form_placeholder: joe@example.com + display_property: dc:creator + display_snippet: email.html + display_email_name_field: author + +- field_name: maintainer + label: Maintainer + form_placeholder: Joe Bloggs + display_property: dc:contributor + +- field_name: maintainer_email + label: Maintainer Email + form_placeholder: joe@example.com + display_property: dc:contributor + display_snippet: email.html + display_email_name_field: maintainer + +- preset: resource_visibility_de_identified_data + + +resource_fields: + +- field_name: url + label: URL + preset: resource_url_upload + +- field_name: name + label: Name + form_placeholder: eg. January 2011 Gold Prices + +- field_name: description + label: Description + form_snippet: markdown.html + form_placeholder: Some useful notes about the data + +- field_name: format + label: Format + preset: resource_format_autocomplete + +- preset: resource_visibility_resource_visible + +- preset: resource_visibility_governance_acknowledgement + +- preset: resource_visibility_request_privacy_assessment + +- preset: resource_visibility_privacy_assessment_result + From 0ba2600d500106258aa88bbc0b5fbafc9b474243 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Wed, 19 Jul 2023 12:53:28 +1000 Subject: [PATCH 4/6] [QOLSVC-2464] update help text for privacy assessment - Also set the 'help_allow_html' flag so link markup is preserved --- ckanext/resource_visibility/schema/presets.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ckanext/resource_visibility/schema/presets.json b/ckanext/resource_visibility/schema/presets.json index 9228903..e0c2c2f 100644 --- a/ckanext/resource_visibility/schema/presets.json +++ b/ckanext/resource_visibility/schema/presets.json @@ -84,7 +84,8 @@ "label": "YES" } ], - "help_text": "Where the dataset contains de-identified data, selecting ‘YES’ will hide this resource, pending a privacy assessment. Assessments will not be completed where the dataset does not contain de-identified data. Select ‘NO’ where the dataset does not contain de-identified data or where a privacy assessment is not required." + "help_text": "Privacy risk assessment prior to public release might assist the publishing decision-making process. Refer to the Privacy assessment guidance for further information. A ‘YES’ value will hide this resource from the public and users within other organisations.", + "help_allow_html": true } }, { From 36b5fd174ae1c3b1f2b93e462ba6f949713ee4ba Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Wed, 19 Jul 2023 14:09:57 +1000 Subject: [PATCH 5/6] [QOLSVC-2464] add scenario test for new help text --- test/features/resource_availability.feature | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/features/resource_availability.feature b/test/features/resource_availability.feature index 7909d16..1185007 100644 --- a/test/features/resource_availability.feature +++ b/test/features/resource_availability.feature @@ -15,7 +15,7 @@ Feature: Re-identification risk governance acknowledgement or Resource visibilit Then I should see "Package with invisible resource" And I should not see "invisible-resource" - Scenario: As an unprivileged user, I cannot see resources with privacy assessment requested and risk governance completed + Scenario: As an unprivileged user, I cannot see resources with privacy assessment requested and risk governance completed Given "TestOrgEditor" as the persona When I log in And I create a dataset and resource with key-value parameters "name=package-with-assessed-resource::notes=Package with assessed resource::de_identified_data=NO::private=False" and "name=resource-for-assessment::request_privacy_assessment=YES::governance_acknowledgement=YES::resource_visible=TRUE" @@ -78,6 +78,8 @@ Feature: Re-identification risk governance acknowledgement or Resource visibilit And I press "Manage" Then I should not see an element with xpath "//label[@for="field-request_privacy_assessment"]//*[@class="control-required"]" And I should see an element with xpath "//select[@id="field-request_privacy_assessment"]//option[@value="" or @value="YES" or @value="NO"]" + And I should see "Privacy risk assessment prior to public release might assist the publishing decision-making process" + And I should see an element with xpath "//a[contains(@href, 'download') and contains(string(), 'Privacy assessment guidance')]" When I press the element with xpath "//button[string()='Update Resource']" Then I should see an element with xpath "//th[string()='Request privacy assessment']/following-sibling::td[not(string())]" From 23a18e722f2624f63085829c857bd0d4cf891f59 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Wed, 19 Jul 2023 14:21:24 +1000 Subject: [PATCH 6/6] [QOLSVC-2464] adjust test steps for faster build - Initialise database while waiting for CKAN instance to start --- .ahoy.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.ahoy.yml b/.ahoy.yml index b265775..cb6bad7 100644 --- a/.ahoy.yml +++ b/.ahoy.yml @@ -35,13 +35,11 @@ commands: usage: Build and start Docker containers. cmd: | docker-compose up -d "$@" - sleep 10 - docker-compose logs + ahoy cli '$APP_DIR/bin/init.sh' ahoy cli "dockerize -wait tcp://ckan:5000 -timeout 1m" - if docker-compose logs | grep -q "\[Error\]"; then docker-compose logs; exit 1; fi - if docker-compose logs | grep -q "Exception"; then docker-compose logs; exit 1; fi + if docker-compose logs | grep -q "\[Error\]"; then exit 1; fi + if docker-compose logs | grep -q "Exception"; then exit 1; fi docker ps -a --filter name=^/${COMPOSE_PROJECT_NAME}_ - ahoy cli '$APP_DIR/bin/init.sh' export DOCTOR_CHECK_CLI=0 down: