Merge pull request #21 from qld-gov-au/develop

Develop to master

.ahoy.yml

@@ -0,0 +1,197 @@
+---
+ahoyapi: v2
+
+commands:
+
+  # Docker commands.
+  build:
+    usage: Build or rebuild project.
+    cmd: |
+      ahoy title "Building project"
+      ahoy pre-flight
+      ahoy clean
+      ahoy build-network
+      ahoy up -- --build --force-recreate
+      ahoy title "Build complete"
+      ahoy doctor
+      ahoy info 1
+
+  build-network:
+    usage: Ensure that the amazeeio network exists.
+    cmd: |
+      docker network prune -f > /dev/null
+      docker network inspect amazeeio-network > /dev/null || docker network create amazeeio-network
+
+  info:
+    usage: Print information about this project.
+    cmd: |
+      ahoy line "Project                  : " ${PROJECT}
+      ahoy line "Site local URL           : " ${LAGOON_LOCALDEV_URL}
+      ahoy line "DB port on host          : " $(docker port $(docker-compose ps -q postgres) 5432 | cut -d : -f 2)
+      ahoy line "Solr port on host        : " $(docker port $(docker-compose ps -q solr) 8983 | cut -d : -f 2)
+      ahoy line "Mailhog URL              : " http://mailhog.docker.amazee.io/
+
+  up:
+    usage: Build and start Docker containers.
+    cmd: |
+      docker-compose up -d "$@"
+      ahoy cli '$APP_DIR/bin/init.sh'
+      ahoy cli "dockerize -wait tcp://ckan:5000 -timeout 1m"
+      if docker-compose logs | grep -q "\[Error\]"; then exit 1; fi
+      if docker-compose logs | grep -q "Exception"; then exit 1; fi
+      docker ps -a --filter name=^/${COMPOSE_PROJECT_NAME}_
+      export DOCTOR_CHECK_CLI=0
+
+  down:
+    usage: Stop Docker containers and remove container, images, volumes and networks.
+    cmd: 'if [ -f "docker-compose.yml" ]; then docker-compose down --volumes; fi'
+
+  start:
+    usage: Start existing Docker containers.
+    cmd: docker-compose start "$@"
+
+  stop:
+    usage: Stop running Docker containers.
+    cmd: docker-compose stop "$@"
+
+  restart:
+    usage: Restart all stopped and running Docker containers.
+    cmd: docker-compose restart "$@"
+
+  logs:
+    usage: Show Docker logs.
+    cmd: docker-compose logs "$@"
+
+  pull:
+    usage: Pull latest docker images.
+    cmd: if [ ! -z "$(docker image ls -q)" ]; then docker image ls --format \"{{.Repository}}:{{.Tag}}\" | grep ckan/ckan- | grep -v none | xargs -n1 docker pull | cat; fi
+
+  cli:
+    usage: Start a shell inside CLI container or run a command.
+    cmd: if \[ "${#}" -ne 0 \]; then docker exec $(docker-compose ps -q ckan) sh -c '. ${APP_DIR}/bin/activate; cd $APP_DIR;'" $*"; else docker exec $(docker-compose ps -q ckan) sh -c '. ${APP_DIR}/bin/activate && cd $APP_DIR && sh'; fi
+
+  doctor:
+    usage: Find problems with current project setup.
+    cmd: bin/doctor.sh "$@"
+
+  install-site:
+    usage: Install test site data.
+    cmd: |
+      ahoy title "Installing a fresh site"
+      ahoy cli '$APP_DIR/bin/init.sh && $APP_DIR/bin/create-test-data.sh'
+
+  clean:
+    usage: Remove containers and all build files.
+    cmd: |
+      ahoy down
+      # Remove other directories.
+      # @todo: Add destinations below.
+      rm -rf \
+        ./ckan
+
+  reset:
+    usage: "Reset environment: remove containers, all build, manually created and Drupal-Dev files."
+    cmd: |
+      ahoy clean
+      git ls-files --others -i --exclude-from=.git/info/exclude | xargs chmod 777
+      git ls-files --others -i --exclude-from=.git/info/exclude | xargs rm -Rf
+      find . -type d -not -path "./.git/*" -empty -delete
+
+  flush-redis:
+    usage: Flush Redis cache.
+    cmd: docker exec -i $(docker-compose ps -q redis) redis-cli flushall > /dev/null
+
+  lint:
+    usage: Lint code.
+    cmd: |
+      ahoy cli "flake8 ${@:-ckanext}" || \
+      [ "${ALLOW_LINT_FAIL:-0}" -eq 1 ]
+
+  copy-local-files:
+    usage: Update files from local repo.
+    cmd: |
+      docker cp . $(docker-compose ps -q ckan):/srv/app/
+      docker cp bin/ckan_cli $(docker-compose ps -q ckan):/usr/bin/
+      ahoy cli 'chmod -v u+x /usr/bin/ckan_cli $APP_DIR/bin/*; cp -v .docker/test.ini $CKAN_INI; $APP_DIR/bin/process-config.sh'
+
+  pip-list:
+    usage: List pip install version details
+    cmd: |
+      ahoy cli 'pip list'
+
+  test-unit:
+    usage: Run unit tests.
+    cmd: |
+      ahoy cli 'pytest --ckan-ini=${CKAN_INI} --cov=ckanext $APP_DIR/ckanext' || \
+      [ "${ALLOW_UNIT_FAIL:-0}" -eq 1 ]
+
+  test-bdd:
+    usage: Run BDD tests.
+    cmd: |
+      ahoy cli "rm -f test/screenshots/*"
+      ahoy start-mailmock &
+      sleep 5 &&
+      if [ "$BEHAVE_TAG" = "" ]; then
+        # no tag specified, probably running locally
+        (ahoy cli "behave -k ${*:-test/features} --tags=smoke" \
+         && ahoy cli "behave -k ${*:-test/features} --tags=-smoke" \
+        ) || [ "${ALLOW_BDD_FAIL:-0}" -eq 1 ]
+      elif [ "$BEHAVE_TAG" = "authenticated" ]; then
+        # run any tests that don't have a specific tag
+        ahoy cli "behave -k ${*:-test/features} --tags=-unauthenticated --tags=-smoke" \
+          || [ "${ALLOW_BDD_FAIL:-0}" -eq 1 ]
+      else
+        # run tests with the specified tag
+        ahoy cli "behave -k ${*:-test/features} --tags=$BEHAVE_TAG" \
+          || [ "${ALLOW_BDD_FAIL:-0}" -eq 1 ]
+      fi
+      ahoy stop-mailmock
+
+  start-mailmock:
+    usage: Starts email mock server used for email BDD tests
+    cmd: |
+      ahoy title 'Starting mailmock'
+      ahoy cli 'mailmock -p 8025 -o ${APP_DIR}/test/emails' # for debugging mailmock email output remove --no-stdout
+
+  stop-mailmock:
+    usage: Stops email mock server used for email BDD tests
+    cmd: |
+      ahoy title 'Stopping mailmock'
+      ahoy cli "killall -2 mailmock"
+
+  # Utilities.
+  title:
+    cmd: printf "$(tput -Txterm setaf 4)==> ${1}$(tput -Txterm sgr0)\n"
+    hide: true
+
+  line:
+    cmd: printf "$(tput -Txterm setaf 2)${1}$(tput -Txterm sgr0)${2}\n"
+    hide: true
+
+  getvar:
+    cmd: eval echo "${@}"
+    hide: true
+
+  pre-flight:
+    cmd: |
+      export DOCTOR_CHECK_DB=${DOCTOR_CHECK_DB:-1}
+      export DOCTOR_CHECK_TOOLS=${DOCTOR_CHECK_TOOLS:-1}
+      export DOCTOR_CHECK_PORT=${DOCTOR_CHECK_PORT:-0}
+      export DOCTOR_CHECK_PYGMY=${DOCTOR_CHECK_PYGMY:-1}
+      export DOCTOR_CHECK_CLI=${DOCTOR_CHECK_CLI:-0}
+      export DOCTOR_CHECK_SSH=${DOCTOR_CHECK_SSH:-0}
+      export DOCTOR_CHECK_WEBSERVER=${DOCTOR_CHECK_WEBSERVER:-0}
+      export DOCTOR_CHECK_BOOTSTRAP=${DOCTOR_CHECK_BOOTSTRAP:-0}
+      ahoy doctor
+    hide: true
+
+entrypoint:
+  - bash
+  - "-c"
+  - "-e"
+  - |
+    export LAGOON_LOCALDEV_URL=http://$PROJECT.docker.amazee.io
+    [ -f .env ] && [ -s .env ] && export $(grep -v '^#' .env | xargs) && if [ -f .env.local ] && [ -s .env.local ]; then export $(grep -v '^#' .env.local | xargs); fi
+    bash -e -c "$0" "$@"
+  - "{{cmd}}"
+  - "{{name}}"

.docker/Dockerfile-template.ckan

@@ -0,0 +1,36 @@
+FROM openknowledge/ckan-dev:{CKAN_VERSION}
+
+ARG SITE_URL=http://ckan:5000/
+ENV PYTHON_VERSION={PYTHON_VERSION}
+ENV CKAN_VERSION={CKAN_VERSION}
+ENV CKAN_SITE_URL="${SITE_URL}"
+ENV PYTHON={PYTHON}
+
+WORKDIR "${APP_DIR}"
+
+ENV DOCKERIZE_VERSION v0.6.1
+RUN apk add --no-cache build-base \
+    && curl -sL https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \
+        | tar -C /usr/local/bin -xzvf -
+
+# Install CKAN.
+
+RUN cd $SRC_DIR/ckan \
+    && git config --global --add safe.directory "$SRC_DIR/ckan" \
+    && git remote set-url origin 'https://github.com/{CKAN_GIT_ORG}/ckan.git' \
+    && git fetch \
+    && git reset --hard && git clean -f \
+    && git checkout '{CKAN_GIT_VERSION}'
+
+COPY .docker/test.ini $CKAN_INI
+
+COPY . ${APP_DIR}/
+
+COPY bin/ckan_cli /usr/bin/
+
+RUN chmod +x ${APP_DIR}/bin/*.sh /usr/bin/ckan_cli
+
+# Init current extension.
+RUN ${APP_DIR}/bin/init-ext.sh
+
+CMD ["/srv/app/bin/serve.sh"]

.docker/test.ini

@@ -0,0 +1,162 @@
+#UNIT TEST CONFIG
+#
+# CKAN - Pylons configuration
+#
+# These are some of the configuration options available for your CKAN
+# instance. Check the documentation in 'doc/configuration.rst' or at the
+# following URL for a description of what they do and the full list of
+# available options:
+#
+# http://docs.ckan.org/en/latest/maintaining/configuration.html
+#
+# The %(here)s variable will be replaced with the parent directory of this file
+#
+
+[DEFAULT]
+debug = false
+smtp_server = localhost:8025
+error_email_from = paste@localhost
+
+[server:main]
+use = egg:Paste#http
+host = 0.0.0.0
+port = 5000
+
+[app:main]
+ckan.devserver.host = 0.0.0.0
+ckan.devserver.port = 5000
+
+use = egg:ckan
+full_stack = true
+cache_dir = /tmp/%(ckan.site_id)s/
+beaker.session.key = ckan
+
+# This is the secret token that the beaker library uses to hash the cookie sent
+# to the client. `paster make-config` generates a unique value for this each
+# time it generates a config file.
+beaker.session.secret = bSmgPpaxg2M+ZRes3u1TXwIcE
+
+# `paster make-config` generates a unique value for this each time it generates
+# a config file.
+app_instance_uuid = 6e3daf8e-1c6b-443b-911f-c7ab4c5f9605
+
+# repoze.who config
+who.config_file = %(here)s/who.ini
+who.log_level = warning
+who.log_file = %(cache_dir)s/who_log.ini
+
+## Database Settings
+sqlalchemy.url = postgresql://ckan_default:pass@postgres/ckan_test
+
+# PostgreSQL' full-text search parameters
+ckan.datastore.default_fts_lang = english
+ckan.datastore.default_fts_index_method = gist
+
+## Site Settings.
+ckan.site_url = http://ckan:5000/
+
+## Search Settings
+
+ckan.site_id = default
+solr_url = http://solr:8983/solr/ckan
+
+## Redis Settings
+
+# URL to your Redis instance, including the database to be used.
+ckan.redis.url = redis://redis:6379
+
+## Authorization Settings
+ckan.auth.anon_create_dataset = false
+ckan.auth.create_unowned_dataset = false
+ckan.auth.create_dataset_if_not_in_organization = false
+ckan.auth.user_create_groups = false
+ckan.auth.user_create_organizations = false
+ckan.auth.user_delete_groups = true
+ckan.auth.user_delete_organizations = true
+ckan.auth.create_user_via_api = false
+ckan.auth.create_user_via_web = true
+ckan.auth.roles_that_cascade_to_sub_groups = admin
+ckan.auth.public_user_details = False
+
+## Plugins Settings
+ckan.plugins =
+    scheming_datasets
+    resource_visibility
+
+
+# Define which views should be created by default
+# (plugins must be loaded in ckan.plugins)
+ckan.views.default_views = image_view text_view recline_view
+
+# Customize which text formats the text_view plugin will show
+#ckan.preview.json_formats = json
+#ckan.preview.xml_formats = xml rdf rdf+xml owl+xml atom rss
+#ckan.preview.text_formats = text plain text/plain
+
+# Customize which image formats the image_view plugin will show
+#ckan.preview.image_formats = png jpeg jpg gif
+
+## Internationalisation Settings
+ckan.locale_default = en_AU
+ckan.locale_order = en pt_BR ja it cs_CZ ca es fr el sv sr sr@latin no sk fi ru de pl nl bg ko_KR hu sa sl lv
+ckan.locales_offered =
+ckan.locales_filtered_out = en_AU
+ckan.display_timezone = Australia/Queensland
+
+## Storage Settings
+
+ckan.storage_path = /app/filestore
+
+## Activity Streams Settings
+
+ckan.hide_activity_from_users = %(ckan.site_id)s
+
+
+## Email settings
+
+smtp.server = localhost:8025
+smtp.test_server = localhost:8025
+smtp.mail_from = [email protected]
+
+## ckanext-scheming settings
+# see https://github.com/ckan/ckanext-scheming#configuration
+scheming.dataset_schemas = ckanext.resource_visibility:ckan_dataset.yaml
+scheming.presets =
+    ckanext.scheming:presets.json
+    ckanext.resource_visibility:schema/presets.json
+scheming.dataset_fallback = false
+
+## Logging configuration
+[loggers]
+keys = root, ckan, ckanext
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARNING
+handlers = console
+
+[logger_ckan]
+level = INFO
+handlers = console
+qualname = ckan
+propagate = 0
+
+[logger_ckanext]
+level = DEBUG
+handlers = console
+qualname = ckanext
+propagate = 0
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s