[QOLDEV-982] pin test Solr version to CKAN version

- Also sync test config with Open Data theme

.ahoy.yml

@@ -40,7 +40,15 @@ commands:
       echo "Initialising database schema"
       ahoy cli '"${APP_DIR}"/bin/init.sh'
       echo "Waiting for containers to start listening..."
-      ahoy cli "dockerize -wait tcp://ckan:5000 -timeout 1m"
+      for i in `seq 1 60`; do
+        if (ahoy cli "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/ckan/5000'"); then
+          echo "CKAN became ready on attempt $i"
+          break
+        else
+          echo "CKAN not yet ready, retrying (attempt $i)..."
+          sleep 1
+        fi
+      done
       if sh bin/docker-compose.sh logs | grep -q "\[Error\]"; then exit 1; fi
       if sh bin/docker-compose.sh logs | grep -q "Exception"; then exit 1; fi
       docker ps -a --filter name=^/${COMPOSE_PROJECT_NAME}_
@@ -68,7 +76,13 @@ commands:
     usage: Show Docker logs.
     cmd: |
         ahoy title "Output logs"
-        sh bin/docker-compose.sh logs "$@"
+        # Loop through each container and wrap with github log groups
+        services=$(bin/docker-compose.sh ps --services)
+        for service in $services; do
+          echo "::group::$service"
+          sh bin/docker-compose.sh logs "$service"
+          echo "::endgroup::"
+        done
 
   pull:
     usage: Pull latest docker images.

.docker/Dockerfile-template.ckan

@@ -7,6 +7,7 @@ RUN ORIGINAL_USER=$(id -un)
 ARG SITE_URL=http://ckan:5000/
 ENV PYTHON_VERSION={PYTHON_VERSION}
 ENV CKAN_VERSION={CKAN_VERSION}
+ENV SOLR_VERSION={SOLR_VERSION}
 ENV CKAN_SITE_URL="${SITE_URL}"
 ENV PYTHON={PYTHON}
 
@@ -22,12 +23,13 @@ COPY bin/ckan_cli /usr/bin/
 
 RUN chmod +x "${APP_DIR}"/bin/*.sh /usr/bin/ckan_cli
 
-ENV DOCKERIZE_VERSION=v0.6.1
-RUN wget -O - https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \
-        | tar -C /usr/local/bin -xzvf -
-
 RUN which ps || apt-get install -y procps
 
+# Install setuptools conditionally
+RUN if [ "$CKAN_VERSION" = "2.9" ]; then \
+        pip install "setuptools>=44.1.0,<71"; \
+    fi
+
 # Install CKAN.
 
 RUN cd $SRC_DIR/ckan \

.docker/test.ini

@@ -51,6 +51,7 @@ ckan.auth.create_user_via_web = true
 ckan.auth.roles_that_cascade_to_sub_groups = admin
 ckan.auth.public_user_details = False
 ckan.auth.reveal_private_datasets = True
+ckan.auth.reveal_deleted_datasets = False
 
 
 ## Search Settings

.github/workflows/test.yml

@@ -27,18 +27,22 @@ jobs:
       fail-fast: false
       matrix:
         ckan-version: ["2.10", 2.9]
+        solr-version: ["8"]
         experimental: [false]
         include:
           - ckan-version: '2.11'
-            experimental: true  # Need to wait for 2.11.1 to fix org page
+            solr-version: "9"
+            experimental: true
           - ckan-version: 'master'
+            solr-version: "9"
             experimental: true  #master is unstable, good to know if we are compatible or not
 
     name: Test on CKAN ${{ matrix.ckan-version }}
     runs-on: ubuntu-latest
     container: drevops/ci-runner:23.12.0
     env:
       CKAN_VERSION: ${{ matrix.ckan-version }}
+      SOLR_VERSION: ${{ matrix.solr-version }}
 
     steps:
       # Patch https://github.com/actions/runner/issues/863
@@ -50,38 +54,38 @@ jobs:
         timeout-minutes: 2
 
       - name: Build
-        continue-on-error: ${{ matrix.experimental }}
         run: bin/build.sh
         timeout-minutes: 15
+        continue-on-error: ${{ matrix.experimental }}
 
       - name: Lint
-        continue-on-error: ${{ matrix.experimental }}
         run: ahoy lint
         timeout-minutes: 10
+        continue-on-error: ${{ matrix.experimental }}
 
       - name: Unit test
-        continue-on-error: ${{ matrix.experimental }}
         run: ahoy test-unit
         timeout-minutes: 10
+        continue-on-error: ${{ matrix.experimental }}
 
       - name: Scenario test
-        continue-on-error: ${{ matrix.experimental }}
         run: |
           ahoy install-site
           ahoy test-bdd
         timeout-minutes: 40
+        continue-on-error: ${{ matrix.experimental }}
 
       - name: Retrieve logs
         if: always()
         run: ahoy logs
-        continue-on-error: ${{ matrix.experimental }}
         timeout-minutes: 1
+        continue-on-error: ${{ matrix.experimental }}
 
       - name: Retrieve results
         if: always()
         run: bin/process-artifacts.sh
-        continue-on-error: ${{ matrix.experimental }}
         timeout-minutes: 1
+        continue-on-error: ${{ matrix.experimental }}
 
       - name: Test Summary
         uses: test-summary/action@v2
@@ -93,8 +97,8 @@ jobs:
       - name: Upload screenshots
         if: always()
         uses: actions/upload-artifact@v4
-        continue-on-error: ${{ matrix.experimental }}
         with:
           name: CKAN ${{ matrix.ckan-version }} screenshots
           path: /tmp/artifacts/behave/screenshots
         timeout-minutes: 1
+        continue-on-error: ${{ matrix.experimental }}

bin/build.sh

@@ -21,9 +21,9 @@ CKAN_GIT_VERSION=$CKAN_VERSION
 CKAN_GIT_ORG=qld-gov-au
 
 if [ "$CKAN_VERSION" = "2.11" ]; then
-    CKAN_GIT_VERSION=ckan-2.11.0
+    CKAN_GIT_VERSION=ckan-2.11.1
 elif [ "$CKAN_VERSION" = "2.10" ]; then
-    CKAN_GIT_VERSION=ckan-2.10.5-qgov.4
+    CKAN_GIT_VERSION=ckan-2.10.5-qgov.5
 elif [ "$CKAN_VERSION" = "2.9" ]; then
     CKAN_GIT_VERSION=ckan-2.9.9-qgov.4
 elif [ "$CKAN_VERSION" = "master" ]; then

bin/create-test-data.sh

@@ -127,4 +127,4 @@ echo ${group_create}
 # END.
 #
 
-. ${APP_DIR}/bin/deactivate
+. "${APP_DIR}"/bin/deactivate

docker-compose.yml

@@ -63,7 +63,7 @@ services:
       - default
 
   solr:
-    image: ckan/ckan-solr:${CKAN_VERSION}-solr9
+    image: ckan/ckan-solr:${CKAN_VERSION}-solr${SOLR_VERSION}
     ports:
       - "8983"
     environment:
@@ -73,7 +73,7 @@ services:
       - default
 
   chrome:
-    image: selenium/standalone-chrome:3.141.59-oxygen
+    image: selenium/standalone-chrome:131.0-chromedriver-131.0-grid-4.27.0-20241225
     shm_size: '1gb'
     depends_on:
       - ckan

test/features/data_qld_theme.feature

@@ -1,3 +1,4 @@
+@theme
 Feature: Theme customisations
 
     @unauthenticated
@@ -65,28 +66,6 @@ Feature: Theme customisations
         When I go to organisation page
         Then I should see "Organisations are Queensland Government departments, other agencies or legislative entities responsible for publishing open data on this portal."
 
-    Scenario: Register user password must be 10 characters or longer
-        Given "Unauthenticated" as the persona
-        When I go to register page
-        And I fill in "name" with "name"
-        And I fill in "fullname" with "fullname"
-        And I fill in "email" with "[email protected]"
-        And I fill in "password1" with "pass"
-        And I fill in "password2" with "pass"
-        And I press "Create Account"
-        Then I should see "Password: Your password must be 10 characters or longer"
-
-    Scenario: Register user password must contain at least one number, lowercase letter, capital letter, and symbol
-        Given "Unauthenticated" as the persona
-        When I go to register page
-        And I fill in "name" with "name"
-        And I fill in "fullname" with "fullname"
-        And I fill in "email" with "[email protected]"
-        And I fill in "password1" with "password1234"
-        And I fill in "password2" with "password1234"
-        And I press "Create Account"
-        Then I should see "Password: Must contain at least one number, lowercase letter, capital letter, and symbol"
-
     @unauthenticated
     @Publications
     Scenario: Publications - Menu items are present and correct

test/features/groups.feature

@@ -1,4 +1,4 @@
-@users
+@groups
 Feature: Group APIs
 
     Scenario Outline: Group membership is accessible to admins of the group

test/features/organisations.feature

@@ -34,12 +34,23 @@ Feature: Organization APIs
     Scenario: Organisation overview is accessible to everyone
         Given "Unauthenticated" as the persona
         When I go to organisation page
-        Then I should see "Test Organisation"
+        And I expand the browser height
+        Then I should see "Food Standards Agency"
         And I should not see an element with xpath "//a[contains(@href, '?action=read')]"
-        And I should see an element with xpath "//a[contains(@href, '/organization/test-organisation')]"
-        When I press "Test Organisation"
+        And I should see an element with xpath "//a[contains(@href, '/organization/food-standards-agency')]"
+        When I press "Food Standards Agency"
+        And I take a debugging screenshot
         And I press "Activity Stream"
         Then I should see "created the org"
 
-        When I view the "test-organisation" organisation API "not including" users
-        Then I should see an element with xpath "//*[contains(string(), '"success": true') and contains(string(), '"name": "test-organisation"')]"
+        When I view the "food-standards-agency" organisation API "not including" users
+        Then I should see an element with xpath "//*[contains(string(), '"success": true') and contains(string(), '"name": "food-standards-agency"')]"
+
+    Scenario: Organisation list is accessible via the dashboard
+        Given "SysAdmin" as the persona
+        When I log in
+        And I go to the dashboard
+        And I press "My Organisations"
+        Then I should see "Test Organisation"
+        And I should see an element with xpath "//a[contains(@href, 'organization/new') and contains(string(), 'Add Organisation')]"
+

test/features/steps/steps.py

@@ -109,7 +109,7 @@ def request_reset(context):
 @when(u'I fill in "{name}" with "{value}" if present')
 def fill_in_field_if_present(context, name, value):
     context.execute_steps(u"""
-        When I execute the script "field = $('#field-{0}'); if (!field.length) field = $('#{0}'); if (!field.length) field = $('[name={0}]'); field.val('{1}'); field.keyup();"
+        When I execute the script "field = $('#{0}'); if (!field.length) field = $('[name={0}]'); if (!field.length) field = $('#field-{0}'); field.val('{1}'); field.keyup();"
     """.format(name, value))
 
 
@@ -162,6 +162,11 @@ def go_to_new_resource_form(context, name):
         context.execute_steps(u"""
             When I press "Next:"
         """)
+    elif context.browser.is_element_present_by_xpath("//*[contains(string(), 'Add new resource')]"):
+        # Existing dataset, browse to the resource form
+        context.execute_steps(u"""
+            When I press "Add new resource"
+        """)
     else:
         # Existing dataset, browse to the resource form
         if context.browser.is_element_present_by_xpath(
@@ -177,13 +182,24 @@ def go_to_new_resource_form(context, name):
 
 @when(u'I fill in title with random text')
 def title_random_text(context):
-    assert context.persona
     context.execute_steps(u"""
-        When I fill in "title" with "Test Title {0}"
-        And I fill in "name" with "test-title-{0}" if present
-        And I set "last_generated_title" to "Test Title {0}"
-        And I set "last_generated_name" to "test-title-{0}"
-    """.format(uuid.uuid4()))
+        When I fill in title with random text starting with "Test Title "
+    """)
+
+
+@when(u'I fill in title with random text starting with "{prefix}"')
+def title_random_text_with_prefix(context, prefix):
+    random_text = str(uuid.uuid4())
+    title = prefix + random_text
+    name = prefix.lower().replace(" ", "-") + random_text
+    assert context.persona
+    context.execute_steps(f"""
+        When I fill in "title" with "{title}"
+        And I fill in "name" with "{name}" if present
+        And I set "last_generated_title" to "{title}"
+        And I set "last_generated_name" to "{name}"
+        And I take a debugging screenshot
+    """)
 
 
 @when(u'I go to dataset page')
@@ -222,10 +238,18 @@ def show_more_fields(context):
 @when(u'I edit the "{name}" dataset')
 def edit_dataset(context, name):
     context.execute_steps(u"""
-        When I visit "/dataset/edit/{0}"
+        When I go to dataset "{0}"
+        And I press the element with xpath "//div[contains(@class, 'action')]//a[contains(@href, '/dataset/edit/')]"
     """.format(name))
 
 
+@when(u'I press the resource edit button')
+def press_edit_resource(context):
+    context.execute_steps(u"""
+        When I press the element with xpath "//div[contains(@class, 'action')]//a[contains(@href, '/resource/') and contains(@href, '/edit')]"
+    """)
+
+
 @when(u'I select the "{licence_id}" licence')
 def select_licence(context, licence_id):
     # Licence requires special interaction due to fancy JavaScript
@@ -234,6 +258,15 @@ def select_licence(context, licence_id):
     """.format(licence_id))
 
 
+@when(u'I select the organisation with title "{title}"')
+def select_organisation(context, title):
+    # Organisation requires special interaction due to fancy JavaScript
+    context.execute_steps(u"""
+        When I execute the script "org_uuid=$('#field-organizations').find('option:contains({0})').val(); $('#field-organizations').val(org_uuid).trigger('change')"
+        And I take a debugging screenshot
+    """.format(title))
+
+
 @when(u'I enter the resource URL "{url}"')
 def enter_resource_url(context, url):
     if url != "default":
@@ -276,7 +309,7 @@ def fill_in_default_link_resource_fields(context):
 @when(u'I upload "{file_name}" of type "{file_format}" to resource')
 def upload_file_to_resource(context, file_name, file_format):
     context.execute_steps(u"""
-        When I execute the script "$('#resource-upload-button').trigger('click');"
+        When I execute the script "$('.resource-upload-field .btn-remove-url').trigger('click'); $('#resource-upload-button').trigger('click');"
         And I attach the file "{file_name}" to "upload"
         # Don't quote the injected string since it can have trailing spaces
         And I execute the script "document.getElementById('field-format').value='{file_format}'"
@@ -391,8 +424,8 @@ def _create_dataset_from_params(context, params):
         if key == "owner_org":
             # Owner org uses UUIDs as its values, so we need to rely on displayed text
             context.execute_steps(u"""
-                When I select by text "{1}" from "{0}"
-            """.format(key, value))
+                When I select the organisation with title "{0}"
+            """.format(value))
         elif key in ["update_frequency", "request_privacy_assessment", "private"]:
             context.execute_steps(u"""
                 When I select "{1}" from "{0}"
@@ -499,11 +532,7 @@ def filter_contents(mail):
         payload_bytes = quopri.decodestring(payload)
         if len(payload_bytes) > 0:
             payload_bytes += b'='  # do fix the padding error issue
-        if six.PY2:
-            decoded_payload = payload_bytes.decode('base64')
-        else:
-            import base64
-            decoded_payload = six.ensure_text(base64.b64decode(six.ensure_binary(payload_bytes)))
+        decoded_payload = six.ensure_text(base64.b64decode(six.ensure_binary(payload_bytes)))
         print('Searching for', text, ' and ', text2, ' in decoded_payload: ', decoded_payload)
         return text in decoded_payload and (not text2 or text2 in decoded_payload)
 
@@ -552,5 +581,5 @@ def lock_account(context):
     for x in range(11):
         context.execute_steps(u"""
             When I attempt to log in with password "incorrect password"
-            Then I should see "Bad username or password or reCAPTCHA."
+            Then I should see "Bad username or password or CAPTCHA."
         """)