Add integration test for OIDC authentication

Dockerfile

@@ -121,13 +121,15 @@ RUN set -ex \
 USER archivematica
 
 COPY --chown=${USER_ID}:${GROUP_ID} --from=pyenv-builder --link ${PYENV_DIR} ${PYENV_DIR}
-COPY --link ./install/storage-service.gunicorn-config.py /etc/archivematica/storage-service.gunicorn-config.py
-COPY --chown=${USER_ID}:${GROUP_ID} --link . /src/
+COPY --chown=${USER_ID}:${GROUP_ID} --link ./install/storage-service.gunicorn-config.py /etc/archivematica/storage-service.gunicorn-config.py
 
 # -----------------------------------------------------------------------------
 
 FROM base AS archivematica-storage-service
 
+ARG USER_ID=1000
+ARG GROUP_ID=1000
+
 WORKDIR /src/storage_service
 
 ENV DJANGO_SETTINGS_MODULE=storage_service.settings.local
@@ -138,6 +140,8 @@ ENV SS_GUNICORN_ACCESSLOG=-
 ENV SS_GUNICORN_ERRORLOG=-
 ENV FORWARDED_ALLOW_IPS=*
 
+COPY --chown=${USER_ID}:${GROUP_ID} --link . /src/
+
 RUN set -ex \
 	&& export SS_DB_URL=mysql://ne:ver@min/d \
 	&& pyenv exec python3 ./manage.py collectstatic --noinput --clear \
@@ -148,3 +152,30 @@ ENV DJANGO_SETTINGS_MODULE=storage_service.settings.production
 EXPOSE 8000
 
 ENTRYPOINT ["pyenv", "exec", "python3", "-m", "gunicorn", "--config=/etc/archivematica/storage-service.gunicorn-config.py", "storage_service.wsgi:application"]
+
+# -----------------------------------------------------------------------------
+
+FROM base AS archivematica-storage-service-tests
+
+ARG USER_ID=1000
+ARG GROUP_ID=1000
+
+USER root
+
+RUN set -ex \
+	&& python3 -m playwright install-deps firefox \
+	&& mkdir -p /var/archivematica/.cache/ms-playwright \
+	&& chown -R archivematica:archivematica /var/archivematica/
+
+USER archivematica
+
+RUN set -ex \
+	&& python3 -m playwright install firefox
+
+ENV PYTHONPATH=/src/storage_service
+
+COPY --chown=${USER_ID}:${GROUP_ID} --link . /src/
+
+# -----------------------------------------------------------------------------
+
+FROM ${TARGET}

pyproject.toml

@@ -93,7 +93,7 @@ ignore_errors = true
 
 [[tool.mypy.overrides]]
 module = [
-    "tests.integration.test_integration",
+    "tests.integration.*",
     "tests.storage_service.test_oidc",
 ]
 ignore_errors = false

requirements-dev.in

@@ -6,6 +6,12 @@ pip-tools
 pytest-cov
 pytest-django
 pytest-mock
+pytest-playwright
 pytest-randomly
 pytest
 tox
+
+# playwright requires specific versions of greenlet which may clash with our
+# gevent dependency in requirements.txt.
+# See https://github.com/microsoft/playwright-python/issues/2190
+git+https://github.com/microsoft/playwright-python.git@d9cdfbb1e178b6770625e9f857139aff77516af0#egg=playwright

requirements-dev.txt

@@ -12,9 +12,9 @@ asgiref==3.8.1
     #   django
 bagit==1.8.1
     # via -r requirements.txt
-boto3==1.35.23
+boto3==1.35.29
     # via -r requirements.txt
-botocore==1.35.23
+botocore==1.35.29
     # via
     #   -r requirements.txt
     #   boto3
@@ -92,10 +92,11 @@ filelock==3.16.1
     #   virtualenv
 gevent==24.2.1
     # via -r requirements.txt
-greenlet==3.1.0
+greenlet==3.1.1
     # via
     #   -r requirements.txt
     #   gevent
+    #   playwright
 gunicorn==23.0.0
     # via -r requirements.txt
 httplib2==0.22.0
@@ -209,11 +210,15 @@ platformdirs==4.3.6
     # via
     #   tox
     #   virtualenv
+playwright @ git+https://github.com/microsoft/playwright-python.git@d9cdfbb1e178b6770625e9f857139aff77516af0
+    # via
+    #   -r requirements-dev.in
+    #   pytest-playwright
 pluggy==1.5.0
     # via
     #   pytest
     #   tox
-prometheus-client==0.20.0
+prometheus-client==0.21.0
     # via
     #   -r requirements.txt
     #   django-prometheus
@@ -230,6 +235,8 @@ pycparser==2.22
     # via
     #   -r requirements.txt
     #   cffi
+pyee==12.0.0
+    # via playwright
 pyopenssl==24.2.1
     # via
     #   -r requirements.txt
@@ -248,16 +255,22 @@ pyproject-hooks==1.1.0
 pytest==8.3.3
     # via
     #   -r requirements-dev.in
+    #   pytest-base-url
     #   pytest-cov
     #   pytest-django
     #   pytest-mock
+    #   pytest-playwright
     #   pytest-randomly
+pytest-base-url==2.1.0
+    # via pytest-playwright
 pytest-cov==5.0.0
     # via -r requirements-dev.in
 pytest-django==4.9.0
     # via -r requirements-dev.in
 pytest-mock==3.14.0
     # via -r requirements-dev.in
+pytest-playwright==0.5.2
+    # via -r requirements-dev.in
 pytest-randomly==3.15.0
     # via -r requirements-dev.in
 python-cas==1.6.0
@@ -269,7 +282,7 @@ python-dateutil==2.9.0.post0
     #   -r requirements.txt
     #   botocore
     #   django-tastypie
-python-gnupg==0.5.2
+python-gnupg==0.5.3
     # via -r requirements.txt
 python-keystoneclient==5.5.0
     # via -r requirements.txt
@@ -281,6 +294,8 @@ python-mimeparse==2.0.0
     # via
     #   -r requirements.txt
     #   django-tastypie
+python-slugify==8.0.4
+    # via pytest-playwright
 python-swiftclient==4.6.0
     # via -r requirements.txt
 pyyaml==6.0.2
@@ -295,6 +310,7 @@ requests==2.32.3
     #   keystoneauth1
     #   mozilla-django-oidc
     #   oslo-config
+    #   pytest-base-url
     #   python-cas
     #   python-keystoneclient
     #   python-swiftclient
@@ -323,6 +339,8 @@ stevedore==5.3.0
     #   python-keystoneclient
 sword2 @ git+https://github.com/artefactual-labs/python-client-sword2.git@619ee44467dcdb2ab75fab16864ea2e4ded7ffe4
     # via -r requirements.txt
+text-unidecode==1.3
+    # via python-slugify
 tomli==2.0.1
     # via
     #   build
@@ -338,7 +356,8 @@ typing-extensions==4.12.2
     #   -r requirements.txt
     #   asgiref
     #   dj-database-url
-tzdata==2024.1
+    #   pyee
+tzdata==2024.2
     # via
     #   -r requirements.txt
     #   oslo-serialization
@@ -348,7 +367,7 @@ urllib3==1.26.20
     #   -r requirements.txt
     #   botocore
     #   requests
-virtualenv==20.26.5
+virtualenv==20.26.6
     # via tox
 wheel==0.44.0
     # via pip-tools

requirements.txt

@@ -10,9 +10,9 @@ asgiref==3.8.1
     # via django
 bagit==1.8.1
     # via -r requirements.in
-boto3==1.35.23
+boto3==1.35.29
     # via -r requirements.in
-botocore==1.35.23
+botocore==1.35.29
     # via
     #   boto3
     #   s3transfer
@@ -59,7 +59,7 @@ django-tastypie==0.14.7
     # via -r requirements.in
 gevent==24.2.1
     # via -r requirements.in
-greenlet==3.1.0
+greenlet==3.1.1
     # via gevent
 gunicorn==23.0.0
     # via -r requirements.in
@@ -131,7 +131,7 @@ pbr==6.1.0
     #   oslo-serialization
     #   python-keystoneclient
     #   stevedore
-prometheus-client==0.20.0
+prometheus-client==0.21.0
     # via
     #   -r requirements.in
     #   django-prometheus
@@ -155,7 +155,7 @@ python-dateutil==2.9.0.post0
     # via
     #   botocore
     #   django-tastypie
-python-gnupg==0.5.2
+python-gnupg==0.5.3
     # via -r requirements.in
 python-keystoneclient==5.5.0
     # via -r requirements.in
@@ -202,7 +202,7 @@ typing-extensions==4.12.2
     # via
     #   asgiref
     #   dj-database-url
-tzdata==2024.1
+tzdata==2024.2
     # via
     #   oslo-serialization
     #   oslo-utils

tests/integration/docker-compose.yml

@@ -8,16 +8,20 @@ services:
       context: "../../"
       dockerfile: "Dockerfile"
       args:
+        TARGET: "archivematica-storage-service-tests"
         USER_ID: ${USER_ID:-1000}
         GROUP_ID: ${GROUP_ID:-1000}
         UBUNTU_VERSION: ${UBUNTU_VERSION:-22.04}
         PYTHON_VERSION: ${PYTHON_VERSION:-3.9}
     entrypoint: ""
     working_dir: "/src"
-    command: ["pytest"]
+    command: ["pytest", "--browser", "firefox"]
+    hostname: "archivematica-storage-service"
     environment:
       PYTEST_ADDOPTS: ${PYTEST_ADDOPTS:-}
       RUN_INTEGRATION_TESTS: "true"
+      DJANGO_LIVE_TEST_SERVER_ADDRESS: "archivematica-storage-service:8000"
+      DJANGO_ALLOW_ASYNC_UNSAFE: true
       FORWARDED_ALLOW_IPS: "*"
       SS_GUNICORN_ACCESSLOG: "/dev/null"
       DJANGO_SETTINGS_MODULE: "storage_service.settings.testmysql"
@@ -28,6 +32,15 @@ services:
       RCLONE_CONFIG_MYS3_ACCESS_KEY_ID: "minio"
       RCLONE_CONFIG_MYS3_SECRET_ACCESS_KEY: "minio123"
       RCLONE_CONFIG_MYS3_REGION: "planet-earth"
+      SS_OIDC_AUTHENTICATION: "true"
+      OIDC_RP_CLIENT_ID: "am-storage-service"
+      OIDC_RP_CLIENT_SECRET: "example-secret"
+      OIDC_OP_AUTHORIZATION_ENDPOINT: "http://keycloak:8080/realms/demo/protocol/openid-connect/auth"
+      OIDC_OP_TOKEN_ENDPOINT: "http://keycloak:8080/realms/demo/protocol/openid-connect/token"
+      OIDC_OP_USER_ENDPOINT: "http://keycloak:8080/realms/demo/protocol/openid-connect/userinfo"
+      OIDC_OP_JWKS_ENDPOINT: "http://keycloak:8080/realms/demo/protocol/openid-connect/certs"
+      OIDC_OP_LOGOUT_ENDPOINT: "http://keycloak:8080/realms/demo/protocol/openid-connect/logout"
+      OIDC_RP_SIGN_ALGO: "RS256"
     volumes:
       - "../../:/src"
     depends_on:
@@ -36,6 +49,7 @@ services:
     links:
       - "minio"
       - "mysql"
+      - "keycloak"
 
   minio:
     image: "minio/minio:RELEASE.2024-01-31T20-20-33Z"
@@ -62,3 +76,17 @@ services:
       timeout: 5s
       retries: 5
       start_period: 15s
+
+  keycloak:
+    image: "quay.io/keycloak/keycloak:latest"
+    command: ["start-dev", "--import-realm"]
+    restart: "unless-stopped"
+    environment:
+      KEYCLOAK_ADMIN: "admin"
+      KEYCLOAK_ADMIN_PASSWORD: "admin"
+      KC_METRICS_ENABLED: true
+      KC_LOG_LEVEL: "INFO"
+    ports:
+      - 8080:8080
+    volumes:
+      - "./etc/keycloak/realm.json:/opt/keycloak/data/import/realm.json:ro"

tests/integration/etc/keycloak/realm.json

@@ -0,0 +1,52 @@
+[
+  {
+    "id": "demo",
+    "realm": "demo",
+    "sslRequired": "none",
+    "enabled": true,
+    "eventsEnabled": true,
+    "eventsExpiration": 900,
+    "adminEventsEnabled": true,
+    "adminEventsDetailsEnabled": true,
+    "attributes": {
+      "adminEventsExpiration": "900"
+    },
+    "clients": [
+      {
+        "id": "am-storage-service",
+        "clientId": "am-storage-service",
+        "name": "am-storage-service",
+        "enabled": true,
+        "rootUrl": "http://archivematica-storage-service:8000",
+        "adminUrl": "http://archivematica-storage-service:8000",
+        "baseUrl": "http://archivematica-storage-service:8000",
+        "clientAuthenticatorType": "client-secret",
+        "secret": "example-secret",
+        "redirectUris": ["http://archivematica-storage-service:8000/*"],
+        "webOrigins": ["http://archivematica-storage-service:8000"],
+        "standardFlowEnabled": true,
+        "serviceAccountsEnabled": true,
+        "authorizationServicesEnabled": true,
+        "publicClient": false
+      }
+    ],
+    "users": [
+      {
+        "id": "demo",
+        "email": "[email protected]",
+        "username": "demo",
+        "firstName": "Demo",
+        "lastName": "User",
+        "enabled": true,
+        "emailVerified": true,
+        "credentials": [
+          {
+            "temporary": false,
+            "type": "password",
+            "value": "demo"
+          }
+        ]
+      }
+    ]
+  }
+]