feat: implement extra_settings_files

config/crd/bases/galaxy_v1beta1_galaxy_crd.yaml

@@ -73,6 +73,28 @@ spec:
                         type: boolean
                     type: object
                 type: object
+              extra_settings_files:
+                description: Extra ConfigMaps or Secrets of settings files to specify for AWX
+                properties:
+                  configmaps:
+                    items:
+                      properties:
+                        name:
+                          type: string
+                        key:
+                          type: string
+                      type: object
+                    type: array
+                  secrets:
+                    items:
+                      properties:
+                        name:
+                          type: string
+                        key:
+                          type: string
+                      type: object
+                    type: array
+                type: object
               bundle_cacert_secret:
                 description: Secret where the trusted Certificate Authority Bundle is stored
                 type: string

config/manifests/bases/galaxy-operator.clusterserviceversion.yaml

@@ -333,6 +333,11 @@ spec:
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:advanced
         - urn:alm:descriptor:com.tectonic.ui:hidden
+      - displayName: Extra Settings Files
+        path: extra_settings_files
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:advanced
+        - urn:alm:descriptor:com.tectonic.ui:hidden
       - displayName: Redis deployment configuration
         path: redis
         x-descriptors:

docs/roles/galaxy-config.md

@@ -0,0 +1 @@
+../../roles/galaxy-config/README.md

docs/user-guide/advanced-configuration/extra-settings.md

@@ -0,0 +1,112 @@
+# Extra Settings
+
+With `pulp_settings` and `extra_settings_files`, you can pass multiple custom settings to Galaxy NG via the Galaxy Operator.
+
+!!! note
+    If the same setting is set in both `pulp_settings` and `extra_settings_files`, the setting in `extra_settings_files` will take precedence.
+
+## Add extra settings with `pulp_settings`
+
+You can pass extra settings by specifying the pair of the setting name and value as the `pulp_settings` parameter.
+
+The settings passed via `pulp_settings` will be appended to the `/etc/pulp/settings.py`.
+
+| Name           | Description    | Default   |
+| -------------- | -------------- | --------- |
+| pulp_settings  | Extra settings | `[]`      |
+
+Example configuration of `pulp_settings` parameter
+
+```yaml
+spec:
+  pulp_settings:
+    GALAXY_ENABLE_UNAUTHENTICATED_COLLECTION_ACCESS: "True"
+    GALAXY_ENABLE_UNAUTHENTICATED_COLLECTION_DOWNLOAD: "True"
+```
+
+## Add extra settings with `extra_settings_files`
+
+You can pass extra settings by specifying the additional settings files in the ConfigMaps or Secrets as the `extra_settings_files` parameter.
+
+The settings files passed via `extra_settings_files` will be mounted as the files under the `/etc/pulp/conf.d`.
+
+| Name                 | Description          | Default   |
+| -------------------- | -------------------- | --------- |
+| extra_settings_files | Extra settings files | `{}`      |
+
+!!! note
+    If the same setting is set in multiple files in `extra_settings_files`, it would be difficult to predict which would be adopted since these files are loaded in arbitrary order that [`glob`](https://docs.python.org/3/library/glob.html) returns. For a reliable setting, do not include the same key in more than one file.
+
+Create ConfigMaps or Secrets that contain custom settings files (`*.py`).
+
+```python title="custom_feature_flags.py"
+GALAXY_FEATURE_FLAGS = {
+  "execution_environments": False,
+  "dynaconf_merge": True,
+}
+```
+
+```python title="custom_misc_settings.py"
+GALAXY_AUTHENTICATION_CLASSES = [
+    "galaxy_ng.app.auth.session.SessionAuthentication",
+    "rest_framework.authentication.TokenAuthentication",
+    "rest_framework.authentication.BasicAuthentication",
+]
+GALAXY_MINIMUM_PASSWORD_LENGTH = 15
+SESSION_COOKIE_AGE = 3600
+GALAXY_ENABLE_UNAUTHENTICATED_COLLECTION_ACCESS = True
+GALAXY_ENABLE_UNAUTHENTICATED_COLLECTION_DOWNLOAD = True
+GALAXY_COLLECTION_SIGNING_SERVICE = None
+GALAXY_CONTAINER_SIGNING_SERVICE = None
+TOKEN_AUTH_DISABLED = True
+```
+
+```python title="custom_ldap.py"
+AUTH_LDAP_SERVER_URI = "ldap://ldap:10389"
+AUTH_LDAP_BIND_DN = "cn=admin,dc=planetexpress,dc=com"
+AUTH_LDAP_BIND_PASSWORD = "GoodNewsEveryone"
+AUTH_LDAP_USER_SEARCH_BASE_DN = "ou=people,dc=planetexpress,dc=com"
+AUTH_LDAP_USER_SEARCH_SCOPE = "SUBTREE"
+AUTH_LDAP_USER_SEARCH_FILTER = "(uid=%(user)s)"
+AUTH_LDAP_GROUP_SEARCH_BASE_DN = "ou=people,dc=planetexpress,dc=com"
+AUTH_LDAP_GROUP_SEARCH_SCOPE = "SUBTREE"
+AUTH_LDAP_GROUP_SEARCH_FILTER = "(objectClass=Group)"
+AUTH_LDAP_GROUP_TYPE_CLASS = "django_auth_ldap.config:GroupOfNamesType"
+```
+
+```bash title="Create ConfigMap and Secret"
+# Create ConfigMap
+kubectl create configmap my-custom-settings \
+  --from-file /PATH/TO/YOUR/custom_feature_flags.py \
+  --from-file /PATH/TO/YOUR/custom_misc_settings.py
+
+# Create Secret
+kubectl create secret generic my-custom-passwords \
+  --from-file /PATH/TO/YOUR/custom_ldap.py
+```
+
+Then specify them in the Galaxy CR spec. Here is an example configuration of `extra_settings_files` parameter.
+
+```yaml
+spec:
+  extra_settings_files:
+    configmaps:
+      - name: my-custom-settings         # The name of the ConfigMap
+        key: custom_feature_flags.py     # The key in the ConfigMap, which means the file name
+      - name: my-custom-settings
+        key: custom_misc_settings.py
+    secrets:
+      - name: my-custom-passwords        # The name of the Secret
+        key: custom_ldap.py              # The key in the Secret, which means the file name
+```
+
+!!! warning "Restriction"
+    There are some restrictions on the ConfigMaps or Secrets used in `extra_settings_files`.
+
+    - The keys in ConfigMaps or Secrets MUST be the name of python files and MUST end with `.py`
+    - The keys in ConfigMaps or Secrets MUST consists of alphanumeric characters, `-`, `_` or `.`
+    - The keys in ConfigMaps or Secrets are converted to the following strings, which MUST not exceed 63 characters
+        - Keys in ConfigMaps: `<instance name>-<KEY>-configmap`
+        - Keys in Secrets: `<instance name>-<KEY>-secret`
+
+    Refer to the Kubernetes documentations ([[1]](https://kubernetes.io/docs/reference/kubernetes-api/config-and-storage-resources/config-map-v1/), [[2]](https://kubernetes.io/docs/reference/kubernetes-api/config-and-storage-resources/secret-v1/), [[3]](https://kubernetes.io/docs/reference/kubernetes-api/config-and-storage-resources/volume/), [[4]](https://kubernetes.io/docs/concepts/overview/working-with-objects/names/)) for more information about character types and length restrictions.

docs/user-guide/database-configuration.md

@@ -1,12 +1,12 @@
-### Database Configuration
+# Database Configuration
 
-#### PostgreSQL Version
+## PostgreSQL Version
 
-The default PostgreSQL version for the version of Galaxy bundled with the latest version of the galaxy-operator is PostgreSQL 15. You can find this default for a given version by at the default value for [supported_pg_version](https://github.com/ansible/galaxy-operator/tree/main/roles/installer/vars/main.yml#L7).
+The default PostgreSQL version for the version of Galaxy bundled with the latest version of the galaxy-operator is PostgreSQL 15. You can find this default for a given version by at the default value for [supported_pg_version](https://github.com/ansible/galaxy-operator/blob/main/roles/common/vars/main.yml#L3).
 
 We only have coverage for the default version of PostgreSQL. Newer versions of PostgreSQL will likely work, but should only be configured as an external database. If your database is managed by the galaxy-operator (default if you don't specify a `postgres_configuration_secret`), then you should not override the default version as this may cause issues when the operator tries to upgrade your postgresql pod.
 
-#### External PostgreSQL Service
+## External PostgreSQL Service
 
 To configure Galaxy to use an external database, the Custom Resource needs to know about the connection details. To do this, create a k8s secret with those connection details and specify the name of the secret as `postgres_configuration_secret` at the CR spec level.
 
@@ -46,7 +46,7 @@ spec:
   postgres_configuration_secret: <name-of-your-secret>
 ```
 
-#### Managed PostgreSQL Service
+## Managed PostgreSQL Service
 
 If you don't have access to an external PostgreSQL service, the galaxy-operator can deploy one for you along side the Galaxy instance itself.
 
@@ -86,15 +86,15 @@ spec:
 
 **Note**: If `postgres_storage_class` is not defined, PostgreSQL will store it's data on a volume using the default storage class for your cluster.
 
-#### Note about overriding the postgres image
+## Note about overriding the postgres image
 
 We recommend you use the default image sclorg image. If you are coming from a deployment using the old postgres image from dockerhub (postgres:13), upgrading from galaxy-operator version 2024.02.29 to a newer version will handle migrating your data to the new postgresql image (postgresql-15-c9s).
 
 You can no longer configure a custom `postgres_data_path` because it is hardcoded in the quay.io/sclorg/postgresql-15-c9s image.
 
 If you override the postgres image to use a custom postgres image like postgres:15 for example, the default data directory path may be different. These images cannot be used interchangeably.
 
-#### Initialize Postgres data volume
+## Initialize Postgres data volume
 
 When using a hostPath backed PVC and some other storage classes like longhorn storagfe, the postgres data directory needs to be accessible by the user in the postgres pod (UID 26).
 

mkdocs.yaml

@@ -42,11 +42,16 @@ nav:
       - Getting Started:
         - Quickstart: quickstart.md
         - Build: build.md
+      - User Guide:
+        - Database Configuration: user-guide/database-configuration.md
+      - Advanced Configuration:
+        - Extra Settings: user-guide/advanced-configuration/extra-settings.md
   - Containers: container.md
   - Custom Resources:
       - Galaxy:
         - API: roles/galaxy-api.md
         - Common: roles/galaxy-common.md
+        - Config: roles/galaxy-config.md
         - Content: roles/galaxy-content.md
         - Route: roles/galaxy-route.md
         - Worker: roles/galaxy-worker.md

roles/galaxy-api/templates/galaxy-api.deployment.yaml.j2

@@ -68,6 +68,7 @@ spec:
             items:
               - path: settings.py
                 key: settings.py
+        {{ lookup("template", "../galaxy-config/templates/common/volumes/extra_settings_files.yaml.j2")  | indent(width=8) | trim }}
         - name: {{ ansible_operator_meta.name }}-db-fields-encryption
           secret:
             secretName: {{ db_fields_encryption_secret }}
@@ -203,6 +204,7 @@ spec:
               mountPath: "/etc/pulp/settings.py"
               subPath: settings.py
               readOnly: true
+            {{ lookup("template", "../galaxy-config/templates/common/volume_mounts/extra_settings_files.yaml.j2")  | indent(width=12) | trim }}
             - name: {{ ansible_operator_meta.name }}-db-fields-encryption
               mountPath: "/etc/pulp/keys/database_fields.symmetric.key"
               subPath: database_fields.symmetric.key
@@ -283,6 +285,7 @@ spec:
               mountPath: "/etc/pulp/settings.py"
               subPath: settings.py
               readOnly: true
+            {{ lookup("template", "../galaxy-config/templates/common/volume_mounts/extra_settings_files.yaml.j2")  | indent(width=12) | trim }}
             - name: {{ ansible_operator_meta.name }}-db-fields-encryption
               mountPath: "/etc/pulp/keys/database_fields.symmetric.key"
               subPath: database_fields.symmetric.key
@@ -332,6 +335,7 @@ spec:
               mountPath: "/etc/pulp/settings.py"
               subPath: settings.py
               readOnly: true
+            {{ lookup("template", "../galaxy-config/templates/common/volume_mounts/extra_settings_files.yaml.j2")  | indent(width=12) | trim }}
             - name: {{ ansible_operator_meta.name }}-db-fields-encryption
               mountPath: "/etc/pulp/keys/database_fields.symmetric.key"
               subPath: database_fields.symmetric.key

roles/galaxy-config/README.md

@@ -0,0 +1,34 @@
+Galaxy Config
+============
+
+A role to setup configuration files for Galaxy, yielding the following objects:
+
+* Secret
+
+Role Variables
+--------------
+
+* `extra_settings_files`: A dictionary to specify extra settings files. Refer to [the Advanced Configuration in the User Guide](../user-guide/advanced-configuration/extra-settings.md) for more information. Default: `{}`
+
+Requirements
+------------
+
+Requires the `kubernetes` Python library to interact with Kubernetes: `pip install kubernetes`.
+
+Dependencies
+------------
+
+collections:
+
+  - kubernetes.core
+  - operator_sdk.util
+
+License
+-------
+
+GPLv2+
+
+Author Information
+------------------
+
+[Galaxy-Operator Team](https://github.com/ansible/galaxy-operator)

roles/galaxy-config/defaults/main.yml

@@ -1 +1,2 @@
 ---
+extra_settings_files: {}

roles/galaxy-config/templates/common/volume_mounts/extra_settings_files.yaml.j2

@@ -0,0 +1,16 @@
+{% if extra_settings_files.configmaps is defined and extra_settings_files.configmaps | length %}
+{% for configmap in extra_settings_files.configmaps %}
+- name: {{ ansible_operator_meta.name }}-{{ configmap.key | replace('_', '-') | replace('.', '-') | lower }}-configmap
+  mountPath: "/etc/pulp/conf.d/{{ configmap.key }}"
+  subPath: {{ configmap.key }}
+  readOnly: true
+{% endfor %}
+{% endif %}
+{% if extra_settings_files.secrets is defined and extra_settings_files.secrets | length %}
+{% for secret in extra_settings_files.secrets %}
+- name: {{ ansible_operator_meta.name }}-{{ secret.key | replace('_', '-') | replace('.', '-') | lower }}-secret
+  mountPath: "/etc/pulp/conf.d/{{ secret.key }}"
+  subPath: {{ secret.key }}
+  readOnly: true
+{% endfor %}
+{% endif %}

roles/galaxy-config/templates/common/volumes/extra_settings_files.yaml.j2

@@ -0,0 +1,20 @@
+{% if extra_settings_files.configmaps is defined and extra_settings_files.configmaps | length %}
+{% for configmap in extra_settings_files.configmaps %}
+- name: {{ ansible_operator_meta.name }}-{{ configmap.key | replace('_', '-') | replace('.', '-') | lower }}-configmap
+  configMap:
+    name: {{ configmap.name }}
+    items:
+      - key: {{ configmap.key }}
+        path: {{ configmap.key }}
+{% endfor %}
+{% endif %}
+{% if extra_settings_files.secrets is defined and extra_settings_files.secrets | length %}
+{% for secret in extra_settings_files.secrets %}
+- name: {{ ansible_operator_meta.name }}-{{ secret.key | replace('_', '-') | replace('.', '-') | lower }}-secret
+  secret:
+    secretName: {{ secret.name }}
+    items:
+      - key: {{ secret.key }}
+        path: {{ secret.key }}
+{% endfor %}
+{% endif %}

roles/galaxy-config/templates/galaxy-server.secret.yaml.j2

@@ -6,6 +6,20 @@ metadata:
   namespace: "{{ ansible_operator_meta.namespace }}"
 stringData:
   settings.py: |
-    {% for setting, value in pulp_combined_settings.items() %}
+    import os
+    import traceback
+
+    from split_settings.tools import optional, include
+
+{% for setting, value in pulp_combined_settings.items() %}
     {{ setting | upper }} = {{ value | string | capitalize if value | string in ["True", "False"] else value | to_json }}
-    {% endfor %}
+{% endfor %}
+
+    # Attempt to load settings from /etc/pulp/conf.d/*.py.
+    settings_dir = os.environ.get('PULP_SETTINGS_DIR', '/etc/pulp/conf.d/')
+    settings_files = os.path.join(settings_dir, '*.py')
+    try:
+        include(optional(settings_files), scope=locals())
+    except ImportError:
+        traceback.print_exc()
+        sys.exit(1)

roles/galaxy-content/templates/galaxy-content.deployment.yaml.j2

@@ -81,6 +81,7 @@ spec:
             items:
               - path: settings.py
                 key: settings.py
+        {{ lookup("template", "../galaxy-config/templates/common/volumes/extra_settings_files.yaml.j2")  | indent(width=8) | trim }}
         - name: {{ ansible_operator_meta.name }}-db-fields-encryption
           secret:
             secretName: {{ db_fields_encryption_secret }}
@@ -148,6 +149,7 @@ spec:
               mountPath: "/etc/pulp/settings.py"
               subPath: settings.py
               readOnly: true
+            {{ lookup("template", "../galaxy-config/templates/common/volume_mounts/extra_settings_files.yaml.j2")  | indent(width=12) | trim }}
             - name: {{ ansible_operator_meta.name }}-db-fields-encryption
               mountPath: "/etc/pulp/keys/database_fields.symmetric.key"
               subPath: database_fields.symmetric.key
@@ -227,6 +229,7 @@ spec:
               mountPath: "/etc/pulp/settings.py"
               subPath: settings.py
               readOnly: true
+            {{ lookup("template", "../galaxy-config/templates/common/volume_mounts/extra_settings_files.yaml.j2")  | indent(width=12) | trim }}
             - name: {{ ansible_operator_meta.name }}-db-fields-encryption
               mountPath: "/etc/pulp/keys/database_fields.symmetric.key"
               subPath: database_fields.symmetric.key

roles/galaxy-worker/templates/galaxy-worker.deployment.yaml.j2

@@ -69,6 +69,7 @@ spec:
             items:
               - path: settings.py
                 key: settings.py
+        {{ lookup("template", "../galaxy-config/templates/common/volumes/extra_settings_files.yaml.j2")  | indent(width=8) | trim }}
         - name: {{ ansible_operator_meta.name }}-db-fields-encryption
           secret:
             secretName: {{ db_fields_encryption_secret }}
@@ -153,6 +154,7 @@ spec:
               mountPath: "/etc/pulp/settings.py"
               subPath: settings.py
               readOnly: true
+            {{ lookup("template", "../galaxy-config/templates/common/volume_mounts/extra_settings_files.yaml.j2")  | indent(width=12) | trim }}
             - name: {{ ansible_operator_meta.name }}-db-fields-encryption
               mountPath: "/etc/pulp/keys/database_fields.symmetric.key"
               subPath: database_fields.symmetric.key
@@ -220,6 +222,7 @@ spec:
               mountPath: "/etc/pulp/settings.py"
               subPath: settings.py
               readOnly: true
+            {{ lookup("template", "../galaxy-config/templates/common/volume_mounts/extra_settings_files.yaml.j2")  | indent(width=12) | trim }}
             - name: {{ ansible_operator_meta.name }}-db-fields-encryption
               mountPath: "/etc/pulp/keys/database_fields.symmetric.key"
               subPath: database_fields.symmetric.key