diff --git a/docker-jans-auth-server/Dockerfile b/docker-jans-auth-server/Dockerfile index 90ca4bea757..3657952071a 100644 --- a/docker-jans-auth-server/Dockerfile +++ b/docker-jans-auth-server/Dockerfile @@ -195,7 +195,7 @@ ENV CN_SECRET_ADAPTER=vault \ # =============== ENV CN_PERSISTENCE_TYPE=ldap \ - CN_PERSISTENCE_LDAP_MAPPING=default \ + CN_HYBRID_MAPPING="{}" \ CN_LDAP_URL=localhost:1636 \ CN_LDAP_USE_SSL=true \ CN_COUCHBASE_URL=localhost \ diff --git a/docker-jans-auth-server/README.md b/docker-jans-auth-server/README.md index f3db6754d9f..36393a4df6f 100644 --- a/docker-jans-auth-server/README.md +++ b/docker-jans-auth-server/README.md @@ -47,13 +47,13 @@ The following environment variables are supported by the container: - `CN_MAX_RAM_PERCENTAGE`: Value passed to Java option `-XX:MaxRAMPercentage`. - `CN_DEBUG_PORT`: port of remote debugging (if omitted, remote debugging will be disabled). - `CN_PERSISTENCE_TYPE`: Persistence backend being used (one of `ldap`, `couchbase`, or `hybrid`; default to `ldap`). -- `CN_PERSISTENCE_LDAP_MAPPING`: Specify data that should be saved in LDAP (one of `default`, `user`, `cache`, `site`, `token`, or `session`; default to `default`). Note this environment only takes effect when `CN_PERSISTENCE_TYPE` is set to `hybrid`. -- `CN_LDAP_URL`: Address and port of LDAP server (default to `localhost:1636`); required if `CN_PERSISTENCE_TYPE` is set to `ldap` or `hybrid`. +- `CN_HYBRID_MAPPING`: Specify data mapping for each persistence (default to `"{}"`). Note this environment only takes effect when `CN_PERSISTENCE_TYPE` is set to `hybrid`. See [hybrid mapping](#hybrid-mapping) section for details. +- `CN_LDAP_URL`: Address and port of LDAP server (default to `localhost:1636`). - `CN_LDAP_USE_SSL`: Whether to use SSL connection to LDAP server (default to `true`). -- `CN_COUCHBASE_URL`: Address of Couchbase server (default to `localhost`); required if `CN_PERSISTENCE_TYPE` is set to `couchbase` or `hybrid`. -- `CN_COUCHBASE_USER`: Username of Couchbase server (default to `admin`); required if `CN_PERSISTENCE_TYPE` is set to `couchbase` or `hybrid`. -- `CN_COUCHBASE_CERT_FILE`: Couchbase root certificate location (default to `/etc/certs/couchbase.crt`); required if `CN_PERSISTENCE_TYPE` is set to `couchbase` or `hybrid`. -- `CN_COUCHBASE_PASSWORD_FILE`: Path to file contains Couchbase password (default to `/etc/jans/conf/couchbase_password`); required if `CN_PERSISTENCE_TYPE` is set to `couchbase` or `hybrid`. +- `CN_COUCHBASE_URL`: Address of Couchbase server (default to `localhost`). +- `CN_COUCHBASE_USER`: Username of Couchbase server (default to `admin`). +- `CN_COUCHBASE_CERT_FILE`: Couchbase root certificate location (default to `/etc/certs/couchbase.crt`). +- `CN_COUCHBASE_PASSWORD_FILE`: Path to file contains Couchbase password (default to `/etc/jans/conf/couchbase_password`). - `CN_COUCHBASE_CONN_TIMEOUT`: Connect timeout used when a bucket is opened (default to `10000` milliseconds). - `CN_COUCHBASE_CONN_MAX_WAIT`: Maximum time to wait before retrying connection (default to `20000` milliseconds). - `CN_COUCHBASE_SCAN_CONSISTENCY`: Default scan consistency; one of `not_bounded`, `request_plus`, or `statement_plus` (default to `not_bounded`). @@ -114,3 +114,35 @@ The following key-value pairs are the defaults: "audit_log_level": "INFO" } ``` + +### Hybrid mapping + +As per v1.0.1, hybrid persistence supports all available persistence types. To configure hybrid persistence and its data mapping, follow steps below: + +1. Set `CN_PERSISTENCE_TYPE` environment variable to `hybrid` + +1. Set `CN_HYBRID_MAPPING` with the following format: + + ``` + { + "default": "", + "user": "", + "site": "", + "cache": "", + "token": "", + "session": "", + } + ``` + + Example: + + ``` + { + "default": "sql", + "user": "spanner", + "site": "ldap", + "cache": "sql", + "token": "couchbase", + "session": "spanner", + } + ``` diff --git a/docker-jans-auth-server/requirements.txt b/docker-jans-auth-server/requirements.txt index 73df51bf183..a578f57ccc0 100644 --- a/docker-jans-auth-server/requirements.txt +++ b/docker-jans-auth-server/requirements.txt @@ -1,4 +1,4 @@ # pinned to py3-grpcio version to avoid failure on native extension build grpcio==1.41.0 libcst<0.4 -git+https://github.com/JanssenProject/jans@89859286d69e7de7885bd9da9f50720c8371e797#egg=jans-pycloudlib&subdirectory=jans-pycloudlib +git+https://github.com/JanssenProject/jans@a0b7eac21ab3e405ad60e913f966476c34420c62#egg=jans-pycloudlib&subdirectory=jans-pycloudlib diff --git a/docker-jans-auth-server/scripts/bootstrap.py b/docker-jans-auth-server/scripts/bootstrap.py index d5987925d40..fd4f0725780 100644 --- a/docker-jans-auth-server/scripts/bootstrap.py +++ b/docker-jans-auth-server/scripts/bootstrap.py @@ -16,6 +16,7 @@ from jans.pycloudlib.persistence import sync_ldap_truststore from jans.pycloudlib.persistence import render_sql_properties from jans.pycloudlib.persistence import render_spanner_properties +from jans.pycloudlib.persistence.utils import PersistenceMapper from jans.pycloudlib.utils import cert_to_truststore from jans.pycloudlib.utils import get_server_certificate from jans.pycloudlib.utils import generate_keystore @@ -71,7 +72,13 @@ def main(): render_salt(manager, "/app/templates/salt.tmpl", "/etc/jans/conf/salt") render_base_properties("/app/templates/jans.properties.tmpl", "/etc/jans/conf/jans.properties") - if persistence_type in ("ldap", "hybrid"): + mapper = PersistenceMapper() + persistence_groups = mapper.groups().keys() + + if persistence_type == "hybrid": + render_hybrid_properties("/etc/jans/conf/jans-hybrid.properties") + + if "ldap" in persistence_groups: render_ldap_properties( manager, "/app/templates/jans-ldap.properties.tmpl", @@ -79,7 +86,7 @@ def main(): ) sync_ldap_truststore(manager) - if persistence_type in ("couchbase", "hybrid"): + if "couchbase" in persistence_groups: render_couchbase_properties( manager, "/app/templates/jans-couchbase.properties.tmpl", @@ -89,17 +96,14 @@ def main(): # sync_couchbase_cert(manager) sync_couchbase_truststore(manager) - if persistence_type == "hybrid": - render_hybrid_properties("/etc/jans/conf/jans-hybrid.properties") - - if persistence_type == "sql": + if "sql" in persistence_groups: render_sql_properties( manager, "/app/templates/jans-sql.properties.tmpl", "/etc/jans/conf/jans-sql.properties", ) - if persistence_type == "spanner": + if "spanner" in persistence_groups: render_spanner_properties( manager, "/app/templates/jans-spanner.properties.tmpl", diff --git a/docker-jans-auth-server/scripts/keystore_mod.py b/docker-jans-auth-server/scripts/keystore_mod.py index 126cfbd5384..e0794d917cf 100644 --- a/docker-jans-auth-server/scripts/keystore_mod.py +++ b/docker-jans-auth-server/scripts/keystore_mod.py @@ -5,6 +5,7 @@ from jans.pycloudlib.persistence.ldap import LdapClient from jans.pycloudlib.persistence.sql import SqlClient from jans.pycloudlib.persistence.spanner import SpannerClient +from jans.pycloudlib.persistence.utils import PersistenceMapper class BasePersistence: @@ -122,17 +123,8 @@ def __init__(self, manager): def modify_keystore_path(manager, path, jwks_uri): - persistence_type = os.environ.get("CN_PERSISTENCE_TYPE", "ldap") - ldap_mapping = os.environ.get("CN_PERSISTENCE_LDAP_MAPPING", "default") - - if persistence_type in ("ldap", "couchbase", "sql", "spanner"): - backend_type = persistence_type - else: - # persistence_type is hybrid - if ldap_mapping == "default": - backend_type = "ldap" - else: - backend_type = "couchbase" + mapper = PersistenceMapper() + backend_type = mapper.mapping["default"] # resolve backend backend = _backend_classes[backend_type](manager) diff --git a/docker-jans-auth-server/scripts/wait.py b/docker-jans-auth-server/scripts/wait.py index 06d0e18247a..75169617850 100644 --- a/docker-jans-auth-server/scripts/wait.py +++ b/docker-jans-auth-server/scripts/wait.py @@ -1,25 +1,19 @@ -import logging -import logging.config import os from jans.pycloudlib import get_manager from jans.pycloudlib import wait_for +from jans.pycloudlib import wait_for_persistence from jans.pycloudlib.validators import validate_persistence_type -from jans.pycloudlib.validators import validate_persistence_ldap_mapping +from jans.pycloudlib.validators import validate_persistence_hybrid_mapping from jans.pycloudlib.validators import validate_persistence_sql_dialect -from settings import LOGGING_CONFIG - -logging.config.dictConfig(LOGGING_CONFIG) -logger = logging.getLogger("wait") - def main(): persistence_type = os.environ.get("CN_PERSISTENCE_TYPE", "ldap") validate_persistence_type(persistence_type) - ldap_mapping = os.environ.get("CN_PERSISTENCE_LDAP_MAPPING", "default") - validate_persistence_ldap_mapping(persistence_type, ldap_mapping) + if persistence_type == "hybrid": + validate_persistence_hybrid_mapping() if persistence_type == "sql": sql_dialect = os.environ.get("CN_SQL_DB_DIALECT", "mysql") @@ -27,13 +21,8 @@ def main(): manager = get_manager() deps = ["config", "secret"] - - if persistence_type == "hybrid": - deps += ["ldap", "couchbase"] - else: - deps.append(persistence_type) - wait_for(manager, deps) + wait_for_persistence(manager) if __name__ == "__main__": diff --git a/docker-jans-certmanager/Dockerfile b/docker-jans-certmanager/Dockerfile index 91c50249530..319f34ab1e4 100644 --- a/docker-jans-certmanager/Dockerfile +++ b/docker-jans-certmanager/Dockerfile @@ -104,7 +104,7 @@ ENV CN_SECRET_ADAPTER=vault \ # =============== ENV CN_PERSISTENCE_TYPE=ldap \ - CN_PERSISTENCE_LDAP_MAPPING=default \ + CN_HYBRID_MAPPING="{}" \ CN_LDAP_URL=localhost:1636 \ CN_LDAP_USE_SSL=true \ CN_COUCHBASE_URL=localhost \ diff --git a/docker-jans-certmanager/README.md b/docker-jans-certmanager/README.md index 1e9da0f03a4..8e5689727fc 100644 --- a/docker-jans-certmanager/README.md +++ b/docker-jans-certmanager/README.md @@ -47,13 +47,13 @@ The following environment variables are supported by the container: - `CN_SECRET_GOOGLE_SECRET_NAME_PREFIX`: Prefix for Google Secret Manager name (default to `jans`). - `CN_SECRET_GOOGLE_SECRET_MANAGER_PASSPHRASE`: Passphrase for Google Secret Manager (default to `secret`). - `CN_PERSISTENCE_TYPE`: Persistence backend being used (one of `ldap`, `couchbase`, or `hybrid`; default to `ldap`). -- `CN_PERSISTENCE_LDAP_MAPPING`: Specify data that should be saved in LDAP (one of `default`, `user`, `cache`, `site`, or `token`; default to `default`). Note this environment only takes effect when `CN_PERSISTENCE_TYPE` is set to `hybrid`. -- `CN_LDAP_URL`: Address and port of LDAP server (default to `localhost:1636`); required if `CN_PERSISTENCE_TYPE` is set to `ldap` or `hybrid`. +- `CN_HYBRID_MAPPING`: Specify data mapping for each persistence (default to `"{}"`). Note this environment only takes effect when `CN_PERSISTENCE_TYPE` is set to `hybrid`. See [hybrid mapping](#hybrid-mapping) section for details. +- `CN_LDAP_URL`: Address and port of LDAP server (default to `localhost:1636`). - `CN_LDAP_USE_SSL`: Whether to use SSL connection to LDAP server (default to `true`). -- `CN_COUCHBASE_URL`: Address of Couchbase server (default to `localhost`); required if `CN_PERSISTENCE_TYPE` is set to `couchbase` or `hybrid`. -- `CN_COUCHBASE_USER`: Username of Couchbase server (default to `admin`); required if `CN_PERSISTENCE_TYPE` is set to `couchbase` or `hybrid`. -- `CN_COUCHBASE_CERT_FILE`: Couchbase root certificate location (default to `/etc/certs/couchbase.crt`); required if `CN_PERSISTENCE_TYPE` is set to `couchbase` or `hybrid`. -- `CN_COUCHBASE_PASSWORD_FILE`: Path to file contains Couchbase password (default to `/etc/jans/conf/couchbase_password`); required if `CN_PERSISTENCE_TYPE` is set to `couchbase` or `hybrid`. +- `CN_COUCHBASE_URL`: Address of Couchbase server (default to `localhost`). +- `CN_COUCHBASE_USER`: Username of Couchbase server (default to `admin`). +- `CN_COUCHBASE_CERT_FILE`: Couchbase root certificate location (default to `/etc/certs/couchbase.crt`). +- `CN_COUCHBASE_PASSWORD_FILE`: Path to file contains Couchbase password (default to `/etc/jans/conf/couchbase_password`). - `CN_COUCHBASE_CONN_TIMEOUT`: Connect timeout used when a bucket is opened (default to `10000` milliseconds). - `CN_COUCHBASE_CONN_MAX_WAIT`: Maximum time to wait before retrying connection (default to `20000` milliseconds). - `CN_COUCHBASE_SCAN_CONSISTENCY`: Default scan consistency; one of `not_bounded`, `request_plus`, or `statement_plus` (default to `not_bounded`). @@ -243,3 +243,35 @@ spec: args: ["patch", "auth", "--opts", "interval:48"] restartPolicy: Never ``` + +### Hybrid mapping + +As per v1.0.1, hybrid persistence supports all available persistence types. To configure hybrid persistence and its data mapping, follow steps below: + +1. Set `CN_PERSISTENCE_TYPE` environment variable to `hybrid` + +1. Set `CN_HYBRID_MAPPING` with the following format: + + ``` + { + "default": "", + "user": "", + "site": "", + "cache": "", + "token": "", + "session": "", + } + ``` + + Example: + + ``` + { + "default": "sql", + "user": "spanner", + "site": "ldap", + "cache": "sql", + "token": "couchbase", + "session": "spanner", + } + ``` diff --git a/docker-jans-certmanager/requirements.txt b/docker-jans-certmanager/requirements.txt index 2fbab4bca78..07d3dc06710 100644 --- a/docker-jans-certmanager/requirements.txt +++ b/docker-jans-certmanager/requirements.txt @@ -2,4 +2,4 @@ grpcio==1.41.0 click==6.7 libcst<0.4 -git+https://github.com/JanssenProject/jans@89859286d69e7de7885bd9da9f50720c8371e797#egg=jans-pycloudlib&subdirectory=jans-pycloudlib +git+https://github.com/JanssenProject/jans@a0b7eac21ab3e405ad60e913f966476c34420c62#egg=jans-pycloudlib&subdirectory=jans-pycloudlib diff --git a/docker-jans-certmanager/scripts/auth_handler.py b/docker-jans-certmanager/scripts/auth_handler.py index c334cc45e27..c4c944d6753 100644 --- a/docker-jans-certmanager/scripts/auth_handler.py +++ b/docker-jans-certmanager/scripts/auth_handler.py @@ -10,6 +10,7 @@ from jans.pycloudlib.persistence.ldap import LdapClient from jans.pycloudlib.persistence.sql import SqlClient from jans.pycloudlib.persistence.spanner import SpannerClient +from jans.pycloudlib.persistence.utils import PersistenceMapper from jans.pycloudlib.utils import encode_text from jans.pycloudlib.utils import exec_cmd from jans.pycloudlib.utils import generate_base64_contents @@ -113,7 +114,7 @@ def __init__(self, manager): def get_auth_config(self): bucket = os.environ.get("CN_COUCHBASE_BUCKET_PREFIX", "jans") req = self.client.exec_query( - "SELECT jansRevision, jansConfDyn, jansConfWebKeys " + "SELECT jansRevision, jansConfDyn, jansConfWebKeys " # nosec: B608 f"FROM `{bucket}` " "USE KEYS 'configuration_jans-auth'", ) @@ -187,20 +188,11 @@ class AuthHandler(BaseHandler): def __init__(self, manager, dry_run, **opts): super().__init__(manager, dry_run, **opts) - persistence_type = os.environ.get("CN_PERSISTENCE_TYPE", "ldap") - ldap_mapping = os.environ.get("CN_PERSISTENCE_LDAP_MAPPING", "default") - - if persistence_type in ("ldap", "couchbase", "sql", "spanner"): - backend_type = persistence_type - else: - # persistence_type is hybrid - if ldap_mapping == "default": - backend_type = "ldap" - else: - backend_type = "couchbase" - # resolve backend + mapper = PersistenceMapper() + backend_type = mapper.mapping["default"] self.backend = _backend_classes[backend_type](manager) + self.rotation_interval = opts.get("interval", 48) self.push_keys = as_boolean(opts.get("push-to-container", True)) self.key_strategy = opts.get("key-strategy", "OLDER") diff --git a/docker-jans-client-api/Dockerfile b/docker-jans-client-api/Dockerfile index 16742846a46..dd3d25dab85 100644 --- a/docker-jans-client-api/Dockerfile +++ b/docker-jans-client-api/Dockerfile @@ -98,7 +98,7 @@ ENV CN_SECRET_ADAPTER=vault \ # =============== ENV CN_PERSISTENCE_TYPE=ldap \ - CN_PERSISTENCE_LDAP_MAPPING=default \ + CN_HYBRID_MAPPING="{}" \ CN_LDAP_URL=localhost:1636 \ CN_LDAP_USE_SSL=true \ CN_COUCHBASE_URL=localhost \ diff --git a/docker-jans-client-api/README.md b/docker-jans-client-api/README.md index 2718735660a..f795e91008d 100644 --- a/docker-jans-client-api/README.md +++ b/docker-jans-client-api/README.md @@ -46,13 +46,13 @@ The following environment variables are supported by the container: - `CN_WAIT_SLEEP_DURATION`: Delay between startup "health checks" (default to `10` seconds). - `CN_MAX_RAM_PERCENTAGE`: Value passed to Java option `-XX:MaxRAMPercentage`. - `CN_PERSISTENCE_TYPE`: Persistence backend being used (one of `ldap`, `couchbase`, or `hybrid`; default to `ldap`). -- `CN_PERSISTENCE_LDAP_MAPPING`: Specify data that should be saved in LDAP (one of `default`, `user`, `cache`, `site`, or `token`; default to `default`). Note this environment only takes effect when `CN_PERSISTENCE_TYPE` is set to `hybrid`. -- `CN_LDAP_URL`: Address and port of LDAP server (default to `localhost:1636`); required if `CN_PERSISTENCE_TYPE` is set to `ldap` or `hybrid`. +- `CN_HYBRID_MAPPING`: Specify data mapping for each persistence (default to `"{}"`). Note this environment only takes effect when `CN_PERSISTENCE_TYPE` is set to `hybrid`. See [hybrid mapping](#hybrid-mapping) section for details. +- `CN_LDAP_URL`: Address and port of LDAP server (default to `localhost:1636`). - `CN_LDAP_USE_SSL`: Whether to use SSL connection to LDAP server (default to `true`). -- `CN_COUCHBASE_URL`: Address of Couchbase server (default to `localhost`); required if `CN_PERSISTENCE_TYPE` is set to `couchbase` or `hybrid`. -- `CN_COUCHBASE_USER`: Username of Couchbase server (default to `admin`); required if `CN_PERSISTENCE_TYPE` is set to `couchbase` or `hybrid`. -- `CN_COUCHBASE_CERT_FILE`: Couchbase root certificate location (default to `/etc/certs/couchbase.crt`); required if `CN_PERSISTENCE_TYPE` is set to `couchbase` or `hybrid`. -- `CN_COUCHBASE_PASSWORD_FILE`: Path to file contains Couchbase password (default to `/etc/jans/conf/couchbase_password`); required if `CN_PERSISTENCE_TYPE` is set to `couchbase` or `hybrid`. +- `CN_COUCHBASE_URL`: Address of Couchbase server (default to `localhost`). +- `CN_COUCHBASE_USER`: Username of Couchbase server (default to `admin`). +- `CN_COUCHBASE_CERT_FILE`: Couchbase root certificate location (default to `/etc/certs/couchbase.crt`). +- `CN_COUCHBASE_PASSWORD_FILE`: Path to file contains Couchbase password (default to `/etc/jans/conf/couchbase_password`). - `CN_COUCHBASE_CONN_TIMEOUT`: Connect timeout used when a bucket is opened (default to `10000` milliseconds). - `CN_COUCHBASE_CONN_MAX_WAIT`: Maximum time to wait before retrying connection (default to `20000` milliseconds). - `CN_COUCHBASE_SCAN_CONSISTENCY`: Default scan consistency; one of `not_bounded`, `request_plus`, or `statement_plus` (default to `not_bounded`). @@ -95,3 +95,34 @@ The following key-value pairs are the defaults: } ``` +### Hybrid mapping + +As per v1.0.1, hybrid persistence supports all available persistence types. To configure hybrid persistence and its data mapping, follow steps below: + +1. Set `CN_PERSISTENCE_TYPE` environment variable to `hybrid` + +1. Set `CN_HYBRID_MAPPING` with the following format: + + ``` + { + "default": "", + "user": "", + "site": "", + "cache": "", + "token": "", + "session": "", + } + ``` + + Example: + + ``` + { + "default": "sql", + "user": "spanner", + "site": "ldap", + "cache": "sql", + "token": "couchbase", + "session": "spanner", + } + ``` diff --git a/docker-jans-client-api/requirements.txt b/docker-jans-client-api/requirements.txt index 1aba3514ef5..072c2d07b52 100644 --- a/docker-jans-client-api/requirements.txt +++ b/docker-jans-client-api/requirements.txt @@ -2,4 +2,4 @@ grpcio==1.41.0 ruamel.yaml==0.16.10 libcst<0.4 -git+https://github.com/JanssenProject/jans@89859286d69e7de7885bd9da9f50720c8371e797#egg=jans-pycloudlib&subdirectory=jans-pycloudlib +git+https://github.com/JanssenProject/jans@a0b7eac21ab3e405ad60e913f966476c34420c62#egg=jans-pycloudlib&subdirectory=jans-pycloudlib diff --git a/docker-jans-client-api/scripts/bootstrap.py b/docker-jans-client-api/scripts/bootstrap.py index 081a493f6e5..ceb67388957 100644 --- a/docker-jans-client-api/scripts/bootstrap.py +++ b/docker-jans-client-api/scripts/bootstrap.py @@ -14,6 +14,8 @@ from jans.pycloudlib.persistence import sync_couchbase_truststore from jans.pycloudlib.persistence import sync_ldap_truststore from jans.pycloudlib.persistence import render_sql_properties +from jans.pycloudlib.persistence import render_spanner_properties +from jans.pycloudlib.persistence.utils import PersistencMapper from jans.pycloudlib.utils import cert_to_truststore from jans.pycloudlib.utils import get_random_chars from jans.pycloudlib.utils import exec_cmd @@ -186,7 +188,13 @@ def main(): render_salt(manager, "/app/templates/salt.tmpl", "/etc/jans/conf/salt") render_base_properties("/app/templates/jans.properties.tmpl", "/etc/jans/conf/jans.properties") - if persistence_type in ("ldap", "hybrid"): + mapper = PersistencMapper() + persistence_groups = mapper.groups() + + if persistence_type == "hybrid": + render_hybrid_properties("/etc/jans/conf/jans-hybrid.properties") + + if "ldap" in persistence_groups: render_ldap_properties( manager, "/app/templates/jans-ldap.properties.tmpl", @@ -194,7 +202,7 @@ def main(): ) sync_ldap_truststore(manager) - if persistence_type in ("couchbase", "hybrid"): + if "couchbase" in persistence_groups: render_couchbase_properties( manager, "/app/templates/jans-couchbase.properties.tmpl", @@ -202,16 +210,20 @@ def main(): ) sync_couchbase_truststore(manager) - if persistence_type == "hybrid": - render_hybrid_properties("/etc/jans/conf/jans-hybrid.properties") - - if persistence_type == "sql": + if "sql" in persistence_groups: render_sql_properties( manager, "/app/templates/jans-sql.properties.tmpl", "/etc/jans/conf/jans-sql.properties", ) + if "spanner" in persistence_groups: + render_spanner_properties( + manager, + "/app/templates/jans-spanner.properties.tmpl", + "/etc/jans/conf/jans-spanner.properties", + ) + get_web_cert() # if not os.path.isfile("/opt/client-api/client-api-server.yml"): diff --git a/docker-jans-client-api/scripts/wait.py b/docker-jans-client-api/scripts/wait.py index 9600c871fde..75169617850 100644 --- a/docker-jans-client-api/scripts/wait.py +++ b/docker-jans-client-api/scripts/wait.py @@ -1,25 +1,19 @@ -import logging -import logging.config import os from jans.pycloudlib import get_manager from jans.pycloudlib import wait_for +from jans.pycloudlib import wait_for_persistence from jans.pycloudlib.validators import validate_persistence_type -from jans.pycloudlib.validators import validate_persistence_ldap_mapping +from jans.pycloudlib.validators import validate_persistence_hybrid_mapping from jans.pycloudlib.validators import validate_persistence_sql_dialect -from settings import LOGGING_CONFIG - -logging.config.dictConfig(LOGGING_CONFIG) -logger = logging.getLogger("wait") - def main(): persistence_type = os.environ.get("CN_PERSISTENCE_TYPE", "ldap") validate_persistence_type(persistence_type) - ldap_mapping = os.environ.get("CN_PERSISTENCE_LDAP_MAPPING", "default") - validate_persistence_ldap_mapping(persistence_type, ldap_mapping) + if persistence_type == "hybrid": + validate_persistence_hybrid_mapping() if persistence_type == "sql": sql_dialect = os.environ.get("CN_SQL_DB_DIALECT", "mysql") @@ -27,12 +21,8 @@ def main(): manager = get_manager() deps = ["config", "secret"] - - if persistence_type == "hybrid": - deps += ["ldap", "couchbase"] - else: - deps.append(persistence_type) wait_for(manager, deps) + wait_for_persistence(manager) if __name__ == "__main__": diff --git a/docker-jans-client-api/templates/jans-spanner.properties.tmpl b/docker-jans-client-api/templates/jans-spanner.properties.tmpl new file mode 100644 index 00000000000..73db25b7d54 --- /dev/null +++ b/docker-jans-client-api/templates/jans-spanner.properties.tmpl @@ -0,0 +1,30 @@ +connection.project=%(spanner_project)s +connection.instance=%(spanner_instance)s +connection.database=%(spanner_database)s + +# Prefix connection.client-property.key=value will be coverterd to key=value +# This is reserved for future usage +#connection.client-property=clientPropertyValue + +# spanner creds or emulator +%(spanner_creds)s + +# Password hash method +password.encryption.method=SSHA-256 + +# Connection pool size +#connection.pool.max-sessions=400 +#connection.pool.min-sessions=100 +#connection.pool.inc-step=25 + +# Max time needed to create connection pool in milliseconds +connection.pool.create-max-wait-time-millis=20000 + +# Maximum allowed statement result set size +statement.limit.default-maximum-result-size=1000 + +# Maximum allowed delete statement result set size +statement.limit.maximum-result-delete-size=10000 + +binaryAttributes=objectGUID +certificateAttributes=userCertificate diff --git a/docker-jans-config-api/Dockerfile b/docker-jans-config-api/Dockerfile index 1686fc6c351..4da7ee56394 100644 --- a/docker-jans-config-api/Dockerfile +++ b/docker-jans-config-api/Dockerfile @@ -142,7 +142,7 @@ ENV CN_SECRET_ADAPTER=vault \ # =============== ENV CN_PERSISTENCE_TYPE=ldap \ - CN_PERSISTENCE_LDAP_MAPPING=default \ + CN_HYBRID_MAPPING="{}" \ CN_LDAP_URL=localhost:1636 \ CN_LDAP_USE_SSL=true \ CN_COUCHBASE_URL=localhost \ diff --git a/docker-jans-config-api/README.md b/docker-jans-config-api/README.md index 4aeb86d6feb..4fb1a99cf17 100644 --- a/docker-jans-config-api/README.md +++ b/docker-jans-config-api/README.md @@ -45,14 +45,14 @@ The following environment variables are supported by the container: - `CN_WAIT_MAX_TIME`: How long the startup "health checks" should run (default to `300` seconds). - `CN_WAIT_SLEEP_DURATION`: Delay between startup "health checks" (default to `10` seconds). - `CN_MAX_RAM_PERCENTAGE`: Value passed to Java option `-XX:MaxRAMPercentage`. -- `CN_PERSISTENCE_TYPE`: Persistence backend being used (one of `ldap`, `couchbase`, or `hybrid`; default to `ldap`). -- `CN_PERSISTENCE_LDAP_MAPPING`: Specify data that should be saved in LDAP (one of `default`, `user`, `cache`, `site`, or `token`; default to `default`). Note this environment only takes effect when `CN_PERSISTENCE_TYPE` is set to `hybrid`. -- `CN_LDAP_URL`: Address and port of LDAP server (default to `localhost:1636`); required if `CN_PERSISTENCE_TYPE` is set to `ldap` or `hybrid`. +- `CN_PERSISTENCE_TYPE`: Persistence backend being used (one of `ldap`, `couchbase`, `sql`, `spanner`, or `hybrid`; default to `ldap`). +- `CN_HYBRID_MAPPING`: Specify data mapping for each persistence (default to `"{}"`). Note this environment only takes effect when `CN_PERSISTENCE_TYPE` is set to `hybrid`. See [hybrid mapping](#hybrid-mapping) section for details. +- `CN_LDAP_URL`: Address and port of LDAP server (default to `localhost:1636`). - `CN_LDAP_USE_SSL`: Whether to use SSL connection to LDAP server (default to `true`). -- `CN_COUCHBASE_URL`: Address of Couchbase server (default to `localhost`); required if `CN_PERSISTENCE_TYPE` is set to `couchbase` or `hybrid`. -- `CN_COUCHBASE_USER`: Username of Couchbase server (default to `admin`); required if `CN_PERSISTENCE_TYPE` is set to `couchbase` or `hybrid`. -- `CN_COUCHBASE_CERT_FILE`: Couchbase root certificate location (default to `/etc/certs/couchbase.crt`); required if `CN_PERSISTENCE_TYPE` is set to `couchbase` or `hybrid`. -- `CN_COUCHBASE_PASSWORD_FILE`: Path to file contains Couchbase password (default to `/etc/jans/conf/couchbase_password`); required if `CN_PERSISTENCE_TYPE` is set to `couchbase` or `hybrid`. +- `CN_COUCHBASE_URL`: Address of Couchbase server (default to `localhost`). +- `CN_COUCHBASE_USER`: Username of Couchbase server (default to `admin`). +- `CN_COUCHBASE_CERT_FILE`: Couchbase root certificate location (default to `/etc/certs/couchbase.crt`). +- `CN_COUCHBASE_PASSWORD_FILE`: Path to file contains Couchbase password (default to `/etc/jans/conf/couchbase_password`). - `CN_COUCHBASE_CONN_TIMEOUT`: Connect timeout used when a bucket is opened (default to `10000` milliseconds). - `CN_COUCHBASE_CONN_MAX_WAIT`: Maximum time to wait before retrying connection (default to `20000` milliseconds). - `CN_COUCHBASE_SCAN_CONSISTENCY`: Default scan consistency; one of `not_bounded`, `request_plus`, or `statement_plus` (default to `not_bounded`). @@ -135,3 +135,35 @@ The following key-value pairs are the defaults: "admin_ui_audit_log_level": "INFO" } ``` + +### Hybrid mapping + +As per v1.0.1, hybrid persistence supports all available persistence types. To configure hybrid persistence and its data mapping, follow steps below: + +1. Set `CN_PERSISTENCE_TYPE` environment variable to `hybrid` + +1. Set `CN_HYBRID_MAPPING` with the following format: + + ``` + { + "default": "", + "user": "", + "site": "", + "cache": "", + "token": "", + "session": "", + } + ``` + + Example: + + ``` + { + "default": "sql", + "user": "spanner", + "site": "ldap", + "cache": "sql", + "token": "couchbase", + "session": "spanner", + } + ``` diff --git a/docker-jans-config-api/requirements.txt b/docker-jans-config-api/requirements.txt index 73df51bf183..a578f57ccc0 100644 --- a/docker-jans-config-api/requirements.txt +++ b/docker-jans-config-api/requirements.txt @@ -1,4 +1,4 @@ # pinned to py3-grpcio version to avoid failure on native extension build grpcio==1.41.0 libcst<0.4 -git+https://github.com/JanssenProject/jans@89859286d69e7de7885bd9da9f50720c8371e797#egg=jans-pycloudlib&subdirectory=jans-pycloudlib +git+https://github.com/JanssenProject/jans@a0b7eac21ab3e405ad60e913f966476c34420c62#egg=jans-pycloudlib&subdirectory=jans-pycloudlib diff --git a/docker-jans-config-api/scripts/bootstrap.py b/docker-jans-config-api/scripts/bootstrap.py index 3b6adac9ff5..11a113f20c1 100644 --- a/docker-jans-config-api/scripts/bootstrap.py +++ b/docker-jans-config-api/scripts/bootstrap.py @@ -14,6 +14,7 @@ from jans.pycloudlib.persistence import sync_ldap_truststore from jans.pycloudlib.persistence import render_sql_properties from jans.pycloudlib.persistence import render_spanner_properties +from jans.pycloudlib.persistence.utils import PersistenceMapper from jans.pycloudlib.utils import cert_to_truststore from settings import LOGGING_CONFIG @@ -31,7 +32,13 @@ def main(): render_salt(manager, "/app/templates/salt.tmpl", "/etc/jans/conf/salt") render_base_properties("/app/templates/jans.properties.tmpl", "/etc/jans/conf/jans.properties") - if persistence_type in ("ldap", "hybrid"): + mapper = PersistenceMapper() + persistence_groups = mapper.groups().keys() + + if persistence_type == "hybrid": + render_hybrid_properties("/etc/jans/conf/jans-hybrid.properties") + + if "ldap" in persistence_groups: render_ldap_properties( manager, "/app/templates/jans-ldap.properties.tmpl", @@ -39,7 +46,7 @@ def main(): ) sync_ldap_truststore(manager) - if persistence_type in ("couchbase", "hybrid"): + if "couchbase" in persistence_groups: render_couchbase_properties( manager, "/app/templates/jans-couchbase.properties.tmpl", @@ -48,17 +55,14 @@ def main(): # need to resolve whether we're using default or user-defined couchbase cert sync_couchbase_truststore(manager) - if persistence_type == "hybrid": - render_hybrid_properties("/etc/jans/conf/jans-hybrid.properties") - - if persistence_type == "sql": + if "sql" in persistence_groups: render_sql_properties( manager, "/app/templates/jans-sql.properties.tmpl", "/etc/jans/conf/jans-sql.properties", ) - if persistence_type == "spanner": + if "spanner" in persistence_groups: render_spanner_properties( manager, "/app/templates/jans-spanner.properties.tmpl", diff --git a/docker-jans-config-api/scripts/utils.py b/docker-jans-config-api/scripts/utils.py index 7ab758a6d92..23eeb16cc30 100644 --- a/docker-jans-config-api/scripts/utils.py +++ b/docker-jans-config-api/scripts/utils.py @@ -7,6 +7,7 @@ from jans.pycloudlib.persistence.sql import SqlClient from jans.pycloudlib.persistence.ldap import LdapClient from jans.pycloudlib.persistence.spanner import SpannerClient +from jans.pycloudlib.persistence.utils import PersistenceMapper class LdapPersistence: @@ -30,7 +31,7 @@ def __init__(self, manager): def get_auth_config(self): bucket = os.environ.get("CN_COUCHBASE_BUCKET_PREFIX", "jans") req = self.client.exec_query( - f"SELECT jansConfDyn FROM `{bucket}` USE KEYS 'configuration_jans-auth'" + f"SELECT jansConfDyn FROM `{bucket}` USE KEYS 'configuration_jans-auth'" # nosec: B608 ) if not req.ok: return {} @@ -85,19 +86,9 @@ def transform_url(url): def get_injected_urls(): manager = get_manager() - persistence_type = os.environ.get("CN_PERSISTENCE_TYPE", "ldap") - ldap_mapping = os.environ.get("CN_PERSISTENCE_LDAP_MAPPING", "default") - - if persistence_type in ("ldap", "couchbase", "sql", "spanner"): - backend_type = persistence_type - else: - # maybe hybrid - if ldap_mapping == "default": - backend_type = "ldap" - else: - backend_type = "couchbase" - # resolve backend + mapping = PersistenceMapper().mapping + backend_type = mapping["default"] backend = _backend_classes[backend_type](manager) auth_config = backend.get_auth_config() diff --git a/docker-jans-config-api/scripts/wait.py b/docker-jans-config-api/scripts/wait.py index 06d0e18247a..75169617850 100644 --- a/docker-jans-config-api/scripts/wait.py +++ b/docker-jans-config-api/scripts/wait.py @@ -1,25 +1,19 @@ -import logging -import logging.config import os from jans.pycloudlib import get_manager from jans.pycloudlib import wait_for +from jans.pycloudlib import wait_for_persistence from jans.pycloudlib.validators import validate_persistence_type -from jans.pycloudlib.validators import validate_persistence_ldap_mapping +from jans.pycloudlib.validators import validate_persistence_hybrid_mapping from jans.pycloudlib.validators import validate_persistence_sql_dialect -from settings import LOGGING_CONFIG - -logging.config.dictConfig(LOGGING_CONFIG) -logger = logging.getLogger("wait") - def main(): persistence_type = os.environ.get("CN_PERSISTENCE_TYPE", "ldap") validate_persistence_type(persistence_type) - ldap_mapping = os.environ.get("CN_PERSISTENCE_LDAP_MAPPING", "default") - validate_persistence_ldap_mapping(persistence_type, ldap_mapping) + if persistence_type == "hybrid": + validate_persistence_hybrid_mapping() if persistence_type == "sql": sql_dialect = os.environ.get("CN_SQL_DB_DIALECT", "mysql") @@ -27,13 +21,8 @@ def main(): manager = get_manager() deps = ["config", "secret"] - - if persistence_type == "hybrid": - deps += ["ldap", "couchbase"] - else: - deps.append(persistence_type) - wait_for(manager, deps) + wait_for_persistence(manager) if __name__ == "__main__": diff --git a/docker-jans-configurator/requirements.txt b/docker-jans-configurator/requirements.txt index ffa90fb14a8..b32ca2f82ce 100644 --- a/docker-jans-configurator/requirements.txt +++ b/docker-jans-configurator/requirements.txt @@ -4,4 +4,4 @@ click==6.7 marshmallow==3.10.0 fqdn==1.4.0 libcst<0.4 -git+https://github.com/JanssenProject/jans@89859286d69e7de7885bd9da9f50720c8371e797#egg=jans-pycloudlib&subdirectory=jans-pycloudlib +git+https://github.com/JanssenProject/jans@a0b7eac21ab3e405ad60e913f966476c34420c62#egg=jans-pycloudlib&subdirectory=jans-pycloudlib diff --git a/docker-jans-fido2/Dockerfile b/docker-jans-fido2/Dockerfile index a59d7e57cb0..3ee2d17159a 100644 --- a/docker-jans-fido2/Dockerfile +++ b/docker-jans-fido2/Dockerfile @@ -155,7 +155,7 @@ ENV CN_SECRET_ADAPTER=vault \ # =============== ENV CN_PERSISTENCE_TYPE=ldap \ - CN_PERSISTENCE_LDAP_MAPPING=default \ + CN_HYBRID_MAPPING="{}" \ CN_LDAP_URL=localhost:1636 \ CN_LDAP_USE_SSL=true \ CN_COUCHBASE_URL=localhost \ diff --git a/docker-jans-fido2/README.md b/docker-jans-fido2/README.md index 1885efd91b0..c389000fc8a 100644 --- a/docker-jans-fido2/README.md +++ b/docker-jans-fido2/README.md @@ -46,13 +46,13 @@ The following environment variables are supported by the container: - `CN_WAIT_SLEEP_DURATION`: Delay between startup "health checks" (default to `10` seconds). - `CN_MAX_RAM_PERCENTAGE`: Value passed to Java option `-XX:MaxRAMPercentage`. - `CN_PERSISTENCE_TYPE`: Persistence backend being used (one of `ldap`, `couchbase`, or `hybrid`; default to `ldap`). -- `CN_PERSISTENCE_LDAP_MAPPING`: Specify data that should be saved in LDAP (one of `default`, `user`, `cache`, `site`, or `token`; default to `default`). Note this environment only takes effect when `CN_PERSISTENCE_TYPE` is set to `hybrid`. -- `CN_LDAP_URL`: Address and port of LDAP server (default to `localhost:1636`); required if `CN_PERSISTENCE_TYPE` is set to `ldap` or `hybrid`. +- `CN_HYBRID_MAPPING`: Specify data mapping for each persistence (default to `"{}"`). Note this environment only takes effect when `CN_PERSISTENCE_TYPE` is set to `hybrid`. See [hybrid mapping](#hybrid-mapping) section for details. +- `CN_LDAP_URL`: Address and port of LDAP server (default to `localhost:1636`). - `CN_LDAP_USE_SSL`: Whether to use SSL connection to LDAP server (default to `true`). -- `CN_COUCHBASE_URL`: Address of Couchbase server (default to `localhost`); required if `CN_PERSISTENCE_TYPE` is set to `couchbase` or `hybrid`. -- `CN_COUCHBASE_USER`: Username of Couchbase server (default to `admin`); required if `CN_PERSISTENCE_TYPE` is set to `couchbase` or `hybrid`. -- `CN_COUCHBASE_CERT_FILE`: Couchbase root certificate location (default to `/etc/certs/couchbase.crt`); required if `CN_PERSISTENCE_TYPE` is set to `couchbase` or `hybrid`. -- `CN_COUCHBASE_PASSWORD_FILE`: Path to file contains Couchbase password (default to `/etc/jans/conf/couchbase_password`); required if `CN_PERSISTENCE_TYPE` is set to `couchbase` or `hybrid`. +- `CN_COUCHBASE_URL`: Address of Couchbase server (default to `localhost`). +- `CN_COUCHBASE_USER`: Username of Couchbase server (default to `admin`). +- `CN_COUCHBASE_CERT_FILE`: Couchbase root certificate location (default to `/etc/certs/couchbase.crt`). +- `CN_COUCHBASE_PASSWORD_FILE`: Path to file contains Couchbase password (default to `/etc/jans/conf/couchbase_password`). - `CN_COUCHBASE_CONN_TIMEOUT`: Connect timeout used when a bucket is opened (default to `10000` milliseconds). - `CN_COUCHBASE_CONN_MAX_WAIT`: Maximum time to wait before retrying connection (default to `20000` milliseconds). - `CN_COUCHBASE_SCAN_CONSISTENCY`: Default scan consistency; one of `not_bounded`, `request_plus`, or `statement_plus` (default to `not_bounded`). @@ -93,3 +93,35 @@ The following key-value pairs are the defaults: "persistence_log_level": "INFO" } ``` + +### Hybrid mapping + +As per v1.0.1, hybrid persistence supports all available persistence types. To configure hybrid persistence and its data mapping, follow steps below: + +1. Set `CN_PERSISTENCE_TYPE` environment variable to `hybrid` + +1. Set `CN_HYBRID_MAPPING` with the following format: + + ``` + { + "default": "", + "user": "", + "site": "", + "cache": "", + "token": "", + "session": "", + } + ``` + + Example: + + ``` + { + "default": "sql", + "user": "spanner", + "site": "ldap", + "cache": "sql", + "token": "couchbase", + "session": "spanner", + } + ``` diff --git a/docker-jans-fido2/requirements.txt b/docker-jans-fido2/requirements.txt index 73df51bf183..a578f57ccc0 100644 --- a/docker-jans-fido2/requirements.txt +++ b/docker-jans-fido2/requirements.txt @@ -1,4 +1,4 @@ # pinned to py3-grpcio version to avoid failure on native extension build grpcio==1.41.0 libcst<0.4 -git+https://github.com/JanssenProject/jans@89859286d69e7de7885bd9da9f50720c8371e797#egg=jans-pycloudlib&subdirectory=jans-pycloudlib +git+https://github.com/JanssenProject/jans@a0b7eac21ab3e405ad60e913f966476c34420c62#egg=jans-pycloudlib&subdirectory=jans-pycloudlib diff --git a/docker-jans-fido2/scripts/bootstrap.py b/docker-jans-fido2/scripts/bootstrap.py index 390796ec2d3..60f5b3acee7 100644 --- a/docker-jans-fido2/scripts/bootstrap.py +++ b/docker-jans-fido2/scripts/bootstrap.py @@ -14,6 +14,7 @@ from jans.pycloudlib.persistence import sync_ldap_truststore from jans.pycloudlib.persistence import render_sql_properties from jans.pycloudlib.persistence import render_spanner_properties +from jans.pycloudlib.persistence.utils import PersistenceMapper from jans.pycloudlib.utils import cert_to_truststore from settings import LOGGING_CONFIG @@ -64,7 +65,13 @@ def main(): render_salt(manager, "/app/templates/salt.tmpl", "/etc/jans/conf/salt") render_base_properties("/app/templates/jans.properties.tmpl", "/etc/jans/conf/jans.properties") - if persistence_type in ("ldap", "hybrid"): + mapper = PersistenceMapper() + persistence_groups = mapper.groups() + + if persistence_type == "hybrid": + render_hybrid_properties("/etc/jans/conf/jans-hybrid.properties") + + if "ldap" in persistence_groups: render_ldap_properties( manager, "/app/templates/jans-ldap.properties.tmpl", @@ -72,7 +79,7 @@ def main(): ) sync_ldap_truststore(manager) - if persistence_type in ("couchbase", "hybrid"): + if "couchbase" in persistence_groups: render_couchbase_properties( manager, "/app/templates/jans-couchbase.properties.tmpl", @@ -80,17 +87,14 @@ def main(): ) sync_couchbase_truststore(manager) - if persistence_type == "hybrid": - render_hybrid_properties("/etc/jans/conf/jans-hybrid.properties") - - if persistence_type == "sql": + if "sql" in persistence_groups: render_sql_properties( manager, "/app/templates/jans-sql.properties.tmpl", "/etc/jans/conf/jans-sql.properties", ) - if persistence_type == "spanner": + if "spanner" in persistence_groups: render_spanner_properties( manager, "/app/templates/jans-spanner.properties.tmpl", diff --git a/docker-jans-fido2/scripts/wait.py b/docker-jans-fido2/scripts/wait.py index 06d0e18247a..75169617850 100644 --- a/docker-jans-fido2/scripts/wait.py +++ b/docker-jans-fido2/scripts/wait.py @@ -1,25 +1,19 @@ -import logging -import logging.config import os from jans.pycloudlib import get_manager from jans.pycloudlib import wait_for +from jans.pycloudlib import wait_for_persistence from jans.pycloudlib.validators import validate_persistence_type -from jans.pycloudlib.validators import validate_persistence_ldap_mapping +from jans.pycloudlib.validators import validate_persistence_hybrid_mapping from jans.pycloudlib.validators import validate_persistence_sql_dialect -from settings import LOGGING_CONFIG - -logging.config.dictConfig(LOGGING_CONFIG) -logger = logging.getLogger("wait") - def main(): persistence_type = os.environ.get("CN_PERSISTENCE_TYPE", "ldap") validate_persistence_type(persistence_type) - ldap_mapping = os.environ.get("CN_PERSISTENCE_LDAP_MAPPING", "default") - validate_persistence_ldap_mapping(persistence_type, ldap_mapping) + if persistence_type == "hybrid": + validate_persistence_hybrid_mapping() if persistence_type == "sql": sql_dialect = os.environ.get("CN_SQL_DB_DIALECT", "mysql") @@ -27,13 +21,8 @@ def main(): manager = get_manager() deps = ["config", "secret"] - - if persistence_type == "hybrid": - deps += ["ldap", "couchbase"] - else: - deps.append(persistence_type) - wait_for(manager, deps) + wait_for_persistence(manager) if __name__ == "__main__": diff --git a/docker-jans-persistence-loader/Dockerfile b/docker-jans-persistence-loader/Dockerfile index 6e8877f3c60..314cc4a2379 100644 --- a/docker-jans-persistence-loader/Dockerfile +++ b/docker-jans-persistence-loader/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.15.4 +FROM alpine:3.16.0 # =============== # Alpine packages @@ -128,7 +128,7 @@ ENV CN_SECRET_ADAPTER=vault \ # =============== ENV CN_PERSISTENCE_TYPE=couchbase \ - CN_PERSISTENCE_LDAP_MAPPING=default \ + CN_HYBRID_MAPPING="{}" \ CN_COUCHBASE_URL=localhost \ CN_COUCHBASE_USER=admin \ CN_COUCHBASE_CERT_FILE=/etc/certs/couchbase.crt \ diff --git a/docker-jans-persistence-loader/README.md b/docker-jans-persistence-loader/README.md index 0bf82aa1eb6..7dcf9e2ff0d 100644 --- a/docker-jans-persistence-loader/README.md +++ b/docker-jans-persistence-loader/README.md @@ -52,16 +52,16 @@ The following environment variables are supported by the container: - `CN_REDIS_TYPE`: Redis service type, either `STANDALONE` or `CLUSTER` (optional; default to `STANDALONE`). - `CN_MEMCACHED_URL`: URL of Memcache server, format is host:port (optional; default to `localhost:11211`). - `CN_PERSISTENCE_TYPE`: Persistence backend being used (one of `ldap`, `couchbase`, or `hybrid`; default to `ldap`). -- `CN_PERSISTENCE_LDAP_MAPPING`: Specify data that should be saved in LDAP (one of `default`, `user`, `cache`, `site`, `token`, or `session`; default to `default`). Note this environment only takes effect when `CN_PERSISTENCE_TYPE` is set to `hybrid`. +- `CN_HYBRID_MAPPING`: Specify data mapping for each persistence (default to `"{}"`). Note this environment only takes effect when `CN_PERSISTENCE_TYPE` is set to `hybrid`. See [hybrid mapping](#hybrid-mapping) section for details. - `CN_PERSISTENCE_SKIP_INITIALIZED`: skip initialization if backend already initialized (default to `false`). -- `CN_LDAP_URL`: Address and port of LDAP server (default to `localhost:1636`); required if `CN_PERSISTENCE_TYPE` is set to `ldap` or `hybrid`. +- `CN_LDAP_URL`: Address and port of LDAP server (default to `localhost:1636`). - `CN_LDAP_USE_SSL`: Whether to use SSL connection to LDAP server (default to `true`). -- `CN_COUCHBASE_URL`: Address of Couchbase server (default to `localhost`); required if `CN_PERSISTENCE_TYPE` is set to `couchbase` or `hybrid`. -- `CN_COUCHBASE_USER`: Username of Couchbase server (default to `admin`); required if `CN_PERSISTENCE_TYPE` is set to `couchbase` or `hybrid`. -- `CN_COUCHBASE_SUPERUSER`: Superuser of Couchbase server (default to empty-string); required if `CN_PERSISTENCE_TYPE` is set to `couchbase` or `hybrid`. Fallback to `CN_COUCHBASE_USER`. -- `CN_COUCHBASE_CERT_FILE`: Couchbase root certificate location (default to `/etc/certs/couchbase.crt`); required if `CN_PERSISTENCE_TYPE` is set to `couchbase` or `hybrid`. -- `CN_COUCHBASE_PASSWORD_FILE`: Path to file contains Couchbase password (default to `/etc/jans/conf/couchbase_password`); required if `CN_PERSISTENCE_TYPE` is set to `couchbase` or `hybrid`. -- `CN_COUCHBASE_SUPERUSER_PASSWORD_FILE`: Path to file contains Couchbase superuser password (default to `/etc/jans/conf/couchbase_superuser_password`); required if `CN_PERSISTENCE_TYPE` is set to `couchbase` or `hybrid`. +- `CN_COUCHBASE_URL`: Address of Couchbase server (default to `localhost`). +- `CN_COUCHBASE_USER`: Username of Couchbase server (default to `admin`). +- `CN_COUCHBASE_SUPERUSER`: Superuser of Couchbase server (default to empty-string). +- `CN_COUCHBASE_CERT_FILE`: Couchbase root certificate location (default to `/etc/certs/couchbase.crt`). +- `CN_COUCHBASE_PASSWORD_FILE`: Path to file contains Couchbase password (default to `/etc/jans/conf/couchbase_password`). +- `CN_COUCHBASE_SUPERUSER_PASSWORD_FILE`: Path to file contains Couchbase superuser password (default to `/etc/jans/conf/couchbase_superuser_password`). - `CN_DOCUMENT_STORE_TYPE`: Document store type (one of `LOCAL` or `JCA`; default to `LOCAL`). - `CN_JACKRABBIT_URL`: URL to remote repository (default to `http://localhost:8080`). - `CN_JACKRABBIT_ADMIN_ID_FILE`: Absolute path to file contains ID for admin user (default to `/etc/jans/conf/jackrabbit_admin_id`). @@ -73,143 +73,34 @@ The following environment variables are supported by the container: - `CN_AUTH_SERVER_URL`: Base URL of Janssen Auth server, i.e. `auth-server:8080` (default to empty string). - `CN_TOKEN_SERVER_BASE_HOSTNAME`: Hostname of token server (default to empty string). -## Initializing Data +### Hybrid mapping -### LDAP +As per v1.0.1, hybrid persistence supports all available persistence types. To configure hybrid persistence and its data mapping, follow steps below: -Deploy Wren:DS container: +1. Set `CN_PERSISTENCE_TYPE` environment variable to `hybrid` -```sh -docker run -d \ - --network container:consul \ - --name ldap \ - -e CN_CONFIG_ADAPTER=consul \ - -e CN_CONFIG_CONSUL_HOST=consul \ - -e CN_SECRET_ADAPTER=vault \ - -e CN_SECRET_VAULT_HOST=vault \ - -v /path/to/opendj/config:/opt/opendj/config \ - -v /path/to/opendj/db:/opt/opendj/db \ - -v /path/to/opendj/logs:/opt/opendj/logs \ - -v /path/to/opendj/ldif:/opt/opendj/ldif \ - -v /path/to/opendj/backup:/opt/opendj/bak \ - -v /path/to/vault_role_id.txt:/etc/certs/vault_role_id \ - -v /path/to/vault_secret_id.txt:/etc/certs/vault_secret_id \ - gluufederation/opendj:4.2.1_02 -``` +1. Set `CN_HYBRID_MAPPING` with the following format: -Run the following command to initialize data and save it to LDAP: - -```sh -docker run --rm \ - --network container:consul \ - --name persistence \ - -e CN_CONFIG_ADAPTER=consul \ - -e CN_CONFIG_CONSUL_HOST=consul \ - -e CN_SECRET_ADAPTER=vault \ - -e CN_SECRET_VAULT_HOST=vault \ - -e CN_PERSISTENCE_TYPE=ldap \ - -e CN_LDAP_URL=ldap:1636 \ - -v /path/to/vault_role_id.txt:/etc/certs/vault_role_id \ - -v /path/to/vault_secret_id.txt:/etc/certs/vault_secret_id \ - janssenproject/persistence:1.0.1_dev -``` - -The process may take awhile, check the output of the `persistence` container log. - -### Couchbase - -Assuming there is Couchbase instance running hosted at `192.168.100.2` address, setup the cluster: - -1. Set the username and password of Couchbase cluster -1. Configure the instance to use Query, Data, and Index services - -Once cluster has been configured successfully, do the following steps: - -1. Pass the address of Couchbase server in `CN_COUCHBASE_URL` (omit the port) -1. Pass the Couchbase superuser in `CN_COUCHBASE_SUPERUSER` -1. Save the password into `/path/to/couchbase_superuser_password` file -1. Get the certificate root of Couchbase and save it into `/path/to/couchbase.crt` file - -Run the following command to initialize data and save it to Couchbase: - -```sh -docker run --rm \ - --network container:consul \ - --name persistence \ - -e CN_CONFIG_ADAPTER=consul \ - -e CN_CONFIG_CONSUL_HOST=consul \ - -e CN_SECRET_ADAPTER=vault \ - -e CN_SECRET_VAULT_HOST=vault \ - -e CN_PERSISTENCE_TYPE=couchbase \ - -e CN_COUCHBASE_URL=192.168.100.2 \ - -e CN_COUCHBASE_SUPERUSER=admin \ - -v /path/to/couchbase.crt:/etc/certs/couchbase.crt \ - -v /path/to/couchbase_superuser_password:/etc/jans/conf/couchbase_superuser_password \ - -v /path/to/vault_role_id.txt:/etc/certs/vault_role_id \ - -v /path/to/vault_secret_id.txt:/etc/certs/vault_secret_id \ - janssenproject/persistence:1.0.1_dev -``` - -The process may take awhile, check the output of the `persistence` container log. - -### Hybrid - -Hybrid is a mix of LDAP and Couchbase persistence backend. To initialize data for this type of persistence: - -1. Deploy LDAP container: - - ```sh - docker run -d \ - --network container:consul \ - --name ldap \ - -e CN_CONFIG_ADAPTER=consul \ - -e CN_CONFIG_CONSUL_HOST=consul \ - -e CN_SECRET_ADAPTER=vault \ - -e CN_SECRET_VAULT_HOST=vault \ - -v /path/to/opendj/config:/opt/opendj/config \ - -v /path/to/opendj/db:/opt/opendj/db \ - -v /path/to/opendj/logs:/opt/opendj/logs \ - -v /path/to/opendj/ldif:/opt/opendj/ldif \ - -v /path/to/opendj/backup:/opt/opendj/bak \ - -v /path/to/vault_role_id.txt:/etc/certs/vault_role_id \ - -v /path/to/vault_secret_id.txt:/etc/certs/vault_secret_id \ - gluufederation/opendj:4.2.1_02 + ``` + { + "default": "", + "user": "", + "site": "", + "cache": "", + "token": "", + "session": "", + } ``` -1. Prepare Couchbase cluster. - - Assuming there is Couchbase instance running hosted at `192.168.100.2` address, setup the cluster: - - 1. Set the username and password of Couchbase cluster - 1. Configure the instance to use Query, Data, and Index services - - Once cluster has been configured successfully, do the following steps: - - 1. Pass the address of Couchbase server in `CN_COUCHBASE_URL` (omit the port) - 1. Pass the Couchbase user in `CN_COUCHBASE_SUPERUSER` - 1. Save the password into `/path/to/couchbase_superuser_password` file - 1. Get the certificate root of Couchbase and save it into `/path/to/couchbase.crt` file - -1. Determine which data goes to LDAP backend by specifying it using `CN_PERSISTENCE_LDAP_MAPPING` environment variable. For example, if `user` data should be saved into LDAP, set `CN_PERSISTENCE_LDAP_MAPPING=user`. This will make other data saved into Couchbase. - -1. Run the following command to initialize data and save it to LDAP and Couchbase: + Example: - ```sh - docker run --rm \ - --network container:consul \ - --name persistence \ - -e CN_CONFIG_ADAPTER=consul \ - -e CN_CONFIG_CONSUL_HOST=consul \ - -e CN_SECRET_ADAPTER=vault \ - -e CN_SECRET_VAULT_HOST=vault \ - -e CN_PERSISTENCE_TYPE=hybrid \ - -e CN_PERSISTENCE_LDAP_MAPPING=user \ - -e CN_LDAP_URL=ldap:1636 \ - -e CN_COUCHBASE_URL=192.168.100.2 \ - -e CN_COUCHBASE_SUPERUSER=admin \ - -v /path/to/couchbase.crt:/etc/certs/couchbase.crt \ - -v /path/to/couchbase_superuser_password:/etc/jans/conf/couchbase_superuser_password \ - -v /path/to/vault_role_id.txt:/etc/certs/vault_role_id \ - -v /path/to/vault_secret_id.txt:/etc/certs/vault_secret_id \ - janssenproject/persistence:1.0.1_dev + ``` + { + "default": "sql", + "user": "spanner", + "site": "ldap", + "cache": "sql", + "token": "couchbase", + "session": "spanner", + } ``` diff --git a/docker-jans-persistence-loader/requirements.txt b/docker-jans-persistence-loader/requirements.txt index 4509ed86301..21b8bcc8f63 100644 --- a/docker-jans-persistence-loader/requirements.txt +++ b/docker-jans-persistence-loader/requirements.txt @@ -2,4 +2,4 @@ grpcio==1.41.0 libcst<0.4 ruamel.yaml==0.16.10 -git+https://github.com/JanssenProject/jans@89859286d69e7de7885bd9da9f50720c8371e797#egg=jans-pycloudlib&subdirectory=jans-pycloudlib +git+https://github.com/JanssenProject/jans@a0b7eac21ab3e405ad60e913f966476c34420c62#egg=jans-pycloudlib&subdirectory=jans-pycloudlib diff --git a/docker-jans-persistence-loader/scripts/couchbase_setup.py b/docker-jans-persistence-loader/scripts/couchbase_setup.py index acabde44a4e..07586fb3bc2 100644 --- a/docker-jans-persistence-loader/scripts/couchbase_setup.py +++ b/docker-jans-persistence-loader/scripts/couchbase_setup.py @@ -11,57 +11,43 @@ from utils import get_ldif_mappings logging.config.dictConfig(LOGGING_CONFIG) -logger = logging.getLogger("entrypoint") +logger = logging.getLogger("couchbase_setup") def get_bucket_mappings(manager): prefix = os.environ.get("CN_COUCHBASE_BUCKET_PREFIX", "jans") - bucket_mappings = { + _mappings = { "default": { "bucket": prefix, "mem_alloc": 100, - # "document_key_prefix": [], }, "user": { "bucket": f"{prefix}_user", "mem_alloc": 300, - # "document_key_prefix": ["groups_", "people_", "authorizations_"], }, "site": { "bucket": f"{prefix}_site", "mem_alloc": 100, - # "document_key_prefix": ["site_", "cache-refresh_"], }, "token": { "bucket": f"{prefix}_token", "mem_alloc": 300, - # "document_key_prefix": ["tokens_"], }, "cache": { "bucket": f"{prefix}_cache", "mem_alloc": 100, - # "document_key_prefix": ["cache_"], }, "session": { "bucket": f"{prefix}_session", "mem_alloc": 200, - # "document_key_prefix": [], }, } optional_scopes = json.loads(manager.config.get("optional_scopes", "[]")) - ldif_mappings = get_ldif_mappings(optional_scopes) - - for name, files in ldif_mappings.items(): - bucket_mappings[name]["files"] = files - - persistence_type = os.environ.get("CN_PERSISTENCE_TYPE", "ldap") - ldap_mapping = os.environ.get("CN_PERSISTENCE_LDAP_MAPPING", "default") - if persistence_type == "hybrid": - bucket_mappings = { - name: mapping for name, mapping in bucket_mappings.items() - if name != ldap_mapping - } + bucket_mappings = { + mapping: {"files": files} | _mappings[mapping] + for mapping, files in get_ldif_mappings("couchbase", optional_scopes).items() + } return bucket_mappings @@ -89,18 +75,16 @@ def create_buckets(self, bucket_mappings, bucket_type="couchbase"): if total_mem < min_mem: logger.warning("Available quota on couchbase node is less than {} MB".format(min_mem)) - persistence_type = os.environ.get("CN_PERSISTENCE_TYPE", "ldap") - ldap_mapping = os.environ.get("CN_PERSISTENCE_LDAP_MAPPING", "default") - - # always create `jans` bucket even when `default` mapping stored in LDAP - if persistence_type == "hybrid" and ldap_mapping == "default": + # always create `jans` bucket even when `default` mapping stored in another persistence + if "default" not in bucket_mappings: memsize = 100 + bucket = os.environ.get("CN_COUCHBASE_BUCKET_PREFIX", "jans") + + logger.info(f"Creating bucket {bucket} with type {bucket_type} and RAM size {memsize}") - logger.info("Creating bucket {0} with type {1} and RAM size {2}".format("jans", bucket_type, memsize)) - prefix = os.environ.get("CN_COUCHBASE_BUCKET_PREFIX", "jans") - req = self.client.add_bucket(prefix, memsize, bucket_type) + req = self.client.add_bucket(bucket, memsize, bucket_type) if not req.ok: - logger.warning("Failed to create bucket {}; reason={}".format("jans", req.text)) + logger.warning(f"Failed to create bucket {bucket}; reason={req.text}") req = self.client.get_buckets() if req.ok: @@ -114,10 +98,11 @@ def create_buckets(self, bucket_mappings, bucket_type="couchbase"): memsize = int((mapping["mem_alloc"] / float(min_mem)) * total_mem) - logger.info("Creating bucket {0} with type {1} and RAM size {2}".format(mapping["bucket"], bucket_type, memsize)) + logger.info(f"Creating bucket {mapping['bucket']} with type {bucket_type} and RAM size {memsize}") + req = self.client.add_bucket(mapping["bucket"], memsize, bucket_type) if not req.ok: - logger.warning("Failed to create bucket {}; reason={}".format(mapping["bucket"], req.text)) + logger.warning(f"Failed to create bucket {mapping['bucket']}; reason={req.text}") def create_indexes(self, bucket_mappings): buckets = [mapping["bucket"] for _, mapping in bucket_mappings.items()] @@ -131,65 +116,61 @@ def create_indexes(self, bucket_mappings): if bucket not in indexes: continue - query_file = "/app/tmp/index_{}.n1ql".format(bucket) - logger.info("Running Couchbase index creation for {} bucket (if not exist)".format(bucket)) - with open(query_file, "w") as f: - index_list = indexes.get(bucket, {}) - index_names = [] - - for index in index_list.get("attributes", []): - if '(' in ''.join(index): - attr_ = index[0] - index_name_ = index[0].replace('(', '_').replace(')', '_').replace('`', '').lower() - if index_name_.endswith('_'): - index_name_ = index_name_[:-1] - index_name = 'def_{0}_{1}'.format(bucket, index_name_) + queries = [] + index_list = indexes.get(bucket, {}) + index_names = [] + + for index in index_list.get("attributes", []): + if '(' in ''.join(index): + attr_ = index[0] + index_name_ = index[0].replace('(', '_').replace(')', '_').replace('`', '').lower() + if index_name_.endswith('_'): + index_name_ = index_name_[:-1] + index_name = 'def_{0}_{1}'.format(bucket, index_name_) + else: + attr_ = ','.join(['`{}`'.format(a) for a in index]) + index_name = "def_{0}_{1}".format(bucket, '_'.join(index)) + + queries.append( + 'CREATE INDEX %s ON `%s`(%s) USING GSI WITH {"defer_build":true,"num_replica": %s};\n' % (index_name, bucket, attr_, self.index_num_replica) + ) + + index_names.append(index_name) + + if index_names: + queries.append('BUILD INDEX ON `%s` (%s) USING GSI;\n' % (bucket, ', '.join(index_names))) + + sic = 1 + for attribs, wherec in index_list.get("static", []): + attrquoted = [] + + for a in attribs: + if '(' not in a: + attrquoted.append('`{}`'.format(a)) else: - attr_ = ','.join(['`{}`'.format(a) for a in index]) - index_name = "def_{0}_{1}".format(bucket, '_'.join(index)) - - f.write( - 'CREATE INDEX %s ON `%s`(%s) USING GSI WITH {"defer_build":true,"num_replica": %s};\n' % (index_name, bucket, attr_, self.index_num_replica) - ) - - index_names.append(index_name) - - if index_names: - f.write('BUILD INDEX ON `%s` (%s) USING GSI;\n' % (bucket, ', '.join(index_names))) - - sic = 1 - for attribs, wherec in index_list.get("static", []): - attrquoted = [] - - for a in attribs: - if '(' not in a: - attrquoted.append('`{}`'.format(a)) - else: - attrquoted.append(a) - attrquoteds = ', '.join(attrquoted) - - f.write( - 'CREATE INDEX `{0}_static_{1:02d}` ON `{0}`({2}) WHERE ({3}) WITH {{ "num_replica": {4} }}\n'.format(bucket, sic, attrquoteds, wherec, self.index_num_replica) - ) - sic += 1 - - # exec query - with open(query_file) as f: - for line in f: - query = line.strip() - if not query: + attrquoted.append(a) + attrquoteds = ', '.join(attrquoted) + + queries.append( + 'CREATE INDEX `{0}_static_{1:02d}` ON `{0}`({2}) WHERE ({3}) WITH {{ "num_replica": {4} }}\n'.format(bucket, sic, attrquoteds, wherec, self.index_num_replica) + ) + sic += 1 + + for query in queries: + query = query.strip() + if not query: + continue + + req = self.client.exec_query(query) + if not req.ok: + # the following code should be ignored + # - 4300: index already exists + error = req.json()["errors"][0] + if error["code"] in (4300,): continue - - req = self.client.exec_query(query) - if not req.ok: - # the following code should be ignored - # - 4300: index already exists - error = req.json()["errors"][0] - if error["code"] in (4300,): - continue - logger.warning(f"Failed to execute query, reason={error['msg'].strip()}") # .format(error["msg"])) + logger.warning(f"Failed to execute query, reason={error['msg'].strip()}") # .format(error["msg"])) def import_builtin_ldif(self, bucket_mappings, ctx): for _, mapping in bucket_mappings.items(): @@ -215,7 +196,11 @@ def initialize(self): time.sleep(5) ctx = prepare_template_ctx(self.manager) + + logger.info("Importing builtin LDIF files") self.import_builtin_ldif(bucket_mappings, ctx) + + logger.info("Importing custom LDIF files (if any)") self.import_custom_ldif(ctx) time.sleep(5) diff --git a/docker-jans-persistence-loader/scripts/hybrid_setup.py b/docker-jans-persistence-loader/scripts/hybrid_setup.py index 6922c67e04c..e99cd645f3e 100644 --- a/docker-jans-persistence-loader/scripts/hybrid_setup.py +++ b/docker-jans-persistence-loader/scripts/hybrid_setup.py @@ -1,12 +1,27 @@ +from jans.pycloudlib.persistence.utils import PersistenceMapper + from ldap_setup import LDAPBackend from couchbase_setup import CouchbaseBackend +from sql_setup import SQLBackend +from spanner_setup import SpannerBackend + + +_backend_classes = { + "ldap": LDAPBackend, + "couchbase": CouchbaseBackend, + "sql": SQLBackend, + "spanner": SpannerBackend, +} class HybridBackend: def __init__(self, manager): - self.ldap_backend = LDAPBackend(manager) - self.couchbase_backend = CouchbaseBackend(manager) + mapper = PersistenceMapper() + self.backends = [ + _backend_classes[type_] for type_ in mapper.groups() + ] + self.manager = manager def initialize(self): - self.ldap_backend.initialize() - self.couchbase_backend.initialize() + for backend in self.backends: + backend(self.manager).initialize() diff --git a/docker-jans-persistence-loader/scripts/ldap_setup.py b/docker-jans-persistence-loader/scripts/ldap_setup.py index 015b9306e81..4f3cea79860 100644 --- a/docker-jans-persistence-loader/scripts/ldap_setup.py +++ b/docker-jans-persistence-loader/scripts/ldap_setup.py @@ -1,6 +1,5 @@ import json import logging.config -import os import time from pathlib import Path @@ -8,13 +7,14 @@ from ldap3.core.exceptions import LDAPSocketOpenError from jans.pycloudlib.persistence.ldap import LdapClient +from jans.pycloudlib.persistence.utils import PersistenceMapper from settings import LOGGING_CONFIG from utils import prepare_template_ctx from utils import get_ldif_mappings logging.config.dictConfig(LOGGING_CONFIG) -logger = logging.getLogger("entrypoint") +logger = logging.getLogger("ldap_setup") class LDAPBackend: @@ -52,23 +52,16 @@ def check_indexes(self, mapping): time.sleep(sleep_duration) def import_builtin_ldif(self, ctx): - optional_scopes = json.loads(self.manager.config.get("optional_scopes", "[]")) - ldif_mappings = get_ldif_mappings(optional_scopes) - - # hybrid means only a subsets of ldif are needed - persistence_type = os.environ.get("CN_PERSISTENCE_TYPE", "ldap") - ldap_mapping = os.environ.get("CN_PERSISTENCE_LDAP_MAPPING", "default") - if persistence_type == "hybrid": - mapping = ldap_mapping - ldif_mappings = {mapping: ldif_mappings[mapping]} - - # these mappings require `base.ldif` - # opt_mappings = ("user", "token",) - - # `user` mapping requires `o=jans` which available in `base.ldif` - # if mapping in opt_mappings and "base.ldif" not in ldif_mappings[mapping]: - if "base.ldif" not in ldif_mappings[mapping]: - ldif_mappings[mapping].insert(0, "base.ldif") + optional_scopes = json.loads( + self.manager.config.get("optional_scopes", "[]") + ) + ldif_mappings = get_ldif_mappings("ldap", optional_scopes) + + # ensure base.ldif (contains base RDNs) is in list of ldif files + if ldif_mappings and "default" not in ldif_mappings: + # insert base.ldif into the first mapping found + mapping = next(iter(ldif_mappings)) + ldif_mappings[mapping].insert(0, "base.ldif") for mapping, files in ldif_mappings.items(): self.check_indexes(mapping) diff --git a/docker-jans-persistence-loader/scripts/settings.py b/docker-jans-persistence-loader/scripts/settings.py index cf717246383..e7ccc066049 100644 --- a/docker-jans-persistence-loader/scripts/settings.py +++ b/docker-jans-persistence-loader/scripts/settings.py @@ -22,6 +22,27 @@ "level": "INFO", "propagate": False, }, + "ldap_setup": { + "handlers": ["console"], + "level": "INFO", + "propagate": False, + }, + "sql_setup": { + "handlers": ["console"], + "level": "INFO", + "propagate": False, + }, + "spanner_setup": { + "handlers": ["console"], + "level": "INFO", + "propagate": False, + }, + "couchbase_setup": { + "handlers": ["console"], + "level": "INFO", + "propagate": False, + }, + }, # "root": { # "level": "INFO", diff --git a/docker-jans-persistence-loader/scripts/spanner_setup.py b/docker-jans-persistence-loader/scripts/spanner_setup.py index 3affe909ad7..7fbb0f41e21 100644 --- a/docker-jans-persistence-loader/scripts/spanner_setup.py +++ b/docker-jans-persistence-loader/scripts/spanner_setup.py @@ -13,7 +13,7 @@ FIELD_RE = re.compile(r"[^0-9a-zA-Z\s]+") logging.config.dictConfig(LOGGING_CONFIG) -logger = logging.getLogger("entrypoint") +logger = logging.getLogger("spanner_setup") class SpannerBackend: @@ -158,7 +158,7 @@ def create_indexes(self): def import_builtin_ldif(self, ctx): optional_scopes = json.loads(self.manager.config.get("optional_scopes", "[]")) - ldif_mappings = get_ldif_mappings(optional_scopes) + ldif_mappings = get_ldif_mappings("spanner", optional_scopes) for _, files in ldif_mappings.items(): for file_ in files: diff --git a/docker-jans-persistence-loader/scripts/sql_setup.py b/docker-jans-persistence-loader/scripts/sql_setup.py index 7f9ff23ac6d..54091e34321 100644 --- a/docker-jans-persistence-loader/scripts/sql_setup.py +++ b/docker-jans-persistence-loader/scripts/sql_setup.py @@ -16,7 +16,7 @@ FIELD_RE = re.compile(r"[^0-9a-zA-Z\s]+") logging.config.dictConfig(LOGGING_CONFIG) -logger = logging.getLogger("entrypoint") +logger = logging.getLogger("sql_setup") class SQLBackend: @@ -222,7 +222,7 @@ def create_indexes(self): def import_builtin_ldif(self, ctx): optional_scopes = json.loads(self.manager.config.get("optional_scopes", "[]")) - ldif_mappings = get_ldif_mappings(optional_scopes) + ldif_mappings = get_ldif_mappings("sql", optional_scopes) for _, files in ldif_mappings.items(): for file_ in files: diff --git a/docker-jans-persistence-loader/scripts/upgrade.py b/docker-jans-persistence-loader/scripts/upgrade.py index 2a1da4be349..3d6c076b60e 100644 --- a/docker-jans-persistence-loader/scripts/upgrade.py +++ b/docker-jans-persistence-loader/scripts/upgrade.py @@ -13,6 +13,7 @@ from jans.pycloudlib.persistence import SqlClient from jans.pycloudlib.persistence import doc_id_from_dn from jans.pycloudlib.persistence import id_from_dn +from jans.pycloudlib.persistence import PersistenceMapper from jans.pycloudlib.utils import as_boolean from settings import LOGGING_CONFIG @@ -364,21 +365,26 @@ def modify_entry(self, key, attrs=None, **kwargs): return self.client.update(table_name, key, attrs), "" +BACKEND_CLASSES = { + "sql": SQLBackend, + "couchbase": CouchbaseBackend, + "spanner": SpannerBackend, + "ldap": LDAPBackend, +} + + class Upgrade: def __init__(self, manager): self.manager = manager - persistence_type = os.environ.get("CN_PERSISTENCE_TYPE", "ldap") - if persistence_type == "sql": - backend_cls = SQLBackend - elif persistence_type == "couchbase": - backend_cls = CouchbaseBackend - elif persistence_type == "spanner": - backend_cls = SpannerBackend - else: - backend_cls = LDAPBackend + mapper = PersistenceMapper() + + backend_cls = BACKEND_CLASSES[mapper.mapping["default"]] self.backend = backend_cls(manager) + user_backend_cls = BACKEND_CLASSES[mapper.mapping["user"]] + self.user_backend = user_backend_cls(manager) + def invoke(self): logger.info("Running upgrade process (if required)") @@ -577,15 +583,15 @@ def update_people_entries(self): id_ = f"inum={admin_inum},ou=people,o=jans" kwargs = {} - if self.backend.type in ("sql", "spanner"): + if self.user_backend.type in ("sql", "spanner"): id_ = doc_id_from_dn(id_) kwargs = {"table_name": "jansPerson"} - elif self.backend.type == "couchbase": + elif self.user_backend.type == "couchbase": id_ = id_from_dn(id_) bucket = os.environ.get("CN_COUCHBASE_BUCKET_PREFIX", "jans") kwargs = {"bucket": f"{bucket}_user"} - entry = self.backend.get_entry(id_, **kwargs) + entry = self.user_backend.get_entry(id_, **kwargs) if not entry: return @@ -593,10 +599,10 @@ def update_people_entries(self): # add jansAdminUIRole to default admin user should_update = False - if self.backend.type == "sql" and not entry.attrs["jansAdminUIRole"]["v"]: + if self.user_backend.type == "sql" and not entry.attrs["jansAdminUIRole"]["v"]: entry.attrs["jansAdminUIRole"] = {"v": ["api-admin"]} should_update = True - elif self.backend.type == "spanner" and not entry.attrs["jansAdminUIRole"]: + elif self.user_backend.type == "spanner" and not entry.attrs["jansAdminUIRole"]: entry.attrs["jansAdminUIRole"] = ["api-admin"] should_update = True else: # ldap and couchbase @@ -605,7 +611,7 @@ def update_people_entries(self): should_update = True if should_update: - self.backend.modify_entry(entry.id, entry.attrs, **kwargs) + self.user_backend.modify_entry(entry.id, entry.attrs, **kwargs) def update_clients_entries(self): # modify redirect UI of config-api client diff --git a/docker-jans-persistence-loader/scripts/utils.py b/docker-jans-persistence-loader/scripts/utils.py index 36f9b1cc232..13a36f21798 100644 --- a/docker-jans-persistence-loader/scripts/utils.py +++ b/docker-jans-persistence-loader/scripts/utils.py @@ -316,7 +316,9 @@ def prepare_template_ctx(manager): return ctx -def get_ldif_mappings(optional_scopes=None): +def get_ldif_mappings(group, optional_scopes=None): + from jans.pycloudlib.persistence.utils import PersistenceMapper + optional_scopes = optional_scopes or [] dist = os.environ.get("CN_DISTRIBUTION", "default") @@ -393,6 +395,12 @@ def site_files(): "token": [], "session": [], } + + mapper = PersistenceMapper() + ldif_mappings = { + mapping: files for mapping, files in ldif_mappings.items() + if mapping in mapper.groups()[group] + } return ldif_mappings diff --git a/docker-jans-persistence-loader/scripts/wait.py b/docker-jans-persistence-loader/scripts/wait.py index 4f71e437453..8b06e30aa39 100644 --- a/docker-jans-persistence-loader/scripts/wait.py +++ b/docker-jans-persistence-loader/scripts/wait.py @@ -4,8 +4,9 @@ from jans.pycloudlib import get_manager from jans.pycloudlib import wait_for +from jans.pycloudlib import wait_for_persistence_conn from jans.pycloudlib.validators import validate_persistence_type -from jans.pycloudlib.validators import validate_persistence_ldap_mapping +from jans.pycloudlib.validators import validate_persistence_hybrid_mapping from jans.pycloudlib.validators import validate_persistence_sql_dialect from settings import LOGGING_CONFIG @@ -18,22 +19,17 @@ def main(): persistence_type = os.environ.get("CN_PERSISTENCE_TYPE", "ldap") validate_persistence_type(persistence_type) - ldap_mapping = os.environ.get("CN_PERSISTENCE_LDAP_MAPPING", "default") - validate_persistence_ldap_mapping(persistence_type, ldap_mapping) - if persistence_type == "sql": sql_dialect = os.environ.get("CN_SQL_DB_DIALECT", "mysql") validate_persistence_sql_dialect(sql_dialect) - manager = get_manager() - deps = ["config", "secret"] - if persistence_type == "hybrid": - deps += ["ldap_conn", "couchbase_conn"] - else: - deps.append("{}_conn".format(persistence_type)) + validate_persistence_hybrid_mapping() + manager = get_manager() + deps = ["config", "secret"] wait_for(manager, deps) + wait_for_persistence_conn(manager) if __name__ == "__main__": diff --git a/docker-jans-scim/Dockerfile b/docker-jans-scim/Dockerfile index 2acc81fc0a9..dc7b5cfcb7d 100644 --- a/docker-jans-scim/Dockerfile +++ b/docker-jans-scim/Dockerfile @@ -45,7 +45,7 @@ RUN wget -q https://ox.gluu.org/maven/org/gluufederation/jython-installer/${JYTH # ==== ENV CN_VERSION=1.0.1-SNAPSHOT -ENV CN_BUILD_DATE='2022-06-01 08:15' +ENV CN_BUILD_DATE='2022-06-02 14:20' ENV CN_SOURCE_URL=https://jenkins.jans.io/maven/io/jans/jans-scim-server/${CN_VERSION}/jans-scim-server-${CN_VERSION}.war # Install SCIM @@ -130,7 +130,7 @@ ENV CN_SECRET_ADAPTER=vault \ # =============== ENV CN_PERSISTENCE_TYPE=ldap \ - CN_PERSISTENCE_LDAP_MAPPING=default \ + CN_HYBRID_MAPPING="{}" \ CN_LDAP_URL=localhost:1636 \ CN_LDAP_USE_SSL=true \ CN_COUCHBASE_URL=localhost \ diff --git a/docker-jans-scim/README.md b/docker-jans-scim/README.md index 4bcce5655ba..eba39e821a3 100644 --- a/docker-jans-scim/README.md +++ b/docker-jans-scim/README.md @@ -46,13 +46,13 @@ The following environment variables are supported by the container: - `CN_WAIT_SLEEP_DURATION`: Delay between startup "health checks" (default to `10` seconds). - `CN_MAX_RAM_PERCENTAGE`: Value passed to Java option `-XX:MaxRAMPercentage`. - `CN_PERSISTENCE_TYPE`: Persistence backend being used (one of `ldap`, `couchbase`, or `hybrid`; default to `ldap`). -- `CN_PERSISTENCE_LDAP_MAPPING`: Specify data that should be saved in LDAP (one of `default`, `user`, `cache`, `site`, or `token`; default to `default`). Note this environment only takes effect when `CN_PERSISTENCE_TYPE` is set to `hybrid`. -- `CN_LDAP_URL`: Address and port of LDAP server (default to `localhost:1636`); required if `CN_PERSISTENCE_TYPE` is set to `ldap` or `hybrid`. +- `CN_HYBRID_MAPPING`: Specify data mapping for each persistence (default to `"{}"`). Note this environment only takes effect when `CN_PERSISTENCE_TYPE` is set to `hybrid`. See [hybrid mapping](#hybrid-mapping) section for details. +- `CN_LDAP_URL`: Address and port of LDAP server (default to `localhost:1636`). - `CN_LDAP_USE_SSL`: Whether to use SSL connection to LDAP server (default to `true`). -- `CN_COUCHBASE_URL`: Address of Couchbase server (default to `localhost`); required if `CN_PERSISTENCE_TYPE` is set to `couchbase` or `hybrid`. -- `CN_COUCHBASE_USER`: Username of Couchbase server (default to `admin`); required if `CN_PERSISTENCE_TYPE` is set to `couchbase` or `hybrid`. -- `CN_COUCHBASE_CERT_FILE`: Couchbase root certificate location (default to `/etc/certs/couchbase.crt`); required if `CN_PERSISTENCE_TYPE` is set to `couchbase` or `hybrid`. -- `CN_COUCHBASE_PASSWORD_FILE`: Path to file contains Couchbase password (default to `/etc/jans/conf/couchbase_password`); required if `CN_PERSISTENCE_TYPE` is set to `couchbase` or `hybrid`. +- `CN_COUCHBASE_URL`: Address of Couchbase server (default to `localhost`). +- `CN_COUCHBASE_USER`: Username of Couchbase server (default to `admin`). +- `CN_COUCHBASE_CERT_FILE`: Couchbase root certificate location (default to `/etc/certs/couchbase.crt`). +- `CN_COUCHBASE_PASSWORD_FILE`: Path to file contains Couchbase password (default to `/etc/jans/conf/couchbase_password`). - `CN_COUCHBASE_CONN_TIMEOUT`: Connect timeout used when a bucket is opened (default to `10000` milliseconds). - `CN_COUCHBASE_CONN_MAX_WAIT`: Maximum time to wait before retrying connection (default to `20000` milliseconds). - `CN_COUCHBASE_SCAN_CONSISTENCY`: Default scan consistency; one of `not_bounded`, `request_plus`, or `statement_plus` (default to `not_bounded`). @@ -99,3 +99,35 @@ The following key-value pairs are the defaults: "script_log_level": "INFO", } ``` + +### Hybrid mapping + +As per v1.0.1, hybrid persistence supports all available persistence types. To configure hybrid persistence and its data mapping, follow steps below: + +1. Set `CN_PERSISTENCE_TYPE` environment variable to `hybrid` + +1. Set `CN_HYBRID_MAPPING` with the following format: + + ``` + { + "default": "", + "user": "", + "site": "", + "cache": "", + "token": "", + "session": "", + } + ``` + + Example: + + ``` + { + "default": "sql", + "user": "spanner", + "site": "ldap", + "cache": "sql", + "token": "couchbase", + "session": "spanner", + } + ``` diff --git a/docker-jans-scim/requirements.txt b/docker-jans-scim/requirements.txt index 73df51bf183..a578f57ccc0 100644 --- a/docker-jans-scim/requirements.txt +++ b/docker-jans-scim/requirements.txt @@ -1,4 +1,4 @@ # pinned to py3-grpcio version to avoid failure on native extension build grpcio==1.41.0 libcst<0.4 -git+https://github.com/JanssenProject/jans@89859286d69e7de7885bd9da9f50720c8371e797#egg=jans-pycloudlib&subdirectory=jans-pycloudlib +git+https://github.com/JanssenProject/jans@a0b7eac21ab3e405ad60e913f966476c34420c62#egg=jans-pycloudlib&subdirectory=jans-pycloudlib diff --git a/docker-jans-scim/scripts/bootstrap.py b/docker-jans-scim/scripts/bootstrap.py index ec6380f127f..a9f6bffb3a7 100644 --- a/docker-jans-scim/scripts/bootstrap.py +++ b/docker-jans-scim/scripts/bootstrap.py @@ -13,6 +13,7 @@ from jans.pycloudlib.persistence import sync_ldap_truststore from jans.pycloudlib.persistence import render_sql_properties from jans.pycloudlib.persistence import render_spanner_properties +from jans.pycloudlib.persistence.utils import PersistenceMapper from jans.pycloudlib.utils import cert_to_truststore import logging.config @@ -64,7 +65,13 @@ def main(): render_salt(manager, "/app/templates/salt.tmpl", "/etc/jans/conf/salt") render_base_properties("/app/templates/jans.properties.tmpl", "/etc/jans/conf/jans.properties") - if persistence_type in ("ldap", "hybrid"): + mapper = PersistenceMapper() + persistence_groups = mapper.groups() + + if persistence_type == "hybrid": + render_hybrid_properties("/etc/jans/conf/jans-hybrid.properties") + + if "ldap" in persistence_groups: render_ldap_properties( manager, "/app/templates/jans-ldap.properties.tmpl", @@ -72,7 +79,7 @@ def main(): ) sync_ldap_truststore(manager) - if persistence_type in ("couchbase", "hybrid"): + if "couchbase" in persistence_groups: render_couchbase_properties( manager, "/app/templates/jans-couchbase.properties.tmpl", @@ -80,17 +87,14 @@ def main(): ) sync_couchbase_truststore(manager) - if persistence_type == "hybrid": - render_hybrid_properties("/etc/jans/conf/jans-hybrid.properties") - - if persistence_type == "sql": + if "sql" in persistence_groups: render_sql_properties( manager, "/app/templates/jans-sql.properties.tmpl", "/etc/jans/conf/jans-sql.properties", ) - if persistence_type == "spanner": + if "spanner" in persistence_groups: render_spanner_properties( manager, "/app/templates/jans-spanner.properties.tmpl", diff --git a/docker-jans-scim/scripts/wait.py b/docker-jans-scim/scripts/wait.py index 06d0e18247a..75169617850 100644 --- a/docker-jans-scim/scripts/wait.py +++ b/docker-jans-scim/scripts/wait.py @@ -1,25 +1,19 @@ -import logging -import logging.config import os from jans.pycloudlib import get_manager from jans.pycloudlib import wait_for +from jans.pycloudlib import wait_for_persistence from jans.pycloudlib.validators import validate_persistence_type -from jans.pycloudlib.validators import validate_persistence_ldap_mapping +from jans.pycloudlib.validators import validate_persistence_hybrid_mapping from jans.pycloudlib.validators import validate_persistence_sql_dialect -from settings import LOGGING_CONFIG - -logging.config.dictConfig(LOGGING_CONFIG) -logger = logging.getLogger("wait") - def main(): persistence_type = os.environ.get("CN_PERSISTENCE_TYPE", "ldap") validate_persistence_type(persistence_type) - ldap_mapping = os.environ.get("CN_PERSISTENCE_LDAP_MAPPING", "default") - validate_persistence_ldap_mapping(persistence_type, ldap_mapping) + if persistence_type == "hybrid": + validate_persistence_hybrid_mapping() if persistence_type == "sql": sql_dialect = os.environ.get("CN_SQL_DB_DIALECT", "mysql") @@ -27,13 +21,8 @@ def main(): manager = get_manager() deps = ["config", "secret"] - - if persistence_type == "hybrid": - deps += ["ldap", "couchbase"] - else: - deps.append(persistence_type) - wait_for(manager, deps) + wait_for_persistence(manager) if __name__ == "__main__": diff --git a/jans-pycloudlib/jans/pycloudlib/__init__.py b/jans-pycloudlib/jans/pycloudlib/__init__.py index e0b1a3703c7..f79c9a360d7 100644 --- a/jans-pycloudlib/jans/pycloudlib/__init__.py +++ b/jans-pycloudlib/jans/pycloudlib/__init__.py @@ -1,4 +1,4 @@ from jans.pycloudlib.manager import get_manager # noqa: F401 from jans.pycloudlib.wait import wait_for # noqa: F401 -from jans.pycloudlib.constants import PERSISTENCE_TYPES # noqa: F401 -from jans.pycloudlib.constants import PERSISTENCE_LDAP_MAPPINGS # noqa: F401 +from jans.pycloudlib.wait import wait_for_persistence # noqa: F401 +from jans.pycloudlib.wait import wait_for_persistence_conn # noqa: F401 diff --git a/jans-pycloudlib/jans/pycloudlib/constants.py b/jans-pycloudlib/jans/pycloudlib/constants.py deleted file mode 100644 index 32665f54d5c..00000000000 --- a/jans-pycloudlib/jans/pycloudlib/constants.py +++ /dev/null @@ -1,29 +0,0 @@ -""" -jans.pycloudlib.constants -~~~~~~~~~~~~~~~~~~~~~~~~~ - -This module contains values that are not supposedly modified. -""" - -PERSISTENCE_TYPES = ( - "ldap", - "couchbase", - "hybrid", - "sql", - "spanner", -) - -PERSISTENCE_LDAP_MAPPINGS = ( - "default", - "user", - "site", - "cache", - "token", - "session", -) - -# Supported SQL dialects -PERSISTENCE_SQL_DIALECTS = ( - "mysql", - "pgsql", -) diff --git a/jans-pycloudlib/jans/pycloudlib/persistence/__init__.py b/jans-pycloudlib/jans/pycloudlib/persistence/__init__.py index e2344023806..21ea8590d53 100644 --- a/jans-pycloudlib/jans/pycloudlib/persistence/__init__.py +++ b/jans-pycloudlib/jans/pycloudlib/persistence/__init__.py @@ -1,5 +1,3 @@ -import os - from jans.pycloudlib.persistence.couchbase import render_couchbase_properties # noqa: F401 from jans.pycloudlib.persistence.couchbase import sync_couchbase_truststore # noqa: F401 from jans.pycloudlib.persistence.couchbase import id_from_dn # noqa: F401 @@ -13,42 +11,8 @@ from jans.pycloudlib.persistence.sql import SqlClient # noqa: F401 from jans.pycloudlib.persistence.spanner import render_spanner_properties # noqa: F401 from jans.pycloudlib.persistence.spanner import SpannerClient # noqa: F401 - - -def render_salt(manager, src: str, dest: str) -> None: - """Render file contains salt string. - - The generated file has the following contents: - - .. code-block:: text - - encode_salt = random-salt-string - - :params manager: An instance of :class:`~jans.pycloudlib.manager._Manager`. - :params src: Absolute path to the template. - :params dest: Absolute path where generated file is located. - """ - encode_salt = manager.secret.get("encoded_salt") - - with open(src) as f: - txt = f.read() - - with open(dest, "w") as f: - rendered_txt = txt % {"encode_salt": encode_salt} - f.write(rendered_txt) - - -def render_base_properties(src: str, dest: str) -> None: - """Render file contains properties for Janssen Server. - - :params src: Absolute path to the template. - :params dest: Absolute path where generated file is located. - """ - with open(src) as f: - txt = f.read() - - with open(dest, "w") as f: - rendered_txt = txt % { - "persistence_type": os.environ.get("CN_PERSISTENCE_TYPE", "ldap"), - } - f.write(rendered_txt) +from jans.pycloudlib.persistence.utils import PersistenceMapper # noqa: F401 +from jans.pycloudlib.persistence.utils import PERSISTENCE_TYPES # noqa: F401 +from jans.pycloudlib.persistence.utils import PERSISTENCE_SQL_DIALECTS # noqa: F401 +from jans.pycloudlib.persistence.utils import render_salt # noqa: F401 +from jans.pycloudlib.persistence.utils import render_base_properties # noqa: F401 diff --git a/jans-pycloudlib/jans/pycloudlib/persistence/couchbase.py b/jans-pycloudlib/jans/pycloudlib/persistence/couchbase.py index f6d581af670..c258c19cea8 100644 --- a/jans-pycloudlib/jans/pycloudlib/persistence/couchbase.py +++ b/jans-pycloudlib/jans/pycloudlib/persistence/couchbase.py @@ -18,6 +18,8 @@ from requests_toolbelt.adapters.host_header_ssl import HostHeaderSSLAdapter from ldif import LDIFParser +from jans.pycloudlib.persistence.utils import PersistenceMapper +from jans.pycloudlib.persistence.utils import RDN_MAPPING from jans.pycloudlib.utils import encode_text from jans.pycloudlib.utils import cert_to_truststore from jans.pycloudlib.utils import as_boolean @@ -75,7 +77,7 @@ def get_couchbase_password(manager) -> str: :params manager: An instance of :class:`~jans.pycloudlib.manager._Manager`. :returns: Plaintext password. """ - secret_name = "couchbase_password" + secret_name = "couchbase_password" # nosec: B105 password_file = os.environ.get("CN_COUCHBASE_PASSWORD_FILE", "/etc/jans/conf/couchbase_password") return _get_cb_password(manager, password_file, secret_name) @@ -99,57 +101,11 @@ def get_couchbase_superuser_password(manager) -> str: :params manager: An instance of :class:`~jans.pycloudlib.manager._Manager`. :returns: Plaintext password. """ - secret_name = "couchbase_superuser_password" + secret_name = "couchbase_superuser_password" # nosec: B105 password_file = os.environ.get("CN_COUCHBASE_SUPERUSER_PASSWORD_FILE", "/etc/jans/conf/couchbase_superuser_password") return _get_cb_password(manager, password_file, secret_name) -def prefixed_couchbase_mappings(): - prefix = os.environ.get("CN_COUCHBASE_BUCKET_PREFIX", "jans") - mappings = { - "default": {"bucket": prefix, "mapping": ""}, - "user": {"bucket": f"{prefix}_user", "mapping": "people, groups, authorizations"}, - "cache": {"bucket": f"{prefix}_cache", "mapping": "cache"}, - "site": {"bucket": f"{prefix}_site", "mapping": "cache-refresh"}, - "token": {"bucket": f"{prefix}_token", "mapping": "tokens"}, - "session": {"bucket": f"{prefix}_session", "mapping": "sessions"}, - } - return mappings - - -def get_couchbase_mappings(persistence_type: str, ldap_mapping: str) -> dict: - """Get mappings of Couchbase buckets. - - Supported persistence types: - - - ``ldap`` - - ``couchbase`` - - ``hybrid`` - - Supported LDAP mappings: - - - ``default`` - - ``user`` - - ``token`` - - ``site`` - - ``cache`` - - ``session`` - - :params persistence_type: Type of persistence. - :params ldap_mapping: Mapping that stored in LDAP persistence. - :returns: A map of Couchbase buckets. - """ - mappings = prefixed_couchbase_mappings() - - if persistence_type == "hybrid": - return { - name: mapping - for name, mapping in mappings.items() - if name != ldap_mapping - } - return mappings - - def get_couchbase_conn_timeout() -> int: """Get connection timeout to Couchbase server. @@ -211,24 +167,38 @@ def render_couchbase_properties(manager, src: str, dest: str) -> None: :params src: Absolute path to the template. :params dest: Absolute path where generated file is located. """ - persistence_type = os.environ.get("CN_PERSISTENCE_TYPE", "couchbase") - ldap_mapping = os.environ.get("CN_PERSISTENCE_LDAP_MAPPING", "default") hostname = os.environ.get("CN_COUCHBASE_URL", "localhost") bucket_prefix = os.environ.get("CN_COUCHBASE_BUCKET_PREFIX", "jans") - _couchbase_mappings = get_couchbase_mappings(persistence_type, ldap_mapping) + mapper = PersistenceMapper() + groups = mapper.groups()["couchbase"] + + mappings = {} + for mapping, rdn in RDN_MAPPING.items(): + if mapping not in groups: + continue + + if mapping == "default": + bucket = "" + else: + bucket = f"{bucket_prefix}_{mapping}" + + mappings[mapping] = { + "bucket": bucket, + "rdn": rdn, + } + couchbase_buckets = [] couchbase_mappings = [] - for _, mapping in _couchbase_mappings.items(): - couchbase_buckets.append(mapping["bucket"]) - - if not mapping["mapping"]: - continue + for mapping in mappings.values(): + if mapping["bucket"]: + couchbase_buckets.append(mapping["bucket"]) - couchbase_mappings.append( - f"bucket.{mapping['bucket']}.mapping: {mapping['mapping']}" - ) + if mapping["rdn"]: + couchbase_mappings.append( + f"bucket.{mapping['bucket']}.mapping: {mapping['rdn']}" + ) # always have _default_ bucket if bucket_prefix not in couchbase_buckets: @@ -237,33 +207,33 @@ def render_couchbase_properties(manager, src: str, dest: str) -> None: with open(src) as fr: txt = fr.read() - with open(dest, "w") as fw: - rendered_txt = txt % { - "hostname": hostname, - "couchbase_server_user": get_couchbase_user(manager), - "encoded_couchbase_server_pw": encode_text( - get_couchbase_password(manager), - manager.secret.get("encoded_salt"), - ).decode(), - "couchbase_buckets": ", ".join(couchbase_buckets), - "default_bucket": bucket_prefix, - "couchbase_mappings": "\n".join(couchbase_mappings), - "encryption_method": "SSHA-256", - "ssl_enabled": str(as_boolean( - os.environ.get("CN_COUCHBASE_TRUSTSTORE_ENABLE", True) - )).lower(), - "couchbaseTrustStoreFn": manager.config.get("couchbaseTrustStoreFn") or "/etc/certs/couchbase.pkcs12", - "encoded_couchbaseTrustStorePass": encode_text( - CN_COUCHBASE_TRUSTSTORE_PASSWORD, - manager.secret.get("encoded_salt"), - ).decode(), - "couchbase_conn_timeout": get_couchbase_conn_timeout(), - "couchbase_conn_max_wait": get_couchbase_conn_max_wait(), - "couchbase_scan_consistency": get_couchbase_scan_consistency(), - "couchbase_keepalive_interval": get_couchbase_keepalive_interval(), - "couchbase_keepalive_timeout": get_couchbase_keepalive_timeout(), - } - fw.write(rendered_txt) + with open(dest, "w") as fw: + rendered_txt = txt % { + "hostname": hostname, + "couchbase_server_user": get_couchbase_user(manager), + "encoded_couchbase_server_pw": encode_text( + get_couchbase_password(manager), + manager.secret.get("encoded_salt"), + ).decode(), + "couchbase_buckets": ", ".join(couchbase_buckets), + "default_bucket": bucket_prefix, + "couchbase_mappings": "\n".join(couchbase_mappings), + "encryption_method": "SSHA-256", + "ssl_enabled": str(as_boolean( + os.environ.get("CN_COUCHBASE_TRUSTSTORE_ENABLE", True) + )).lower(), + "couchbaseTrustStoreFn": manager.config.get("couchbaseTrustStoreFn") or "/etc/certs/couchbase.pkcs12", + "encoded_couchbaseTrustStorePass": encode_text( + CN_COUCHBASE_TRUSTSTORE_PASSWORD, + manager.secret.get("encoded_salt"), + ).decode(), + "couchbase_conn_timeout": get_couchbase_conn_timeout(), + "couchbase_conn_max_wait": get_couchbase_conn_max_wait(), + "couchbase_scan_consistency": get_couchbase_scan_consistency(), + "couchbase_keepalive_interval": get_couchbase_keepalive_interval(), + "couchbase_keepalive_timeout": get_couchbase_keepalive_timeout(), + } + fw.write(rendered_txt) # DEPRECATED @@ -742,12 +712,38 @@ def create_from_ldif(self, filepath, ctx): data = json.dumps(entry) # using INSERT will cause duplication error, but the data is left intact - query = 'INSERT INTO `%s` (KEY, VALUE) VALUES ("%s", %s)' % (bucket, key, data) + query = 'INSERT INTO `%s` (KEY, VALUE) VALUES ("%s", %s)' % (bucket, key, data) # nosec: B608 req = self.exec_query(query) if not req.ok: logger.warning("Failed to execute query, reason={}".format(req.json())) + def doc_exists(self, bucket: str, id_: str) -> bool: + """ + Check if certain document exists in a bucket. + + :param bucket: Bucket name. + :param id_: ID of document. + """ + req = self.exec_query( + f"SELECT objectClass FROM {bucket} USE KEYS $key", # nosec: B608 + key=id_, + ) + + if not req.ok: + try: + data = json.loads(req.text) + err = data["errors"][0]["msg"] + except (ValueError, KeyError, IndexError): + err = req.reason + logger.warning(f"Unable to find document {id_} in bucket {bucket}; reason={err}") + return False + + if not req.json()["results"]: + logger.warning(f"Missing document {id_} in bucket {bucket}") + return False + return True + # backward-compat def suppress_verification_warning(): diff --git a/jans-pycloudlib/jans/pycloudlib/persistence/hybrid.py b/jans-pycloudlib/jans/pycloudlib/persistence/hybrid.py index 255f07a96bb..85ab1720b1a 100644 --- a/jans-pycloudlib/jans/pycloudlib/persistence/hybrid.py +++ b/jans-pycloudlib/jans/pycloudlib/persistence/hybrid.py @@ -5,41 +5,44 @@ This module contains various helpers related to hybrid (LDAP + Couchbase) persistence. """ -import os - -from jans.pycloudlib.persistence.couchbase import get_couchbase_mappings -from jans.pycloudlib.persistence.couchbase import prefixed_couchbase_mappings +from jans.pycloudlib.persistence.utils import PersistenceMapper def render_hybrid_properties(dest: str) -> None: """Render file contains properties to connect to hybrid persistence, i.e. ``/etc/jans/conf/jans-hybrid.properties``. - :params dest: Absolute path where generated file is located. + :param dest: Absolute path where generated file is located. """ - persistence_type = os.environ.get("CN_PERSISTENCE_TYPE", "couchbase") - ldap_mapping = os.environ.get("CN_PERSISTENCE_LDAP_MAPPING", "default") - - if ldap_mapping == "default": - default_storage = "ldap" - else: - default_storage = "couchbase" - - _couchbase_mappings = get_couchbase_mappings(persistence_type, ldap_mapping) - - couchbase_mappings = ", ".join([ - mapping["mapping"] - for name, mapping in _couchbase_mappings.items() - if mapping["mapping"] and name != ldap_mapping - ]) - ldap_mappings = prefixed_couchbase_mappings().get(ldap_mapping, {}).get("mapping") or "default" + hybrid_storages = resolve_hybrid_storages(PersistenceMapper()) out = "\n".join([ - "storages: ldap, couchbase", - f"storage.default: {default_storage}", - f"storage.ldap.mapping: {ldap_mappings}", - f"storage.couchbase.mapping: {couchbase_mappings}", + f"{k}: {v}" for k, v in hybrid_storages.items() ]) with open(dest, "w") as fw: fw.write(out) + + +def resolve_hybrid_storages(mapper: PersistenceMapper) -> dict: + """ + Resolve hybrid storage configuration. + + :param data_mapping: Persistence data mapping, usually generated by + ``.utils.resolve_persistence_data_mapping``. + """ + ctx = { + # unique storage names + "storages": ", ".join(sorted(set( + mapper.mapping.values()) + )), + "storage.default": mapper.mapping["default"], + } + + for k, v in mapper.groups_with_rdn().items(): + # remove empty value (if any) + values = [val for val in v if val] + if not values: + continue + ctx[f"storage.{k}.mapping"] = ", ".join(values) + return ctx diff --git a/jans-pycloudlib/jans/pycloudlib/persistence/utils.py b/jans-pycloudlib/jans/pycloudlib/persistence/utils.py new file mode 100644 index 00000000000..b3948fdf4f6 --- /dev/null +++ b/jans-pycloudlib/jans/pycloudlib/persistence/utils.py @@ -0,0 +1,231 @@ +""" +jans.pycloudlib.persistence.utils +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This module consists of common utilities to work with persistence. +""" +import json +import os +from collections import defaultdict +from typing import Dict + + +def render_salt(manager, src: str, dest: str) -> None: + """Render file contains salt string. + + The generated file has the following contents: + + .. code-block:: text + + encode_salt = random-salt-string + + :params manager: An instance of :class:`~jans.pycloudlib.manager._Manager`. + :params src: Absolute path to the template. + :params dest: Absolute path where generated file is located. + """ + encode_salt = manager.secret.get("encoded_salt") + + with open(src) as f: + txt = f.read() + + with open(dest, "w") as f: + rendered_txt = txt % {"encode_salt": encode_salt} + f.write(rendered_txt) + + +def render_base_properties(src: str, dest: str) -> None: + """Render file contains properties for Janssen Server. + + :params src: Absolute path to the template. + :params dest: Absolute path where generated file is located. + """ + with open(src) as f: + txt = f.read() + + with open(dest, "w") as f: + rendered_txt = txt % { + "persistence_type": os.environ.get("CN_PERSISTENCE_TYPE", "ldap"), + } + f.write(rendered_txt) + + +#: Supported persistence types +PERSISTENCE_TYPES = ( + "ldap", + "couchbase", + "sql", + "spanner", + "hybrid", +) + +#: Data mapping of persistence, ordered by priority +PERSISTENCE_DATA_KEYS = ( + "default", + "user", + "site", + "cache", + "token", + "session", +) + +#: Supported SQL dialects +PERSISTENCE_SQL_DIALECTS = ( + "mysql", + "pgsql", +) + +RDN_MAPPING = { + "default": "", + "user": "people, groups, authorizations", + "cache": "cache", + "site": "cache-refresh", + "token": "tokens", + "session": "sessions", +} + + +class PersistenceMapper: + """ + This class creates persistence data mapping. + + Example of data mapping when using ``sql`` persistence type: + + .. codeblock:: python + + os.environ["CN_PERSISTENCE_TYPE"] = "sql" + + mapper = PersistenceMapper() + mapper.validate_hybrid_mapping() + print(mapper.mapping) + + The output will be: + + .. codeblock:: python + + { + "default": "sql", + "user": "sql", + "site": "sql", + "cache": "sql", + "token": "sql", + "session": "sql", + } + + The same rule applies to any supported persistence types, except for ``hybrid`` + where each key can have different value. To customize the mapping, additional environment + variable is required. + + .. codeblock:: python + + os.environ["CN_PERSISTENCE_TYPE"] = "hybrid" + os.environ["CN_HYBRID_MAPPING"] = json.loads({ + "default": "sql", "user": "spanner", "site": "sql", "cache": "sql", "token": "sql", "session": "sql" + }) + + mapper = PersistenceMapper() + mapper.validate_hybrid_mapping() + print(mapper.mapping) + + The output will be: + + .. codeblock:: python + + { + "default": "sql", + "user": "spanner", + "site": "sql", + "cache": "sql", + "token": "sql", + "session": "sql", + } + + Note that when using ``hybrid``, all mapping must be defined explicitly. + """ + + def __init__(self) -> None: + self.type = os.environ.get("CN_PERSISTENCE_TYPE", "ldap") + self._mapping = {} + + @property + def mapping(self) -> Dict[str, str]: + """Pre-populate mapping (if empty). + + Example of pre-populated mapping: + + .. codeblock:: python + + { + "default": "sql", + "user": "spanner", + "site": "sql", + "cache": "sql", + "token": "sql", + "session": "sql", + } + """ + if not self._mapping: + if self.type != "hybrid": + self._mapping = dict.fromkeys(PERSISTENCE_DATA_KEYS, self.type) + else: + self._mapping = self.validate_hybrid_mapping() + return self._mapping + + def groups(self) -> Dict[str, list]: + """Pre-populate mapping groupped by persistence type. + + Example of pre-populated groupped mapping: + + .. codeblock:: python + + { + "sql": ["cache", "default", "session"], + "couchbase": ["user"], + "spanner": ["token"], + "ldap": ["site"], + } + """ + mapper = defaultdict(list) + + for k, v in self.mapping.items(): + mapper[v].append(k) + return dict(sorted(mapper.items())) + + def groups_with_rdn(self) -> Dict[str, list]: + """Pre-populate mapping groupped by persistence type and its values replaced by RDN. + + Example of pre-populated groupped mapping: + + .. codeblock:: python + + { + "sql": ["cache", "", "sessions"], + "couchbase": ["people, groups, authorizations"], + "spanner": ["tokens"], + "ldap": ["cache-refresh"], + } + """ + + mapper = defaultdict(list) + for k, v in self.mapping.items(): + mapper[v].append(RDN_MAPPING[k]) + return dict(sorted(mapper.items())) + + @classmethod + def validate_hybrid_mapping(cls) -> Dict[str, list]: + """Validate the value of ``hybrid_mapping`` attribute. + """ + mapping = json.loads(os.environ.get("CN_HYBRID_MAPPING", "{}")) + + # build whitelisted mapping based on supported PERSISTENCE_DATA_KEYS and PERSISTENCE_TYPES + try: + sanitized_mapping = { + key: type_ for key, type_ in mapping.items() + if key in PERSISTENCE_DATA_KEYS and type_ in PERSISTENCE_TYPES + } + except AttributeError: + # likely not a dict + raise ValueError(f"Invalid hybrid mapping {mapping}") + + if sorted(sanitized_mapping.keys()) != sorted(PERSISTENCE_DATA_KEYS): + raise ValueError(f"Missing key(s) in hybrid mapping {mapping}") + return sanitized_mapping diff --git a/jans-pycloudlib/jans/pycloudlib/validators.py b/jans-pycloudlib/jans/pycloudlib/validators.py index 5c9f9660adb..2589660a518 100644 --- a/jans-pycloudlib/jans/pycloudlib/validators.py +++ b/jans-pycloudlib/jans/pycloudlib/validators.py @@ -5,14 +5,18 @@ This module contains helpers to validate things. """ -from typing import NoReturn +from jans.pycloudlib.persistence.utils import PERSISTENCE_TYPES +from jans.pycloudlib.persistence.utils import PERSISTENCE_SQL_DIALECTS +from jans.pycloudlib.persistence.utils import PersistenceMapper -from jans.pycloudlib.constants import PERSISTENCE_TYPES -from jans.pycloudlib.constants import PERSISTENCE_LDAP_MAPPINGS -from jans.pycloudlib.constants import PERSISTENCE_SQL_DIALECTS +def validate_persistence_type(type_: str) -> None: + """ + Validates persistence type. + + :param type\\_: Persistence type. + """ -def validate_persistence_type(type_: str) -> NoReturn: if type_ not in PERSISTENCE_TYPES: types = ", ".join(PERSISTENCE_TYPES) @@ -22,17 +26,18 @@ def validate_persistence_type(type_: str) -> NoReturn: ) -def validate_persistence_ldap_mapping(type_: str, ldap_mapping: str) -> NoReturn: - if type_ == "hybrid" and ldap_mapping not in PERSISTENCE_LDAP_MAPPINGS: - mappings = ", ".join(PERSISTENCE_LDAP_MAPPINGS) +# deprecated function +def validate_persistence_ldap_mapping(type_: str, ldap_mapping: str) -> None: + import warnings - raise ValueError( - f"Unsupported persistence ldap mapping {ldap_mapping}; " - f"please choose one of {mappings}" - ) + warnings.warn( + "'validate_persistence_ldap_mapping' function is now a no-op function " + "and no longer required by hybrid persistence; " + "use 'validate_persistence_hybrid_mapping' function instead", + DeprecationWarning) -def validate_persistence_sql_dialect(dialect: str) -> NoReturn: +def validate_persistence_sql_dialect(dialect: str) -> None: """ Validates SQL dialect. @@ -44,3 +49,11 @@ def validate_persistence_sql_dialect(dialect: str) -> NoReturn: f"Unsupported persistence sql dialects; " f"please choose one of {dialects}" ) + + +def validate_persistence_hybrid_mapping() -> None: + """ + Validate hybrid mapping. + """ + mapper = PersistenceMapper() + mapper.validate_hybrid_mapping() diff --git a/jans-pycloudlib/jans/pycloudlib/wait.py b/jans-pycloudlib/jans/pycloudlib/wait.py index 59674172b37..8bf64d3e69b 100644 --- a/jans-pycloudlib/jans/pycloudlib/wait.py +++ b/jans-pycloudlib/jans/pycloudlib/wait.py @@ -4,8 +4,14 @@ This module consists of startup order utilities. """ +from __future__ import annotations +from typing import TYPE_CHECKING + +if TYPE_CHECKING: # pragma: no cover + # imported objects for function type hint, completion, etc. + # these won't be executed in runtime + from jans.pycloudlib.manager import _Manager -import json import logging import os import sys @@ -13,10 +19,13 @@ import backoff from jans.pycloudlib.persistence.couchbase import CouchbaseClient +from jans.pycloudlib.persistence.couchbase import id_from_dn from jans.pycloudlib.persistence.sql import SqlClient +from jans.pycloudlib.persistence.sql import doc_id_from_dn from jans.pycloudlib.persistence.spanner import SpannerClient from jans.pycloudlib.utils import as_boolean from jans.pycloudlib.persistence.ldap import LdapClient +from jans.pycloudlib.persistence.utils import PersistenceMapper logger = logging.getLogger(__name__) @@ -157,40 +166,37 @@ def wait_for_secret(manager, **kwargs): raise WaitError("Secret 'ssl_cert' is not available") +#: DN of admin group +_ADMIN_GROUP_DN = "inum=60B7,ou=groups,o=jans" + + @retry_on_exception -def wait_for_ldap(manager, **kwargs): +def wait_for_ldap(manager: _Manager, **kwargs) -> None: """Wait for readiness/availability of LDAP server based on existing entry. :param manager: An instance of :class:`~jans.pycloudlib.manager._Manager`. """ - persistence_type = os.environ.get("CN_PERSISTENCE_TYPE", "ldap") - ldap_mapping = os.environ.get("CN_PERSISTENCE_LDAP_MAPPING", "default") - - # a minimum service stack is having config-api client jca_client_id = manager.config.get("jca_client_id") - default_search = ( - f"inum={jca_client_id},ou=clients,o=jans", - "(objectClass=jansClnt)", - ) - - if persistence_type == "hybrid": - # `cache` and `token` mapping only have base entries - search_mapping = { - "default": default_search, - "user": ("inum=60B7,ou=groups,o=jans", "(objectClass=jansGrp)"), - "site": ("ou=cache-refresh,o=site", "(ou=cache-refresh)"), - "cache": ("ou=cache,o=jans", "(ou=cache)"), - "token": ("ou=tokens,o=jans", "(ou=tokens)"), - "session": ("ou=sessions,o=jans", "(ou=sessions)"), - } - search = search_mapping[ldap_mapping] - else: - search = default_search + search_mapping = { + "default": (f"inum={jca_client_id},ou=clients,o=jans", "(objectClass=jansClnt)"), + "user": (_ADMIN_GROUP_DN, "(objectClass=jansGrp)"), + "site": ("ou=cache-refresh,o=site", "(ou=cache-refresh)"), + "cache": ("ou=cache,o=jans", "(ou=cache)"), + "token": ("ou=tokens,o=jans", "(ou=tokens)"), + "session": ("ou=sessions,o=jans", "(ou=sessions)"), + } client = LdapClient(manager) - entries = client.search(search[0], search[1], attributes=["objectClass"], limit=1) - if not entries: + try: + # get the first data key + key = PersistenceMapper().groups().get("ldap", [])[0] + search_base, search_filter = search_mapping[key] + init = client.search(search_base, search_filter, attributes=["objectClass"], limit=1) + except (IndexError, KeyError): + init = client.is_connected() + + if not init: raise WaitError("LDAP is not fully initialized") @@ -212,40 +218,24 @@ def wait_for_couchbase(manager, **kwargs): :param manager: An instance of :class:`~jans.pycloudlib.manager._Manager`. """ - - persistence_type = os.environ.get("CN_PERSISTENCE_TYPE", "couchbase") - ldap_mapping = os.environ.get("CN_PERSISTENCE_LDAP_MAPPING", "default") bucket_prefix = os.environ.get("CN_COUCHBASE_BUCKET_PREFIX", "jans") - - # only default and user buckets buckets that may have initial data; - # these data also affected by LDAP mapping selection; jca_client_id = manager.config.get("jca_client_id") - bucket, key = bucket_prefix, f"clients_{jca_client_id}" - - # if `hybrid` is selected and default mapping is stored in LDAP, - # the default bucket won't have data, hence we check the user bucket instead - if persistence_type == "hybrid" and ldap_mapping == "default": - bucket, key = f"{bucket_prefix}_user", "groups_60B7" - - cb_client = CouchbaseClient(manager) - - req = cb_client.exec_query( - f"SELECT objectClass FROM {bucket} USE KEYS $key", - key=key, - ) + search_mapping = { + "default": (id_from_dn(f"inum={jca_client_id},ou=clients,o=jans"), f"{bucket_prefix}"), + "user": (id_from_dn(_ADMIN_GROUP_DN), f"{bucket_prefix}_user"), + } - if not req.ok: - try: - data = json.loads(req.text) - err = data["errors"][0]["msg"] - except (ValueError, KeyError, IndexError): - err = req.reason - raise WaitError(err) + client = CouchbaseClient(manager) + try: + # get the first data key + key = PersistenceMapper().groups().get("couchbase", [])[0] + id_, bucket = search_mapping[key] + init = client.doc_exists(bucket, id_) + except (IndexError, KeyError): + init = client.get_buckets().ok - # request is OK, but result is not found - data = req.json() - if not data["results"]: - raise WaitError(f"Missing document {key} in bucket {bucket}") + if not init: + raise WaitError("Couchbase backend is not fully initialized") @retry_on_exception @@ -272,13 +262,26 @@ def wait_for_sql_conn(manager, **kwargs): @retry_on_exception -def wait_for_sql(manager, **kwargs): +def wait_for_sql(manager: _Manager, **kwargs) -> None: """Wait for readiness/liveness of an SQL database. """ - init = SqlClient(manager).row_exists("jansClnt", manager.config.get("jca_client_id")) + jca_client_id = manager.config.get("jca_client_id") + search_mapping = { + "default": (doc_id_from_dn(f"inum={jca_client_id},ou=clients,o=jans"), "jansClnt"), + "user": (doc_id_from_dn(_ADMIN_GROUP_DN), "jansGrp"), + } + + client = SqlClient(manager) + try: + # get the first data key + key = PersistenceMapper().groups().get("sql", [])[0] + doc_id, table_name = search_mapping[key] + init = client.row_exists(table_name, doc_id) + except (IndexError, KeyError): + init = client.connected() if not init: - raise WaitError("SQL is not fully initialized") + raise WaitError("SQL backend is not fully initialized") @retry_on_exception @@ -292,13 +295,26 @@ def wait_for_spanner_conn(manager, **kwargs): @retry_on_exception -def wait_for_spanner(manager, **kwargs): +def wait_for_spanner(manager: _Manager, **kwargs) -> None: """Wait for readiness/liveness of an Spanner database. """ - init = SpannerClient(manager).row_exists("jansClnt", manager.config.get("jca_client_id")) + jca_client_id = manager.config.get("jca_client_id") + search_mapping = { + "default": (doc_id_from_dn(f"inum={jca_client_id},ou=clients,o=jans"), "jansClnt"), + "user": (doc_id_from_dn(_ADMIN_GROUP_DN), "jansGrp"), + } + + client = SpannerClient(manager) + try: + # get the first data key + key = PersistenceMapper().groups().get("spanner", [])[0] + doc_id, table_name = search_mapping[key] + init = client.row_exists(table_name, doc_id) + except (IndexError, KeyError): + init = client.connected() if not init: - raise WaitError("Spanner is not fully initialized") + raise WaitError("Spanner backend is not fully initialized") def wait_for(manager, deps=None): @@ -362,3 +378,24 @@ def wait_for(manager, deps=None): logger.warning(f"Unsupported callback for {dep} dependency") continue callback["func"](manager, **callback["kwargs"]) + + +def wait_for_persistence(manager: _Manager) -> None: + """Wait for defined persistence(s). + + :param manager: An instance of :class:`~jans.pycloudlib.manager._Manager`. + """ + mapper = PersistenceMapper() + # cast ``dict_keys`` to ``list`` + deps = list(mapper.groups().keys()) + wait_for(manager, deps) + + +def wait_for_persistence_conn(manager: _Manager) -> None: + """Wait for defined persistence(s) connection. + + :param manager: An instance of :class:`~jans.pycloudlib.manager._Manager`. + """ + mapper = PersistenceMapper() + deps = [f"{type_}_conn" for type_ in mapper.groups().keys()] + wait_for(manager, deps) diff --git a/jans-pycloudlib/tests/conftest.py b/jans-pycloudlib/tests/conftest.py index 8a7ee480d63..ea07d4cfd9d 100644 --- a/jans-pycloudlib/tests/conftest.py +++ b/jans-pycloudlib/tests/conftest.py @@ -71,6 +71,7 @@ def get_config(key, default=""): ctx = { "ldap_binddn": "cn=Directory Manager", "couchbase_server_user": "admin", + "jca_client_id": "1234", } return ctx.get(key) or default diff --git a/jans-pycloudlib/tests/test_persistence.py b/jans-pycloudlib/tests/test_persistence.py index a901d71a3ae..adb05ae2e69 100644 --- a/jans-pycloudlib/tests/test_persistence.py +++ b/jans-pycloudlib/tests/test_persistence.py @@ -1,3 +1,4 @@ +import json import os import shutil from collections import namedtuple @@ -366,12 +367,16 @@ def test_get_couchbase_keepalive_timeout(monkeypatch, timeout, expected): assert get_couchbase_keepalive_timeout() == expected -def test_render_couchbase_properties(monkeypatch, tmpdir, gmanager): +@pytest.mark.parametrize("bucket_prefix", ["jans", "myprefix"]) +def test_render_couchbase_properties(monkeypatch, tmpdir, gmanager, bucket_prefix): from jans.pycloudlib.persistence.couchbase import render_couchbase_properties passwd = tmpdir.join("couchbase_password") passwd.write("secret") + monkeypatch.setenv("CN_COUCHBASE_PASSWORD_FILE", str(passwd)) + monkeypatch.setenv("CN_PERSISTENCE_TYPE", "couchbase") + monkeypatch.setenv("CN_COUCHBASE_BUCKET_PREFIX", bucket_prefix) tmpl = """ connection.connect-timeout: %(couchbase_conn_timeout)s @@ -379,6 +384,9 @@ def test_render_couchbase_properties(monkeypatch, tmpdir, gmanager): connection.scan-consistency: %(couchbase_scan_consistency)s connection.keep-alive-interval: %(couchbase_keepalive_interval)s connection.keep-alive-timeout: %(couchbase_keepalive_timeout)s +buckets: %(couchbase_buckets)s +bucket.default: %(default_bucket)s +%(couchbase_mappings)s """.strip() expected = """ @@ -387,6 +395,61 @@ def test_render_couchbase_properties(monkeypatch, tmpdir, gmanager): connection.scan-consistency: not_bounded connection.keep-alive-interval: 30000 connection.keep-alive-timeout: 2500 +buckets: {bucket_prefix}, {bucket_prefix}_user, {bucket_prefix}_cache, {bucket_prefix}_site, {bucket_prefix}_token, {bucket_prefix}_session +bucket.default: {bucket_prefix} +bucket.{bucket_prefix}_user.mapping: people, groups, authorizations +bucket.{bucket_prefix}_cache.mapping: cache +bucket.{bucket_prefix}_site.mapping: cache-refresh +bucket.{bucket_prefix}_token.mapping: tokens +bucket.{bucket_prefix}_session.mapping: sessions +""".strip().format(bucket_prefix=bucket_prefix) + + src = tmpdir.join("jans-couchbase.properties.tmpl") + src.write(tmpl) + dest = tmpdir.join("jans-couchbase.properties") + + render_couchbase_properties(gmanager, str(src), str(dest)) + assert dest.read() == expected + + +def test_render_couchbase_properties_hybrid(monkeypatch, tmpdir, gmanager): + from jans.pycloudlib.persistence.couchbase import render_couchbase_properties + + passwd = tmpdir.join("couchbase_password") + passwd.write("secret") + + monkeypatch.setenv("CN_COUCHBASE_PASSWORD_FILE", str(passwd)) + monkeypatch.setenv("CN_PERSISTENCE_TYPE", "hybrid") + monkeypatch.setenv("CN_HYBRID_MAPPING", json.dumps({ + "default": "ldap", + "user": "couchbase", + "site": "ldap", + "cache": "ldap", + "token": "couchbase", + "session": "ldap", + })) + + tmpl = """ +connection.connect-timeout: %(couchbase_conn_timeout)s +connection.connection-max-wait-time: %(couchbase_conn_max_wait)s +connection.scan-consistency: %(couchbase_scan_consistency)s +connection.keep-alive-interval: %(couchbase_keepalive_interval)s +connection.keep-alive-timeout: %(couchbase_keepalive_timeout)s +buckets: %(couchbase_buckets)s +bucket.default: %(default_bucket)s +%(couchbase_mappings)s +""".strip() + + expected = """ +connection.connect-timeout: 10000 +connection.connection-max-wait-time: 20000 +connection.scan-consistency: not_bounded +connection.keep-alive-interval: 30000 +connection.keep-alive-timeout: 2500 +buckets: jans, jans_user, jans_token +bucket.default: jans +bucket.jans_user.mapping: people, groups, authorizations +bucket.jans_token.mapping: tokens """.strip() src = tmpdir.join("jans-couchbase.properties.tmpl") @@ -423,106 +486,53 @@ def test_get_bucket_for_key(key, bucket): # ====== -def test_render_hybrid_properties_default(monkeypatch, tmpdir): - from jans.pycloudlib.persistence.hybrid import render_hybrid_properties - - monkeypatch.setenv("CN_PERSISTENCE_TYPE", "hybrid") - - expected = """ -storages: ldap, couchbase -storage.default: ldap -storage.ldap.mapping: default -storage.couchbase.mapping: people, groups, authorizations, cache, cache-refresh, tokens, sessions -""".strip() - - dest = tmpdir.join("jans-hybrid.properties") - render_hybrid_properties(str(dest)) - assert dest.read() == expected - - -def test_render_hybrid_properties_user(monkeypatch, tmpdir): - from jans.pycloudlib.persistence.hybrid import render_hybrid_properties - - monkeypatch.setenv("CN_PERSISTENCE_TYPE", "hybrid") - monkeypatch.setenv("CN_PERSISTENCE_LDAP_MAPPING", "user") - - expected = """ -storages: ldap, couchbase -storage.default: couchbase -storage.ldap.mapping: people, groups, authorizations -storage.couchbase.mapping: cache, cache-refresh, tokens, sessions -""".strip() - - dest = tmpdir.join("jans-hybrid.properties") - render_hybrid_properties(str(dest)) - assert dest.read() == expected - - -def test_render_hybrid_properties_token(monkeypatch, tmpdir): - from jans.pycloudlib.persistence.hybrid import render_hybrid_properties - - monkeypatch.setenv("CN_PERSISTENCE_TYPE", "hybrid") - monkeypatch.setenv("CN_PERSISTENCE_LDAP_MAPPING", "token") - - expected = """ -storages: ldap, couchbase -storage.default: couchbase -storage.ldap.mapping: tokens -storage.couchbase.mapping: people, groups, authorizations, cache, cache-refresh, sessions -""".strip() - - dest = tmpdir.join("jans-hybrid.properties") - render_hybrid_properties(str(dest)) - assert dest.read() == expected - - -def test_render_hybrid_properties_session(monkeypatch, tmpdir): - from jans.pycloudlib.persistence.hybrid import render_hybrid_properties - - monkeypatch.setenv("CN_PERSISTENCE_TYPE", "hybrid") - monkeypatch.setenv("CN_PERSISTENCE_LDAP_MAPPING", "session") - - expected = """ -storages: ldap, couchbase -storage.default: couchbase -storage.ldap.mapping: sessions -storage.couchbase.mapping: people, groups, authorizations, cache, cache-refresh, tokens -""".strip() - - dest = tmpdir.join("jans-hybrid.properties") - render_hybrid_properties(str(dest)) - assert dest.read() == expected - - -def test_render_hybrid_properties_cache(monkeypatch, tmpdir): - from jans.pycloudlib.persistence.hybrid import render_hybrid_properties +def test_resolve_hybrid_storages(monkeypatch): + from jans.pycloudlib.persistence.hybrid import resolve_hybrid_storages + from jans.pycloudlib.persistence.utils import PersistenceMapper monkeypatch.setenv("CN_PERSISTENCE_TYPE", "hybrid") - monkeypatch.setenv("CN_PERSISTENCE_LDAP_MAPPING", "cache") - - expected = """ -storages: ldap, couchbase -storage.default: couchbase -storage.ldap.mapping: cache -storage.couchbase.mapping: people, groups, authorizations, cache-refresh, tokens, sessions -""".strip() - - dest = tmpdir.join("jans-hybrid.properties") - render_hybrid_properties(str(dest)) - assert dest.read() == expected - - -def test_render_hybrid_properties_site(monkeypatch, tmpdir): + monkeypatch.setenv("CN_HYBRID_MAPPING", json.dumps({ + "default": "sql", + "user": "spanner", + "site": "couchbase", + "cache": "ldap", + "token": "sql", + "session": "sql", + })) + expected = { + "storages": "couchbase, ldap, spanner, sql", + "storage.default": "sql", + "storage.couchbase.mapping": "cache-refresh", + "storage.ldap.mapping": "cache", + "storage.spanner.mapping": "people, groups, authorizations", + "storage.sql.mapping": "tokens, sessions", + } + mapper = PersistenceMapper() + assert resolve_hybrid_storages(mapper) == expected + + +def test_render_hybrid_properties(monkeypatch, tmpdir): from jans.pycloudlib.persistence.hybrid import render_hybrid_properties monkeypatch.setenv("CN_PERSISTENCE_TYPE", "hybrid") - monkeypatch.setenv("CN_PERSISTENCE_LDAP_MAPPING", "site") + monkeypatch.setenv( + "CN_HYBRID_MAPPING", + json.dumps({ + "default": "ldap", + "user": "couchbase", + "site": "sql", + "cache": "sql", + "token": "spanner", + "session": "sql", + }) + ) expected = """ -storages: ldap, couchbase -storage.default: couchbase -storage.ldap.mapping: cache-refresh -storage.couchbase.mapping: people, groups, authorizations, cache, tokens, sessions +storages: couchbase, ldap, spanner, sql +storage.default: ldap +storage.couchbase.mapping: people, groups, authorizations +storage.spanner.mapping: tokens +storage.sql.mapping: cache-refresh, cache, sessions """.strip() dest = tmpdir.join("jans-hybrid.properties") @@ -837,3 +847,108 @@ def test_spanner_sub_tables(gmanager, monkeypatch): client = SpannerClient(gmanager) assert isinstance(client.sub_tables, dict) + + +# ===== +# utils +# ===== + + +@pytest.mark.parametrize("type_", [ + "ldap", + "couchbase", + "sql", + "spanner", +]) +def test_persistence_mapper_mapping(monkeypatch, type_): + from jans.pycloudlib.persistence import PersistenceMapper + + monkeypatch.setenv("CN_PERSISTENCE_TYPE", type_) + expected = dict.fromkeys([ + "default", + "user", + "site", + "cache", + "token", + "session", + ], type_) + assert PersistenceMapper().mapping == expected + + +def test_persistence_mapper_hybrid_mapping(monkeypatch): + from jans.pycloudlib.persistence import PersistenceMapper + + mapping = { + "default": "sql", + "user": "spanner", + "site": "ldap", + "cache": "sql", + "token": "couchbase", + "session": "sql", + } + monkeypatch.setenv("CN_PERSISTENCE_TYPE", "hybrid") + monkeypatch.setenv("CN_HYBRID_MAPPING", json.dumps(mapping)) + assert PersistenceMapper().mapping == mapping + + +@pytest.mark.parametrize("mapping", [ + "ab", + "1", + "[]", + "{}", # empty dict + {"user": "sql"}, # missing remaining keys + {"default": "sql", "user": "spanner", "cache": "ldap", "site": "couchbase", "token": "sql", "session": "random"}, # invalid type + {"default": "sql", "user": "spanner", "cache": "ldap", "site": "couchbase", "token": "sql", "foo": "sql"}, # invalid key +]) +def test_persistence_mapper_validate_hybrid_mapping(monkeypatch, mapping): + from jans.pycloudlib.persistence.utils import PersistenceMapper + + monkeypatch.setenv("CN_PERSISTENCE_TYPE", "hybrid") + monkeypatch.setenv("CN_HYBRID_MAPPING", json.dumps(mapping)) + + with pytest.raises(ValueError): + PersistenceMapper().validate_hybrid_mapping() + + +def test_persistence_mapper_groups(monkeypatch): + from jans.pycloudlib.persistence.utils import PersistenceMapper + + monkeypatch.setenv("CN_PERSISTENCE_TYPE", "hybrid") + monkeypatch.setenv("CN_HYBRID_MAPPING", json.dumps({ + "default": "sql", + "user": "spanner", + "site": "ldap", + "cache": "sql", + "token": "couchbase", + "session": "sql", + })) + + groups = { + "couchbase": ["token"], + "ldap": ["site"], + "spanner": ["user"], + "sql": ["default", "cache", "session"], + } + assert PersistenceMapper().groups() == groups + + +def test_persistence_mapper_groups_rdn(monkeypatch): + from jans.pycloudlib.persistence.utils import PersistenceMapper + + monkeypatch.setenv("CN_PERSISTENCE_TYPE", "hybrid") + monkeypatch.setenv("CN_HYBRID_MAPPING", json.dumps({ + "default": "sql", + "user": "spanner", + "site": "ldap", + "cache": "sql", + "token": "couchbase", + "session": "sql", + })) + + groups = { + "couchbase": ["tokens"], + "ldap": ["cache-refresh"], + "spanner": ["people, groups, authorizations"], + "sql": ["", "cache", "sessions"], + } + assert PersistenceMapper().groups_with_rdn() == groups diff --git a/jans-pycloudlib/tests/test_validators.py b/jans-pycloudlib/tests/test_validators.py index 1d1b342ff8c..e88a36754ae 100644 --- a/jans-pycloudlib/tests/test_validators.py +++ b/jans-pycloudlib/tests/test_validators.py @@ -6,6 +6,7 @@ "couchbase", "hybrid", "sql", + "spanner", ]) def test_validate_persistence_type(type_): from jans.pycloudlib.validators import validate_persistence_type @@ -19,26 +20,6 @@ def test_validate_persistence_type_invalid(): validate_persistence_type("random") -@pytest.mark.parametrize("mapping", [ - "default", - "user", - "site", - "cache", - "token", - "session", -]) -def test_validate_persistence_ldap_mapping(mapping): - from jans.pycloudlib.validators import validate_persistence_ldap_mapping - assert validate_persistence_ldap_mapping("hybrid", mapping) is None - - -def test_validate_persistence_ldap_mapping_invalid(): - from jans.pycloudlib.validators import validate_persistence_ldap_mapping - - with pytest.raises(ValueError): - validate_persistence_ldap_mapping("hybrid", "random") - - @pytest.mark.parametrize("dialect", [ "mysql", # "pgsql", @@ -54,3 +35,22 @@ def test_validate_persistence_sql_dialect_invalid(): with pytest.raises(ValueError): validate_persistence_sql_dialect("random") + + +def test_validate_persistence_ldap_mapping(): + from jans.pycloudlib.validators import validate_persistence_ldap_mapping + + with pytest.deprecated_call(): + validate_persistence_ldap_mapping("hybrid", "default") + + +def test_validate_persistence_hybrid_mapping(monkeypatch): + from jans.pycloudlib.validators import validate_persistence_hybrid_mapping + + monkeypatch.setattr( + "jans.pycloudlib.persistence.utils.PersistenceMapper.validate_hybrid_mapping", + lambda cls: True, + ) + + # asserts PersistenceMapper.validate_hybrid_mapping is called + assert validate_persistence_hybrid_mapping() is None diff --git a/jans-pycloudlib/tests/test_wait.py b/jans-pycloudlib/tests/test_wait.py index 8eb9d28b756..290c06573d4 100644 --- a/jans-pycloudlib/tests/test_wait.py +++ b/jans-pycloudlib/tests/test_wait.py @@ -1,3 +1,7 @@ +import json +from dataclasses import dataclass +from unittest.mock import patch + import pytest @@ -69,23 +73,309 @@ def test_wait_for_secret(gmanager, monkeypatch): wait_for_secret(gmanager) -@pytest.mark.parametrize("persistence_type", ["ldap", "hybrid"]) -def test_wait_for_ldap(gmanager, monkeypatch, persistence_type): +def test_wait_for_ldap(gmanager, monkeypatch): from jans.pycloudlib.wait import wait_for_ldap monkeypatch.setenv("CN_WAIT_MAX_TIME", "0") - monkeypatch.setenv("CN_PERSISTENCE_TYPE", persistence_type) + monkeypatch.setenv("CN_PERSISTENCE_TYPE", "ldap") + + monkeypatch.setattr( + "jans.pycloudlib.persistence.ldap.LdapClient.search", + lambda cls, base, filter_, attrs, limit: None + ) + + with pytest.raises(Exception): + wait_for_ldap(gmanager) + + +_PERSISTENCE_MAPPER_GROUP_FUNC = "jans.pycloudlib.persistence.utils.PersistenceMapper.groups" + + +def test_wait_for_ldap_no_search_mapping(gmanager, monkeypatch): + from jans.pycloudlib.wait import wait_for_ldap + + monkeypatch.setenv("CN_WAIT_MAX_TIME", "0") + monkeypatch.setenv("CN_PERSISTENCE_TYPE", "ldap") + + monkeypatch.setattr( + "jans.pycloudlib.persistence.ldap.LdapClient.is_connected", + lambda cls: False + ) + + monkeypatch.setattr( + _PERSISTENCE_MAPPER_GROUP_FUNC, + lambda cls: {"ldap": ["random"]} + ) with pytest.raises(Exception): wait_for_ldap(gmanager) -@pytest.mark.parametrize("persistence_type", ["couchbase", "hybrid"]) -def test_wait_for_couchbase(gmanager, monkeypatch, persistence_type): +def test_wait_for_ldap_conn(gmanager, monkeypatch): + from jans.pycloudlib.wait import wait_for_ldap_conn + + monkeypatch.setenv("CN_WAIT_MAX_TIME", "0") + monkeypatch.setenv("CN_PERSISTENCE_TYPE", "ldap") + + monkeypatch.setattr( + "jans.pycloudlib.persistence.ldap.LdapClient.is_connected", + lambda cls: False + ) + + with pytest.raises(Exception): + wait_for_ldap_conn(gmanager) + + +def test_wait_for_couchbase(gmanager, monkeypatch): from jans.pycloudlib.wait import wait_for_couchbase monkeypatch.setenv("CN_WAIT_MAX_TIME", "0") - monkeypatch.setenv("CN_PERSISTENCE_TYPE", persistence_type) + monkeypatch.setenv("CN_PERSISTENCE_TYPE", "couchbase") + + monkeypatch.setattr( + "jans.pycloudlib.persistence.couchbase.CouchbaseClient.doc_exists", + lambda cls, b, i: False + ) + + with pytest.raises(Exception): + wait_for_couchbase(gmanager) + + +def test_wait_for_couchbase_no_search_mapping(gmanager, monkeypatch): + from jans.pycloudlib.wait import wait_for_couchbase + + monkeypatch.setenv("CN_WAIT_MAX_TIME", "0") + monkeypatch.setenv("CN_PERSISTENCE_TYPE", "couchbase") + + monkeypatch.setattr( + "jans.pycloudlib.persistence.couchbase.CouchbaseClient.doc_exists", + lambda cls, b, i: False + ) + + monkeypatch.setattr( + _PERSISTENCE_MAPPER_GROUP_FUNC, + lambda cls: {"couchbase": ["random"]} + ) + + @dataclass + class FakeResponse: + ok: bool + + monkeypatch.setattr( + "jans.pycloudlib.persistence.couchbase.CouchbaseClient.get_buckets", + lambda cls: FakeResponse(False), + ) with pytest.raises(Exception): wait_for_couchbase(gmanager) + + +def test_wait_for_couchbase_conn(gmanager, monkeypatch): + from jans.pycloudlib.wait import wait_for_couchbase_conn + + monkeypatch.setenv("CN_WAIT_MAX_TIME", "0") + monkeypatch.setenv("CN_PERSISTENCE_TYPE", "couchbase") + + @dataclass + class FakeResponse: + ok: bool + + monkeypatch.setattr( + "jans.pycloudlib.persistence.couchbase.CouchbaseClient.get_buckets", + lambda cls: FakeResponse(False), + ) + + with pytest.raises(Exception): + wait_for_couchbase_conn(gmanager) + + +def test_wait_for_sql(monkeypatch, gmanager): + from jans.pycloudlib.wait import wait_for_sql + + monkeypatch.setenv("CN_WAIT_MAX_TIME", "0") + monkeypatch.setenv("CN_PERSISTENCE_TYPE", "sql") + + monkeypatch.setattr( + "jans.pycloudlib.persistence.sql.SqlClient.row_exists", + lambda cls, t, i: False + ) + + with pytest.raises(Exception): + wait_for_sql(gmanager) + + +def test_wait_for_sql_no_search_mapping(monkeypatch, gmanager): + from jans.pycloudlib.wait import wait_for_sql + + monkeypatch.setenv("CN_WAIT_MAX_TIME", "0") + monkeypatch.setenv("CN_PERSISTENCE_TYPE", "sql") + + monkeypatch.setattr( + _PERSISTENCE_MAPPER_GROUP_FUNC, + lambda cls: {"sql": ["random"]} + ) + + monkeypatch.setattr( + "jans.pycloudlib.persistence.sql.SqlClient.connected", + lambda cls: False + ) + + with pytest.raises(Exception): + wait_for_sql(gmanager) + + +def test_wait_for_sql_conn(monkeypatch, gmanager): + from jans.pycloudlib.wait import wait_for_sql_conn + + monkeypatch.setenv("CN_WAIT_MAX_TIME", "0") + monkeypatch.setenv("CN_PERSISTENCE_TYPE", "sql") + + monkeypatch.setattr( + "jans.pycloudlib.persistence.sql.SqlClient.connected", + lambda cls: False + ) + + with pytest.raises(Exception): + wait_for_sql_conn(gmanager) + + +def test_wait_for_spanner(monkeypatch, gmanager): + from jans.pycloudlib.wait import wait_for_spanner + + monkeypatch.setenv("CN_WAIT_MAX_TIME", "0") + monkeypatch.setenv("CN_PERSISTENCE_TYPE", "spanner") + + monkeypatch.setattr( + "jans.pycloudlib.persistence.spanner.SpannerClient.row_exists", + lambda cls, t, i: False + ) + + with pytest.raises(Exception): + wait_for_spanner(gmanager) + + +def test_wait_for_spanner_no_search_mapping(monkeypatch, gmanager): + from jans.pycloudlib.wait import wait_for_spanner + + monkeypatch.setenv("CN_WAIT_MAX_TIME", "0") + monkeypatch.setenv("CN_PERSISTENCE_TYPE", "spanner") + + monkeypatch.setattr( + _PERSISTENCE_MAPPER_GROUP_FUNC, + lambda cls: {"spanner": ["random"]} + ) + + monkeypatch.setattr( + "jans.pycloudlib.persistence.spanner.SpannerClient.connected", + lambda cls: False + ) + + with pytest.raises(Exception): + wait_for_spanner(gmanager) + + +def test_wait_for_spanner_conn(monkeypatch, gmanager): + from jans.pycloudlib.wait import wait_for_spanner_conn + + monkeypatch.setenv("CN_WAIT_MAX_TIME", "0") + monkeypatch.setenv("CN_PERSISTENCE_TYPE", "spanner") + + monkeypatch.setattr( + "jans.pycloudlib.persistence.spanner.SpannerClient.connected", + lambda cls: False + ) + + with pytest.raises(Exception): + wait_for_spanner_conn(gmanager) + + +_WAIT_FOR_FUNC = "jans.pycloudlib.wait.wait_for" + + +@pytest.mark.parametrize("persistence_type, deps", [ + ("ldap", ["ldap"]), + ("couchbase", ["couchbase"]), + ("sql", ["sql"]), + ("spanner", ["spanner"]), +]) +def test_wait_for_persistence(monkeypatch, gmanager, persistence_type, deps): + from jans.pycloudlib.wait import wait_for_persistence + + monkeypatch.setenv("CN_PERSISTENCE_TYPE", persistence_type) + + with patch(_WAIT_FOR_FUNC, autospec=True) as patched: + wait_for_persistence(gmanager) + patched.assert_called_with(gmanager, deps) + + +def test_wait_for_persistence_hybrid(monkeypatch, gmanager): + from jans.pycloudlib.wait import wait_for_persistence + + monkeypatch.setenv("CN_PERSISTENCE_TYPE", "hybrid") + monkeypatch.setenv( + "CN_HYBRID_MAPPING", + json.dumps({ + "default": "sql", + "user": "spanner", + "site": "ldap", + "cache": "sql", + "token": "couchbase", + "session": "sql", + }), + ) + + with patch(_WAIT_FOR_FUNC, autospec=True) as patched: + wait_for_persistence(gmanager) + patched.assert_called_with(gmanager, ["couchbase", "ldap", "spanner", "sql"]) + + +@pytest.mark.parametrize("persistence_type, deps", [ + ("ldap", ["ldap_conn"]), + ("couchbase", ["couchbase_conn"]), + ("sql", ["sql_conn"]), + ("spanner", ["spanner_conn"]), +]) +def test_wait_for_persistence_conn(monkeypatch, gmanager, persistence_type, deps): + from jans.pycloudlib.wait import wait_for_persistence_conn + + monkeypatch.setenv("CN_PERSISTENCE_TYPE", persistence_type) + + with patch(_WAIT_FOR_FUNC, autospec=True) as patched: + wait_for_persistence_conn(gmanager) + patched.assert_called_with(gmanager, deps) + + +def test_wait_for_persistence_conn_hybrid(monkeypatch, gmanager): + from jans.pycloudlib.wait import wait_for_persistence_conn + + monkeypatch.setenv("CN_PERSISTENCE_TYPE", "hybrid") + monkeypatch.setenv( + "CN_HYBRID_MAPPING", + json.dumps({ + "default": "sql", + "user": "spanner", + "site": "ldap", + "cache": "sql", + "token": "couchbase", + "session": "sql", + }), + ) + + with patch(_WAIT_FOR_FUNC, autospec=True) as patched: + wait_for_persistence_conn(gmanager) + patched.assert_called_with(gmanager, ["couchbase_conn", "ldap_conn", "spanner_conn", "sql_conn"]) + + +def test_wait_for(gmanager): + from jans.pycloudlib.wait import wait_for + + with patch("jans.pycloudlib.wait.wait_for_config") as patched: + wait_for(gmanager, ["config"]) + patched.assert_called() + + +def test_wait_for_invalid_deps(gmanager, caplog): + from jans.pycloudlib.wait import wait_for + + wait_for(gmanager, ["random"]) + assert "Unsupported callback for random dependency" in caplog.records[0].message diff --git a/jans-pycloudlib/tox.ini b/jans-pycloudlib/tox.ini index df8c8fcc621..727ff448b70 100644 --- a/jans-pycloudlib/tox.ini +++ b/jans-pycloudlib/tox.ini @@ -9,7 +9,7 @@ deps = pytest-localserver distro commands = - pytest -v --cov-config=.coveragerc --cov=jans.pycloudlib --cov-report=term-missing:skip-covered --cov-report=xml tests/ + pytest -vv --cov-config=.coveragerc --cov=jans.pycloudlib --cov-report=term-missing:skip-covered --cov-report=xml {posargs:tests/} [flake8] # A003 class attribute "X" is shadowing a python builtin