From 17f9e3ee292caf03dcc249f84461361cf7ed935e Mon Sep 17 00:00:00 2001 From: Neville Samuell Date: Wed, 22 Feb 2023 13:04:08 -0500 Subject: [PATCH 01/10] Add support for redis.user for the cache --- src/fides/api/ops/util/cache.py | 1 + src/fides/core/config/redis_settings.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/fides/api/ops/util/cache.py b/src/fides/api/ops/util/cache.py index 3d2b8f4e46..99e5ffdd54 100644 --- a/src/fides/api/ops/util/cache.py +++ b/src/fides/api/ops/util/cache.py @@ -172,6 +172,7 @@ def get_cache() -> FidesopsRedis: host=CONFIG.redis.host, port=CONFIG.redis.port, db=CONFIG.redis.db_index, + username=CONFIG.redis.user, password=CONFIG.redis.password, ssl=CONFIG.redis.ssl, ssl_cert_reqs=CONFIG.redis.ssl_cert_reqs, diff --git a/src/fides/core/config/redis_settings.py b/src/fides/core/config/redis_settings.py index be89a97613..cedf81ca5f 100644 --- a/src/fides/core/config/redis_settings.py +++ b/src/fides/core/config/redis_settings.py @@ -13,7 +13,7 @@ class RedisSettings(FidesSettings): host: str = "redis" port: int = 6379 - user: Optional[str] = "" + user: Optional[str] = None password: str = "testpassword" charset: str = "utf8" decode_responses: bool = True From 2373dc80eaad3bb443e43b8275d91cc140b3cff6 Mon Sep 17 00:00:00 2001 From: Neville Samuell Date: Wed, 22 Feb 2023 15:44:10 -0500 Subject: [PATCH 02/10] Fix handling of optional redis.user setting --- src/fides/core/config/redis_settings.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/fides/core/config/redis_settings.py b/src/fides/core/config/redis_settings.py index cedf81ca5f..74c687640f 100644 --- a/src/fides/core/config/redis_settings.py +++ b/src/fides/core/config/redis_settings.py @@ -37,17 +37,18 @@ def assemble_connection_url( # If the whole URL is provided via the config, preference that return v - db_index = values.get("db_index") if values.get("db_index") is not None else "" + db_index = values.get("db_index") or "" connection_protocol = "redis" params = "" use_tls = values.get("ssl", False) + user = values.get("user") or "" if use_tls: # If using TLS update the connection URL format connection_protocol = "rediss" cert_reqs = values.get("ssl_cert_reqs", "none") params = f"?ssl_cert_reqs={quote_plus(cert_reqs)}" - return f"{connection_protocol}://{quote_plus(values.get('user', ''))}:{quote_plus(values.get('password', ''))}@{values.get('host', '')}:{values.get('port', '')}/{db_index}{params}" + return f"{connection_protocol}://{quote_plus(user)}:{quote_plus(values.get('password', ''))}@{values.get('host', '')}:{values.get('port', '')}/{db_index}{params}" class Config: env_prefix = ENV_PREFIX From caf1b3e7b3a3e4e9a0da11cd5aa9be5aa9a2de98 Mon Sep 17 00:00:00 2001 From: Neville Samuell Date: Wed, 22 Feb 2023 15:44:52 -0500 Subject: [PATCH 03/10] Add more detailed usage for nox -s dev --- noxfiles/dev_nox.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/noxfiles/dev_nox.py b/noxfiles/dev_nox.py index 8ed905ba06..044a9833fc 100644 --- a/noxfiles/dev_nox.py +++ b/noxfiles/dev_nox.py @@ -21,7 +21,26 @@ @nox_session() def dev(session: Session) -> None: - """Spin up the application. Uses positional arguments for additional features.""" + """ + Spin up the Fides webserver in development mode alongside it's Postgres + database and Redis cache. Use positional arguments to run other services + like privacy center, shell, admin UI, etc. (see usage for examples) + + Usage: + 'nox -s dev' - runs the Fides weserver, database, and cache + 'nox -s dev -- shell' - also open a shell on the Fides webserver + 'nox -s dev -- ui' - also build and run the Admin UI + 'nox -s dev -- pc' - also build and run the Privacy Center + 'nox -s dev -- remote_debug' - run with remote debugging enabled (see docker-compose.remote-debug.yml) + 'nox -s dev -- worker' - also run a Fides worker + 'nox -s dev -- child' - also run a Fides child node + 'nox -s dev -- ' - also run a test datastore (e.g. 'mssql', 'mongodb') + + Note that you can combine any of the above arguments together, for example: + 'nox -s dev -- shell ui pc' + + See noxfiles/dev_nox.py for more info + """ build(session, "dev") session.notify("teardown") From c5b896907086080a65b1a7804e15db2a50739286 Mon Sep 17 00:00:00 2001 From: Neville Samuell Date: Wed, 22 Feb 2023 18:20:05 -0500 Subject: [PATCH 04/10] Add a redis.conf for testing --- .fides/fides.toml | 2 +- docker-compose.yml | 16 +++++++++++++--- docker/redis/redis.conf | 15 +++++++++++++++ 3 files changed, 29 insertions(+), 4 deletions(-) create mode 100644 docker/redis/redis.conf diff --git a/.fides/fides.toml b/.fides/fides.toml index 794da4f5b5..bb21d7f107 100644 --- a/.fides/fides.toml +++ b/.fides/fides.toml @@ -22,7 +22,7 @@ analytics_opt_out = false [redis] host = "redis" -password = "testpassword" +password = "redispassword" port = 6379 charset = "utf8" default_ttl_seconds = 604800 diff --git a/docker-compose.yml b/docker-compose.yml index a891fcf613..7d3e052307 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -127,13 +127,23 @@ services: redis: image: "redis:6.2.5-alpine" - command: redis-server --requirepass testpassword - environment: - - REDIS_PASSWORD=testpassword + # AUTH option #1: no authentication at all + # command: redis-server + # AUTH option #2: require password + command: redis-server --requirepass redispassword + # AUTH option #3: Redis ACL defined in redis.conf + # command: redis-server /usr/local/etc/redis/redis.conf expose: - 6379 ports: - "0.0.0.0:6379:6379" + volumes: + # Mount a redis.conf file for configuration + # NOTE: Only used by "AUTH option #3" above! + - type: bind + source: ./docker/redis + target: /usr/local/etc/redis + read_only: False volumes: postgres: null diff --git a/docker/redis/redis.conf b/docker/redis/redis.conf new file mode 100644 index 0000000000..18dec3e0c6 --- /dev/null +++ b/docker/redis/redis.conf @@ -0,0 +1,15 @@ +# Redis configuration file for local Fides development +# +# Note that this file is not loaded by default, and it is checked in here for +# manual testing in the future. To use this redis.conf file, do the following: +# 1) Check docker-compose.yml is mounting this to /usr/local/etc/redis +# 2) Edit docker-compose.yml to swap in the `command` for "AUTH option #3", +# which should look like this: +# ``` +# command: redis-server /usr/local/etc/redis/redis.conf +# ``` +# 3) Make any edits to this file and bring up redis with `nox -s dev` or similar + +# Enable an ACL that gives access to all keys and all commands, but requires +# a login with user="redisadmin" and password="redispassword" +user redisadmin on ~* +@all >redispassword \ No newline at end of file From ca607c6daa7bbf09c836101964599f470d2a1600 Mon Sep 17 00:00:00 2001 From: Neville Samuell Date: Wed, 22 Feb 2023 18:20:39 -0500 Subject: [PATCH 05/10] Remove default redis password --- src/fides/core/config/redis_settings.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/fides/core/config/redis_settings.py b/src/fides/core/config/redis_settings.py index 74c687640f..62eff068e9 100644 --- a/src/fides/core/config/redis_settings.py +++ b/src/fides/core/config/redis_settings.py @@ -14,7 +14,7 @@ class RedisSettings(FidesSettings): host: str = "redis" port: int = 6379 user: Optional[str] = None - password: str = "testpassword" + password: str = None charset: str = "utf8" decode_responses: bool = True default_ttl_seconds: int = 604800 @@ -42,13 +42,20 @@ def assemble_connection_url( params = "" use_tls = values.get("ssl", False) user = values.get("user") or "" + password = values.get("password") or "" if use_tls: # If using TLS update the connection URL format connection_protocol = "rediss" cert_reqs = values.get("ssl_cert_reqs", "none") params = f"?ssl_cert_reqs={quote_plus(cert_reqs)}" - return f"{connection_protocol}://{quote_plus(user)}:{quote_plus(values.get('password', ''))}@{values.get('host', '')}:{values.get('port', '')}/{db_index}{params}" + # Configure a basic auth prefix if either user or password is provided, e.g. + # redis://:@ + auth_prefix = "" + if password or user: + auth_prefix = f"{quote_plus(user)}:{quote_plus(password)}@" + + return f"{connection_protocol}://{auth_prefix}{values.get('host', '')}:{values.get('port', '')}/{db_index}{params}" class Config: env_prefix = ENV_PREFIX From 6825e7f3aec87c9a48549db52802e2bf7446b375 Mon Sep 17 00:00:00 2001 From: Neville Samuell Date: Wed, 22 Feb 2023 18:25:50 -0500 Subject: [PATCH 06/10] Update CHANGELOG --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2760f4fa7d..db6b606741 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,11 +17,17 @@ The types of changes are: ## [Unreleased](https://github.com/ethyca/fides/compare/2.7.0...main) ### Added + * Add API support for messaging config properties [#2551](https://github.com/ethyca/fides/pull/2551) ### Changed * Add warning to 'fides deploy' when installed outside of a virtual environment [#2641](https://github.com/ethyca/fides/pull/2641) +* Removed unexpected default Redis password [#2666](https://github.com/ethyca/fides/pull/2666) + +### Fixed + +* Fix support for "redis.user" setting when authenticating to the Redis cache [#2666](https://github.com/ethyca/fides/pull/2666) ## [2.7.0](https://github.com/ethyca/fides/compare/2.6.6...2.7.0) From 57cc04bddb7f7178b5c807af48047d14acd3a6b8 Mon Sep 17 00:00:00 2001 From: Neville Samuell Date: Wed, 22 Feb 2023 18:34:51 -0500 Subject: [PATCH 07/10] Update test code that relied on default redis password --- docker-compose.child-env.yml | 4 +--- docker/docker-compose.minimal-config.yml | 5 ++--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/docker-compose.child-env.yml b/docker-compose.child-env.yml index 926b4209bf..7488fbd508 100644 --- a/docker-compose.child-env.yml +++ b/docker-compose.child-env.yml @@ -66,9 +66,7 @@ services: redis-child: image: "redis:6.2.5-alpine" - command: redis-server --requirepass testpassword - environment: - - REDIS_PASSWORD=testpassword + command: redis-server --requirepass redispassword expose: - 6379 ports: diff --git a/docker/docker-compose.minimal-config.yml b/docker/docker-compose.minimal-config.yml index c86c4e064f..15e49ea9e6 100644 --- a/docker/docker-compose.minimal-config.yml +++ b/docker/docker-compose.minimal-config.yml @@ -22,6 +22,7 @@ services: FIDES__DATABASE__PASSWORD: "fides" FIDES__DATABASE__PORT: "5432" FIDES__DATABASE__DB: "fides" + FIDES__REDIS__PASSWORD: "redispassword" FIDES__USER__ANALYTICS_OPT_OUT: "True" FIDES__SECURITY__APP_ENCRYPTION_KEY: "OLMkv91j8DHiDAULnK5Lxx3kSCov30b3" FIDES__SECURITY__OAUTH_ROOT_CLIENT_ID: "fidesadmin" @@ -52,9 +53,7 @@ services: redis: image: "redis:6.2.5-alpine" - command: redis-server --requirepass testpassword - environment: - - REDIS_PASSWORD=testpassword + command: redis-server --requirepass redispassword expose: - 6379 ports: From 060e72f13331fca5cec416c6f7b3c7bcb4172c66 Mon Sep 17 00:00:00 2001 From: Sean Preston Date: Wed, 22 Feb 2023 18:43:41 -0500 Subject: [PATCH 08/10] adds comment --- src/fides/core/config/redis_settings.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/fides/core/config/redis_settings.py b/src/fides/core/config/redis_settings.py index 62eff068e9..05a89e03c5 100644 --- a/src/fides/core/config/redis_settings.py +++ b/src/fides/core/config/redis_settings.py @@ -37,12 +37,16 @@ def assemble_connection_url( # If the whole URL is provided via the config, preference that return v - db_index = values.get("db_index") or "" connection_protocol = "redis" params = "" use_tls = values.get("ssl", False) + + # These vars are intentionally fetched with `or ""` as the default to account + # for the edge case where `None` is explicitly set in `values` by Pydantic because + # it is not overridden by the config file or an env var user = values.get("user") or "" password = values.get("password") or "" + db_index = values.get("db_index") or "" if use_tls: # If using TLS update the connection URL format connection_protocol = "rediss" From 91b667468cf118f59b2b71b896161fecc1b82820 Mon Sep 17 00:00:00 2001 From: Neville Samuell Date: Wed, 22 Feb 2023 18:44:37 -0500 Subject: [PATCH 09/10] Fix mypy warning --- src/fides/core/config/redis_settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fides/core/config/redis_settings.py b/src/fides/core/config/redis_settings.py index 05a89e03c5..f23bcdc1f9 100644 --- a/src/fides/core/config/redis_settings.py +++ b/src/fides/core/config/redis_settings.py @@ -14,7 +14,7 @@ class RedisSettings(FidesSettings): host: str = "redis" port: int = 6379 user: Optional[str] = None - password: str = None + password: Optional[str] = None charset: str = "utf8" decode_responses: bool = True default_ttl_seconds: int = 604800 From e2ae769ce544f1112572a51fc388f898ef67958c Mon Sep 17 00:00:00 2001 From: Neville Samuell Date: Wed, 22 Feb 2023 20:00:49 -0500 Subject: [PATCH 10/10] Update test_env config with new redis password --- src/fides/data/test_env/fides.test_env.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fides/data/test_env/fides.test_env.toml b/src/fides/data/test_env/fides.test_env.toml index f29ced3c8a..855c5a2afb 100644 --- a/src/fides/data/test_env/fides.test_env.toml +++ b/src/fides/data/test_env/fides.test_env.toml @@ -8,7 +8,7 @@ db = "fides" [redis] host = "redis" -password = "testpassword" +password = "redispassword" port = 6379 db_index = 0