From da4057238a8f528632d70df04b6f5350d4d74095 Mon Sep 17 00:00:00 2001 From: Neville Samuell Date: Wed, 12 Apr 2023 08:35:20 -0400 Subject: [PATCH 1/4] Changed test environment (`nox -s fides_env`) to run `fides deploy` for local testing (#3017) Co-authored-by: Thomas --- .github/workflows/cli_checks.yml | 13 +- CHANGELOG.md | 1 + docker-compose.child-env.yml | 1 - docker-compose.integration-tests.yml | 1 + docker-compose.test-env.yml | 33 ---- docker-compose.yml | 2 +- .../docs/development/release_checklist.md | 19 +- .../docs/development/testing_environment.md | 46 +++-- .../docs/development/vscode_debugging.md | 2 +- noxfiles/constants_nox.py | 33 ++-- noxfiles/dev_nox.py | 164 +++++++----------- noxfiles/docker_nox.py | 15 +- noxfiles/docs_nox.py | 1 - noxfiles/utils_nox.py | 62 +++---- scripts/load_examples.py | 9 +- scripts/setup/user.py | 65 ------- src/fides/api/main.py | 5 +- src/fides/cli/commands/util.py | 28 ++- src/fides/core/config/__init__.py | 34 ++++ src/fides/core/config/helpers.py | 50 +----- src/fides/core/config/redis_settings.py | 2 +- src/fides/core/deploy.py | 28 +-- .../data/sample_project/docker-compose.yml | 37 ++-- src/fides/data/sample_project/fides.toml | 14 +- .../privacy_center/config/config.json | 29 +++- src/fides/data/sample_project/sample.env | 5 + src/fides/data/test_env/fides.test_env.toml | 43 ----- .../test_env/privacy_center_config/config.css | 19 -- .../privacy_center_config/config.json | 80 --------- tests/ctl/core/config/test_config.py | 58 ++++++- tests/ctl/core/config/test_config_helpers.py | 26 --- tests/ctl/test_default_config.toml | 1 + 32 files changed, 351 insertions(+), 575 deletions(-) delete mode 100644 docker-compose.test-env.yml delete mode 100644 scripts/setup/user.py create mode 100644 src/fides/data/sample_project/sample.env delete mode 100644 src/fides/data/test_env/fides.test_env.toml delete mode 100644 src/fides/data/test_env/privacy_center_config/config.css delete mode 100644 src/fides/data/test_env/privacy_center_config/config.json create mode 100644 tests/ctl/test_default_config.toml diff --git a/.github/workflows/cli_checks.yml b/.github/workflows/cli_checks.yml index 2c18efb9e5..2b39411fc2 100644 --- a/.github/workflows/cli_checks.yml +++ b/.github/workflows/cli_checks.yml @@ -17,7 +17,8 @@ env: DEFAULT_PYTHON_VERSION: "3.10.11" jobs: - Fides-Deploy: + # Basic smoke test of a local install of the fides Python CLI + Fides-Install: runs-on: ubuntu-latest timeout-minutes: 20 steps: @@ -32,14 +33,8 @@ jobs: - name: Install Nox run: pip install nox>=2022 - - name: Build the sample image - run: nox -s "build(sample)" - - name: Install fides run: pip install . - - name: Start the sample application - run: fides deploy up --no-pull --no-init - - - name: Stop the sample application - run: fides deploy down + - name: Run `fides --version` + run: fides --version diff --git a/CHANGELOG.md b/CHANGELOG.md index 3fce1c246f..b4b8f70b7f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,6 +51,7 @@ The types of changes are: ### Developer Experience - Nox commands for git tagging to support feature branch builds [#2979](https://github.com/ethyca/fides/pull/2979) +- Changed test environment (`nox -s fides_env`) to run `fides deploy` for local testing [#3071](https://github.com/ethyca/fides/pull/3017) ### Removed diff --git a/docker-compose.child-env.yml b/docker-compose.child-env.yml index 7488fbd508..68f823c84a 100644 --- a/docker-compose.child-env.yml +++ b/docker-compose.child-env.yml @@ -27,7 +27,6 @@ services: FIDES__DATABASE__DB: "child_test_db" FIDES__DATABASE__TEST_DB: "child_test_db" FIDES__DEV_MODE: "True" - FIDES__REDIS__ENABLED: "True" FIDES__REDIS__HOST: "redis-child" FIDES__TEST_MODE: "True" FIDES__USER__ANALYTICS_OPT_OUT: "True" diff --git a/docker-compose.integration-tests.yml b/docker-compose.integration-tests.yml index 7cc46817ac..0a66191bc0 100644 --- a/docker-compose.integration-tests.yml +++ b/docker-compose.integration-tests.yml @@ -1,5 +1,6 @@ services: fides: + image: ethyca/fides:local depends_on: - postgres-test - mysql-test diff --git a/docker-compose.test-env.yml b/docker-compose.test-env.yml deleted file mode 100644 index 7a2556ddca..0000000000 --- a/docker-compose.test-env.yml +++ /dev/null @@ -1,33 +0,0 @@ -services: - fides: - depends_on: - - postgres-test - - mongodb-test - - postgres-test: - image: postgres:12 - environment: - - POSTGRES_USER=postgres - - POSTGRES_PASSWORD=postgres - - POSTGRES_DB=postgres_example - expose: - - 6432 - ports: - - "0.0.0.0:6432:5432" - volumes: - - ./src/fides/data/sample_project/postgres_sample.sql:/docker-entrypoint-initdb.d/postgres_example.sql:ro - - mongodb-test: - image: mongo:5.0.3 - environment: - - MONGO_INITDB_DATABASE=mongo_test - - MONGO_INITDB_ROOT_USERNAME=mongo_user - - MONGO_INITDB_ROOT_PASSWORD=mongo_pass - expose: - - 27017 - ports: - - "27017:27017" - # Because we're using the "-f" flag from a parent directory, this relative path needs - # to be from the parent directory as well - volumes: - - ./src/fides/data/sample_project/mongo_sample.js:/docker-entrypoint-initdb.d/mongo-init.js:ro diff --git a/docker-compose.yml b/docker-compose.yml index 22869abb19..8093a93c2b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,6 @@ services: fides: + container_name: fides image: ethyca/fides:local command: uvicorn --host 0.0.0.0 --port 8080 --reload --reload-dir src fides.api.main:app healthcheck: @@ -25,7 +26,6 @@ services: FIDES__CLI__SERVER_PORT: "8080" FIDES__DATABASE__SERVER: "fides-db" FIDES__DEV_MODE: "True" - FIDES__REDIS__ENABLED: "True" FIDES__USER__ANALYTICS_OPT_OUT: "True" FIDES__SECURITY__ALLOW_CUSTOM_CONNECTOR_FUNCTIONS: "True" VAULT_ADDR: ${VAULT_ADDR-} diff --git a/docs/fides/docs/development/release_checklist.md b/docs/fides/docs/development/release_checklist.md index 34857ee5ad..19db462afb 100644 --- a/docs/fides/docs/development/release_checklist.md +++ b/docs/fides/docs/development/release_checklist.md @@ -11,23 +11,8 @@ This checklist should be copy/pasted into the final pre-release PR, and checked From the release branch, confirm the following: - [ ] Quickstart works: `nox -s quickstart` (verify you can complete the interactive prompts from the command-line) - [ ] Test environment works: `nox -s "fides_env(test)"` (verify the admin UI on localhost:8080, privacy center on localhost:3001, CLI and webserver) -- [ ] Building the sample app images works: `nox -s "build(sample)"` (creates the sample images, which is also prereq for `fides deploy up --no-pull` next) -- [ ] Running the CLI deploy works: `fides deploy up --no-pull` (see instructions below...) - -``` -mkdir ~/fides-deploy-test -cd ~/fides-deploy-test -python3 -m venv venv -source venv/bin/activate -pip install git+https://github.com/ethyca/fides.git@ -fides deploy up --no-pull -fides status -fides deploy down -rm -rf ~/fides-deploy-test -exit -``` - -Next, run the following checks using the test environment (`nox -s "fides_env(test)"`): + +Next, run the following checks via the test environment: ### API diff --git a/docs/fides/docs/development/testing_environment.md b/docs/fides/docs/development/testing_environment.md index 320abd9e6a..0a40e998cf 100644 --- a/docs/fides/docs/development/testing_environment.md +++ b/docs/fides/docs/development/testing_environment.md @@ -1,29 +1,43 @@ # Testing Environment -To facilitate thorough manual testing of the application, there is a comprehensive testing environment that can be set up via a single `nox` command. +## Quickstart +1. Use `nox -s "fides_env(test)"` to launch the test environment +2. Read the terminal output for details +3. Customize Fides ENV variables by editing `.env` -## Configuration +## Overview + +To facilitate thorough manual testing of the application, there is a comprehensive testing environment that can be set up via a single `nox` command: `nox -s "fides_env(test)"`. -The environment will configure the `fides` server and CLI using the TOML configuration set in `src/fides/data/test_env/fides.test_env.toml`. To test out other configurations, you can edit this file and reload the test env; however, don't commit these changes unless you are sure that the default configuration for testing should change for everyone! +This test environment includes: +* Fides Server +* Fides Admin UI +* Fides Postgres Database & Redis Cache +* Sample "Cookie House" Application +* Test Postgres Database +* Test Redis Database +* Sample Resources +* Sample Connectors +* etc. -## Secrets Management +This test environment is exactly the same environment that users can launch themselves using `fides deploy up`, and you can find all the configuration and settings in `src/fides/data/sample_project`. + +## Configuration -The environment will work "out of the box", but can also be configured with secrets needed to configure other features like S3 storage, Mailgun notifications, etc. To configure this, you'll need to create the `.env` file, place it at the root of the repository directory, and provide some secrets. There is an `example.env` file you can reference to see what secrets are supported. +There are two ways to configure the `fides` server and CLI: +1. Editing the ENV file in the project root: `.env` +2. Editing the TOML file in the sample project files: `src/fides/data/sample_project/fides.toml` -This `.env` file is ignored by git and therefore safe to keep in your local repo during development. +The `.env` file is safest to add secrets and local customizations, since it is `.gitignore`'d and will not be accidentally committed to version control. -For Ethyca-internal engineers, you can also grab a fully populated `.env` file from 1Password (called `Fides .env`). +The `fides.toml` file should be used for configurations that should be present for all users testing out the application. -## Spinning up the Environment +## Advanced Usage -Running `nox -s fides_env(test)` will spin up a comprehensive testing environment that does the following: +The environment will work "out of the box", but can also be configured to enable other features like S3 storage, email notifications, etc. -1. Builds the Webserver, Admin UI and Privacy Center. -1. Downloads all required images. -1. Spins up the entire application, including external Docker-based datastores. -1. Runs various commands and scripts to seed the application with example data, create a user, etc. -1. Opens a shell with the CLI loaded and available for use. +To configure these, you'll need to edit the `.env` file and provide some secrets - see `example.env` for what is supported. -Just before the shell is opened, a `Fides Test Environment` banner will be displayed along with various information about the testing environment and how to access various parts of the application. +## Automated Cypress E2E Tests -From here, everything has been configured and you may commence testing. +The test environment is also used to run automated end-to-end (E2E) tests via Cypress. Use `nox -s e2e_test` to run this locally. \ No newline at end of file diff --git a/docs/fides/docs/development/vscode_debugging.md b/docs/fides/docs/development/vscode_debugging.md index c94d20bbf1..3dfb988d07 100644 --- a/docs/fides/docs/development/vscode_debugging.md +++ b/docs/fides/docs/development/vscode_debugging.md @@ -18,7 +18,7 @@ nox -s dev -- remote_debug postgres timescale With those commands, the `fides` Docker Compose service that's running the Fides server locally is able to accept incoming remote debugging connections. -Note that, at this point, the `remote_debug` flag is not enabled for other `nox` sessions, e.g. `test_env`, `pytest_ops`, etc. +Note that, at this point, the `remote_debug` flag is not enabled for other `nox` sessions, e.g. `fides_env`, `pytest_ops`, etc. ### Attach a Remote Debugger to the Fides Server diff --git a/noxfiles/constants_nox.py b/noxfiles/constants_nox.py index ec0c54ca19..320ce0ae39 100644 --- a/noxfiles/constants_nox.py +++ b/noxfiles/constants_nox.py @@ -3,16 +3,27 @@ # Files COMPOSE_FILE = "docker-compose.yml" -INTEGRATION_COMPOSE_FILE = "docker-compose.integration-tests.yml" -INTEGRATION_POSTGRES_COMPOSE_FILE = "docker/docker-compose.integration-postgres.yml" -TEST_ENV_COMPOSE_FILE = "docker-compose.test-env.yml" +INTEGRATION_COMPOSE_FILE = "./docker-compose.integration-tests.yml" +INTEGRATION_POSTGRES_COMPOSE_FILE = "./docker/docker-compose.integration-postgres.yml" REMOTE_DEBUG_COMPOSE_FILE = "docker-compose.remote-debug.yml" +SAMPLE_PROJECT_COMPOSE_FILE = "./src/fides/data/sample_project/docker-compose.yml" WITH_TEST_CONFIG = ("-f", "tests/ctl/test_config.toml") +COMPOSE_FILE_LIST = { + COMPOSE_FILE, + SAMPLE_PROJECT_COMPOSE_FILE, + INTEGRATION_COMPOSE_FILE, + "docker/docker-compose.integration-mariadb.yml", + "docker/docker-compose.integration-mongodb.yml", + "docker/docker-compose.integration-mysql.yml", + "docker/docker-compose.integration-postgres.yml", + "docker/docker-compose.integration-mssql.yml", +} + # Image Names & Tags REGISTRY = "ethyca" IMAGE_NAME = "fides" -CONTAINER_NAME = "fides-fides-1" +CONTAINER_NAME = "fides" COMPOSE_SERVICE_NAME = "fides" # Image Names & Tags @@ -22,7 +33,6 @@ IMAGE_LOCAL = f"{IMAGE}:local" IMAGE_LOCAL_UI = f"{IMAGE}:local-ui" IMAGE_DEV = f"{IMAGE}:dev" -IMAGE_SAMPLE = f"{IMAGE}:sample" IMAGE_LATEST = f"{IMAGE}:latest" # Image names for the secondary apps @@ -49,7 +59,7 @@ LOGIN = ( "docker", "exec", - "fides-fides-1", + CONTAINER_NAME, "fides", "user", "login", @@ -97,17 +107,6 @@ "--wait", COMPOSE_SERVICE_NAME, ) -START_TEST_ENV = ( - "docker", - "compose", - "-f", - COMPOSE_FILE, - "-f", - TEST_ENV_COMPOSE_FILE, - "up", - "--wait", - COMPOSE_SERVICE_NAME, -) START_APP_REMOTE_DEBUG = ( "docker", "compose", diff --git a/noxfiles/dev_nox.py b/noxfiles/dev_nox.py index 83ff1b34bf..20201c604c 100644 --- a/noxfiles/dev_nox.py +++ b/noxfiles/dev_nox.py @@ -1,23 +1,21 @@ """Contains the nox sessions for running development environments.""" +import time +from pathlib import Path from typing import Literal -from nox import Session, param, parametrize -from nox import session as nox_session -from nox.command import CommandFailed - from constants_nox import ( COMPOSE_SERVICE_NAME, - EXEC, EXEC_IT, - LOGIN, RUN_CYPRESS_TESTS, START_APP, START_APP_REMOTE_DEBUG, - START_TEST_ENV, ) from docker_nox import build +from nox import Session, param, parametrize +from nox import session as nox_session +from nox.command import CommandFailed from run_infrastructure import ALL_DATASTORES, run_infrastructure -from utils_nox import COMPOSE_DOWN_VOLUMES +from utils_nox import install_requirements, teardown @nox_session() @@ -124,10 +122,10 @@ def cypress_tests(session: Session) -> None: @nox_session() def e2e_test(session: Session) -> None: """ - Spins up the test_env session and runs Cypress E2E tests against it. + Spins up the fides_env session and runs Cypress E2E tests against it. """ session.log("Running end-to-end tests...") - session.notify("fides_env(test)", posargs=["test"]) + session.notify("fides_env(test)", posargs=["keep_alive"]) session.notify("cypress_tests") session.notify("teardown") @@ -145,112 +143,84 @@ def fides_env(session: Session, fides_image: Literal["test", "dev"] = "test") -> Spins up a full fides environment seeded with data. Params: - dev = Spins up a full fides application with a dev-style docker container. This includes hot-reloading and no pre-baked UI. - test = Spins up a full fides application with a production-style docker container. This includes the UI being pre-built as static files. + dev = Spins up a full fides application with a dev-style docker container. + This includes hot-reloading and no pre-baked UI. + + test = Spins up a full fides application with a production-style docker + container. This includes the UI being pre-built as static files. Posargs: - test = instead of running 'bin/bash', runs 'fides' to verify the CLI and provide a zero exit code keep_alive = does not automatically call teardown after the session """ - - is_test = "test" in session.posargs keep_alive = "keep_alive" in session.posargs - - exec_command = EXEC if any([is_test, keep_alive]) else EXEC_IT - shell_command = "fides" if any([is_test, keep_alive]) else "/bin/bash" - - # Temporarily override some ENV vars as needed. To set local secrets, see 'example.env' - test_env_vars = { - "FIDES__CONFIG_PATH": "/fides/src/fides/data/test_env/fides.test_env.toml", - } - - session.log( - "Tearing down existing containers & volumes to prepare test environment..." - ) - try: - session.run(*COMPOSE_DOWN_VOLUMES, external=True, env=test_env_vars) - except CommandFailed: + if fides_image == "dev": session.error( - "Failed to cleanly teardown existing containers & volumes. Please exit out of all other and try again" + "'fides_env(dev)' is not currently implemented! Use 'nox -s dev' to run the server in dev mode. " + "Currently unclear how to (cleanly) mount the source code into the running container..." ) - if not keep_alive: - session.notify("teardown", posargs=["volumes"]) - session.log("Building images...") - build(session, fides_image) - build(session, "admin_ui") - build(session, "privacy_center") - - session.log( - "Starting the application with example databases defined in docker-compose.integration-tests.yml..." - ) - session.run( - *START_TEST_ENV, "fides-ui", "fides-pc", external=True, env=test_env_vars - ) - session.log("Logging in...") - session.run(*LOGIN, external=True) - - session.log( - "Running example setup scripts for DSR Automation tests... (scripts/load_examples.py)" - ) - session.run( - *EXEC, - "python", - "/fides/scripts/load_examples.py", - external=True, - env=test_env_vars, - ) + # Record timestamps along the way, so we can generate a build-time report + timestamps = [] + timestamps.append({"time": time.monotonic(), "label": "Start"}) + session.log("Tearing down existing containers & volumes...") + try: + teardown(session) + except CommandFailed: + session.error("Failed to cleanly teardown. Please try again!") + timestamps.append({"time": time.monotonic(), "label": "Docker Teardown"}) + + session.log("Building production images with 'build(test)'...") + build(session, "test") + timestamps.append({"time": time.monotonic(), "label": "Docker Build"}) + + session.log("Installing ethyca-fides locally...") + install_requirements(session) + session.install("-e", ".", "--no-deps") + session.run("fides", "--version") + timestamps.append({"time": time.monotonic(), "label": "pip install"}) + + # Configure the args for 'fides deploy up' for testing + env_file_path = Path(__file__, "../../.env").resolve() + fides_deploy_args = [ + "--no-pull", + "--no-init", + "--env-file", + str(env_file_path), + ] + + session.log("Deploying test environment with 'fides deploy up'...") session.log( - "Pushing example resources for Data Mapping tests... (demo_resources/*)" + f"NOTE: Customize your local Fides configuration via ENV file here: {env_file_path}" ) session.run( - *EXEC, "fides", - "push", - "demo_resources/", - external=True, - env=test_env_vars, - ) - - # Make spaces in the info message line up - title = ( - "FIDES TEST ENVIRONMENT" if fides_image == "test" else "FIDES DEV ENVIRONMENT " - ) - - session.log("****************************************") - session.log("* *") - session.log(f"* {title} *") - session.log("* *") - session.log("****************************************") - session.log("") - # Print out some helpful tips for using the test_env! - # NOTE: These constants are defined in scripts/setup/constants.py, docker-compose.yml, and docker-compose.integration-tests.yml - session.log( - "Using secrets set in '.env' for example setup scripts (see 'example.env' for options)" + "deploy", + "up", + *fides_deploy_args, ) - if fides_image == "test": + timestamps.append({"time": time.monotonic(), "label": "fides deploy"}) + + # Log a quick build-time report to help troubleshoot slow builds + session.log("[fides_env]: Ready! Build time report:") + session.log(f"{'Step':5} | {'Label':20} | Time") + session.log("------+----------------------+------") + for index, value in enumerate(timestamps): + if index == 0: + continue session.log( - "Fides Admin UI (production build) running at http://localhost:8080 (user: 'root_user', pass: 'Testpassword1!')" + f"{index:5} | {value['label']:20} | {value['time'] - timestamps[index-1]['time']:.2f}s" ) session.log( - "Run 'fides user login' to authenticate the CLI (user: 'root_user', pass: 'Testpassword1!')" - ) - session.log( - "Fides Admin UI (dev) running at http://localhost:3000 (user: 'root_user', pass: 'Testpassword1!')" - ) - session.log( - "Fides Privacy Center (production build) running at http://localhost:3001 (user: 'jane@example.com')" + f" | {'Total':20} | {timestamps[-1]['time'] - timestamps[0]['time']:.2f}s" ) - session.log( - "Example Postgres Database running at localhost:6432 (user: 'postgres', pass: 'postgres', db: 'postgres_example')" - ) - session.log( - "Example Mongo Database running at localhost:27017 (user: 'mongo_test', pass: 'mongo_pass', db: 'mongo_test')" - ) - session.log("Opening Fides CLI shell... (press CTRL+D to exit)") + session.log("------+----------------------+------\n") + + # Start a shell session unless 'keep_alive' is provided as a posarg if not keep_alive: - session.run(*exec_command, shell_command, external=True, env=test_env_vars) + session.log("Opening Fides CLI shell... (press CTRL+D to exit)") + session.run(*EXEC_IT, "/bin/bash", external=True, success_codes=[0, 1]) + session.run("fides", "deploy", "down") @nox_session() diff --git a/noxfiles/docker_nox.py b/noxfiles/docker_nox.py index cf9dbd55a4..d9da99b5ad 100644 --- a/noxfiles/docker_nox.py +++ b/noxfiles/docker_nox.py @@ -3,14 +3,12 @@ from typing import List import nox - from constants_nox import ( IMAGE, IMAGE_DEV, IMAGE_LATEST, IMAGE_LOCAL, IMAGE_LOCAL_UI, - IMAGE_SAMPLE, PRIVACY_CENTER_IMAGE, SAMPLE_APP_IMAGE, ) @@ -48,7 +46,6 @@ def get_platform(posargs: List[str]) -> str: nox.param("dev", id="dev"), nox.param("privacy_center", id="privacy-center"), nox.param("prod", id="prod"), - nox.param("sample", id="sample"), nox.param("test", id="test"), ], ) @@ -61,8 +58,7 @@ def build(session: nox.Session, image: str, machine_type: str = "") -> None: dev = Build the fides webserver/CLI, tagged as `local`. privacy-center = Build the Next.js Privacy Center application. prod = Build the fides webserver/CLI and tag it as the current application version. - sample = Builds all components required for the sample application. - test = Build the fides webserver/CLI the same as `prod`, but tag is as `local`. + test = Build the fides webserver/CLI the same as `prod`, but tag it as `local`. """ build_platform = get_platform(session.posargs) @@ -79,20 +75,19 @@ def build(session: nox.Session, image: str, machine_type: str = "") -> None: # This allows the dev deployment to run without requirements build_matrix = { "prod": {"tag": get_current_image, "target": "prod"}, - "dev": {"tag": lambda: IMAGE_LOCAL, "target": "dev"}, - "sample": {"tag": lambda: IMAGE_SAMPLE, "target": "prod"}, "test": {"tag": lambda: IMAGE_LOCAL, "target": "prod"}, + "dev": {"tag": lambda: IMAGE_LOCAL, "target": "dev"}, "admin_ui": {"tag": lambda: IMAGE_LOCAL_UI, "target": "frontend"}, } # When building for release, there are additional images that need # to get built. These images are outside of the primary `ethyca/fides` # image so some additional logic is required. - if image in ("sample", "prod"): + if image in ("test", "prod"): if image == "prod": tag_name = get_current_tag() - if image == "sample": - tag_name = "sample" + if image == "test": + tag_name = "local" privacy_center_image_tag = f"{PRIVACY_CENTER_IMAGE}:{tag_name}" sample_app_image_tag = f"{SAMPLE_APP_IMAGE}:{tag_name}" diff --git a/noxfiles/docs_nox.py b/noxfiles/docs_nox.py index e47906c8f5..286cabd9b6 100644 --- a/noxfiles/docs_nox.py +++ b/noxfiles/docs_nox.py @@ -1,6 +1,5 @@ """Contains the nox sessions for developing docs.""" import nox - from constants_nox import CI_ARGS diff --git a/noxfiles/utils_nox.py b/noxfiles/utils_nox.py index 0422f92e87..ecb1818356 100644 --- a/noxfiles/utils_nox.py +++ b/noxfiles/utils_nox.py @@ -2,35 +2,9 @@ from pathlib import Path import nox - -from constants_nox import COMPOSE_FILE, INTEGRATION_COMPOSE_FILE, TEST_ENV_COMPOSE_FILE +from constants_nox import COMPOSE_FILE_LIST from run_infrastructure import run_infrastructure -COMPOSE_DOWN = ( - "docker", - "compose", - "-f", - COMPOSE_FILE, - "-f", - INTEGRATION_COMPOSE_FILE, - "-f", - TEST_ENV_COMPOSE_FILE, - "-f", - "docker/docker-compose.integration-mariadb.yml", - "-f", - "docker/docker-compose.integration-mongodb.yml", - "-f", - "docker/docker-compose.integration-mysql.yml", - "-f", - "docker/docker-compose.integration-postgres.yml", - "-f", - "docker/docker-compose.integration-mssql.yml", - "down", - "--remove-orphans", -) -COMPOSE_DOWN_VOLUMES = COMPOSE_DOWN + ("--volumes",) - - @nox.session() def seed_test_data(session: nox.Session) -> None: """Seed test data in the Postgres application database.""" @@ -43,20 +17,36 @@ def clean(session: nox.Session) -> None: Clean up docker containers, remove orphans, remove volumes and prune images related to this project. """ - clean_command = (*COMPOSE_DOWN, "--volumes", "--rmi", "all") - session.run(*clean_command, external=True) + teardown(session, volumes=True, images=True) session.run("docker", "system", "prune", "--force", "--all", external=True) print("Clean Complete!") @nox.session() -def teardown(session: nox.Session) -> None: - """Tear down the docker dev environment.""" - if "volumes" in session.posargs: - session.run(*COMPOSE_DOWN_VOLUMES, external=True) - else: - session.run(*COMPOSE_DOWN, external=True) - print("Teardown complete") +def teardown(session: nox.Session, volumes: bool = False, images: bool = False) -> None: + """Tear down all docker environments.""" + for compose_file in COMPOSE_FILE_LIST: + teardown_command = ( + "docker", + "compose", + "-f", + compose_file, + "down", + "--remove-orphans", + ) + + if volumes or "volumes" in session.posargs: + teardown_command = (*teardown_command, "--volumes") + + if images: + teardown_command = (*teardown_command, "--rmi", "all") + + try: + session.run(*teardown_command, external=True) + except nox.command.CommandFailed: + session.warn(f"Teardown failed: '{teardown_command}'") + + session.log("Teardown complete") def install_requirements(session: nox.Session) -> None: diff --git a/scripts/load_examples.py b/scripts/load_examples.py index a5a4a13a8c..de4a990bd8 100644 --- a/scripts/load_examples.py +++ b/scripts/load_examples.py @@ -16,7 +16,6 @@ from setup.privacy_request import create_privacy_request from setup.s3_storage import create_s3_storage from setup.stripe_connector import create_stripe_connector -from setup.user import create_user print("Generating example data for local Fides test environment...") @@ -28,12 +27,10 @@ ) raise -# Start by creating an OAuth client and user for testing +# Start by creating an OAuth client and authenticating auth_header = get_auth_header() -create_user( - auth_header=auth_header, -) +# TODO: update to use default configs # Create an S3 storage config to store DSR results if get_secret("AWS_SECRETS")["access_key_id"]: print("AWS secrets provided, attempting to configure S3 storage...") @@ -41,7 +38,6 @@ # Edit the default DSR policies to use for testing privacy requests # NOTE: We use the default policies to test the default privacy center -# TODO: change this to edit the default policies instead, so the default privacy center can be used create_dsr_policy(auth_header=auth_header, key=constants.DEFAULT_ACCESS_POLICY) create_dsr_policy(auth_header=auth_header, key=constants.DEFAULT_ERASURE_POLICY) create_rule( @@ -57,6 +53,7 @@ action_type="erasure", ) +# TODO: update to use default configs # Configure the email integration to use for identity verification and notifications if get_secret("MAILGUN_SECRETS")["api_key"]: print("Mailgun secrets provided, attempting to configure email...") diff --git a/scripts/setup/user.py b/scripts/setup/user.py deleted file mode 100644 index 56ec171275..0000000000 --- a/scripts/setup/user.py +++ /dev/null @@ -1,65 +0,0 @@ -from typing import Dict - -import requests -from loguru import logger - -from fides.api.ops.api.v1 import urn_registry as urls -from fides.core.config import CONFIG - -from . import constants - - -def create_user( - auth_header: Dict[str, str], - username=constants.FIDES_USERNAME, - password=constants.FIDES_PASSWORD, -): - """Adds a user with full permissions - all scopes and admin role""" - login_response = requests.post( - f"{constants.BASE_URL}{urls.LOGIN}", - headers=auth_header, - json={ - "username": username, - "password": password, - }, - ) - - if login_response.ok: - logger.info(f"Successfully logged in as {username}") - return - - response = requests.post( - f"{constants.BASE_URL}{urls.USERS}", - headers=auth_header, - json={ - "first_name": "Atest", - "last_name": "User", - "username": username, - "password": password, - }, - ) - - if not response.ok: - raise RuntimeError( - f"fides user creation failed! response.status_code={response.status_code}, response.json()={response.json()}" - ) - - user_id = response.json()["id"] - - user_permissions_url = urls.USER_PERMISSIONS.format(user_id=user_id) - response = requests.put( - f"{constants.BASE_URL}{user_permissions_url}", - headers=auth_header, - json={ - "id": user_id, - "scopes": CONFIG.security.root_user_scopes, - "roles": CONFIG.security.root_user_roles, - }, - ) - - if not response.ok: - raise RuntimeError( - f"fides user permissions creation failed! response.status_code={response.status_code}, response.json()={response.json()}" - ) - else: - logger.info(f"Created user with username: {username} and password: {password}") diff --git a/src/fides/api/main.py b/src/fides/api/main.py index 39e0556feb..3e4a4992d2 100644 --- a/src/fides/api/main.py +++ b/src/fides/api/main.py @@ -66,8 +66,7 @@ verify_oauth_client_for_system_from_fides_key_cli, verify_oauth_client_for_system_from_request_body_cli, ) -from fides.core.config import CONFIG -from fides.core.config.helpers import check_required_webserver_config_values +from fides.core.config import CONFIG, check_required_webserver_config_values from fides.lib.oauth.api.routes.user_endpoints import router as user_router VERSION = fides.__version__ @@ -379,7 +378,7 @@ def read_other_paths(request: Request) -> Response: def start_webserver(port: int = 8080) -> None: """Run the webserver.""" - check_required_webserver_config_values() + check_required_webserver_config_values(config=CONFIG) server = Server(Config(app, host="0.0.0.0", port=port, log_level=WARNING)) logger.info( diff --git a/src/fides/cli/commands/util.py b/src/fides/cli/commands/util.py index f0e10df194..1235af6094 100644 --- a/src/fides/cli/commands/util.py +++ b/src/fides/cli/commands/util.py @@ -1,6 +1,8 @@ """Contains all of the Utility-type CLI commands for fides.""" from datetime import datetime, timezone +from os import environ from subprocess import CalledProcessError +from typing import Optional import rich_click as click @@ -123,8 +125,22 @@ def deploy(ctx: click.Context) -> None: is_flag=True, help="Disable the initialization of the Fides CLI, to run in headless mode.", ) +@click.option( + "--env-file", + type=click.Path(exists=True), + help="Use a custom ENV file for the Fides container to override settings.", +) +@click.option( + "--image", + type=str, + help="Use a custom image for the Fides container instead of the default ('ethyca/fides').", +) def up( - ctx: click.Context, no_pull: bool = False, no_init: bool = False + ctx: click.Context, + no_pull: bool = False, + no_init: bool = False, + env_file: Optional[click.Path] = None, + image: Optional[str] = None, ) -> None: # pragma: no cover """ Starts a sample project via docker compose. @@ -138,11 +154,19 @@ def up( if not no_pull: pull_specific_docker_image() + if env_file: + print(f"> Using custom ENV file from: {env_file}") + environ["FIDES_DEPLOY_ENV_FILE"] = str(env_file) + + if image: + print(f"> Using custom image: {image}") + environ["FIDES_DEPLOY_IMAGE"] = image + try: check_fides_uploads_dir() print("> Starting application...") start_application() - print("> Seeding data...") + print("> Setting up sample data...") seed_example_data() click.clear() diff --git a/src/fides/core/config/__init__.py b/src/fides/core/config/__init__.py index d5298fde24..a45e0ae077 100644 --- a/src/fides/core/config/__init__.py +++ b/src/fides/core/config/__init__.py @@ -215,4 +215,38 @@ def get_config(config_path_override: str = "", verbose: bool = False) -> FidesCo return config +def check_required_webserver_config_values(config: FidesConfig) -> None: + """Check for required config values and print a user-friendly error message.""" + required_config_dict = { + "security": [ + "app_encryption_key", + "oauth_root_client_id", + "oauth_root_client_secret", + ] + } + + missing_required_config_vars = [] + for subsection_key, values in required_config_dict.items(): + for key in values: + subsection_model = dict(config).get(subsection_key, {}) + config_value = dict(subsection_model).get(key) + + if not config_value: + missing_required_config_vars.append(".".join((subsection_key, key))) + + if missing_required_config_vars: + echo_red( + "\nThere are missing required configuration variables. Please add the following config variables to either the " + "`fides.toml` file or your environment variables to start Fides: \n" + ) + for missing_value in missing_required_config_vars: + echo_red(f"- {missing_value}") + echo_red( + "\nVisit the Fides deployment documentation for more information: " + "https://ethyca.github.io/fides/deployment/" + ) + + raise SystemExit(1) + + CONFIG = get_config() diff --git a/src/fides/core/config/helpers.py b/src/fides/core/config/helpers.py index 2bbd9cd4ab..a4a9c69b9e 100644 --- a/src/fides/core/config/helpers.py +++ b/src/fides/core/config/helpers.py @@ -1,6 +1,6 @@ """This module contains logic related to loading/manipulation/writing the config.""" import os -from os import environ, getenv +from os import environ from pathlib import Path from re import compile as regex from typing import Any, Dict, List, Union @@ -9,8 +9,6 @@ from loguru import logger from toml import dump, load -from fides.core.utils import echo_red - DEFAULT_CONFIG_PATH = ".fides/fides.toml" @@ -140,49 +138,3 @@ def handle_deprecated_env_variables(settings: Dict[str, Any]) -> Dict[str, Any]: settings["database"][setting] = val return settings - - -def check_required_webserver_config_values() -> None: - """Check for required env vars and print a user-friendly error message.""" - required_config_dict = { - "app_encryption_key": { - "env_var": "FIDES__SECURITY__APP_ENCRYPTION_KEY", - "config_subsection": "security", - }, - "oauth_root_client_id": { - "env_var": "FIDES__SECURITY__OAUTH_ROOT_CLIENT_ID", - "config_subsection": "security", - }, - "oauth_root_client_secret": { - "env_var": "FIDES__SECURITY__OAUTH_ROOT_CLIENT_SECRET", - "config_subsection": "security", - }, - } - - missing_required_config_vars = [] - for key, value in required_config_dict.items(): - try: - config_value = getenv(value["env_var"]) or get_config_from_file( - "", - value["config_subsection"], - key, - ) - except FileNotFoundError: - config_value = None - - if not config_value: - missing_required_config_vars.append(key) - - if missing_required_config_vars: - echo_red( - "\nThere are missing required configuration variables. Please add the following config variables to either the " - "`fides.toml` file or your environment variables to start Fides: \n" - ) - for missing_value in missing_required_config_vars: - print(f"- {missing_value}") - print( - "\nVisit the Fides deployment documentation for more information: " - "https://ethyca.github.io/fides/deployment/" - ) - - raise SystemExit(1) diff --git a/src/fides/core/config/redis_settings.py b/src/fides/core/config/redis_settings.py index 42fce9730c..9795371e86 100644 --- a/src/fides/core/config/redis_settings.py +++ b/src/fides/core/config/redis_settings.py @@ -28,7 +28,7 @@ class RedisSettings(FidesSettings): description="The number of seconds for which data will live in Redis before automatically expiring.", ) enabled: bool = Field( - default=False, + default=True, description="Whether the application's Redis cache should be enabled. Only set to false for certain narrow uses of the application.", ) host: str = Field( diff --git a/src/fides/core/deploy.py b/src/fides/core/deploy.py index fcf00647b8..b43e82e398 100644 --- a/src/fides/core/deploy.py +++ b/src/fides/core/deploy.py @@ -12,7 +12,7 @@ from fides.cli.utils import FIDES_ASCII_ART from fides.core.utils import echo_green, echo_red -FIDES_UPLOADS_DIR = getcwd() + "/fides_uploads/" +FIDES_DEPLOY_UPLOADS_DIR = getcwd() + "/fides_uploads/" REQUIRED_DOCKER_VERSION = "20.10.17" SAMPLE_PROJECT_DIR = join( dirname(__file__), @@ -133,11 +133,11 @@ def check_virtualenv() -> bool: def seed_example_data() -> None: run_shell( DOCKER_COMPOSE_COMMAND - + "run --no-deps --rm fides fides push src/fides/data/sample_project/sample_resources/" + + """exec fides /bin/bash -c "fides user login && fides push src/fides/data/sample_project/sample_resources/" """ ) run_shell( DOCKER_COMPOSE_COMMAND - + "run --no-deps --rm fides python scripts/load_examples.py" + + """exec fides /bin/bash -c "python scripts/load_examples.py" """ ) @@ -148,21 +148,21 @@ def check_fides_uploads_dir() -> None: This fixes an error that was happening in CI checks related to binding a file that doesn't exist. """ - if not exists(FIDES_UPLOADS_DIR): - makedirs(FIDES_UPLOADS_DIR) + if not exists(FIDES_DEPLOY_UPLOADS_DIR): + makedirs(FIDES_DEPLOY_UPLOADS_DIR) def teardown_application() -> None: """Teardown all of the application containers for fides.""" # This needs to get set, or else it throws an error - environ["FIDES_UPLOADS_DIR"] = FIDES_UPLOADS_DIR + environ["FIDES_DEPLOY_UPLOADS_DIR"] = FIDES_DEPLOY_UPLOADS_DIR run_shell(DOCKER_COMPOSE_COMMAND + "down --remove-orphans --volumes") def start_application() -> None: """Spin up the application via a docker compose file.""" - environ["FIDES_UPLOADS_DIR"] = FIDES_UPLOADS_DIR + environ["FIDES_DEPLOY_UPLOADS_DIR"] = FIDES_DEPLOY_UPLOADS_DIR run_shell( DOCKER_COMPOSE_COMMAND + "up --wait", ) @@ -208,13 +208,13 @@ def pull_specific_docker_image() -> None: run_shell(f"docker pull {current_privacy_center_image}") run_shell(f"docker pull {current_sample_app_image}") run_shell( - f"docker tag {current_fides_image} {fides_image_stub.format('sample')}" + f"docker tag {current_fides_image} {fides_image_stub.format('local')}" ) run_shell( - f"docker tag {current_privacy_center_image} {privacy_center_image_stub.format('sample')}" + f"docker tag {current_privacy_center_image} {privacy_center_image_stub.format('local')}" ) run_shell( - f"docker tag {current_sample_app_image} {sample_app_image_stub.format('sample')}" + f"docker tag {current_sample_app_image} {sample_app_image_stub.format('local')}" ) except CalledProcessError: print("Unable to fetch matching version, defaulting to 'dev' versions...") @@ -229,13 +229,13 @@ def pull_specific_docker_image() -> None: run_shell(f"docker pull {dev_privacy_center_image}") run_shell(f"docker pull {dev_sample_app_image}") run_shell( - f"docker tag {dev_fides_image} {fides_image_stub.format('sample')}" + f"docker tag {dev_fides_image} {fides_image_stub.format('local')}" ) run_shell( - f"docker tag {dev_privacy_center_image} {privacy_center_image_stub.format('sample')}" + f"docker tag {dev_privacy_center_image} {privacy_center_image_stub.format('local')}" ) run_shell( - f"docker tag {dev_sample_app_image} {sample_app_image_stub.format('sample')}" + f"docker tag {dev_sample_app_image} {sample_app_image_stub.format('local')}" ) except CalledProcessError: echo_red("Failed to pull 'dev' versions of docker containers! Aborting...") @@ -277,4 +277,4 @@ def print_deploy_success() -> None: # Open the landing page and DSR directory webbrowser.open("http://localhost:3000/landing") - webbrowser.open(f"file:///{FIDES_UPLOADS_DIR}") + webbrowser.open(f"file:///{FIDES_DEPLOY_UPLOADS_DIR}") diff --git a/src/fides/data/sample_project/docker-compose.yml b/src/fides/data/sample_project/docker-compose.yml index 1dd8ea7592..502b8c7b45 100644 --- a/src/fides/data/sample_project/docker-compose.yml +++ b/src/fides/data/sample_project/docker-compose.yml @@ -1,6 +1,7 @@ services: fides: - image: ethyca/fides:sample + container_name: fides + image: ${FIDES_DEPLOY_IMAGE:-ethyca/fides:local} healthcheck: test: [ "CMD", "curl", "-f", "http://0.0.0.0:8080/health" ] interval: 20s @@ -15,22 +16,31 @@ services: condition: service_healthy mongodb-test: condition: service_started + # WARNING: This env_file option is specified so that we can provide an + # alternate ENV file via 'fides deploy up --env-file ' as a + # convenient way for users to provide their own ENV files to the 'fides' + # at runtime. However, since Docker Compose doesn't support optional + # env_file specifications, we also need to provide a default 'sample.env' + # as a placeholder. + # (see https://github.com/compose-spec/compose-spec/issues/240) + # + # This seems fine, but it also leads to some gotchas when calling + # 'docker compose' from different working directories, like we do in the + # 'fides' nox build commands. Beware! + env_file: + - ${FIDES_DEPLOY_ENV_FILE:-sample.env} environment: FIDES__CONFIG_PATH: "/fides/src/fides/data/sample_project/fides.toml" - # These need to be defined here instead of the config file - # due to the `check_required_webserver_config_values` function - FIDES__SECURITY__APP_ENCRYPTION_KEY: "examplevalidprojectencryptionkey" - FIDES__SECURITY__OAUTH_ROOT_CLIENT_ID: "fidesadmin" - FIDES__SECURITY__OAUTH_ROOT_CLIENT_SECRET: "fidesadminsecret" # Mount a local volume so the user can see their privacy requests volumes: - type: bind - source: ${FIDES_UPLOADS_DIR} + source: ${FIDES_DEPLOY_UPLOADS_DIR:-./fides_uploads} target: /fides/fides_uploads read_only: False sample-app: - image: ethyca/fides-sample-app:sample + container_name: sample-app + image: ethyca/fides-sample-app:local environment: - PORT=3000 - DATABASE_HOST=postgres-test @@ -44,7 +54,8 @@ services: - postgres-test fides-pc: - image: ethyca/fides-privacy-center:sample + container_name: fides-privacy-center + image: ethyca/fides-privacy-center:local ports: - "3001:3000" volumes: @@ -58,6 +69,7 @@ services: read_only: False fides-db: + container_name: fides-db image: postgres:12 healthcheck: test: [ "CMD-SHELL", "pg_isready -U postgres" ] @@ -74,14 +86,14 @@ services: - postgres:/var/lib/postgresql/data redis: + container_name: fides-redis image: redis:6.2.5-alpine - command: redis-server --requirepass redispass - environment: - - REDIS_PASSWORD=redispass + command: redis-server --requirepass redispassword ports: - "7379:6379" postgres-test: + container_name: fides-postgres-example-db image: postgres:12 healthcheck: test: [ "CMD-SHELL", "pg_isready -U postgres" ] @@ -98,6 +110,7 @@ services: - ./postgres_sample.sql:/docker-entrypoint-initdb.d/postgres_sample.sql:ro mongodb-test: + container_name: fides-redis-example-db image: mongo:5.0.3 environment: - MONGO_INITDB_DATABASE=mongo_test diff --git a/src/fides/data/sample_project/fides.toml b/src/fides/data/sample_project/fides.toml index de94fcc332..2e91875113 100644 --- a/src/fides/data/sample_project/fides.toml +++ b/src/fides/data/sample_project/fides.toml @@ -6,14 +6,19 @@ port = "5432" db = "fides" [redis] -enabled = true host = "redis" -password = "redispass" +password = "redispassword" port = 6379 db_index = 0 [security] +env = "prod" cors_origins = [ "http://localhost:8080", "http://localhost:3001",] +app_encryption_key = "examplevalidprojectencryptionkey" +oauth_root_client_id = "fidesadmin" +oauth_root_client_secret = "fidesadminsecret" +root_username = "root_user" +root_password = "Testpassword1!" [execution] require_manual_request_approval = true @@ -24,3 +29,8 @@ server_port = 8080 [user] analytics_opt_out = false +username = "root_user" +password = "Testpassword1!" + +[logging] +level = "DEBUG" \ No newline at end of file diff --git a/src/fides/data/sample_project/privacy_center/config/config.json b/src/fides/data/sample_project/privacy_center/config/config.json index cbe3d75991..08322e6d6e 100644 --- a/src/fides/data/sample_project/privacy_center/config/config.json +++ b/src/fides/data/sample_project/privacy_center/config/config.json @@ -29,25 +29,38 @@ "icon_path": "/assets/consent.svg", "title": "Manage your consent", "description": "Manage your consent preferences, including the option to select 'Do Not Sell My Personal Information'.", - "cookieName": "fides_consent", + "identity_inputs": { + "email": "required" + }, + "policy_key": "default_consent_policy", "consentOptions": [ { "fidesDataUseKey": "advertising", "name": "Data Sales or Sharing", "description": "We may use some of your personal information for behavioral advertising purposes, which may be interpreted as 'Data Sales' or 'Data Sharing' under regulations such as CCPA, CPRA, VCDPA, etc.", "url": "https://example.com/privacy#data-sales", - "default": true, + "default": { + "value": true, + "globalPrivacyControl": false + }, "highlight": false, - "cookieKeys": ["data_sales"] + "cookieKeys": [ + "data_sales" + ] }, { "fidesDataUseKey": "advertising.first_party", "name": "Email Marketing", "description": "We may use some of your personal information to contact you about our products & services.", "url": "https://example.com/privacy#email-marketing", - "default": true, + "default": { + "value": true, + "globalPrivacyControl": false + }, "highlight": false, - "cookieKeys": [] + "cookieKeys": [ + "marketing" + ] }, { "fidesDataUseKey": "improve", @@ -56,8 +69,10 @@ "url": "https://example.com/privacy#analytics", "default": true, "highlight": false, - "cookieKeys": [] + "cookieKeys": [ + "analytics" + ] } ] } -} +} \ No newline at end of file diff --git a/src/fides/data/sample_project/sample.env b/src/fides/data/sample_project/sample.env new file mode 100644 index 0000000000..5688e84976 --- /dev/null +++ b/src/fides/data/sample_project/sample.env @@ -0,0 +1,5 @@ +# This .env file is used to configure default settings for the 'fides' container +# It is empty by default; it exists only as a placeholder. +# +# See comment in src/fides/data/sample_project/docker-compose.yml for details. +FIDES_DEPLOY_SAMPLE_ENV_LOADED=1 \ No newline at end of file diff --git a/src/fides/data/test_env/fides.test_env.toml b/src/fides/data/test_env/fides.test_env.toml deleted file mode 100644 index 00729382d8..0000000000 --- a/src/fides/data/test_env/fides.test_env.toml +++ /dev/null @@ -1,43 +0,0 @@ -# Configuration values used for test environment (see `nox -s fides_env(test)`) -[database] -server = "fides-db" -user = "postgres" -password = "fides" -port = "5432" -db = "fides" - -[redis] -host = "redis" -password = "redispassword" -port = 6379 -db_index = 0 - -[logging] -level = "DEBUG" - -[security] -app_encryption_key = "atestencryptionkeythatisvalidlen" -cors_origins = [ "http://localhost", "http://localhost:8080", "http://localhost:3000", "http://localhost:3001",] -oauth_root_client_id = "fidesadmin" -oauth_root_client_secret = "fidesadminsecret" -root_username = "root_user" -root_password = "Testpassword1!" -env = "prod" - -[execution] -task_retry_count = 0 -task_retry_delay = 1 -task_retry_backoff = 1 -require_manual_request_approval = true -subject_identity_verification_required = false -masking_strict = true - -[cli] -server_host = "localhost" -server_port = 8080 - -[user] -analytics_opt_out = false - -[notifications] -notification_service_type = "mailgun" diff --git a/src/fides/data/test_env/privacy_center_config/config.css b/src/fides/data/test_env/privacy_center_config/config.css deleted file mode 100644 index 191348d8f4..0000000000 --- a/src/fides/data/test_env/privacy_center_config/config.css +++ /dev/null @@ -1,19 +0,0 @@ -/* -Add any global CSS overrides to this file. -Override basic theme colors by uncommenting and editing those below -*/ - -:root:root { - /* Background color */ - /* --chakra-colors-gray-50: #F7FAFC; */ - /* Header & highlight color */ - /* --chakra-colors-gray-100: #EDF2F7; */ - /* Modal text color */ - /* --chakra-colors-gray-500: #718096; */ - /* Body text color */ - /* --chakra-colors-gray-600: #4A5568; */ - /* Primary button hover color */ - /* --chakra-colors-primary-400: #464B83; */ - /* Primary button color */ - /* --chakra-colors-primary-800: #111439; */ -} diff --git a/src/fides/data/test_env/privacy_center_config/config.json b/src/fides/data/test_env/privacy_center_config/config.json deleted file mode 100644 index 33be8cdcc4..0000000000 --- a/src/fides/data/test_env/privacy_center_config/config.json +++ /dev/null @@ -1,80 +0,0 @@ -{ - "title": "Take control of your data", - "description": "When you use our services, you’re trusting us with your information. We understand this is a big responsibility and work hard to protect your information and put you in control.", - "description_subtext": [], - "server_url_development": "http://localhost:8080/api/v1", - "server_url_production": "http://localhost:8080/api/v1", - "logo_path": "/logo.svg", - "actions": [ - { - "policy_key": "default_access_policy", - "icon_path": "/download.svg", - "title": "Access your data", - "description": "We will provide you a report of all your personal data.", - "identity_inputs": { - "email": "required" - } - }, - { - "policy_key": "default_erasure_policy", - "icon_path": "/delete.svg", - "title": "Erase your data", - "description": "We will erase all of your personal data. This action cannot be undone.", - "identity_inputs": { - "email": "required" - } - } - ], - "includeConsent": true, - "consent": { - "icon_path": "/consent.svg", - "title": "Manage your consent", - "description": "Manage your consent preferences, including the option to select 'Do Not Sell My Personal Information'.", - "description_subtext": [ - "When you use our services, you're trusting us with your information. We understand this is a big responsibility and work hard to protect your information and put you in control." - ], - "identity_inputs": { - "email": "required" - }, - "cookieName": "fides_consent", - "policy_key": "default_consent_policy", - "consentOptions": [ - { - "fidesDataUseKey": "advertising", - "name": "Data Sales or Sharing", - "description": "We may use some of your personal information for behavioral advertising purposes, which may be interpreted as 'Data Sales' or 'Data Sharing' under regulations such as CCPA, CPRA, VCDPA, etc.", - "url": "https://example.com/privacy#data-sales", - "default": { - "value": true, - "globalPrivacyControl": false - }, - "highlight": false, - "cookieKeys": ["data_sales"], - "executable": false - }, - { - "fidesDataUseKey": "advertising.first_party", - "name": "Email Marketing", - "description": "We may use some of your personal information to contact you about our products & services.", - "url": "https://example.com/privacy#email-marketing", - "default": { - "value": true, - "globalPrivacyControl": false - }, - "highlight": false, - "cookieKeys": [], - "executable": false - }, - { - "fidesDataUseKey": "improve", - "name": "Product Analytics", - "description": "We may use some of your personal information to collect analytics about how you use our products & services.", - "url": "https://example.com/privacy#analytics", - "default": true, - "highlight": false, - "cookieKeys": [], - "executable": false - } - ] - } -} diff --git a/tests/ctl/core/config/test_config.py b/tests/ctl/core/config/test_config.py index c7db322a1d..37f8cc089c 100644 --- a/tests/ctl/core/config/test_config.py +++ b/tests/ctl/core/config/test_config.py @@ -5,7 +5,7 @@ import pytest from pydantic import ValidationError -from fides.core.config import get_config +from fides.core.config import check_required_webserver_config_values, get_config from fides.core.config.database_settings import DatabaseSettings from fides.core.config.security_settings import SecuritySettings @@ -52,16 +52,21 @@ def test_get_config(test_config_path: str) -> None: ) -# Unit @patch.dict( os.environ, - {}, + { + "FIDES__CONFIG_PATH": "tests/ctl/test_default_config.toml", + }, clear=True, ) @pytest.mark.unit -def test_get_config_fails_missing_required(test_config_path: str) -> None: - """Check that the correct error gets raised if a required value is missing.""" - config = get_config(test_config_path) +def test_get_config_default() -> None: + """Check that get_config loads default values when given an empty TOML.""" + config = get_config() + assert config.database.api_engine_pool_size == 50 + assert config.security.env == "dev" + assert config.security.app_encryption_key == "" + assert config.logging.level == "INFO" @patch.dict( @@ -188,7 +193,7 @@ def test_database_url_test_mode_disabled() -> None: @patch.dict( os.environ, { - "FIDES__CONFIG_PATH": "src/fides/data/test_env/fides.test_env.toml", + "FIDES__CONFIG_PATH": "src/fides/data/sample_project/fides.toml", }, clear=True, ) @@ -347,3 +352,42 @@ def test_validating_included_async_database_uri(self) -> None: ) assert incorrect_value not in database_settings.async_database_uri assert correct_value in database_settings.async_database_uri + + +@pytest.mark.unit +def test_check_required_webserver_config_values_success(test_config_path: str) -> None: + config = get_config(test_config_path) + assert check_required_webserver_config_values(config=config) is None + + +@patch.dict( + os.environ, + { + "FIDES__CONFIG_PATH": "tests/ctl/test_default_config.toml", + }, + clear=True, +) +@pytest.mark.unit +def test_check_required_webserver_config_values_error(capfd) -> None: + config = get_config() + assert config.security.app_encryption_key is "" + + with pytest.raises(SystemExit): + check_required_webserver_config_values(config=config) + + out, _ = capfd.readouterr() + assert "app_encryption_key" in out + assert "oauth_root_client_id" in out + assert "oauth_root_client_secret" in out + + +@patch.dict( + os.environ, + { + "FIDES__CONFIG_PATH": "src/fides/data/sample_project/fides.toml", + }, + clear=True, +) +def test_check_required_webserver_config_values_success_from_path() -> None: + config = get_config() + assert check_required_webserver_config_values(config=config) is None diff --git a/tests/ctl/core/config/test_config_helpers.py b/tests/ctl/core/config/test_config_helpers.py index 9345b83f92..01ef4bad7d 100644 --- a/tests/ctl/core/config/test_config_helpers.py +++ b/tests/ctl/core/config/test_config_helpers.py @@ -34,29 +34,3 @@ def test_get_config_from_file_none(self, section, option, tmp_path): toml.dump({section: {option: "value"}}, f) assert helpers.get_config_from_file(file, "bad", "missing") is None - - @patch("fides.core.config.helpers.get_config_from_file") - def test_check_required_webserver_config_values(self, mock_get_config, capfd): - mock_get_config.return_value = None - - with pytest.raises(SystemExit): - helpers.check_required_webserver_config_values() - out, _ = capfd.readouterr() - - assert "app_encryption_key" in out - assert "oauth_root_client_id" in out - assert "oauth_root_client_secret" in out - - @patch("fides.core.config.helpers.get_config_from_file") - def test_check_required_webserver_config_values_file_not_found( - self, mock_get_config, capfd - ): - mock_get_config.side_effect = FileNotFoundError - - with pytest.raises(SystemExit): - helpers.check_required_webserver_config_values() - out, _ = capfd.readouterr() - - assert "app_encryption_key" in out - assert "oauth_root_client_id" in out - assert "oauth_root_client_secret" in out diff --git a/tests/ctl/test_default_config.toml b/tests/ctl/test_default_config.toml new file mode 100644 index 0000000000..cbf6caae41 --- /dev/null +++ b/tests/ctl/test_default_config.toml @@ -0,0 +1 @@ +# Default (empty) config file \ No newline at end of file From 41ae34942019e76db9981618f1230e106d6b1193 Mon Sep 17 00:00:00 2001 From: Andrew Jackson Date: Wed, 12 Apr 2023 10:17:01 -0400 Subject: [PATCH 2/4] Data Flow Modal (#3008) --- CHANGELOG.md | 1 + clients/admin-ui/cypress/e2e/systems.cy.ts | 32 +++ .../system-data-flow/DataFlowAccordion.tsx | 25 +++ .../DataFlowAccordionForm.tsx | 208 ++++++++++++++++++ .../DataFlowSystemsDeleteTable.tsx | 78 +++++++ .../system-data-flow/DataFlowSystemsModal.tsx | 175 +++++++++++++++ .../system-data-flow/DataFlowSystemsTable.tsx | 98 +++++++++ .../src/features/datamap/SpatialDatamap.tsx | 43 ++-- .../datamap/datamap-drawer/DatamapDrawer.tsx | 13 ++ .../admin-ui/src/features/datamap/types.ts | 3 +- .../src/features/system/SystemFormTabs.tsx | 33 ++- .../src/features/system/system.slice.ts | 3 +- 12 files changed, 693 insertions(+), 19 deletions(-) create mode 100644 clients/admin-ui/src/features/common/system-data-flow/DataFlowAccordion.tsx create mode 100644 clients/admin-ui/src/features/common/system-data-flow/DataFlowAccordionForm.tsx create mode 100644 clients/admin-ui/src/features/common/system-data-flow/DataFlowSystemsDeleteTable.tsx create mode 100644 clients/admin-ui/src/features/common/system-data-flow/DataFlowSystemsModal.tsx create mode 100644 clients/admin-ui/src/features/common/system-data-flow/DataFlowSystemsTable.tsx diff --git a/CHANGELOG.md b/CHANGELOG.md index b4b8f70b7f..e01071dc3f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ The types of changes are: - Add endpoint to retrieve privacy notices grouped by their associated data uses [#2956](https://github.com/ethyca/fides/pull/2956) - Support for uploading custom connector templates via the UI [#2997](https://github.com/ethyca/fides/pull/2997) - Add a backwards-compatible workflow for saving and propagating consent preferences with respect to Privacy Notices [#3016](https://github.com/ethyca/fides/pull/3016) +- Added Data flow modal [#3008](https://github.com/ethyca/fides/pull/3008) ### Changed diff --git a/clients/admin-ui/cypress/e2e/systems.cy.ts b/clients/admin-ui/cypress/e2e/systems.cy.ts index 87ba6e7e27..abd7a039b6 100644 --- a/clients/admin-ui/cypress/e2e/systems.cy.ts +++ b/clients/admin-ui/cypress/e2e/systems.cy.ts @@ -565,4 +565,36 @@ describe("System management page", () => { }); }); }); + + describe("Data flow", () => { + beforeEach(() => { + stubSystemCrud(); + stubTaxonomyEntities(); + cy.fixture("systems/systems.json").then((systems) => { + cy.intercept("GET", "/api/v1/system/*", { + body: systems[1], + }).as("getFidesctlSystem"); + }); + + cy.visit(SYSTEM_ROUTE); + cy.getByTestId("system-fidesctl_system").within(() => { + cy.getByTestId("more-btn").click(); + cy.getByTestId("edit-btn").click(); + }); + cy.getByTestId("tab-Data flow").click(); + }); + + it("Can navigate to the data flow tab", () => { + cy.getByTestId("data-flow-accordion").should("exist"); + }); + + it("Can open both accordion items", () => { + cy.getByTestId("data-flow-accordion").within(()=>{ + cy.getByTestId("data-flow-button-Source").click(); + cy.getByTestId("data-flow-panel-Source").should("exist"); + cy.getByTestId("data-flow-button-Destination").click(); + cy.getByTestId("data-flow-panel-Destination").should("exist"); + }) + }); + }); }); diff --git a/clients/admin-ui/src/features/common/system-data-flow/DataFlowAccordion.tsx b/clients/admin-ui/src/features/common/system-data-flow/DataFlowAccordion.tsx new file mode 100644 index 0000000000..7cc707d3e0 --- /dev/null +++ b/clients/admin-ui/src/features/common/system-data-flow/DataFlowAccordion.tsx @@ -0,0 +1,25 @@ +import { Accordion } from "@fidesui/react"; +import React from "react"; + +import { System } from "~/types/api/models/System"; + +import { DataFlowAccordionForm } from "./DataFlowAccordionForm"; + +type DataFlowFormProps = { + system: System; + isSystemTab?: boolean; +}; + +export const DataFlowAccordion = ({ + system, + isSystemTab, +}: DataFlowFormProps) => ( + + + + +); diff --git a/clients/admin-ui/src/features/common/system-data-flow/DataFlowAccordionForm.tsx b/clients/admin-ui/src/features/common/system-data-flow/DataFlowAccordionForm.tsx new file mode 100644 index 0000000000..5bd00d00c7 --- /dev/null +++ b/clients/admin-ui/src/features/common/system-data-flow/DataFlowAccordionForm.tsx @@ -0,0 +1,208 @@ +import { + AccordionButton, + AccordionIcon, + AccordionItem, + AccordionPanel, + Button, + ButtonGroup, + Flex, + Spacer, + Stack, + Tag, + Text, + useDisclosure, + useToast, +} from "@fidesui/react"; +import { isErrorResult } from "common/helpers"; +import { FormGuard } from "common/hooks/useIsAnyFormDirty"; +import { GearLightIcon } from "common/Icon"; +import { DataFlowSystemsDeleteTable } from "common/system-data-flow/DataFlowSystemsDeleteTable"; +import DataFlowSystemsModal from "common/system-data-flow/DataFlowSystemsModal"; +import { errorToastParams, successToastParams } from "common/toast"; +import { Form, Formik, FormikHelpers } from "formik"; +import React, { useEffect, useMemo, useState } from "react"; + +import { useAppSelector } from "~/app/hooks"; +import { + useGetAllSystemsQuery, + useUpdateSystemMutation, +} from "~/features/system"; +import { selectAllSystems } from "~/features/system/system.slice"; +import { DataFlow, System } from "~/types/api"; + +const defaultInitialValues = { + dataFlowSystems: [] as DataFlow[], +}; + +export type FormValues = typeof defaultInitialValues; + +type DataFlowAccordionItemProps = { + isIngress?: boolean; + system: System; + isSystemTab?: boolean; +}; + +export const DataFlowAccordionForm = ({ + system, + isIngress, + isSystemTab, +}: DataFlowAccordionItemProps) => { + const toast = useToast(); + const flowType = isIngress ? "Source" : "Destination"; + const pluralFlowType = `${flowType}s`; + const dataFlowSystemsModal = useDisclosure(); + const [updateSystemMutationTrigger] = useUpdateSystemMutation(); + + useGetAllSystemsQuery(); + const systems = useAppSelector(selectAllSystems); + + const initialDataFlows = useMemo(() => { + let dataFlows = isIngress ? system.ingress : system.egress; + if (!dataFlows) { + dataFlows = []; + } + const systemFidesKeys = systems ? systems.map((s) => s.fides_key) : []; + + return dataFlows.filter((df) => systemFidesKeys.includes(df.fides_key)); + }, [isIngress, system, systems]); + + const [assignedDataFlow, setAssignedDataFlows] = + useState(initialDataFlows); + + useEffect(() => { + setAssignedDataFlows(initialDataFlows); + }, [initialDataFlows]); + + const handleSubmit = async ( + { dataFlowSystems }: FormValues, + { resetForm }: FormikHelpers + ) => { + const updatedSystem = { + ...system, + ingress: isIngress ? dataFlowSystems : system.ingress, + egress: !isIngress ? dataFlowSystems : system.egress, + }; + const result = await updateSystemMutationTrigger(updatedSystem); + + if (isErrorResult(result)) { + toast(errorToastParams("Failed to update data flows")); + } else { + toast(successToastParams(`${pluralFlowType} updated`)); + } + + resetForm({ values: { dataFlowSystems } }); + }; + + return ( + + + + + {pluralFlowType} + + {/* Commented out until we get copy for the tooltips */} + {/* */} + + + {assignedDataFlow.length} + + + + + + + + + {({ isSubmitting, dirty, resetForm }) => ( +
+ + + + + + + + + {/* By conditionally rendering the modal, we force it to reset its state + whenever it opens */} + {dataFlowSystemsModal.isOpen ? ( + + ) : null} + + )} +
+ + + + ); +}; diff --git a/clients/admin-ui/src/features/common/system-data-flow/DataFlowSystemsDeleteTable.tsx b/clients/admin-ui/src/features/common/system-data-flow/DataFlowSystemsDeleteTable.tsx new file mode 100644 index 0000000000..9a350fb0cb --- /dev/null +++ b/clients/admin-ui/src/features/common/system-data-flow/DataFlowSystemsDeleteTable.tsx @@ -0,0 +1,78 @@ +import { + IconButton, + Table, + Tbody, + Td, + Text, + Th, + Thead, + Tr, +} from "@fidesui/react"; +import { TrashCanSolidIcon } from "common/Icon/TrashCanSolidIcon"; +import { useFormikContext } from "formik"; +import React from "react"; + +import { DataFlow, System } from "~/types/api"; + +type Props = { + systems: System[]; + dataFlows: DataFlow[]; + onDataFlowSystemChange: (systems: DataFlow[]) => void; +}; + +export const DataFlowSystemsDeleteTable = ({ + systems, + dataFlows, + onDataFlowSystemChange, +}: Props) => { + const { setFieldValue } = useFormikContext(); + + const dataFlowKeys = dataFlows.map((f) => f.fides_key); + + const onDelete = (dataFlow: System) => { + const updatedDataFlows = dataFlows.filter( + (dataFlowSystem) => dataFlowSystem.fides_key !== dataFlow.fides_key + ); + setFieldValue("dataFlowSystems", updatedDataFlows); + onDataFlowSystemChange(updatedDataFlows); + }; + + return ( + + + + + + + + {systems + .filter((system) => dataFlowKeys.includes(system.fides_key)) + .map((system) => ( + + + + + ))} + +
System +
+ + {system.name} + + + } + variant="outline" + size="sm" + onClick={() => onDelete(system)} + data-testid="unassign-btn" + /> +
+ ); +}; diff --git a/clients/admin-ui/src/features/common/system-data-flow/DataFlowSystemsModal.tsx b/clients/admin-ui/src/features/common/system-data-flow/DataFlowSystemsModal.tsx new file mode 100644 index 0000000000..baf3543c07 --- /dev/null +++ b/clients/admin-ui/src/features/common/system-data-flow/DataFlowSystemsModal.tsx @@ -0,0 +1,175 @@ +import { + Badge, + Box, + Button, + ButtonGroup, + Flex, + FormControl, + FormLabel, + Modal, + ModalBody, + ModalContent, + ModalFooter, + ModalHeader, + ModalOverlay, + ModalProps, + Stack, + Switch, + Text, +} from "@fidesui/react"; +import SearchBar from "common/SearchBar"; +import { useFormikContext } from "formik"; +import { ChangeEvent, useMemo, useState } from "react"; + +import { SEARCH_FILTER } from "~/features/system/SystemsManagement"; +import { DataFlow, System } from "~/types/api"; + +import DataFlowSystemsTable from "./DataFlowSystemsTable"; + +type Props = { + currentSystem: System; + systems: System[]; + dataFlowSystems: DataFlow[]; + onDataFlowSystemChange: (systems: DataFlow[]) => void; + flowType: string; +}; + +const DataFlowSystemsModal = ({ + currentSystem, + systems, + isOpen, + onClose, + dataFlowSystems, + onDataFlowSystemChange, + flowType, +}: Pick & Props) => { + const { setFieldValue } = useFormikContext(); + const [searchFilter, setSearchFilter] = useState(""); + const [selectedDataFlows, setSelectedDataFlows] = + useState(dataFlowSystems); + + const handleConfirm = async () => { + onDataFlowSystemChange(selectedDataFlows); + onClose(); + }; + + const emptySystems = systems.length === 0; + + const filteredSystems = useMemo(() => { + if (!systems) { + return []; + } + + return systems + .filter((system) => system.fides_key !== currentSystem.fides_key) + .filter((s) => SEARCH_FILTER(s, searchFilter)); + }, [systems, currentSystem.fides_key, searchFilter]); + + const handleToggleAllSystems = (event: ChangeEvent) => { + const { checked } = event.target; + if (checked && systems) { + const updatedDataFlows = filteredSystems.map((fs) => ({ + fides_key: fs.fides_key, + type: "system", + })); + + setFieldValue("dataFlowSystems", updatedDataFlows); + setSelectedDataFlows(updatedDataFlows); + } else { + setSelectedDataFlows([]); + } + }; + + const allSystemsAssigned = useMemo(() => { + const assignedSet = new Set(selectedDataFlows.map((s) => s.fides_key)); + return filteredSystems.every((item) => assignedSet.has(item.fides_key)); + }, [filteredSystems, selectedDataFlows]); + + return ( + + + + + + Configure {flowType.toLocaleLowerCase()} systems + + + Assigned to {selectedDataFlows.length} systems + + + + {emptySystems ? ( + No systems found + ) : ( + + + + Add or remove destination systems from your data map + + + + + Assign all systems + + + + + + + + + )} + + + + + {!emptySystems ? ( + + ) : null} + + + + + ); +}; + +export default DataFlowSystemsModal; diff --git a/clients/admin-ui/src/features/common/system-data-flow/DataFlowSystemsTable.tsx b/clients/admin-ui/src/features/common/system-data-flow/DataFlowSystemsTable.tsx new file mode 100644 index 0000000000..82e004c61c --- /dev/null +++ b/clients/admin-ui/src/features/common/system-data-flow/DataFlowSystemsTable.tsx @@ -0,0 +1,98 @@ +import { + Box, + Switch, + Table, + Tbody, + Td, + Text, + Th, + Thead, + Tr, +} from "@fidesui/react"; +import { useFormikContext } from "formik"; +import React from "react"; + +import { DataFlow, System } from "~/types/api"; + +type Props = { + allSystems: System[]; + dataFlowSystems: DataFlow[]; + onChange: (dataFlows: DataFlow[]) => void; + flowType: string; +}; + +const DataFlowSystemsTable = ({ + allSystems, + dataFlowSystems, + onChange, + flowType, +}: Props) => { + const { setFieldValue } = useFormikContext(); + const handleToggle = (system: System) => { + const isAssigned = !!dataFlowSystems.find( + (assigned) => assigned.fides_key === system.fides_key + ); + if (isAssigned) { + const updatedDataFlows = dataFlowSystems.filter( + (assignedSystem) => assignedSystem.fides_key !== system.fides_key + ); + setFieldValue("dataFlowSystems", updatedDataFlows); + onChange(updatedDataFlows); + } else { + const updatedDataFlows = [ + ...dataFlowSystems, + { fides_key: system.fides_key, type: "system" }, + ]; + + setFieldValue("dataFlowSystems", updatedDataFlows); + onChange(updatedDataFlows); + } + }; + + return ( + + + + + + + + + + {allSystems.map((system) => { + const isAssigned = !!dataFlowSystems.find( + (assigned) => assigned.fides_key === system.fides_key + ); + return ( + + + + + ); + })} + +
SystemSet as {flowType}
+ + {system.name} + + + handleToggle(system)} + data-testid="assign-switch" + /> +
+ + ); +}; + +export default DataFlowSystemsTable; diff --git a/clients/admin-ui/src/features/datamap/SpatialDatamap.tsx b/clients/admin-ui/src/features/datamap/SpatialDatamap.tsx index 289d581754..8e70751bd2 100644 --- a/clients/admin-ui/src/features/datamap/SpatialDatamap.tsx +++ b/clients/admin-ui/src/features/datamap/SpatialDatamap.tsx @@ -24,8 +24,11 @@ const useSpatialDatamap = (rows: Row[]) => { draft[key] = { name: obj.values["system.name"], description: obj.values["system.description"], - dependencies: obj.values["system.system_dependencies"] - ? obj.values["system.system_dependencies"].split(",") + ingress: obj.values["system.ingress"] + ? obj.values["system.ingress"].split(", ") + : [], + egress: obj.values["system.egress"] + ? obj.values["system.egress"].split(", ") : [], id: obj.values["system.fides_key"], }; @@ -34,26 +37,34 @@ const useSpatialDatamap = (rows: Row[]) => { }, {} as Record), [rows] ); - const data = useMemo(() => { let nodes: SystemNode[] = []; - let links: Link[] = []; + const links: Set = new Set([]); if (datamapBySystem) { nodes = Object.values(datamapBySystem); - links = nodes.reduce( - (acc: Link[], system: SystemNode) => [ - ...acc, - ...(system.dependencies - ?.filter((dependency) => datamapBySystem[dependency]) - .map((dependency: string) => ({ + nodes + .map((system) => [ + ...system.ingress + .filter((ingress_system) => datamapBySystem[ingress_system]) + .map((ingress_system) => ({ + source: ingress_system, + target: system.id, + })), + ...system.egress + .filter((egress_system) => datamapBySystem[egress_system]) + .map((egress_system) => ({ source: system.id, - target: dependency, - })) ?? ([] as Link[])), - ], - [] - ); + target: egress_system, + })), + ]) + .flatMap((link) => link) + .forEach((link) => links.add(JSON.stringify(link))); } - return { nodes, links }; + + return { + nodes, + links: Array.from(links).map((l) => JSON.parse(l)) as Link[], + }; }, [datamapBySystem]); return { diff --git a/clients/admin-ui/src/features/datamap/datamap-drawer/DatamapDrawer.tsx b/clients/admin-ui/src/features/datamap/datamap-drawer/DatamapDrawer.tsx index c76a4db9b6..70fa30aa7a 100644 --- a/clients/admin-ui/src/features/datamap/datamap-drawer/DatamapDrawer.tsx +++ b/clients/admin-ui/src/features/datamap/datamap-drawer/DatamapDrawer.tsx @@ -10,6 +10,7 @@ import { } from "@fidesui/react"; import { SerializedError } from "@reduxjs/toolkit"; import { FetchBaseQueryError } from "@reduxjs/toolkit/dist/query/fetchBaseQuery"; +import { DataFlowAccordion } from "common/system-data-flow/DataFlowAccordion"; import React, { useMemo } from "react"; import { getErrorMessage, isErrorResult } from "~/features/common/helpers"; @@ -189,6 +190,18 @@ const DatamapDrawer = ({ {...dataProps} /> + + Data flow + + ) : null} diff --git a/clients/admin-ui/src/features/datamap/types.ts b/clients/admin-ui/src/features/datamap/types.ts index 1ec54fdd50..73a1cfe0ab 100644 --- a/clients/admin-ui/src/features/datamap/types.ts +++ b/clients/admin-ui/src/features/datamap/types.ts @@ -17,7 +17,8 @@ export type SpatialData = { }; export type SystemNode = { - dependencies: string[]; + ingress: string[]; + egress: string[]; description: string; id: string; name: string; diff --git a/clients/admin-ui/src/features/system/SystemFormTabs.tsx b/clients/admin-ui/src/features/system/SystemFormTabs.tsx index 48372bea2a..905bba64c6 100644 --- a/clients/admin-ui/src/features/system/SystemFormTabs.tsx +++ b/clients/admin-ui/src/features/system/SystemFormTabs.tsx @@ -1,4 +1,5 @@ import { Box, Button, Text, useToast } from "@fidesui/react"; +import { DataFlowAccordion } from "common/system-data-flow/DataFlowAccordion"; import NextLink from "next/link"; import { useRouter } from "next/router"; import React, { useEffect, useState } from "react"; @@ -120,7 +121,11 @@ const SystemFormTabs = ({ const checkTabChange = (index: number) => { // While privacy declarations aren't updated yet, only apply the "unsaved changes" modal logic // to the system information tab - if (index === 0) { + if ( + index === 0 || + (index === 1 && tabIndex === 2) || + (index === 2 && tabIndex === 1) + ) { setTabIndex(index); } else { setQueuedIndex(index); @@ -182,6 +187,32 @@ const SystemFormTabs = ({ ) : null, isDisabled: !activeSystem, }, + { + label: "Data flow", + content: activeSystem ? ( + + + + Data flow + + + Data flow describes the flow of data between systems in your Data + Map. Below, you can configure Source and Destination systems and + the corresponding links will be drawn in the Data Map graph. + Source systems are systems that send data to this system while + Destination systems receive data from this system. + + + + + ) : null, + isDisabled: !activeSystem, + }, ]; return ( diff --git a/clients/admin-ui/src/features/system/system.slice.ts b/clients/admin-ui/src/features/system/system.slice.ts index 2cf46d16a1..d43e384409 100644 --- a/clients/admin-ui/src/features/system/system.slice.ts +++ b/clients/admin-ui/src/features/system/system.slice.ts @@ -127,9 +127,10 @@ export const selectActiveClassifySystemFidesKey = createSelector( (state) => state.activeClassifySystemFidesKey ); +const emptySelectAllSystems: System[] = []; export const selectAllSystems = createSelector( systemApi.endpoints.getAllSystems.select(), - ({ data }) => data + ({ data }) => data || emptySelectAllSystems ); export const selectActiveClassifySystem = createSelector( From 9bd4172ddfe42996d1a862f31909df6615d64cfc Mon Sep 17 00:00:00 2001 From: Neville Samuell Date: Wed, 12 Apr 2023 10:20:18 -0400 Subject: [PATCH 3/4] Update Cookie House (sample project) privacy center to new config.json format (#3040) --- .../privacy_center/config/config.json | 98 ++++++++++--------- 1 file changed, 53 insertions(+), 45 deletions(-) diff --git a/src/fides/data/sample_project/privacy_center/config/config.json b/src/fides/data/sample_project/privacy_center/config/config.json index 08322e6d6e..68181fc515 100644 --- a/src/fides/data/sample_project/privacy_center/config/config.json +++ b/src/fides/data/sample_project/privacy_center/config/config.json @@ -1,6 +1,8 @@ { "title": "Take control of your data", "description": "When you use our services, you’re trusting us with your information. We understand this is a big responsibility and work hard to protect your information and put you in control.", + "description_subtext": [], + "addendum": [], "server_url_development": "http://localhost:8080/api/v1", "server_url_production": "http://localhost:8080/api/v1", "logo_path": "/assets/logo.svg", @@ -26,53 +28,59 @@ ], "includeConsent": true, "consent": { - "icon_path": "/assets/consent.svg", - "title": "Manage your consent", - "description": "Manage your consent preferences, including the option to select 'Do Not Sell My Personal Information'.", - "identity_inputs": { - "email": "required" + "button": { + "description": "Manage your consent preferences, including the option to select 'Do Not Sell My Personal Information'.", + "icon_path": "/assets/consent.svg", + "identity_inputs": { + "email": "required" + }, + "title": "Manage your consent" }, - "policy_key": "default_consent_policy", - "consentOptions": [ - { - "fidesDataUseKey": "advertising", - "name": "Data Sales or Sharing", - "description": "We may use some of your personal information for behavioral advertising purposes, which may be interpreted as 'Data Sales' or 'Data Sharing' under regulations such as CCPA, CPRA, VCDPA, etc.", - "url": "https://example.com/privacy#data-sales", - "default": { - "value": true, - "globalPrivacyControl": false + "page": { + "consentOptions": [ + { + "fidesDataUseKey": "advertising", + "name": "Data Sales or Sharing", + "description": "We may use some of your personal information for behavioral advertising purposes, which may be interpreted as 'Data Sales' or 'Data Sharing' under regulations such as CCPA, CPRA, VCDPA, etc.", + "url": "https://example.com/privacy#data-sales", + "default": { + "value": true, + "globalPrivacyControl": false + }, + "highlight": false, + "cookieKeys": ["data_sales"], + "executable": false }, - "highlight": false, - "cookieKeys": [ - "data_sales" - ] - }, - { - "fidesDataUseKey": "advertising.first_party", - "name": "Email Marketing", - "description": "We may use some of your personal information to contact you about our products & services.", - "url": "https://example.com/privacy#email-marketing", - "default": { - "value": true, - "globalPrivacyControl": false + { + "fidesDataUseKey": "advertising.first_party", + "name": "Email Marketing", + "description": "We may use some of your personal information to contact you about our products & services.", + "url": "https://example.com/privacy#email-marketing", + "default": { + "value": true, + "globalPrivacyControl": false + }, + "highlight": false, + "cookieKeys": ["tracking"], + "executable": false }, - "highlight": false, - "cookieKeys": [ - "marketing" - ] - }, - { - "fidesDataUseKey": "improve", - "name": "Product Analytics", - "description": "We may use some of your personal information to collect analytics about how you use our products & services.", - "url": "https://example.com/privacy#analytics", - "default": true, - "highlight": false, - "cookieKeys": [ - "analytics" - ] - } - ] + { + "fidesDataUseKey": "improve", + "name": "Product Analytics", + "description": "We may use some of your personal information to collect analytics about how you use our products & services.", + "url": "https://example.com/privacy#analytics", + "default": true, + "highlight": false, + "cookieKeys": ["tracking"], + "executable": false + } + ], + "description": "Manage your consent preferences, including the option to select 'Do Not Sell My Personal Information'.", + "description_subtext": [ + "When you use our services, you're trusting us with your information. We understand this is a big responsibility and work hard to protect your information and put you in control." + ], + "policy_key": "default_consent_policy", + "title": "Manage your consent" + } } } \ No newline at end of file From 65005e52403c7937199dc91beae109446d1b48b3 Mon Sep 17 00:00:00 2001 From: Andrew Jackson Date: Wed, 12 Apr 2023 10:33:20 -0400 Subject: [PATCH 4/4] Update datamap table export (#3038) --- CHANGELOG.md | 1 + .../src/features/datamap/constants.ts | 31 --- .../features/datamap/modals/ExportModal.tsx | 212 ++---------------- 3 files changed, 23 insertions(+), 221 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e01071dc3f..6ebd376123 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ The types of changes are: - Support for uploading custom connector templates via the UI [#2997](https://github.com/ethyca/fides/pull/2997) - Add a backwards-compatible workflow for saving and propagating consent preferences with respect to Privacy Notices [#3016](https://github.com/ethyca/fides/pull/3016) - Added Data flow modal [#3008](https://github.com/ethyca/fides/pull/3008) +- Update datamap table export [#3038](https://github.com/ethyca/fides/pull/3038) ### Changed diff --git a/clients/admin-ui/src/features/datamap/constants.ts b/clients/admin-ui/src/features/datamap/constants.ts index d738251152..d24ee7f64b 100644 --- a/clients/admin-ui/src/features/datamap/constants.ts +++ b/clients/admin-ui/src/features/datamap/constants.ts @@ -1,42 +1,11 @@ -import { ExportFilterItem } from "./types"; - /** * Enums */ -export enum ExportFilterType { - DEFAULT, - GROUP_BY_PURPOSE_OF_PROCESSING, - GROUP_BY_SYSTEM, -} export const CELL_SIZE = 20; export const DATA_CATEGORY_COLUMN_ID = "unioned_data_categories"; -export const EXPORT_FILTER_MAP: ExportFilterItem[] = [ - { - id: ExportFilterType.GROUP_BY_SYSTEM, - name: `Group by system`, - description: `Export a file grouped by system. All other data within a system will be collapsed in each row.`, - key: `system.name`, - fileName: `report_systems_[timestamp]`, - }, - { - id: ExportFilterType.GROUP_BY_PURPOSE_OF_PROCESSING, - name: `Group by purpose of processing`, - description: `Export a file grouped by purpose of processing. All other data within a purpose of processing will be collapsed in each row.`, - key: `system.privacy_declaration.data_use.name`, - fileName: `report_purposes_processing_[timestamp]`, - }, - { - id: ExportFilterType.DEFAULT, - name: `Default`, - description: `Export a file which retains the format of the table within the Fides application. This can be used if you need to filter on a single value like data category.`, - key: ``, - fileName: `report_[timestamp]`, - }, -]; - export const GRAY_BACKGROUND = "#F7F7F7"; export const ItemTypes = { diff --git a/clients/admin-ui/src/features/datamap/modals/ExportModal.tsx b/clients/admin-ui/src/features/datamap/modals/ExportModal.tsx index dc028d53b3..58bdda1454 100644 --- a/clients/admin-ui/src/features/datamap/modals/ExportModal.tsx +++ b/clients/admin-ui/src/features/datamap/modals/ExportModal.tsx @@ -9,34 +9,13 @@ import { ModalFooter, ModalHeader, ModalOverlay, - Radio, - RadioGroup, - Stack, Text, } from "@fidesui/react"; import { stringify } from "csv-stringify/sync"; import { saveAs } from "file-saver"; -import React, { - useCallback, - useContext, - useEffect, - useMemo, - useRef, - useState, -} from "react"; +import React, { useContext, useRef } from "react"; import { utils, WorkBook, writeFileXLSX } from "xlsx"; -import { useAppSelector } from "~/app/hooks"; -import QuestionTooltip from "~/features/common/QuestionTooltip"; - -import { EXPORT_FILTER_MAP, ExportFilterType } from "../constants"; -import { - DatamapColumn, - DatamapRow, - DatamapTableData, - selectColumns, - useLazyGetDatamapQuery, -} from "../datamap.slice"; import DatamapTableContext from "../datamap-table/DatamapTableContext"; export type ExportFileType = "xlsx" | "csv"; @@ -46,51 +25,13 @@ interface ExportModalProps { onClose: () => void; } -const ExportModal: React.FC = ({ isOpen, onClose }) => { +const ExportModal = ({ isOpen, onClose }: ExportModalProps) => { const initialRef = useRef(null); - const [selectedFilter, setSelectedFilter] = useState( - ExportFilterType.DEFAULT - ); - const tableColumns = useAppSelector(selectColumns); const { tableInstance } = useContext(DatamapTableContext); - const [getDatamap] = useLazyGetDatamapQuery(); - const applyMergeFilter = async (data: DatamapTableData) => { - // eslint-disable-next-line @typescript-eslint/naming-convention - const _ = (await import("lodash")).default; - const key = EXPORT_FILTER_MAP.find( - (item) => item.id === selectedFilter - )?.key; - if (key) { - const DELIMITER = ", "; - return _.chain(data.rows) - .groupBy((element) => element[key]) - .map((rows) => { - let merge: DatamapRow = {}; - rows.forEach((r) => { - merge = _.mergeWith( - merge, - r, - (objValue: string, srcValue: string) => { - if (typeof objValue === "string") { - if (objValue === srcValue || objValue.includes(srcValue)) { - return objValue; - } - const list = objValue.split(DELIMITER); - list.push(srcValue); - return list.sort().join(DELIMITER); - } - return srcValue; - } - ); - }); - return merge; - }) - .sortBy(key) - .value(); - } - return data.rows; - }; + if (!tableInstance) { + return null; + } const generateExportFile = ( data: { @@ -102,7 +43,6 @@ const ExportModal: React.FC = ({ isOpen, onClose }) => { if (!data || !data.columns || !data.rows) { return ""; } - const { columns, rows } = data; // If we are generating a CSV file, do that and return @@ -118,68 +58,25 @@ const ExportModal: React.FC = ({ isOpen, onClose }) => { return workbook; }; - const visibleColumns = useMemo( - () => tableColumns?.filter((column) => column.isVisible) || [], - [tableColumns] - ); + const generateROPAExportData = () => { + const columns = tableInstance.columns + .filter((column) => column.isVisible) + .map((column) => column.Header) as string[]; - const generateROPAExportData = async () => { - const { data } = await getDatamap( - { - organizationName: "default_organization", - }, - true - ); - if (!data) { - return null; - } - const columns = visibleColumns.map((column) => column.text); - const mergeFilter = applyMergeFilter(data); - const rows = (await mergeFilter).map((row) => - visibleColumns.reduce( - (fields, column: DatamapColumn) => [...fields, `${row[column.value]}`], - [] as string[] - ) - ); + const rows = tableInstance.rows + .map((row) => row.subRows) + .flatMap((row) => row) + .map((row) => row.cells.map((cell) => cell.value)) as string[][]; return { columns, rows }; }; - const getFilterItem = useCallback( - (id: ExportFilterType) => EXPORT_FILTER_MAP.find((item) => item.id === id), - [] - ); - - const handleChange = (nextValue: string) => { - setSelectedFilter(Number(nextValue)); - }; - - const isColumnVisible = useCallback( - (key: string) => - key - ? visibleColumns.some( - (column) => column.value.toLowerCase() === key.toLowerCase() - ) - : true, - [visibleColumns] - ); - - const hasDisabledFilter = useMemo( - () => - [ - ExportFilterType.GROUP_BY_PURPOSE_OF_PROCESSING, - ExportFilterType.GROUP_BY_SYSTEM, - ].some((value) => !isColumnVisible(getFilterItem(value)!.key)), - [getFilterItem, isColumnVisible] - ); - const triggerExportFileDownload = ( file: string | WorkBook, fileType: ExportFileType ) => { const now = new Date().toISOString(); - const fileName = `${ - getFilterItem(selectedFilter)!.fileName - }.${fileType}`.replace("[timestamp]", now); + const fileName = `report_${now}.${fileType}`; + if (typeof file === "string") { if (fileType === "csv") { const blob = new Blob([file], { type: "text/csv;charset=utf-8" }); @@ -195,17 +92,11 @@ const ExportModal: React.FC = ({ isOpen, onClose }) => { if (!tableInstance) { return; } - const data = await generateROPAExportData(); + const data = generateROPAExportData(); const file = generateExportFile(data, fileType); triggerExportFileDownload(file, fileType); }; - useEffect(() => { - if (isOpen) { - setSelectedFilter(ExportFilterType.DEFAULT); - } - }, [isOpen]); - return ( = ({ isOpen, onClose }) => { - - Choose a format - {hasDisabledFilter && ( - - )} - - - - - {EXPORT_FILTER_MAP.map((item, index) => ( - { - handleChange(item.id.toString()); - }} - > - - - - {item.name} - - - {item.description} - - - - ))} - - + + + To export the current view of the Data Map table, please select + the appropriate format below. Your export will contain only the + visible columns and rows. +