From e6ad6b360b5a6c1385cc3ec4853a82ffd8cbc69a Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Fri, 23 Dec 2022 15:31:22 -0600 Subject: [PATCH] Remove DatasetConfig.dataset field + New Get DatasetConfig Endpoints [#1763] (#2096) - Remove the DatasetConfig.dataset column - Throw a 404 if the ctl_dataset_id does not existing when creating a DatasetConfig through the new patch_dataset_configs endpoint - Add new GET datasetconfig list and detail endpoints that include both the fides_key and nested ctl_dataset in the response. The DatasetConfig and CTLDataset are different resources and their fides_keys can differ, so both keys need to be in the response. - Add a fix to prevent existing upsert from attempting to update the id of an existing resource. Datasets are referenced by DatasetConfigs now, we need them to stay the same. - Add more validation before linking an existing CTL Dataset to a DatasetConfig - Update the Ops DatasetConfig Admin UI for parity with existing UI Co-authored-by: Allison King --- .fides/db_dataset.yml | 3 - CHANGELOG.md | 8 +- clients/admin-ui/cypress/e2e/connectors.cy.ts | 74 ++ .../fixtures/connectors/connection_types.json | 67 ++ .../fixtures/connectors/datasetconfig.json | 844 ++++++++++++++++++ .../cypress/fixtures/connectors/list.json | 33 + .../connectors/postgres_connector.json | 13 + .../fixtures/connectors/postgres_secret.json | 15 + .../src/features/dataset/dataset.slice.ts | 19 +- .../ConnectionGridItem.tsx | 7 +- .../datastore-connections/ConnectionMenu.tsx | 18 +- .../add-connection/DatasetConfiguration.tsx | 50 +- .../forms/ConnectorParametersForm.tsx | 1 + .../add-connection/forms/YamlEditorForm.tsx | 5 +- .../datastore-connection.slice.ts | 26 +- .../features/datastore-connections/types.ts | 17 +- clients/admin-ui/src/types/api/index.ts | 13 + .../src/types/api/models/AccessLevel.ts | 14 + .../models/BulkPutConnectionConfiguration.ts | 14 + .../src/types/api/models/BulkPutDataset.ts | 14 + .../models/ConnectionConfigurationResponse.ts | 26 + .../types/api/models/ConnectionTestStatus.ts | 12 + .../api/models/DatasetConfigCtlDataset.ts | 11 + .../types/api/models/DatasetConfigSchema.ts | 13 + .../api/models/DatasetTraversalDetails.ts | 12 + .../Page_ConnectionConfigurationResponse_.ts | 12 + .../api/models/Page_DatasetConfigSchema_.ts | 12 + .../src/types/api/models/Page_Dataset_.ts | 12 + .../src/types/api/models/SaaSConfigBase.ts | 12 + .../api/models/ValidateDatasetResponse.ts | 16 + demo_resources/demo_dataset.yml | 3 +- .../postman/Fides.postman_collection.json | 94 +- src/fides/api/ctl/database/crud.py | 5 +- ...c6c6555c86_remove_datasetconfig_dataset.py | 32 + .../ops/api/v1/endpoints/dataset_endpoints.py | 95 +- src/fides/api/ops/api/v1/urn_registry.py | 1 + src/fides/api/ops/models/datasetconfig.py | 7 +- src/fides/api/ops/schemas/dataset.py | 12 + .../ops/service/connectors/email_connector.py | 8 +- .../saas/connector_registry_service.py | 2 +- tests/ctl/core/test_dataset.py | 125 ++- .../test_connection_template_endpoints.py | 3 +- .../v1/endpoints/test_dataset_endpoints.py | 247 ++++- tests/ops/fixtures/application_fixtures.py | 50 -- tests/ops/fixtures/bigquery_fixtures.py | 1 - tests/ops/fixtures/email_fixtures.py | 1 - .../fides_connector_example_fixtures.py | 1 - tests/ops/fixtures/manual_fixtures.py | 1 - tests/ops/fixtures/mariadb_fixtures.py | 1 - tests/ops/fixtures/mssql_fixtures.py | 1 - tests/ops/fixtures/mysql_fixtures.py | 2 - tests/ops/fixtures/postgres_fixtures.py | 2 - tests/ops/fixtures/redshift_fixtures.py | 1 - .../fixtures/saas/adobe_campaign_fixtures.py | 1 - tests/ops/fixtures/saas/auth0_fixtures.py | 1 - tests/ops/fixtures/saas/braze_fixtures.py | 1 - tests/ops/fixtures/saas/datadog_fixtures.py | 1 - tests/ops/fixtures/saas/domo_fixtures.py | 1 - tests/ops/fixtures/saas/doordash_fixtures.py | 2 - tests/ops/fixtures/saas/friendbuy_fixtures.py | 13 +- tests/ops/fixtures/saas/fullstory_fixtures.py | 2 - tests/ops/fixtures/saas/hubspot_fixtures.py | 1 - tests/ops/fixtures/saas/mailchimp_fixtures.py | 1 - tests/ops/fixtures/saas/outreach_fixtures.py | 1 - .../firebase_auth_fixtures.py | 1 - .../mailchimp_override_fixtures.py | 1 - tests/ops/fixtures/saas/rollbar_fixtures.py | 1 - .../ops/fixtures/saas/salesforce_fixtures.py | 1 - tests/ops/fixtures/saas/segment_fixtures.py | 1 - tests/ops/fixtures/saas/sendgrid_fixtures.py | 1 - tests/ops/fixtures/saas/sentry_fixtures.py | 1 - tests/ops/fixtures/saas/shopify_fixtures.py | 1 - .../saas/slack_enterprise_fixtures.py | 1 - tests/ops/fixtures/saas/square_fixtures.py | 1 - tests/ops/fixtures/saas/stripe_fixtures.py | 1 - .../saas/twilio_conversations_fixtures.py | 2 - tests/ops/fixtures/saas/zendesk_fixtures.py | 1 - tests/ops/fixtures/saas_example_fixtures.py | 2 - tests/ops/fixtures/snowflake_fixtures.py | 1 - tests/ops/models/test_datasetconfig.py | 33 +- 80 files changed, 1934 insertions(+), 229 deletions(-) create mode 100644 clients/admin-ui/cypress/e2e/connectors.cy.ts create mode 100644 clients/admin-ui/cypress/fixtures/connectors/connection_types.json create mode 100644 clients/admin-ui/cypress/fixtures/connectors/datasetconfig.json create mode 100644 clients/admin-ui/cypress/fixtures/connectors/list.json create mode 100644 clients/admin-ui/cypress/fixtures/connectors/postgres_connector.json create mode 100644 clients/admin-ui/cypress/fixtures/connectors/postgres_secret.json create mode 100644 clients/admin-ui/src/types/api/models/AccessLevel.ts create mode 100644 clients/admin-ui/src/types/api/models/BulkPutConnectionConfiguration.ts create mode 100644 clients/admin-ui/src/types/api/models/BulkPutDataset.ts create mode 100644 clients/admin-ui/src/types/api/models/ConnectionConfigurationResponse.ts create mode 100644 clients/admin-ui/src/types/api/models/ConnectionTestStatus.ts create mode 100644 clients/admin-ui/src/types/api/models/DatasetConfigCtlDataset.ts create mode 100644 clients/admin-ui/src/types/api/models/DatasetConfigSchema.ts create mode 100644 clients/admin-ui/src/types/api/models/DatasetTraversalDetails.ts create mode 100644 clients/admin-ui/src/types/api/models/Page_ConnectionConfigurationResponse_.ts create mode 100644 clients/admin-ui/src/types/api/models/Page_DatasetConfigSchema_.ts create mode 100644 clients/admin-ui/src/types/api/models/Page_Dataset_.ts create mode 100644 clients/admin-ui/src/types/api/models/SaaSConfigBase.ts create mode 100644 clients/admin-ui/src/types/api/models/ValidateDatasetResponse.ts create mode 100644 src/fides/api/ctl/migrations/versions/d6c6c6555c86_remove_datasetconfig_dataset.py diff --git a/.fides/db_dataset.yml b/.fides/db_dataset.yml index 9e604bf3f4..97f2c2ee1e 100644 --- a/.fides/db_dataset.yml +++ b/.fides/db_dataset.yml @@ -1031,9 +1031,6 @@ dataset: - name: created_at data_categories: [system.operations] data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified - - name: dataset - data_categories: [system.operations] - data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified - name: fides_key data_categories: [system.operations] data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified diff --git a/CHANGELOG.md b/CHANGELOG.md index 29a56e7c6c..89d5e81619 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,11 +21,6 @@ The types of changes are: * Include a pre-check workflow that collects the pytest suite [#2098](https://github.com/ethyca/fides/pull/2098) -### Changed - -* Move the `fides.ctl.core.` and `fides.ctl.connectors` modules into `fides.core` and `fides.connectors` respectively [#2097](https://github.com/ethyca/fides/pull/2097) -* Fides: Skip cypress tests due to nav bar 2.0 [#2102](https://github.com/ethyca/fides/pull/2103) - ### Added * Adds new erasure policy for complete user data masking [#1839](https://github.com/ethyca/fides/pull/1839) * New Fides Home page [#1864](https://github.com/ethyca/fides/pull/2050) @@ -34,8 +29,11 @@ The types of changes are: * Added ability to use Mailgun templates when sending emails. [#2039](https://github.com/ethyca/fides/pull/2039) ### Changed +* Move the `fides.ctl.core.` and `fides.ctl.connectors` modules into `fides.core` and `fides.connectors` respectively [#2097](https://github.com/ethyca/fides/pull/2097) +* Fides: Skip cypress tests due to nav bar 2.0 [#2102](https://github.com/ethyca/fides/pull/2103) * Remove several fidesops schemas for DSR's in favor of updated Fideslang schemas [#2009](https://github.com/ethyca/fides/pull/2009) * New datasetconfig.ctl_dataset_id field to unify fides dataset resources [#2046](https://github.com/ethyca/fides/pull/2046) +* Update UI dataset config routes to use new unified routes [#2113](https://github.com/ethyca/fides/pull/2113) ## [2.3.1](https://github.com/ethyca/fides/compare/2.3.0...2.3.1) diff --git a/clients/admin-ui/cypress/e2e/connectors.cy.ts b/clients/admin-ui/cypress/e2e/connectors.cy.ts new file mode 100644 index 0000000000..cfb5da052d --- /dev/null +++ b/clients/admin-ui/cypress/e2e/connectors.cy.ts @@ -0,0 +1,74 @@ +describe("Connectors", () => { + beforeEach(() => { + cy.login(); + }); + describe("Configuring connectors", () => { + beforeEach(() => { + cy.intercept("GET", "/api/v1/connection*", { + fixture: "connectors/list.json", + }).as("getConnectors"); + cy.intercept("GET", "/api/v1/connection_type*", { + fixture: "connectors/connection_types.json", + }).as("getConnectionTypes"); + cy.intercept("GET", "/api/v1/connection/postgres_connector", { + fixture: "connectors/postgres_connector.json", + }).as("getPostgresConnector"); + cy.intercept("GET", "/api/v1/connection_type/postgres/secret", { + fixture: "connectors/postgres_secret.json", + }).as("getPostgresConnectorSecret"); + cy.intercept( + "GET", + "/api/v1/connection/postgres_connector/datasetconfig", + { + fixture: "connectors/datasetconfig.json", + } + ).as("getPostgresConnectorDatasetconfig"); + + cy.intercept("POST", "/api/v1/dataset/upsert", { body: {} }).as( + "upsertDataset" + ); + cy.intercept( + "PATCH", + "/api/v1/connection/postgres_connector/datasetconfig", + { body: {} } + ).as("patchDatasetconfig"); + }); + + it("Should show data store connections and view configuration", () => { + cy.visit("/datastore-connection"); + cy.getByTestId("connection-grid-item-mongodb_connector"); + cy.getByTestId("connection-grid-item-postgres_connector").within(() => { + cy.getByTestId("connection-menu-btn").click(); + }); + cy.getByTestId("connection-menu-postgres_connector").within(() => { + cy.getByTestId("configure-btn").click(); + }); + cy.getByTestId("input-name").should("have.value", "postgres_connector"); + }); + + it("Should allow saving a dataset configuration", () => { + cy.visit("/datastore-connection/postgres_connector"); + cy.getByTestId("tab-Dataset configuration").click(); + cy.wait("@getPostgresConnectorDatasetconfig"); + // The monaco yaml editor takes a bit to load. Since this is likely going away, + // just wait for now and remove this once the yaml editor is no longer available + // eslint-disable-next-line cypress/no-unnecessary-waiting + cy.wait(1000); + cy.getByTestId("save-btn").click(); + cy.wait("@upsertDataset").then((interception) => { + expect(interception.request.body.length).to.eql(1); + expect(interception.request.body[0].fides_key).to.eql( + "postgres_example_test_dataset" + ); + }); + cy.wait("@patchDatasetconfig").then((interception) => { + expect(interception.request.body).to.eql([ + { + fides_key: "postgres_example_test_dataset", + ctl_dataset_fides_key: "postgres_example_test_dataset", + }, + ]); + }); + }); + }); +}); diff --git a/clients/admin-ui/cypress/fixtures/connectors/connection_types.json b/clients/admin-ui/cypress/fixtures/connectors/connection_types.json new file mode 100644 index 0000000000..7cf2b3afa3 --- /dev/null +++ b/clients/admin-ui/cypress/fixtures/connectors/connection_types.json @@ -0,0 +1,67 @@ +{ + "items": [ + { + "identifier": "bigquery", + "type": "database", + "human_readable": "BigQuery", + "encoded_icon": null + }, + { + "identifier": "mariadb", + "type": "database", + "human_readable": "MariaDB", + "encoded_icon": null + }, + { + "identifier": "mongodb", + "type": "database", + "human_readable": "MongoDB", + "encoded_icon": null + }, + { + "identifier": "mssql", + "type": "database", + "human_readable": "Microsoft SQL Server", + "encoded_icon": null + }, + { + "identifier": "mysql", + "type": "database", + "human_readable": "MySQL", + "encoded_icon": null + }, + { + "identifier": "postgres", + "type": "database", + "human_readable": "PostgreSQL", + "encoded_icon": null + }, + { + "identifier": "redshift", + "type": "database", + "human_readable": "Amazon Redshift", + "encoded_icon": null + }, + { + "identifier": "snowflake", + "type": "database", + "human_readable": "Snowflake", + "encoded_icon": null + }, + { + "identifier": "timescale", + "type": "database", + "human_readable": "TimescaleDB", + "encoded_icon": null + }, + { + "identifier": "manual_webhook", + "type": "manual", + "human_readable": "Manual Webhook", + "encoded_icon": null + } + ], + "total": 10, + "page": 1, + "size": 50 +} diff --git a/clients/admin-ui/cypress/fixtures/connectors/datasetconfig.json b/clients/admin-ui/cypress/fixtures/connectors/datasetconfig.json new file mode 100644 index 0000000000..a7fd4bbe5c --- /dev/null +++ b/clients/admin-ui/cypress/fixtures/connectors/datasetconfig.json @@ -0,0 +1,844 @@ +{ + "items": [ + { + "fides_key": "postgres_example_test_dataset", + "ctl_dataset": { + "fides_key": "postgres_example_test_dataset", + "organization_fides_key": "default_organization", + "tags": null, + "name": "Postgres Example Test Dataset", + "description": "Example of a Postgres dataset containing a variety of related tables like customers, products, addresses, etc.", + "meta": null, + "data_categories": null, + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "fides_meta": null, + "joint_controller": null, + "retention": "No retention or erasure policy", + "third_country_transfers": null, + "collections": [ + { + "name": "address", + "description": null, + "data_categories": null, + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fields": [ + { + "name": "city", + "description": null, + "data_categories": ["user.contact.address.city"], + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fides_meta": null, + "fields": null + }, + { + "name": "house", + "description": null, + "data_categories": ["user.contact.address.street"], + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fides_meta": null, + "fields": null + }, + { + "name": "id", + "description": null, + "data_categories": ["system.operations"], + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fides_meta": { + "references": null, + "identity": null, + "primary_key": true, + "data_type": null, + "length": null, + "return_all_elements": null, + "read_only": null + }, + "fields": null + }, + { + "name": "state", + "description": null, + "data_categories": ["user.contact.address.state"], + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fides_meta": null, + "fields": null + }, + { + "name": "street", + "description": null, + "data_categories": ["user.contact.address.street"], + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fides_meta": null, + "fields": null + }, + { + "name": "zip", + "description": null, + "data_categories": ["user.contact.address.postal_code"], + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fides_meta": null, + "fields": null + } + ], + "fides_meta": null + }, + { + "name": "customer", + "description": null, + "data_categories": null, + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fields": [ + { + "name": "address_id", + "description": null, + "data_categories": ["system.operations"], + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fides_meta": { + "references": [ + { + "dataset": "postgres_example_test_dataset", + "field": "address.id", + "direction": "to" + } + ], + "identity": null, + "primary_key": null, + "data_type": null, + "length": null, + "return_all_elements": null, + "read_only": null + }, + "fields": null + }, + { + "name": "created", + "description": null, + "data_categories": ["system.operations"], + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fides_meta": null, + "fields": null + }, + { + "name": "email", + "description": null, + "data_categories": ["user.contact.email"], + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fides_meta": { + "references": null, + "identity": "email", + "primary_key": null, + "data_type": "string", + "length": null, + "return_all_elements": null, + "read_only": null + }, + "fields": null + }, + { + "name": "id", + "description": null, + "data_categories": ["user.unique_id"], + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fides_meta": { + "references": null, + "identity": null, + "primary_key": true, + "data_type": null, + "length": null, + "return_all_elements": null, + "read_only": null + }, + "fields": null + }, + { + "name": "name", + "description": null, + "data_categories": ["user.name"], + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fides_meta": { + "references": null, + "identity": null, + "primary_key": null, + "data_type": "string", + "length": 40, + "return_all_elements": null, + "read_only": null + }, + "fields": null + } + ], + "fides_meta": null + }, + { + "name": "employee", + "description": null, + "data_categories": null, + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fields": [ + { + "name": "address_id", + "description": null, + "data_categories": ["system.operations"], + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fides_meta": { + "references": [ + { + "dataset": "postgres_example_test_dataset", + "field": "address.id", + "direction": "to" + } + ], + "identity": null, + "primary_key": null, + "data_type": null, + "length": null, + "return_all_elements": null, + "read_only": null + }, + "fields": null + }, + { + "name": "email", + "description": null, + "data_categories": ["user.contact.email"], + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fides_meta": { + "references": null, + "identity": "email", + "primary_key": null, + "data_type": "string", + "length": null, + "return_all_elements": null, + "read_only": null + }, + "fields": null + }, + { + "name": "id", + "description": null, + "data_categories": ["user.unique_id"], + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fides_meta": { + "references": null, + "identity": null, + "primary_key": true, + "data_type": null, + "length": null, + "return_all_elements": null, + "read_only": null + }, + "fields": null + }, + { + "name": "name", + "description": null, + "data_categories": ["user.name"], + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fides_meta": { + "references": null, + "identity": null, + "primary_key": null, + "data_type": "string", + "length": null, + "return_all_elements": null, + "read_only": null + }, + "fields": null + } + ], + "fides_meta": null + }, + { + "name": "login", + "description": null, + "data_categories": null, + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fields": [ + { + "name": "customer_id", + "description": null, + "data_categories": ["user.unique_id"], + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fides_meta": { + "references": [ + { + "dataset": "postgres_example_test_dataset", + "field": "customer.id", + "direction": "from" + } + ], + "identity": null, + "primary_key": null, + "data_type": null, + "length": null, + "return_all_elements": null, + "read_only": null + }, + "fields": null + }, + { + "name": "id", + "description": null, + "data_categories": ["system.operations"], + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fides_meta": { + "references": null, + "identity": null, + "primary_key": true, + "data_type": null, + "length": null, + "return_all_elements": null, + "read_only": null + }, + "fields": null + }, + { + "name": "time", + "description": null, + "data_categories": ["user.sensor"], + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fides_meta": null, + "fields": null + } + ], + "fides_meta": null + }, + { + "name": "order_item", + "description": null, + "data_categories": null, + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fields": [ + { + "name": "order_id", + "description": null, + "data_categories": ["system.operations"], + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fides_meta": { + "references": [ + { + "dataset": "postgres_example_test_dataset", + "field": "orders.id", + "direction": "from" + } + ], + "identity": null, + "primary_key": null, + "data_type": null, + "length": null, + "return_all_elements": null, + "read_only": null + }, + "fields": null + }, + { + "name": "product_id", + "description": null, + "data_categories": ["system.operations"], + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fides_meta": { + "references": [ + { + "dataset": "postgres_example_test_dataset", + "field": "product.id", + "direction": "to" + } + ], + "identity": null, + "primary_key": null, + "data_type": null, + "length": null, + "return_all_elements": null, + "read_only": null + }, + "fields": null + }, + { + "name": "quantity", + "description": null, + "data_categories": ["system.operations"], + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fides_meta": null, + "fields": null + } + ], + "fides_meta": null + }, + { + "name": "orders", + "description": null, + "data_categories": null, + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fields": [ + { + "name": "customer_id", + "description": null, + "data_categories": ["user.unique_id"], + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fides_meta": { + "references": [ + { + "dataset": "postgres_example_test_dataset", + "field": "customer.id", + "direction": "from" + } + ], + "identity": null, + "primary_key": null, + "data_type": null, + "length": null, + "return_all_elements": null, + "read_only": null + }, + "fields": null + }, + { + "name": "id", + "description": null, + "data_categories": ["system.operations"], + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fides_meta": { + "references": null, + "identity": null, + "primary_key": true, + "data_type": null, + "length": null, + "return_all_elements": null, + "read_only": null + }, + "fields": null + }, + { + "name": "shipping_address_id", + "description": null, + "data_categories": ["system.operations"], + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fides_meta": { + "references": [ + { + "dataset": "postgres_example_test_dataset", + "field": "address.id", + "direction": "to" + } + ], + "identity": null, + "primary_key": null, + "data_type": null, + "length": null, + "return_all_elements": null, + "read_only": null + }, + "fields": null + } + ], + "fides_meta": null + }, + { + "name": "payment_card", + "description": null, + "data_categories": null, + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fields": [ + { + "name": "billing_address_id", + "description": null, + "data_categories": ["system.operations"], + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fides_meta": { + "references": [ + { + "dataset": "postgres_example_test_dataset", + "field": "address.id", + "direction": "to" + } + ], + "identity": null, + "primary_key": null, + "data_type": null, + "length": null, + "return_all_elements": null, + "read_only": null + }, + "fields": null + }, + { + "name": "ccn", + "description": null, + "data_categories": ["user.financial.account_number"], + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fides_meta": null, + "fields": null + }, + { + "name": "code", + "description": null, + "data_categories": ["user.financial"], + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fides_meta": null, + "fields": null + }, + { + "name": "customer_id", + "description": null, + "data_categories": ["user.unique_id"], + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fides_meta": { + "references": [ + { + "dataset": "postgres_example_test_dataset", + "field": "customer.id", + "direction": "from" + } + ], + "identity": null, + "primary_key": null, + "data_type": null, + "length": null, + "return_all_elements": null, + "read_only": null + }, + "fields": null + }, + { + "name": "id", + "description": null, + "data_categories": ["system.operations"], + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fides_meta": { + "references": null, + "identity": null, + "primary_key": true, + "data_type": null, + "length": null, + "return_all_elements": null, + "read_only": null + }, + "fields": null + }, + { + "name": "name", + "description": null, + "data_categories": ["user.financial"], + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fides_meta": null, + "fields": null + }, + { + "name": "preferred", + "description": null, + "data_categories": ["user"], + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fides_meta": null, + "fields": null + } + ], + "fides_meta": null + }, + { + "name": "product", + "description": null, + "data_categories": null, + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fields": [ + { + "name": "id", + "description": null, + "data_categories": ["system.operations"], + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fides_meta": { + "references": null, + "identity": null, + "primary_key": true, + "data_type": null, + "length": null, + "return_all_elements": null, + "read_only": null + }, + "fields": null + }, + { + "name": "name", + "description": null, + "data_categories": ["system.operations"], + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fides_meta": null, + "fields": null + }, + { + "name": "price", + "description": null, + "data_categories": ["system.operations"], + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fides_meta": null, + "fields": null + } + ], + "fides_meta": null + }, + { + "name": "report", + "description": null, + "data_categories": null, + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fields": [ + { + "name": "email", + "description": null, + "data_categories": ["user.contact.email"], + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fides_meta": { + "references": null, + "identity": "email", + "primary_key": null, + "data_type": "string", + "length": null, + "return_all_elements": null, + "read_only": null + }, + "fields": null + }, + { + "name": "id", + "description": null, + "data_categories": ["system.operations"], + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fides_meta": { + "references": null, + "identity": null, + "primary_key": true, + "data_type": null, + "length": null, + "return_all_elements": null, + "read_only": null + }, + "fields": null + }, + { + "name": "month", + "description": null, + "data_categories": ["system.operations"], + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fides_meta": null, + "fields": null + }, + { + "name": "name", + "description": null, + "data_categories": ["system.operations"], + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fides_meta": null, + "fields": null + }, + { + "name": "total_visits", + "description": null, + "data_categories": ["system.operations"], + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fides_meta": null, + "fields": null + }, + { + "name": "year", + "description": null, + "data_categories": ["system.operations"], + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fides_meta": null, + "fields": null + } + ], + "fides_meta": null + }, + { + "name": "service_request", + "description": null, + "data_categories": null, + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fields": [ + { + "name": "alt_email", + "description": null, + "data_categories": ["user.contact.email"], + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fides_meta": { + "references": null, + "identity": "email", + "primary_key": null, + "data_type": "string", + "length": null, + "return_all_elements": null, + "read_only": null + }, + "fields": null + }, + { + "name": "closed", + "description": null, + "data_categories": ["system.operations"], + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fides_meta": null, + "fields": null + }, + { + "name": "email", + "description": null, + "data_categories": ["system.operations"], + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fides_meta": { + "references": null, + "identity": "email", + "primary_key": null, + "data_type": "string", + "length": null, + "return_all_elements": null, + "read_only": null + }, + "fields": null + }, + { + "name": "employee_id", + "description": null, + "data_categories": ["user.unique_id"], + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fides_meta": { + "references": [ + { + "dataset": "postgres_example_test_dataset", + "field": "employee.id", + "direction": "from" + } + ], + "identity": null, + "primary_key": null, + "data_type": null, + "length": null, + "return_all_elements": null, + "read_only": null + }, + "fields": null + }, + { + "name": "id", + "description": null, + "data_categories": ["system.operations"], + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fides_meta": { + "references": null, + "identity": null, + "primary_key": true, + "data_type": null, + "length": null, + "return_all_elements": null, + "read_only": null + }, + "fields": null + }, + { + "name": "opened", + "description": null, + "data_categories": ["system.operations"], + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fides_meta": null, + "fields": null + } + ], + "fides_meta": null + }, + { + "name": "visit", + "description": null, + "data_categories": null, + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fields": [ + { + "name": "email", + "description": null, + "data_categories": ["user.contact.email"], + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fides_meta": { + "references": null, + "identity": "email", + "primary_key": null, + "data_type": "string", + "length": null, + "return_all_elements": null, + "read_only": null + }, + "fields": null + }, + { + "name": "last_visit", + "description": null, + "data_categories": ["system.operations"], + "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + "retention": null, + "fides_meta": null, + "fields": null + } + ], + "fides_meta": null + } + ] + } + } + ], + "total": 1, + "page": 1, + "size": 50 +} diff --git a/clients/admin-ui/cypress/fixtures/connectors/list.json b/clients/admin-ui/cypress/fixtures/connectors/list.json new file mode 100644 index 0000000000..c1343a50d9 --- /dev/null +++ b/clients/admin-ui/cypress/fixtures/connectors/list.json @@ -0,0 +1,33 @@ +{ + "items": [ + { + "name": "mongodb_connector", + "key": "mongodb_connector", + "description": null, + "connection_type": "mongodb", + "access": "write", + "created_at": "2022-12-23T15:03:25.163340+00:00", + "updated_at": "2022-12-23T15:03:25.177787+00:00", + "disabled": false, + "last_test_timestamp": "2022-12-23T15:03:25.206745+00:00", + "last_test_succeeded": true, + "saas_config": null + }, + { + "name": "postgres_connector", + "key": "postgres_connector", + "description": null, + "connection_type": "postgres", + "access": "write", + "created_at": "2022-12-23T15:03:25.062083+00:00", + "updated_at": "2022-12-23T15:03:25.087398+00:00", + "disabled": false, + "last_test_timestamp": "2022-12-23T15:03:25.098056+00:00", + "last_test_succeeded": true, + "saas_config": null + } + ], + "total": 2, + "page": 1, + "size": 5 +} diff --git a/clients/admin-ui/cypress/fixtures/connectors/postgres_connector.json b/clients/admin-ui/cypress/fixtures/connectors/postgres_connector.json new file mode 100644 index 0000000000..16e520e5de --- /dev/null +++ b/clients/admin-ui/cypress/fixtures/connectors/postgres_connector.json @@ -0,0 +1,13 @@ +{ + "name": "postgres_connector", + "key": "postgres_connector", + "description": null, + "connection_type": "postgres", + "access": "write", + "created_at": "2022-12-23T15:30:42.274628+00:00", + "updated_at": "2022-12-23T15:30:42.292992+00:00", + "disabled": false, + "last_test_timestamp": "2022-12-23T15:30:42.300853+00:00", + "last_test_succeeded": true, + "saas_config": null +} diff --git a/clients/admin-ui/cypress/fixtures/connectors/postgres_secret.json b/clients/admin-ui/cypress/fixtures/connectors/postgres_secret.json new file mode 100644 index 0000000000..db3dad1c8b --- /dev/null +++ b/clients/admin-ui/cypress/fixtures/connectors/postgres_secret.json @@ -0,0 +1,15 @@ +{ + "title": "PostgreSQLSchema", + "description": "Schema to validate the secrets needed to connect to a PostgreSQL Database", + "type": "object", + "properties": { + "url": { "title": "Url", "type": "string" }, + "username": { "title": "Username", "type": "string" }, + "password": { "title": "Password", "type": "string" }, + "dbname": { "title": "Dbname", "type": "string" }, + "db_schema": { "title": "Db Schema", "type": "string" }, + "host": { "title": "Host", "type": "string" }, + "port": { "title": "Port", "type": "integer" } + }, + "additionalProperties": false +} diff --git a/clients/admin-ui/src/features/dataset/dataset.slice.ts b/clients/admin-ui/src/features/dataset/dataset.slice.ts index f59b44344a..440ba6cfd9 100644 --- a/clients/admin-ui/src/features/dataset/dataset.slice.ts +++ b/clients/admin-ui/src/features/dataset/dataset.slice.ts @@ -2,7 +2,12 @@ import { createSelector, createSlice, PayloadAction } from "@reduxjs/toolkit"; import { createApi, fetchBaseQuery } from "@reduxjs/toolkit/query/react"; import type { RootState } from "~/app/store"; -import { Dataset, GenerateRequestPayload, GenerateResponse } from "~/types/api"; +import { + BulkPutDataset, + Dataset, + GenerateRequestPayload, + GenerateResponse, +} from "~/types/api"; import { EditableType } from "./types"; @@ -59,6 +64,17 @@ export const datasetApi = createApi({ }), invalidatesTags: ["Datasets"], }), + /** + * Also accepts unknown for the same reason as above + */ + upsertDatasets: build.mutation({ + query: (datasets) => ({ + url: `dataset/upsert`, + method: "POST", + body: datasets, + }), + invalidatesTags: ["Datasets"], + }), deleteDataset: build.mutation({ query: (key) => ({ url: `dataset/${key}`, @@ -81,6 +97,7 @@ export const { useGetAllDatasetsQuery, useGetDatasetByKeyQuery, useUpdateDatasetMutation, + useUpsertDatasetsMutation, useCreateDatasetMutation, useDeleteDatasetMutation, useGenerateDatasetMutation, diff --git a/clients/admin-ui/src/features/datastore-connections/ConnectionGridItem.tsx b/clients/admin-ui/src/features/datastore-connections/ConnectionGridItem.tsx index 9fba6ca73c..5cfa9bbe79 100644 --- a/clients/admin-ui/src/features/datastore-connections/ConnectionGridItem.tsx +++ b/clients/admin-ui/src/features/datastore-connections/ConnectionGridItem.tsx @@ -47,7 +47,12 @@ const ConnectionGridItem: React.FC = ({ const [trigger, result] = useLazyGetDatastoreConnectionStatusQuery(); return ( - + = ({ name, }) => ( - + - + - + Configure diff --git a/clients/admin-ui/src/features/datastore-connections/add-connection/DatasetConfiguration.tsx b/clients/admin-ui/src/features/datastore-connections/add-connection/DatasetConfiguration.tsx index acdc7ae63f..bf1bb2dbef 100644 --- a/clients/admin-ui/src/features/datastore-connections/add-connection/DatasetConfiguration.tsx +++ b/clients/admin-ui/src/features/datastore-connections/add-connection/DatasetConfiguration.tsx @@ -2,15 +2,18 @@ import { Box, Center, Spinner, VStack } from "@fidesui/react"; import { useAlert, useAPIHelper } from "common/hooks"; import { selectConnectionTypeState } from "connection-type/connection-type.slice"; import { - useGetDatasetsQuery, - usePatchDatasetMutation, + useGetDatasetConfigsQuery, + usePatchDatasetConfigsMutation, } from "datastore-connections/datastore-connection.slice"; -import { PatchDatasetsRequest } from "datastore-connections/types"; +import { PatchDatasetsConfigRequest } from "datastore-connections/types"; import { useRouter } from "next/router"; import React, { useState } from "react"; import { DATASTORE_CONNECTION_ROUTE } from "src/constants"; import { useAppSelector } from "~/app/hooks"; +import { getErrorMessage } from "~/features/common/helpers"; +import { useUpsertDatasetsMutation } from "~/features/dataset"; +import { Dataset, DatasetConfigCtlDataset } from "~/types/api"; import YamlEditorForm from "./forms/YamlEditorForm"; @@ -20,17 +23,46 @@ const DatasetConfiguration: React.FC = () => { const { handleError } = useAPIHelper(); const [isSubmitting, setIsSubmitting] = useState(false); const { connection } = useAppSelector(selectConnectionTypeState); - const { data, isFetching, isLoading, isSuccess } = useGetDatasetsQuery( + const { data, isFetching, isLoading, isSuccess } = useGetDatasetConfigsQuery( connection!.key ); - const [patchDataset] = usePatchDatasetMutation(); + const [patchDataset] = usePatchDatasetConfigsMutation(); + const [upsertDatasets] = useUpsertDatasetsMutation(); - const handleSubmit = async (value: any) => { + const handleSubmit = async (value: unknown) => { try { setIsSubmitting(true); - const params: PatchDatasetsRequest = { + // First update the datasets + const datasets = Array.isArray(value) ? value : [value]; + const upsertResult = await upsertDatasets(datasets); + if ("error" in upsertResult) { + const errorMessage = getErrorMessage(upsertResult.error); + errorAlert(errorMessage); + return; + } + + // Upsert was successful, so we can cast from unknown to Dataset + const upsertedDatasets = datasets as Dataset[]; + // Then link the updated dataset to the connection config. + // New entries will have matching keys, + let pairs: DatasetConfigCtlDataset[] = upsertedDatasets.map((d) => ({ + fides_key: d.fides_key, + ctl_dataset_fides_key: d.fides_key, + })); + // But existing entries might have their dataset keys changed from under them + if (data && data.items.length) { + const { items: datasetConfigs } = data; + pairs = datasetConfigs.map((d, i) => ({ + fides_key: d.fides_key, + // This will not handle deletions, additions, or even changing order. If we want to support + // those, we should probably have a different UX + ctl_dataset_fides_key: upsertedDatasets[i].fides_key, + })); + } + + const params: PatchDatasetsConfigRequest = { connection_key: connection?.key as string, - datasets: Array.isArray(value) ? value : [value], + dataset_pairs: pairs, }; const payload = await patchDataset(params).unwrap(); if (payload.failed?.length > 0) { @@ -59,7 +91,7 @@ const DatasetConfiguration: React.FC = () => { )} {isSuccess && data!?.items ? ( item.ctl_dataset)} isSubmitting={isSubmitting} onSubmit={handleSubmit} /> diff --git a/clients/admin-ui/src/features/datastore-connections/add-connection/forms/ConnectorParametersForm.tsx b/clients/admin-ui/src/features/datastore-connections/add-connection/forms/ConnectorParametersForm.tsx index 9f989a7027..1a84d6163a 100644 --- a/clients/admin-ui/src/features/datastore-connections/add-connection/forms/ConnectorParametersForm.tsx +++ b/clients/admin-ui/src/features/datastore-connections/add-connection/forms/ConnectorParametersForm.tsx @@ -286,6 +286,7 @@ const ConnectorParametersForm: React.FC = ({ connectionOption!.human_readable } connection`} size="sm" + data-testid="input-name" /> {props.errors.name} diff --git a/clients/admin-ui/src/features/datastore-connections/add-connection/forms/YamlEditorForm.tsx b/clients/admin-ui/src/features/datastore-connections/add-connection/forms/YamlEditorForm.tsx index eb049617b2..d0cee8221f 100644 --- a/clients/admin-ui/src/features/datastore-connections/add-connection/forms/YamlEditorForm.tsx +++ b/clients/admin-ui/src/features/datastore-connections/add-connection/forms/YamlEditorForm.tsx @@ -13,7 +13,6 @@ import { } from "@fidesui/react"; import { useAlert } from "common/hooks/useAlert"; import { ErrorWarningIcon } from "common/Icon"; -import { Dataset } from "datastore-connections/types"; import yaml, { YAMLException } from "js-yaml"; import { narrow } from "narrow-minded"; import dynamic from "next/dynamic"; @@ -22,6 +21,7 @@ import React, { useRef, useState } from "react"; import { DATASTORE_CONNECTION_ROUTE } from "src/constants"; import { useFeatures } from "~/features/common/features.slice"; +import { Dataset } from "~/types/api"; const Editor = dynamic( // @ts-ignore @@ -35,7 +35,7 @@ const isYamlException = (error: unknown): error is YAMLException => type YamlEditorFormProps = { data: Dataset[]; isSubmitting: boolean; - onSubmit: (value: any) => void; + onSubmit: (value: unknown) => void; }; const YamlEditorForm: React.FC = ({ @@ -131,6 +131,7 @@ const YamlEditorForm: React.FC = ({ _active={{ bg: "primary.500" }} _disabled={{ opacity: "inherit" }} _hover={{ bg: "primary.400" }} + data-testid="save-btn" > Save Yaml system diff --git a/clients/admin-ui/src/features/datastore-connections/datastore-connection.slice.ts b/clients/admin-ui/src/features/datastore-connections/datastore-connection.slice.ts index 511fb46e5d..3bf00e00a8 100644 --- a/clients/admin-ui/src/features/datastore-connections/datastore-connection.slice.ts +++ b/clients/admin-ui/src/features/datastore-connections/datastore-connection.slice.ts @@ -2,7 +2,11 @@ import { createSelector, createSlice, PayloadAction } from "@reduxjs/toolkit"; import { createApi, fetchBaseQuery } from "@reduxjs/toolkit/query/react"; import { addCommonHeaders } from "common/CommonHeaders"; -import { SystemType } from "~/types/api"; +import { + BulkPutDataset, + Page_DatasetConfigSchema_, + SystemType, +} from "~/types/api"; import type { RootState } from "../../app/store"; import { BASE_URL, CONNECTION_ROUTE } from "../../constants"; @@ -13,7 +17,6 @@ import { CreateAccessManualWebhookResponse, CreateSassConnectionConfigRequest, CreateSassConnectionConfigResponse, - DatasetsReponse, DatastoreConnection, DatastoreConnectionParams, DatastoreConnectionRequest, @@ -26,7 +29,7 @@ import { GetAllEnabledAccessManualWebhooksResponse, PatchAccessManualWebhookRequest, PatchAccessManualWebhookResponse, - PatchDatasetsRequest, + PatchDatasetsConfigRequest, } from "./types"; function mapFiltersToSearchParams({ @@ -223,9 +226,9 @@ export const datastoreConnectionApi = createApi({ ], keepUnusedDataFor: 1, }), - getDatasets: build.query({ + getDatasetConfigs: build.query({ query: (key) => ({ - url: `${CONNECTION_ROUTE}/${key}/dataset`, + url: `${CONNECTION_ROUTE}/${key}/datasetconfig`, }), providesTags: () => ["DatastoreConnection"], }), @@ -284,11 +287,14 @@ export const datastoreConnectionApi = createApi({ }), invalidatesTags: () => ["DatastoreConnection"], }), - patchDataset: build.mutation({ + patchDatasetConfigs: build.mutation< + BulkPutDataset, + PatchDatasetsConfigRequest + >({ query: (params) => ({ - url: `${CONNECTION_ROUTE}/${params.connection_key}/dataset`, + url: `${CONNECTION_ROUTE}/${params.connection_key}/datasetconfig`, method: "PATCH", - body: params.datasets, + body: params.dataset_pairs, }), invalidatesTags: () => ["DatastoreConnection"], }), @@ -331,12 +337,12 @@ export const { useGetAccessManualHookQuery, useGetAllEnabledAccessManualHooksQuery, useGetAllDatastoreConnectionsQuery, - useGetDatasetsQuery, + useGetDatasetConfigsQuery, useGetDatastoreConnectionByKeyQuery, useDeleteDatastoreConnectionMutation, useLazyGetDatastoreConnectionStatusQuery, usePatchAccessManualWebhookMutation, - usePatchDatasetMutation, + usePatchDatasetConfigsMutation, usePatchDatastoreConnectionMutation, usePatchDatastoreConnectionsMutation, useUpdateDatastoreConnectionSecretsMutation, diff --git a/clients/admin-ui/src/features/datastore-connections/types.ts b/clients/admin-ui/src/features/datastore-connections/types.ts index f6dd210f9e..6309b2ba2d 100644 --- a/clients/admin-ui/src/features/datastore-connections/types.ts +++ b/clients/admin-ui/src/features/datastore-connections/types.ts @@ -1,4 +1,8 @@ -import { ConnectionType, SystemType } from "~/types/api"; +import { + ConnectionType, + DatasetConfigCtlDataset, + SystemType, +} from "~/types/api"; import { AccessLevel, @@ -52,16 +56,9 @@ export type PatchAccessManualWebhookRequest = CreateAccessManualWebhookRequest; export type PatchAccessManualWebhookResponse = CreateAccessManualWebhookResponse; -export type PatchDatasetsRequest = { +export type PatchDatasetsConfigRequest = { connection_key: string; - datasets: Dataset[]; -}; - -export type DatasetsReponse = { - items: Dataset[]; - total: number; - page: number; - size: number; + dataset_pairs: DatasetConfigCtlDataset[]; }; export type Dataset = { diff --git a/clients/admin-ui/src/types/api/index.ts b/clients/admin-ui/src/types/api/index.ts index baa5e4c86f..357ff1a09f 100644 --- a/clients/admin-ui/src/types/api/index.ts +++ b/clients/admin-ui/src/types/api/index.ts @@ -2,11 +2,14 @@ /* tslint:disable */ /* eslint-disable */ +export { AccessLevel } from "./models/AccessLevel"; export type { AccessToken } from "./models/AccessToken"; export type { ActionType } from "./models/ActionType"; export type { AWSConfig } from "./models/AWSConfig"; export type { BigQueryConfig } from "./models/BigQueryConfig"; export type { BulkPostPrivacyRequests } from "./models/BulkPostPrivacyRequests"; +export type { BulkPutConnectionConfiguration } from "./models/BulkPutConnectionConfiguration"; +export type { BulkPutDataset } from "./models/BulkPutDataset"; export type { BulkUpdateFailed } from "./models/BulkUpdateFailed"; export type { CheckpointActionRequiredDetails } from "./models/CheckpointActionRequiredDetails"; export type { Classification } from "./models/Classification"; @@ -25,6 +28,8 @@ export type { ClassifyStatusUpdatePayload } from "./models/ClassifyStatusUpdateP export type { ClassifySystem } from "./models/ClassifySystem"; export { ClusterHealth } from "./models/ClusterHealth"; export type { ConnectionSystemTypeMap } from "./models/ConnectionSystemTypeMap"; +export type { ConnectionConfigurationResponse } from "./models/ConnectionConfigurationResponse"; +export { ConnectionTestStatus } from "./models/ConnectionTestStatus"; export { ConnectionType } from "./models/ConnectionType"; export type { ContactDetails } from "./models/ContactDetails"; export type { CurrentStep } from "./models/CurrentStep"; @@ -36,8 +41,11 @@ export type { DataQualifier } from "./models/DataQualifier"; export { DataResponsibilityTitle } from "./models/DataResponsibilityTitle"; export type { Dataset } from "./models/Dataset"; export type { DatasetCollection } from "./models/DatasetCollection"; +export type { DatasetConfigCtlDataset } from "./models/DatasetConfigCtlDataset"; +export type { DatasetConfigSchema } from "./models/DatasetConfigSchema"; export type { DatasetField } from "./models/DatasetField"; export type { DatasetMetadata } from "./models/DatasetMetadata"; +export type { DatasetTraversalDetails } from "./models/DatasetTraversalDetails"; export type { DatasetSchema } from "./models/DatasetSchema"; export type { DataSubject } from "./models/DataSubject"; export type { DataSubjectRights } from "./models/DataSubjectRights"; @@ -63,6 +71,9 @@ export type { Organization } from "./models/Organization"; export type { OrganizationMetadata } from "./models/OrganizationMetadata"; export type { Page_ConnectionSystemTypeMap_ } from "./models/Page_ConnectionSystemTypeMap_"; export type { Page_UserResponse_ } from "./models/Page_UserResponse_"; +export type { Page_Dataset_ } from "./models/Page_Dataset_"; +export type { Page_DatasetConfigSchema_ } from "./models/Page_DatasetConfigSchema_"; +export type { Page_ConnectionConfigurationResponse_ } from "./models/Page_ConnectionConfigurationResponse_"; export type { Policy } from "./models/Policy"; export type { PolicyMaskingSpecResponse } from "./models/PolicyMaskingSpecResponse"; export type { PolicyResponse } from "./models/PolicyResponse"; @@ -77,6 +88,7 @@ export type { Registry } from "./models/Registry"; export type { ResourceFilter } from "./models/ResourceFilter"; export type { ResponseFormat } from "./models/ResponseFormat"; export type { RuleResponse } from "./models/RuleResponse"; +export type { SaaSConfigBase } from "./models/SaaSConfigBase"; export { SpecialCategoriesEnum } from "./models/SpecialCategoriesEnum"; export { StatusEnum } from "./models/StatusEnum"; export type { StorageDestinationResponse } from "./models/StorageDestinationResponse"; @@ -99,6 +111,7 @@ export type { UserPermissionsEdit } from "./models/UserPermissionsEdit"; export type { UserPermissionsResponse } from "./models/UserPermissionsResponse"; export type { UserResponse } from "./models/UserResponse"; export type { UserUpdate } from "./models/UserUpdate"; +export type { ValidateDatasetResponse } from "./models/ValidateDatasetResponse"; export type { ValidateRequest } from "./models/ValidateRequest"; export type { ValidateResponse } from "./models/ValidateResponse"; export type { ValidationError } from "./models/ValidationError"; diff --git a/clients/admin-ui/src/types/api/models/AccessLevel.ts b/clients/admin-ui/src/types/api/models/AccessLevel.ts new file mode 100644 index 0000000000..78d7c07585 --- /dev/null +++ b/clients/admin-ui/src/types/api/models/AccessLevel.ts @@ -0,0 +1,14 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +/** + * Perms given to the ConnectionConfig. For example, with "read" permissions, fidesops promises + * to not modify the data on a connected application database in any way. + * + * "Write" perms mean we can update/delete items in the connected database. + */ +export enum AccessLevel { + READ = "read", + WRITE = "write", +} diff --git a/clients/admin-ui/src/types/api/models/BulkPutConnectionConfiguration.ts b/clients/admin-ui/src/types/api/models/BulkPutConnectionConfiguration.ts new file mode 100644 index 0000000000..2940505ad6 --- /dev/null +++ b/clients/admin-ui/src/types/api/models/BulkPutConnectionConfiguration.ts @@ -0,0 +1,14 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { BulkUpdateFailed } from "./BulkUpdateFailed"; +import type { ConnectionConfigurationResponse } from "./ConnectionConfigurationResponse"; + +/** + * Schema with mixed success/failure responses for Bulk Create/Update of ConnectionConfiguration responses. + */ +export type BulkPutConnectionConfiguration = { + succeeded: Array; + failed: Array; +}; diff --git a/clients/admin-ui/src/types/api/models/BulkPutDataset.ts b/clients/admin-ui/src/types/api/models/BulkPutDataset.ts new file mode 100644 index 0000000000..60b069d216 --- /dev/null +++ b/clients/admin-ui/src/types/api/models/BulkPutDataset.ts @@ -0,0 +1,14 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { BulkUpdateFailed } from "./BulkUpdateFailed"; +import type { Dataset } from "./Dataset"; + +/** + * Schema with mixed success/failure responses for Bulk Create/Update of Datasets. + */ +export type BulkPutDataset = { + succeeded: Array; + failed: Array; +}; diff --git a/clients/admin-ui/src/types/api/models/ConnectionConfigurationResponse.ts b/clients/admin-ui/src/types/api/models/ConnectionConfigurationResponse.ts new file mode 100644 index 0000000000..df2af50827 --- /dev/null +++ b/clients/admin-ui/src/types/api/models/ConnectionConfigurationResponse.ts @@ -0,0 +1,26 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { AccessLevel } from "./AccessLevel"; +import type { ConnectionType } from "./ConnectionType"; +import type { SaaSConfigBase } from "./SaaSConfigBase"; + +/** + * Describes the returned schema for a ConnectionConfiguration. + * + * Do *NOT* add "secrets" to this schema. + */ +export type ConnectionConfigurationResponse = { + name: string; + key: string; + description?: string; + connection_type: ConnectionType; + access: AccessLevel; + created_at: string; + updated_at?: string; + disabled?: boolean; + last_test_timestamp?: string; + last_test_succeeded?: boolean; + saas_config?: SaaSConfigBase; +}; diff --git a/clients/admin-ui/src/types/api/models/ConnectionTestStatus.ts b/clients/admin-ui/src/types/api/models/ConnectionTestStatus.ts new file mode 100644 index 0000000000..ad3d72251c --- /dev/null +++ b/clients/admin-ui/src/types/api/models/ConnectionTestStatus.ts @@ -0,0 +1,12 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +/** + * Enum for supplying statuses of validating credentials for a Connection Config to the user + */ +export enum ConnectionTestStatus { + SUCCEEDED = "succeeded", + FAILED = "failed", + SKIPPED = "skipped", +} diff --git a/clients/admin-ui/src/types/api/models/DatasetConfigCtlDataset.ts b/clients/admin-ui/src/types/api/models/DatasetConfigCtlDataset.ts new file mode 100644 index 0000000000..47cbc5e461 --- /dev/null +++ b/clients/admin-ui/src/types/api/models/DatasetConfigCtlDataset.ts @@ -0,0 +1,11 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +/** + * A base template for all other Fidesops Schemas to inherit from. + */ +export type DatasetConfigCtlDataset = { + fides_key: string; + ctl_dataset_fides_key: string; +}; diff --git a/clients/admin-ui/src/types/api/models/DatasetConfigSchema.ts b/clients/admin-ui/src/types/api/models/DatasetConfigSchema.ts new file mode 100644 index 0000000000..83a8ea94e0 --- /dev/null +++ b/clients/admin-ui/src/types/api/models/DatasetConfigSchema.ts @@ -0,0 +1,13 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { Dataset } from "./Dataset"; + +/** + * Returns the DatasetConfig fides key and the linked Ctl Dataset + */ +export type DatasetConfigSchema = { + fides_key: string; + ctl_dataset: Dataset; +}; diff --git a/clients/admin-ui/src/types/api/models/DatasetTraversalDetails.ts b/clients/admin-ui/src/types/api/models/DatasetTraversalDetails.ts new file mode 100644 index 0000000000..6541c40eff --- /dev/null +++ b/clients/admin-ui/src/types/api/models/DatasetTraversalDetails.ts @@ -0,0 +1,12 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +/** + * Describes whether or not the parent dataset is traversable; if not, includes + * an error message describing the traversal issues. + */ +export type DatasetTraversalDetails = { + is_traversable: boolean; + msg?: string; +}; diff --git a/clients/admin-ui/src/types/api/models/Page_ConnectionConfigurationResponse_.ts b/clients/admin-ui/src/types/api/models/Page_ConnectionConfigurationResponse_.ts new file mode 100644 index 0000000000..b7df6e63ce --- /dev/null +++ b/clients/admin-ui/src/types/api/models/Page_ConnectionConfigurationResponse_.ts @@ -0,0 +1,12 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { ConnectionConfigurationResponse } from "./ConnectionConfigurationResponse"; + +export type Page_ConnectionConfigurationResponse_ = { + items: Array; + total: number; + page: number; + size: number; +}; diff --git a/clients/admin-ui/src/types/api/models/Page_DatasetConfigSchema_.ts b/clients/admin-ui/src/types/api/models/Page_DatasetConfigSchema_.ts new file mode 100644 index 0000000000..9769ff5d0b --- /dev/null +++ b/clients/admin-ui/src/types/api/models/Page_DatasetConfigSchema_.ts @@ -0,0 +1,12 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { DatasetConfigSchema } from "./DatasetConfigSchema"; + +export type Page_DatasetConfigSchema_ = { + items: Array; + total: number; + page: number; + size: number; +}; diff --git a/clients/admin-ui/src/types/api/models/Page_Dataset_.ts b/clients/admin-ui/src/types/api/models/Page_Dataset_.ts new file mode 100644 index 0000000000..8eb1ef1308 --- /dev/null +++ b/clients/admin-ui/src/types/api/models/Page_Dataset_.ts @@ -0,0 +1,12 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { Dataset } from "./Dataset"; + +export type Page_Dataset_ = { + items: Array; + total: number; + page: number; + size: number; +}; diff --git a/clients/admin-ui/src/types/api/models/SaaSConfigBase.ts b/clients/admin-ui/src/types/api/models/SaaSConfigBase.ts new file mode 100644 index 0000000000..b6050fdfff --- /dev/null +++ b/clients/admin-ui/src/types/api/models/SaaSConfigBase.ts @@ -0,0 +1,12 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +/** + * Used to store base info for a saas config + */ +export type SaaSConfigBase = { + fides_key: string; + name: string; + type: string; +}; diff --git a/clients/admin-ui/src/types/api/models/ValidateDatasetResponse.ts b/clients/admin-ui/src/types/api/models/ValidateDatasetResponse.ts new file mode 100644 index 0000000000..de11372b8b --- /dev/null +++ b/clients/admin-ui/src/types/api/models/ValidateDatasetResponse.ts @@ -0,0 +1,16 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { Dataset } from "./Dataset"; +import type { DatasetTraversalDetails } from "./DatasetTraversalDetails"; + +/** + * Response model for validating a dataset, which includes both the dataset + * itself (if valid) plus a details object describing if the dataset is + * traversable or not. + */ +export type ValidateDatasetResponse = { + dataset: Dataset; + traversal_details: DatasetTraversalDetails; +}; diff --git a/demo_resources/demo_dataset.yml b/demo_resources/demo_dataset.yml index d2400d7001..29e2f9c463 100644 --- a/demo_resources/demo_dataset.yml +++ b/demo_resources/demo_dataset.yml @@ -5,8 +5,7 @@ dataset: description: Data collected about users for our analytics system. meta: null data_categories: [] - data_qualifiers: - - aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified retention: "30 days after account deletion" third_country_transfers: - GBR diff --git a/docs/fides/docs/development/postman/Fides.postman_collection.json b/docs/fides/docs/development/postman/Fides.postman_collection.json index 0be738dca5..08ecc3e27b 100644 --- a/docs/fides/docs/development/postman/Fides.postman_collection.json +++ b/docs/fides/docs/development/postman/Fides.postman_collection.json @@ -1,6 +1,6 @@ { "info": { - "_postman_id": "09d75fa5-5a52-4d25-8e6b-7c36a22d6175", + "_postman_id": "cbd35879-b646-4c71-9c80-92cff0dfab77", "name": "Fidesops", "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" }, @@ -454,7 +454,44 @@ "response": [] }, { - "name": "Create/Update Postgres Dataset", + "name": "Create/Update Postgres CTL Dataset", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "[{\n \"fides_key\": \"postgres_example\",\n \"name\": \"Postgres Example Test Dataset\",\n \"organization_fides_key\": \"default_organization\",\n \"data_qualifier\": \"aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified\",\n \"description\": \"Example of a Postgres dataset containing a variety of related tables like customers, products, addresses, etc.\",\n \"collections\": [\n {\n \"name\": \"address\",\n \"fields\": [\n {\n \"name\": \"city\",\n \"data_categories\": [\n \"user.contact.address.city\"\n ]\n },\n {\n \"name\": \"house\",\n \"data_categories\": [\n \"user.contact.address.street\"\n ]\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"state\",\n \"data_categories\": [\n \"user.contact.address.state\"\n ]\n },\n {\n \"name\": \"street\",\n \"data_categories\": [\n \"user.contact.address.street\"\n ]\n },\n {\n \"name\": \"zip\",\n \"data_categories\": [\n \"user.contact.address.postal_code\"\n ]\n }\n ]\n },\n {\n \"name\": \"customer\",\n \"fields\": [\n {\n \"name\": \"address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"created\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\"\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n\n }\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"user.name\"\n ]\n }\n ]\n },\n {\n \"name\": \"employee\",\n \"fields\": [\n {\n \"name\": \"address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\"\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n\n }\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"user.name\"\n ]\n }\n ]\n },\n {\n \"name\": \"login\",\n \"fields\": [\n {\n \"name\": \"customer_id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"customer.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"time\",\n \"data_categories\": [\n \"user.sensor\"\n ]\n }\n ]\n },\n {\n \"name\": \"orders\",\n \"fields\": [\n {\n \"name\": \"customer_id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"customer.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n\n }\n },\n {\n \"name\": \"shipping_address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n }\n ]\n },\n {\n \"name\": \"order_item\",\n \"fields\": [\n {\n \"name\": \"order_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"orders.id\",\n \"direction\": \"from\"\n }\n ],\n \"primary_key\": true\n }\n },\n {\n \"name\": \"product_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"product.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"quantity\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"payment_card\",\n \"fields\": [\n {\n \"name\": \"billing_address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"ccn\",\n \"data_categories\": [\n \"user.financial.account_number\"\n ]\n },\n {\n \"name\": \"code\",\n \"data_categories\": [\n \"user.financial\"\n ]\n },\n {\n \"name\": \"customer_id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"customer.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n\n }\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"user.financial\"\n ]\n },\n {\n \"name\": \"preferred\",\n \"data_categories\": [\n \"user\"\n ]\n }\n ]\n },\n {\n \"name\": \"product\",\n \"fields\": [\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"price\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"report\",\n \"fields\": [\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\"\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"month\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"total_visits\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"year\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"service_request\",\n \"fields\": [\n {\n \"name\": \"alt_email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\"\n }\n },\n {\n \"name\": \"closed\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\"\n }\n },\n {\n \"name\": \"employee_id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"employee.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"opened\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"visit\",\n \"fields\": [\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\",\n \"primary_key\": true\n }\n },\n {\n \"name\": \"last_visit\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n }\n ]\n}]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/dataset/upsert", + "host": [ + "{{host}}" + ], + "path": [ + "dataset", + "upsert" + ] + } + }, + "response": [] + }, + { + "name": "Create Dataset Config with Postgres CTL Dataset", "request": { "auth": { "type": "bearer", @@ -470,7 +507,7 @@ "header": [], "body": { "mode": "raw", - "raw": "[{\n \"fides_key\": \"postgres_example\",\n \"name\": \"Postgres Example Test Dataset\",\n \"description\": \"Example of a Postgres dataset containing a variety of related tables like customers, products, addresses, etc.\",\n \"collections\": [\n {\n \"name\": \"address\",\n \"fields\": [\n {\n \"name\": \"city\",\n \"data_categories\": [\n \"user.contact.address.city\"\n ]\n },\n {\n \"name\": \"house\",\n \"data_categories\": [\n \"user.contact.address.street\"\n ]\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"state\",\n \"data_categories\": [\n \"user.contact.address.state\"\n ]\n },\n {\n \"name\": \"street\",\n \"data_categories\": [\n \"user.contact.address.street\"\n ]\n },\n {\n \"name\": \"zip\",\n \"data_categories\": [\n \"user.contact.address.postal_code\"\n ]\n }\n ]\n },\n {\n \"name\": \"customer\",\n \"fields\": [\n {\n \"name\": \"address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"created\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\"\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n\n }\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"user.name\"\n ]\n }\n ]\n },\n {\n \"name\": \"employee\",\n \"fields\": [\n {\n \"name\": \"address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\"\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n\n }\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"user.name\"\n ]\n }\n ]\n },\n {\n \"name\": \"login\",\n \"fields\": [\n {\n \"name\": \"customer_id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"customer.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"time\",\n \"data_categories\": [\n \"user.sensor\"\n ]\n }\n ]\n },\n {\n \"name\": \"orders\",\n \"fields\": [\n {\n \"name\": \"customer_id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"customer.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n\n }\n },\n {\n \"name\": \"shipping_address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n }\n ]\n },\n {\n \"name\": \"order_item\",\n \"fields\": [\n {\n \"name\": \"order_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"orders.id\",\n \"direction\": \"from\"\n }\n ],\n \"primary_key\": true\n }\n },\n {\n \"name\": \"product_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"product.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"quantity\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"payment_card\",\n \"fields\": [\n {\n \"name\": \"billing_address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"ccn\",\n \"data_categories\": [\n \"user.financial.account_number\"\n ]\n },\n {\n \"name\": \"code\",\n \"data_categories\": [\n \"user.financial\"\n ]\n },\n {\n \"name\": \"customer_id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"customer.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n\n }\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"user.financial\"\n ]\n },\n {\n \"name\": \"preferred\",\n \"data_categories\": [\n \"user\"\n ]\n }\n ]\n },\n {\n \"name\": \"product\",\n \"fields\": [\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"price\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"report\",\n \"fields\": [\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\"\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"month\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"total_visits\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"year\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"service_request\",\n \"fields\": [\n {\n \"name\": \"alt_email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\"\n }\n },\n {\n \"name\": \"closed\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\"\n }\n },\n {\n \"name\": \"employee_id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"employee.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"opened\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"visit\",\n \"fields\": [\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\",\n \"primary_key\": true\n }\n },\n {\n \"name\": \"last_visit\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n }\n ]\n}]", + "raw": "[{\n \"fides_key\": \"postgres_example\",\n \"ctl_dataset_fides_key\": \"postgres_example\"\n}]", "options": { "raw": { "language": "json" @@ -478,21 +515,59 @@ } }, "url": { - "raw": "{{host}}/connection/{{postgres_key}}/dataset", + "raw": "{{host}}/connection/{{postgres_key}}/datasetconfig/", "host": [ "{{host}}" ], "path": [ "connection", "{{postgres_key}}", - "dataset" + "datasetconfig", + "" ] } }, "response": [] }, { - "name": "Create/Update Dataset Mongo", + "name": "Create/Update Mongo CTL Dataset", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "[{\n \"fides_key\":\"mongo_test\",\n \"name\":\"Mongo Example Test Dataset\",\n \"description\":\"Example of a Mongo dataset that contains 'details' about customers defined in the 'postgres_example'\",\n \"organization_fides_key\": \"default_organization\",\n \"data_qualifier\": \"aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified\",\n \"collections\":[\n {\n \"name\":\"customer_details\",\n \"fields\":[\n {\n \"name\":\"_id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"primary_key\":true\n }\n },\n {\n \"name\":\"customer_id\",\n \"data_categories\":[\n \"user.unique_id\"\n ],\n \"fidesops_meta\":{\n \"references\":[\n {\n \"dataset\":\"postgres_example\",\n \"field\":\"customer.id\",\n \"direction\":\"from\"\n }\n ]\n }\n },\n {\n \"name\":\"gender\",\n \"data_categories\":[\n \"user.gender\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"birthday\",\n \"data_categories\":[\n \"user.date_of_birth\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"workplace_info\",\n \"fidesops_meta\":{\n \"data_type\":\"object\"\n },\n \"fields\":[\n {\n \"name\":\"employer\",\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"position\",\n \"data_categories\":[\n \"user.job_title\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"direct_reports\",\n \"data_categories\":[\n \"user.name\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string[]\"\n }\n }\n ]\n },\n {\n \"name\":\"emergency_contacts\",\n \"fidesops_meta\":{\n \"data_type\":\"object[]\"\n },\n \"fields\":[\n {\n \"name\":\"name\",\n \"data_categories\":[\n \"user.name\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"relationship\",\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"phone\",\n \"data_categories\":[\n \"user.contact.phone_number\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n },\n {\n \"name\":\"children\",\n \"data_categories\":[\n \"user.childrens\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string[]\"\n }\n },\n {\n \"name\":\"travel_identifiers\",\n \"fidesops_meta\":{\n \"data_type\":\"string[]\",\n \"data_categories\":[\n \"system.operations\"\n ]\n }\n },\n {\n \"name\":\"comments\",\n \"fidesops_meta\":{\n \"data_type\":\"object[]\"\n },\n \"fields\":[\n {\n \"name\":\"comment_id\",\n \"fidesops_meta\":{\n \"data_type\":\"string\",\n \"references\":[\n {\n \"dataset\":\"mongo_test\",\n \"field\":\"conversations.thread.comment\",\n \"direction\":\"to\"\n }\n ]\n }\n }\n ]\n }\n ]\n },\n {\n \"name\":\"internal_customer_profile\",\n \"fields\":[\n {\n \"name\":\"_id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"primary_key\":true,\n \"data_type\":\"object_id\"\n }\n },\n {\n \"name\":\"customer_identifiers\",\n \"fields\":[\n {\n \"name\":\"internal_id\",\n \"fidesops_meta\":{\n \"data_type\":\"string\",\n \"references\":[\n {\n \"dataset\":\"mongo_test\",\n \"field\":\"customer_feedback.customer_information.internal_customer_id\",\n \"direction\":\"from\"\n }\n ]\n }\n },\n {\n \"name\":\"derived_emails\",\n \"data_categories\":[\n \"user\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string[]\",\n \"identity\":\"email\"\n }\n },\n {\n \"name\":\"derived_phone\",\n \"data_categories\":[\n \"user\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string[]\",\n \"return_all_elements\":true,\n \"identity\":\"phone_number\"\n }\n }\n ]\n },\n {\n \"name\":\"derived_interests\",\n \"data_categories\":[\n \"user\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string[]\"\n }\n }\n ]\n },\n {\n \"name\":\"customer_feedback\",\n \"fields\":[\n {\n \"name\":\"_id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"primary_key\":true,\n \"data_type\":\"object_id\"\n }\n },\n {\n \"name\":\"customer_information\",\n \"fields\":[\n {\n \"name\":\"email\",\n \"fidesops_meta\":{\n \"identity\":\"email\",\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"phone\",\n \"data_categories\":[\n \"user.contact.phone_number\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"internal_customer_id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n },\n {\n \"name\":\"rating\",\n \"data_categories\":[\n \"user\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"integer\"\n }\n },\n {\n \"name\":\"date\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"message\",\n \"data_categories\":[\n \"user\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n },\n {\n \"name\":\"flights\",\n \"fields\":[\n {\n \"name\":\"_id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"primary_key\":true,\n \"data_type\":\"object_id\"\n }\n },\n {\n \"name\":\"passenger_information\",\n \"fields\":[\n {\n \"name\":\"passenger_ids\",\n \"fidesops_meta\":{\n \"data_type\":\"string[]\",\n \"references\":[\n {\n \"dataset\":\"mongo_test\",\n \"field\":\"customer_details.travel_identifiers\",\n \"direction\":\"from\"\n }\n ]\n }\n },\n {\n \"name\":\"full_name\",\n \"data_categories\":[\n \"user.name\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n },\n {\n \"name\":\"flight_no\"\n },\n {\n \"name\":\"date\"\n },\n {\n \"name\":\"pilots\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string[]\"\n }\n },\n {\n \"name\":\"plane\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"integer\"\n }\n }\n ]\n },\n {\n \"name\":\"conversations\",\n \"fields\":[\n {\n \"name\":\"_id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"primary_key\":true,\n \"data_type\":\"object_id\"\n }\n },\n {\n \"name\":\"thread\",\n \"fidesops_meta\":{\n \"data_type\":\"object[]\"\n },\n \"fields\":[\n {\n \"name\":\"comment\",\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"message\",\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"chat_name\",\n \"data_categories\":[\n \"user.name\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"ccn\",\n \"data_categories\":[\n \"user.financial.account_number\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n }\n ]\n },\n {\n \"name\":\"employee\",\n \"fields\":[\n {\n \"name\":\"_id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"primary_key\":true,\n \"data_type\":\"object_id\"\n }\n },\n {\n \"name\":\"email\",\n \"data_categories\":[\n \"user.contact.email\"\n ],\n \"fidesops_meta\":{\n \"identity\":\"email\",\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"user.unique_id\"\n ],\n \"fidesops_meta\":{\n \"primary_key\":true,\n \"references\":[\n {\n \"dataset\":\"mongo_test\",\n \"field\":\"flights.pilots\",\n \"direction\":\"from\"\n }\n ]\n }\n },\n {\n \"name\":\"name\",\n \"data_categories\":[\n \"user.name\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n },\n {\n \"name\":\"aircraft\",\n \"fields\":[\n {\n \"name\":\"_id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"primary_key\":true,\n \"data_type\":\"object_id\"\n }\n },\n {\n \"name\":\"planes\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string[]\",\n \"references\":[\n {\n \"dataset\":\"mongo_test\",\n \"field\":\"flights.plane\",\n \"direction\":\"from\"\n }\n ]\n }\n },\n {\n \"name\":\"model\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n },\n {\n \"name\":\"payment_card\",\n \"fields\":[\n {\n \"name\":\"_id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"primary_key\":true,\n \"data_type\":\"object_id\"\n }\n },\n {\n \"name\":\"billing_address_id\",\n \"data_categories\":[\n \"system.operations\"\n ]\n },\n {\n \"name\":\"ccn\",\n \"data_categories\":[\n \"user.financial.account_number\"\n ],\n \"fidesops_meta\":{\n \"references\":[\n {\n \"dataset\":\"mongo_test\",\n \"field\":\"conversations.thread.ccn\",\n \"direction\":\"from\"\n }\n ]\n }\n },\n {\n \"name\":\"code\",\n \"data_categories\":[\n \"user.financial\"\n ]\n },\n {\n \"name\":\"customer_id\",\n \"data_categories\":[\n \"user.unique_id\"\n ]\n },\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"primary_key\":true\n }\n },\n {\n \"name\":\"name\",\n \"data_categories\":[\n \"user.financial\"\n ]\n },\n {\n \"name\":\"preferred\",\n \"data_categories\":[\n \"user\"\n ]\n }\n ]\n },\n {\n \"name\":\"rewards\",\n \"fields\":[\n {\n \"name\":\"_id\",\n \"fidesops_meta\":{\n \"primary_key\":true,\n \"data_type\":\"object_id\"\n }\n },\n {\n \"name\":\"owner\",\n \"fidesops_meta\":{\n \"data_type\":\"object[]\",\n \"return_all_elements\":true\n },\n \"fields\":[\n {\n \"name\":\"phone\",\n \"data_categories\":[\n \"user.contact.phone_number\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\",\n \"references\":[\n {\n \"dataset\":\"mongo_test\",\n \"field\":\"internal_customer_profile.customer_identifiers.derived_phone\",\n \"direction\":\"from\"\n }\n ]\n }\n },\n {\n \"name\":\"shopper_name\"\n }\n ]\n },\n {\n \"name\":\"points\",\n \"fidesops_meta\":{\n \"data_type\":\"integer\"\n }\n },\n {\n \"name\":\"expiration_date\"\n }\n ]\n }\n ]\n}]\n", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/dataset/upsert", + "host": [ + "{{host}}" + ], + "path": [ + "dataset", + "upsert" + ] + } + }, + "response": [] + }, + { + "name": "Create Dataset Config with Mongo CTL Dataset", "request": { "auth": { "type": "bearer", @@ -508,7 +583,7 @@ "header": [], "body": { "mode": "raw", - "raw": "[\n{\n \"fides_key\":\"mongo_test\",\n \"name\":\"Mongo Example Test Dataset\",\n \"description\":\"Example of a Mongo dataset that contains 'details' about customers defined in the 'postgres_example'\",\n \"collections\":[\n {\n \"name\":\"customer_details\",\n \"fields\":[\n {\n \"name\":\"_id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"primary_key\":true\n }\n },\n {\n \"name\":\"customer_id\",\n \"data_categories\":[\n \"user.unique_id\"\n ],\n \"fidesops_meta\":{\n \"references\":[\n {\n \"dataset\":\"postgres_example\",\n \"field\":\"customer.id\",\n \"direction\":\"from\"\n }\n ]\n }\n },\n {\n \"name\":\"gender\",\n \"data_categories\":[\n \"user.gender\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"birthday\",\n \"data_categories\":[\n \"user.date_of_birth\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"workplace_info\",\n \"fidesops_meta\":{\n \"data_type\":\"object\"\n },\n \"fields\":[\n {\n \"name\":\"employer\",\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"position\",\n \"data_categories\":[\n \"user.job_title\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"direct_reports\",\n \"data_categories\":[\n \"user.name\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string[]\"\n }\n }\n ]\n },\n {\n \"name\":\"emergency_contacts\",\n \"fidesops_meta\":{\n \"data_type\":\"object[]\"\n },\n \"fields\":[\n {\n \"name\":\"name\",\n \"data_categories\":[\n \"user.name\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"relationship\",\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"phone\",\n \"data_categories\":[\n \"user.contact.phone_number\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n },\n {\n \"name\":\"children\",\n \"data_categories\":[\n \"user.childrens\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string[]\"\n }\n },\n {\n \"name\":\"travel_identifiers\",\n \"fidesops_meta\":{\n \"data_type\":\"string[]\",\n \"data_categories\":[\n \"system.operations\"\n ]\n }\n },\n {\n \"name\":\"comments\",\n \"fidesops_meta\":{\n \"data_type\":\"object[]\"\n },\n \"fields\":[\n {\n \"name\":\"comment_id\",\n \"fidesops_meta\":{\n \"data_type\":\"string\",\n \"references\":[\n {\n \"dataset\":\"mongo_test\",\n \"field\":\"conversations.thread.comment\",\n \"direction\":\"to\"\n }\n ]\n }\n }\n ]\n }\n ]\n },\n {\n \"name\":\"internal_customer_profile\",\n \"fields\":[\n {\n \"name\":\"_id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"primary_key\":true,\n \"data_type\":\"object_id\"\n }\n },\n {\n \"name\":\"customer_identifiers\",\n \"fields\":[\n {\n \"name\":\"internal_id\",\n \"fidesops_meta\":{\n \"data_type\":\"string\",\n \"references\":[\n {\n \"dataset\":\"mongo_test\",\n \"field\":\"customer_feedback.customer_information.internal_customer_id\",\n \"direction\":\"from\"\n }\n ]\n }\n },\n {\n \"name\":\"derived_emails\",\n \"data_categories\":[\n \"user\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string[]\",\n \"identity\":\"email\"\n }\n },\n {\n \"name\":\"derived_phone\",\n \"data_categories\":[\n \"user\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string[]\",\n \"return_all_elements\":true,\n \"identity\":\"phone_number\"\n }\n }\n ]\n },\n {\n \"name\":\"derived_interests\",\n \"data_categories\":[\n \"user\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string[]\"\n }\n }\n ]\n },\n {\n \"name\":\"customer_feedback\",\n \"fields\":[\n {\n \"name\":\"_id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"primary_key\":true,\n \"data_type\":\"object_id\"\n }\n },\n {\n \"name\":\"customer_information\",\n \"fields\":[\n {\n \"name\":\"email\",\n \"fidesops_meta\":{\n \"identity\":\"email\",\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"phone\",\n \"data_categories\":[\n \"user.contact.phone_number\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"internal_customer_id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n },\n {\n \"name\":\"rating\",\n \"data_categories\":[\n \"user\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"integer\"\n }\n },\n {\n \"name\":\"date\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"message\",\n \"data_categories\":[\n \"user\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n },\n {\n \"name\":\"flights\",\n \"fields\":[\n {\n \"name\":\"_id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"primary_key\":true,\n \"data_type\":\"object_id\"\n }\n },\n {\n \"name\":\"passenger_information\",\n \"fields\":[\n {\n \"name\":\"passenger_ids\",\n \"fidesops_meta\":{\n \"data_type\":\"string[]\",\n \"references\":[\n {\n \"dataset\":\"mongo_test\",\n \"field\":\"customer_details.travel_identifiers\",\n \"direction\":\"from\"\n }\n ]\n }\n },\n {\n \"name\":\"full_name\",\n \"data_categories\":[\n \"user.name\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n },\n {\n \"name\":\"flight_no\"\n },\n {\n \"name\":\"date\"\n },\n {\n \"name\":\"pilots\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string[]\"\n }\n },\n {\n \"name\":\"plane\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"integer\"\n }\n }\n ]\n },\n {\n \"name\":\"conversations\",\n \"fields\":[\n {\n \"name\":\"_id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"primary_key\":true,\n \"data_type\":\"object_id\"\n }\n },\n {\n \"name\":\"thread\",\n \"fidesops_meta\":{\n \"data_type\":\"object[]\"\n },\n \"fields\":[\n {\n \"name\":\"comment\",\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"message\",\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"chat_name\",\n \"data_categories\":[\n \"user.name\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"ccn\",\n \"data_categories\":[\n \"user.financial.account_number\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n }\n ]\n },\n {\n \"name\":\"employee\",\n \"fields\":[\n {\n \"name\":\"_id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"primary_key\":true,\n \"data_type\":\"object_id\"\n }\n },\n {\n \"name\":\"email\",\n \"data_categories\":[\n \"user.contact.email\"\n ],\n \"fidesops_meta\":{\n \"identity\":\"email\",\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"user.unique_id\"\n ],\n \"fidesops_meta\":{\n \"primary_key\":true,\n \"references\":[\n {\n \"dataset\":\"mongo_test\",\n \"field\":\"flights.pilots\",\n \"direction\":\"from\"\n }\n ]\n }\n },\n {\n \"name\":\"name\",\n \"data_categories\":[\n \"user.name\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n },\n {\n \"name\":\"aircraft\",\n \"fields\":[\n {\n \"name\":\"_id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"primary_key\":true,\n \"data_type\":\"object_id\"\n }\n },\n {\n \"name\":\"planes\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string[]\",\n \"references\":[\n {\n \"dataset\":\"mongo_test\",\n \"field\":\"flights.plane\",\n \"direction\":\"from\"\n }\n ]\n }\n },\n {\n \"name\":\"model\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n },\n {\n \"name\":\"payment_card\",\n \"fields\":[\n {\n \"name\":\"_id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"primary_key\":true,\n \"data_type\":\"object_id\"\n }\n },\n {\n \"name\":\"billing_address_id\",\n \"data_categories\":[\n \"system.operations\"\n ]\n },\n {\n \"name\":\"ccn\",\n \"data_categories\":[\n \"user.financial.account_number\"\n ],\n \"fidesops_meta\":{\n \"references\":[\n {\n \"dataset\":\"mongo_test\",\n \"field\":\"conversations.thread.ccn\",\n \"direction\":\"from\"\n }\n ]\n }\n },\n {\n \"name\":\"code\",\n \"data_categories\":[\n \"user.financial\"\n ]\n },\n {\n \"name\":\"customer_id\",\n \"data_categories\":[\n \"user.unique_id\"\n ]\n },\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"primary_key\":true\n }\n },\n {\n \"name\":\"name\",\n \"data_categories\":[\n \"user.financial\"\n ]\n },\n {\n \"name\":\"preferred\",\n \"data_categories\":[\n \"user\"\n ]\n }\n ]\n },\n {\n \"name\":\"rewards\",\n \"fields\":[\n {\n \"name\":\"_id\",\n \"fidesops_meta\":{\n \"primary_key\":true,\n \"data_type\":\"object_id\"\n }\n },\n {\n \"name\":\"owner\",\n \"fidesops_meta\":{\n \"data_type\":\"object[]\",\n \"return_all_elements\":true\n },\n \"fields\":[\n {\n \"name\":\"phone\",\n \"data_categories\":[\n \"user.contact.phone_number\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\",\n \"references\":[\n {\n \"dataset\":\"mongo_test\",\n \"field\":\"internal_customer_profile.customer_identifiers.derived_phone\",\n \"direction\":\"from\"\n }\n ]\n }\n },\n {\n \"name\":\"shopper_name\"\n }\n ]\n },\n {\n \"name\":\"points\",\n \"fidesops_meta\":{\n \"data_type\":\"integer\"\n }\n },\n {\n \"name\":\"expiration_date\"\n }\n ]\n }\n ]\n}\n]", + "raw": "[{\n \"fides_key\": \"mongo_test\",\n \"ctl_dataset_fides_key\": \"mongo_test\"\n}]", "options": { "raw": { "language": "json" @@ -516,14 +591,15 @@ } }, "url": { - "raw": "{{host}}/connection/{{mongo_key}}/dataset", + "raw": "{{host}}/connection/{{mongo_key}}/datasetconfig/", "host": [ "{{host}}" ], "path": [ "connection", "{{mongo_key}}", - "dataset" + "datasetconfig", + "" ] } }, diff --git a/src/fides/api/ctl/database/crud.py b/src/fides/api/ctl/database/crud.py index 80ed362416..0ccdb3a268 100644 --- a/src/fides/api/ctl/database/crud.py +++ b/src/fides/api/ctl/database/crud.py @@ -163,10 +163,13 @@ async def upsert_resources( ) ) + excluded = dict(insert_stmt.excluded.items()) # type: ignore[attr-defined] + excluded.pop("id", None) # If updating, don't update the "id" + result = await session.execute( insert_stmt.on_conflict_do_update( index_elements=["fides_key"], - set_=insert_stmt.excluded, + set_=excluded, ) ) diff --git a/src/fides/api/ctl/migrations/versions/d6c6c6555c86_remove_datasetconfig_dataset.py b/src/fides/api/ctl/migrations/versions/d6c6c6555c86_remove_datasetconfig_dataset.py new file mode 100644 index 0000000000..83f302ca2a --- /dev/null +++ b/src/fides/api/ctl/migrations/versions/d6c6c6555c86_remove_datasetconfig_dataset.py @@ -0,0 +1,32 @@ +"""Remove datasetconfig dataset + +Revision ID: d6c6c6555c86 +Revises: 9c6f62e4c9da +Create Date: 2022-12-20 20:44:32.840423 + +""" +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "d6c6c6555c86" +down_revision = "9c6f62e4c9da" +branch_labels = None +depends_on = None + + +def upgrade(): + op.drop_column("datasetconfig", "dataset") + + +def downgrade(): + op.add_column( + "datasetconfig", + sa.Column( + "dataset", + postgresql.JSONB(astext_type=sa.Text()), + autoincrement=False, + nullable=False, + ), + ) diff --git a/src/fides/api/ops/api/v1/endpoints/dataset_endpoints.py b/src/fides/api/ops/api/v1/endpoints/dataset_endpoints.py index f65d28e4c8..c077417922 100644 --- a/src/fides/api/ops/api/v1/endpoints/dataset_endpoints.py +++ b/src/fides/api/ops/api/v1/endpoints/dataset_endpoints.py @@ -33,6 +33,7 @@ DATASET_BY_KEY, DATASET_CONFIGS, DATASET_VALIDATE, + DATASETCONFIG_BY_KEY, DATASETS, V1_URL_PREFIX, YAML_DATASETS, @@ -54,6 +55,7 @@ from fides.api.ops.schemas.dataset import ( BulkPutDataset, DatasetConfigCtlDataset, + DatasetConfigSchema, DatasetTraversalDetails, ValidateDatasetResponse, validate_data_categories_against_db, @@ -64,7 +66,7 @@ X_YAML = "application/x-yaml" -router = APIRouter(tags=["Datasets"], prefix=V1_URL_PREFIX) +router = APIRouter(tags=["Dataset Configs"], prefix=V1_URL_PREFIX) # Helper method to inject the parent ConnectionConfig into these child routes @@ -176,8 +178,8 @@ def patch_dataset_configs( 1) A DatasetConfig fides_key 2) The corresponding CtlDataset fides_key which stores the bulk of the actual dataset - Currently this endpoint looks up the ctl dataset and writes its contents back to the DatasetConfig.dataset - field for backwards compatibility but soon DatasetConfig.dataset will go away. + The CtlDataset contents are retrieved for extra validation before linking this + to the DatasetConfig. """ created_or_updated: List[Dataset] = [] @@ -193,12 +195,23 @@ def patch_dataset_configs( .filter_by(fides_key=dataset_pair.ctl_dataset_fides_key) .first() ) - fetched_dataset: Dataset = Dataset.from_orm(ctl_dataset) + if not ctl_dataset: + raise HTTPException( + status_code=HTTP_404_NOT_FOUND, + detail=f"No ctl dataset with key '{dataset_pair.ctl_dataset_fides_key}'", + ) + + try: + fetched_dataset: Dataset = Dataset.from_orm(ctl_dataset) + except PydanticValidationError as e: + raise HTTPException( + status_code=HTTP_422_UNPROCESSABLE_ENTITY, detail=e.errors() + ) + validate_data_categories(fetched_dataset, db) data = { "connection_config_id": connection_config.id, "fides_key": dataset_pair.fides_key, - "dataset": fetched_dataset.dict(), "ctl_dataset_id": ctl_dataset.id, } @@ -233,7 +246,7 @@ def patch_datasets( Given a list of dataset elements, create or update corresponding Dataset objects or report failure - Use for bulk creating and/or updating DatasetConfig resources. + This endpoint upserts the DatasetConfig and associated CTL Dataset. Will shortly be deprecated. If the fides_key for a given DatasetConfig exists, it will be treated as an update. Otherwise, a new DatasetConfig will be created. @@ -256,7 +269,7 @@ def patch_datasets( data = { "connection_config_id": connection_config.id, "fides_key": dataset.fides_key, - "dataset": dataset.dict(), + "dataset": dataset.dict(), # Currently used for upserting a CTL Dataset } create_or_update_dataset( connection_config, @@ -310,7 +323,7 @@ async def patch_yaml_datasets( data: dict = { "connection_config_id": connection_config.id, "fides_key": dataset["fides_key"], - "dataset": dataset, + "dataset": dataset, # Currently used for upserting a CTL Dataset } create_or_update_dataset( connection_config, @@ -411,7 +424,10 @@ def get_datasets( params: Params = Depends(), connection_config: ConnectionConfig = Depends(_get_connection_config), ) -> AbstractPage[Dataset]: - """Returns all DatasetConfig datasets in the database.""" + """Returns all CTL datasets attached to the ConnectionConfig via the Dataset Config. + + Soon to be deprecated. + """ logger.info( "Finding all datasets for connection '{}' with pagination params {}", @@ -443,7 +459,10 @@ def get_dataset( db: Session = Depends(deps.get_db), connection_config: ConnectionConfig = Depends(_get_connection_config), ) -> Dataset: - """Returns a single dataset based on the given key.""" + """Returns a single ctl dataset linked to the given DatasetConfig. + + Soon to be deprecated + """ logger.info( "Finding dataset '{}' for connection '{}'", fides_key, connection_config.key @@ -463,6 +482,62 @@ def get_dataset( return dataset_config.ctl_dataset +@router.get( + DATASET_CONFIGS, + dependencies=[Security(verify_oauth_client, scopes=[DATASET_READ])], + response_model=Page[DatasetConfigSchema], +) +def get_dataset_configs( + db: Session = Depends(deps.get_db), + params: Params = Depends(), + connection_config: ConnectionConfig = Depends(_get_connection_config), +) -> AbstractPage[DatasetConfig]: + """Returns all Dataset Configs attached to current Connection Config.""" + + logger.info( + "Finding all dataset configs for connection '{}' with pagination params {}", + connection_config.key, + params, + ) + dataset_configs = DatasetConfig.filter( + db=db, conditions=(DatasetConfig.connection_config_id == connection_config.id) + ).order_by(DatasetConfig.created_at.desc()) + + return paginate(dataset_configs, params) + + +@router.get( + DATASETCONFIG_BY_KEY, + dependencies=[Security(verify_oauth_client, scopes=[DATASET_READ])], + response_model=DatasetConfigSchema, +) +def get_dataset_config( + fides_key: FidesKey, + db: Session = Depends(deps.get_db), + connection_config: ConnectionConfig = Depends(_get_connection_config), +) -> DatasetConfig: + """Returns the specific Dataset Config linked to the Connection Config.""" + + logger.info( + "Finding dataset config '{}' for connection '{}'", + fides_key, + connection_config.key, + ) + dataset_config = DatasetConfig.filter( + db=db, + conditions=( + (DatasetConfig.connection_config_id == connection_config.id) + & (DatasetConfig.fides_key == fides_key) + ), + ).first() + if not dataset_config: + raise HTTPException( + status_code=HTTP_404_NOT_FOUND, + detail=f"No dataset config with fides_key '{fides_key}' and connection key {connection_config.key}'", + ) + return dataset_config + + @router.delete( DATASET_BY_KEY, dependencies=[Security(verify_oauth_client, scopes=[DATASET_DELETE])], diff --git a/src/fides/api/ops/api/v1/urn_registry.py b/src/fides/api/ops/api/v1/urn_registry.py index 3d39498a46..be7f72d0ec 100644 --- a/src/fides/api/ops/api/v1/urn_registry.py +++ b/src/fides/api/ops/api/v1/urn_registry.py @@ -112,6 +112,7 @@ DATASETS = CONNECTION_BY_KEY + "/dataset" DATASET_CONFIGS = CONNECTION_BY_KEY + "/datasetconfig" DATASET_BY_KEY = CONNECTION_BY_KEY + "/dataset/{fides_key}" +DATASETCONFIG_BY_KEY = CONNECTION_BY_KEY + "/datasetconfig/{fides_key}" # YAML Collection URLs YAML_DATASETS = YAML + DATASETS diff --git a/src/fides/api/ops/models/datasetconfig.py b/src/fides/api/ops/models/datasetconfig.py index fc40e1a73d..5f1b8e05aa 100644 --- a/src/fides/api/ops/models/datasetconfig.py +++ b/src/fides/api/ops/models/datasetconfig.py @@ -4,8 +4,6 @@ from fideslang.validation import FidesKey from loguru import logger from sqlalchemy import Column, ForeignKey, String -from sqlalchemy.dialects.postgresql import JSONB -from sqlalchemy.ext.mutable import MutableDict from sqlalchemy.orm import Session, relationship from fides.api.ctl.sql_models import Dataset as CtlDataset # type: ignore[attr-defined] @@ -38,9 +36,6 @@ class DatasetConfig(Base): String, ForeignKey(ConnectionConfig.id_field_path), nullable=False ) fides_key = Column(String, index=True, unique=True, nullable=False) - dataset = Column( - MutableDict.as_mutable(JSONB), index=False, unique=False, nullable=False - ) ctl_dataset_id = Column( String, ForeignKey(CtlDataset.id), index=True, nullable=False ) @@ -104,6 +99,7 @@ def upsert_ctl_dataset(ctl_dataset_obj: Optional[CtlDataset]) -> CtlDataset: upsert_ctl_dataset( dataset.ctl_dataset ) # Update existing ctl_dataset first. + data.pop("dataset", None) dataset.update(db=db, data=data) else: fetched_ctl_dataset = ( @@ -117,6 +113,7 @@ def upsert_ctl_dataset(ctl_dataset_obj: Optional[CtlDataset]) -> CtlDataset: fetched_ctl_dataset ) # Create/update existing ctl_dataset first data["ctl_dataset_id"] = ctl_dataset.id + data.pop("dataset", None) dataset = cls.create(db=db, data=data) return dataset diff --git a/src/fides/api/ops/schemas/dataset.py b/src/fides/api/ops/schemas/dataset.py index 2e757b936e..a8a37b89d2 100644 --- a/src/fides/api/ops/schemas/dataset.py +++ b/src/fides/api/ops/schemas/dataset.py @@ -104,6 +104,18 @@ class DatasetConfigCtlDataset(BaseSchema): ctl_dataset_fides_key: FidesKey # The fides_key for the ctl_datasets record +class DatasetConfigSchema(BaseSchema): + """Returns the DatasetConfig fides key and the linked Ctl Dataset""" + + fides_key: FidesKey + ctl_dataset: Dataset + + class Config: + """Set ORM Mode to True.""" + + orm_mode = True + + class BulkPutDataset(BulkResponse): """Schema with mixed success/failure responses for Bulk Create/Update of Datasets.""" diff --git a/src/fides/api/ops/service/connectors/email_connector.py b/src/fides/api/ops/service/connectors/email_connector.py index 6c0c3011ae..f507ccaf5b 100644 --- a/src/fides/api/ops/service/connectors/email_connector.py +++ b/src/fides/api/ops/service/connectors/email_connector.py @@ -183,7 +183,7 @@ def email_connector_erasure_send(db: Session, privacy_request: PrivacyRequest) - if not template_values: logger.info( "No email sent: no template values saved for '{}'", - ds.dataset.get("fides_key"), + ds.ctl_dataset.fides_key, ) return @@ -196,7 +196,7 @@ def email_connector_erasure_send(db: Session, privacy_request: PrivacyRequest) - ) ): logger.info( - "No email sent: no masking needed on '{}'", ds.dataset.get("fides_key") + "No email sent: no masking needed on '{}'", ds.ctl_dataset.fides_key ) return @@ -213,7 +213,7 @@ def email_connector_erasure_send(db: Session, privacy_request: PrivacyRequest) - logger.info( "Email send succeeded for request '{}' for dataset: '{}'", privacy_request.id, - ds.dataset.get("fides_key"), + ds.ctl_dataset.fides_key, ) AuditLog.create( db=db, @@ -221,7 +221,7 @@ def email_connector_erasure_send(db: Session, privacy_request: PrivacyRequest) - "user_id": "system", "privacy_request_id": privacy_request.id, "action": AuditLogAction.email_sent, - "message": f"Erasure email instructions dispatched for '{ds.dataset.get('fides_key')}'", + "message": f"Erasure email instructions dispatched for '{ds.ctl_dataset.fides_key}'", }, ) diff --git a/src/fides/api/ops/service/connectors/saas/connector_registry_service.py b/src/fides/api/ops/service/connectors/saas/connector_registry_service.py index c9ec3c7947..d326158070 100644 --- a/src/fides/api/ops/service/connectors/saas/connector_registry_service.py +++ b/src/fides/api/ops/service/connectors/saas/connector_registry_service.py @@ -129,7 +129,7 @@ def upsert_dataset_config_from_template( data = { "connection_config_id": connection_config.id, "fides_key": template_values.instance_key, - "dataset": dataset_from_template, + "dataset": dataset_from_template, # Currently used for upserting a CTL Dataset } dataset_config = DatasetConfig.upsert_with_ctl_dataset(db, data=data) return dataset_config diff --git a/tests/ctl/core/test_dataset.py b/tests/ctl/core/test_dataset.py index cf623ade7a..c15817b023 100644 --- a/tests/ctl/core/test_dataset.py +++ b/tests/ctl/core/test_dataset.py @@ -2,13 +2,23 @@ import os from typing import Dict, Generator, List from urllib.parse import quote_plus +from uuid import uuid4 import pytest import sqlalchemy from fideslang.manifests import write_manifest from fideslang.models import Dataset, DatasetCollection, DatasetField from py._path.local import LocalPath - +from sqlalchemy.orm import Session + +from fides.api.ctl.database.crud import get_resource +from fides.api.ctl.sql_models import Dataset as CtlDataset +from fides.api.ops.models.connectionconfig import ( + AccessLevel, + ConnectionConfig, + ConnectionType, +) +from fides.api.ops.models.datasetconfig import DatasetConfig from fides.core import api from fides.core import dataset as _dataset from fides.core.config import FidesConfig @@ -132,6 +142,119 @@ def test_find_uncategorized_dataset_fields_all_categorized() -> None: assert total_field_count == 4 +@pytest.fixture(scope="function") +def connection_config( + db: Session, +) -> Generator: + connection_config = ConnectionConfig.create( + db=db, + data={ + "name": str(uuid4()), + "key": "my_postgres_db_1", + "connection_type": ConnectionType.postgres, + "access": AccessLevel.write, + "disabled": False, + "description": "Primary postgres connection", + }, + ) + yield connection_config + connection_config.delete(db) + + +@pytest.mark.unit +async def test_upsert_db_datasets( + test_config: FidesConfig, db: Session, connection_config +) -> None: + """ + Upsert a CTL Dataset, link this to a DatasetConfig and then upsert that CTL Dataset again. + + The id of the CTL Dataset cannot change on upsert, as the DatasetConfig has a FK to this resource. + """ + + dataset = Dataset( + name="ds1", + fides_key="ds", + data_categories=[], + description="Fides Generated Description for Schema: ds", + collections=[ + DatasetCollection( + name="foo", + description="Fides Generated Description for Table: foo", + data_categories=[], + fields=[ + DatasetField( + name=1, + description="Fides Generated Description for Column: 1", + data_categories=[], + ), + DatasetField( + name=2, + description="Fides Generated Description for Column: 2", + data_categories=[], + ), + ], + ), + DatasetCollection( + name="bar", + description="Fides Generated Description for Table: bar", + data_categories=[], + fields=[ + DatasetField( + name=4, + description="Fides Generated Description for Column: 4", + data_categories=[], + ), + DatasetField( + name=5, + description="Fides Generated Description for Column: 5", + data_categories=[], + ), + ], + ), + ], + ) + + resp = api.upsert( + url=test_config.cli.server_url, + resource_type="dataset", + resources=[dataset.dict(exclude_none=True)], + headers=test_config.user.request_headers, + ) + assert resp.status_code == 201 + assert resp.json()["inserted"] == 1 + + ds: CtlDataset = await get_resource(CtlDataset, "ds") + + # Create a DatasetConfig that links to the created CTL Dataset + dataset_config = DatasetConfig.create( + db=db, + data={ + "connection_config_id": connection_config.id, + "fides_key": "new_fides_key", + "ctl_dataset_id": ds.id, + }, + ) + + ctl_dataset_id = ds.id + assert dataset_config.ctl_dataset_id == ctl_dataset_id + + # Do another upsert of the CTL Dataset to update the name + dataset.name = "new name" + resp = api.upsert( + url=test_config.cli.server_url, + resource_type="dataset", + resources=[dataset.dict(exclude_none=True)], + headers=test_config.user.request_headers, + ) + assert resp.status_code == 200 + assert resp.json()["inserted"] == 0 + assert resp.json()["updated"] == 1 + + db.refresh(dataset_config) + assert dataset_config.ctl_dataset.name == "new name" + assert dataset_config.ctl_dataset.id == ctl_dataset_id, "Id unchanged with upsert" + + @pytest.mark.unit def test_find_uncategorized_dataset_fields_uncategorized_fields() -> None: test_resource = {"foo": ["1", "2"]} diff --git a/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py b/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py index a58d88f14b..4cebb93628 100644 --- a/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py @@ -735,7 +735,8 @@ def test_instantiate_connection_from_template( assert connection_config.last_test_succeeded is None assert dataset_config.connection_config_id == connection_config.id - assert dataset_config.dataset is not None + assert dataset_config.ctl_dataset_id is not None dataset_config.delete(db) connection_config.delete(db) + dataset_config.ctl_dataset.delete(db=db) diff --git a/tests/ops/api/v1/endpoints/test_dataset_endpoints.py b/tests/ops/api/v1/endpoints/test_dataset_endpoints.py index 2aebf503b2..2729e5554a 100644 --- a/tests/ops/api/v1/endpoints/test_dataset_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_dataset_endpoints.py @@ -23,6 +23,7 @@ DATASET_BY_KEY, DATASET_CONFIGS, DATASET_VALIDATE, + DATASETCONFIG_BY_KEY, DATASETS, V1_URL_PREFIX, YAML_DATASETS, @@ -433,13 +434,13 @@ def request_body(self, ctl_dataset): } ] - def test_patch_datasets_not_authenticated( + def test_patch_dataset_configs_not_authenticated( self, datasets_url, api_client, request_body ) -> None: response = api_client.patch(datasets_url, headers={}, json=request_body) assert response.status_code == 401 - def test_patch_datasets_wrong_scope( + def test_patch_dataset_configs_wrong_scope( self, request_body, datasets_url, @@ -452,7 +453,7 @@ def test_patch_datasets_wrong_scope( ) assert response.status_code == 403 - def test_patch_create_datasets_by_ctl_dataset_key( + def test_create_dataset_configs_by_ctl_dataset_key( self, ctl_dataset, generate_auth_header, @@ -475,9 +476,6 @@ def test_patch_create_datasets_by_ctl_dataset_key( assert ( dataset_config.ctl_dataset.fides_key == ctl_dataset.fides_key ), "Differs from datasetconfig.fides_key in this case" - assert ( - dataset_config.dataset["fides_key"] == ctl_dataset.fides_key - ), "Differs from datasetconfig.fides_key in this case" succeeded = response.json()["succeeded"][0] assert ( @@ -487,10 +485,41 @@ def test_patch_create_datasets_by_ctl_dataset_key( dataset_config.delete(db) - def test_patch_datasets_invalid_connection_key( + def test_create_datasetconfigs_bad_data_category( + self, + ctl_dataset, + generate_auth_header, + api_client, + datasets_url, + db, + request_body, + ): + ctl_dataset.collections[0]["fields"][0]["data_categories"] = ["bad_category"] + flag_modified(ctl_dataset, "collections") + db.add(ctl_dataset) + db.commit() + db.refresh(ctl_dataset) + + auth_header = generate_auth_header(scopes=[DATASET_CREATE_OR_UPDATE]) + response = api_client.patch( + datasets_url, + headers=auth_header, + json=request_body, + ) + assert response.status_code == 422 + dataset_config = DatasetConfig.get_by( + db=db, field="fides_key", value="test_fides_key" + ) + assert dataset_config is None + assert ( + response.json()["detail"][0]["msg"] + == "The data category bad_category is not supported." + ) + + def test_create_datasets_configs_invalid_connection_key( self, request_body, api_client: TestClient, generate_auth_header ) -> None: - path = V1_URL_PREFIX + DATASETS + path = V1_URL_PREFIX + DATASET_CONFIGS path_params = {"connection_key": "nonexistent_key"} datasets_url = path.format(**path_params) @@ -500,7 +529,23 @@ def test_patch_datasets_invalid_connection_key( ) assert response.status_code == 404 - def test_patch_datasets_bulk_create_limit_exceeded( + def test_patch_dataset_configs_ctl_dataset_id_does_not_exist( + self, request_body, api_client: TestClient, generate_auth_header, datasets_url + ) -> None: + request_body.append( + { + "fides_key": "second_dataset_config", + "ctl_dataset_fides_key": "bad_ctl_dataset_key", + } + ) + + auth_header = generate_auth_header(scopes=[DATASET_CREATE_OR_UPDATE]) + response = api_client.patch( + datasets_url, headers=auth_header, json=request_body + ) + assert response.status_code == 404 + + def test_patch_dataset_configs_bulk_create_limit_exceeded( self, api_client: TestClient, request_body, generate_auth_header, datasets_url ): payload = [] @@ -516,7 +561,7 @@ def test_patch_datasets_bulk_create_limit_exceeded( == "ensure this value has at most 50 items" ) - def test_patch_create_datasets_bulk_create( + def test_patch_create_dataset_configs_bulk_create( self, ctl_dataset, generate_auth_header, @@ -550,28 +595,20 @@ def test_patch_create_datasets_bulk_create( assert first_dataset_config.ctl_dataset == ctl_dataset assert ( response_body["succeeded"][0]["collections"] - == first_dataset_config.dataset["collections"] + == Dataset.from_orm(first_dataset_config.ctl_dataset).collections ) assert response_body["succeeded"][0]["fides_key"] == ctl_dataset.fides_key - assert ( - first_dataset_config.dataset["collections"] - == Dataset.from_orm(ctl_dataset).collections - ) - assert len(first_dataset_config.dataset["collections"]) == 1 + assert len(first_dataset_config.ctl_dataset.collections) == 1 second_dataset_config = DatasetConfig.get_by( db=db, field="fides_key", value="second_dataset_config" ) assert ( response_body["succeeded"][1]["collections"] - == first_dataset_config.dataset["collections"] + == Dataset.from_orm(second_dataset_config.ctl_dataset).collections ) assert response_body["succeeded"][1]["fides_key"] == ctl_dataset.fides_key assert second_dataset_config.ctl_dataset == ctl_dataset - assert ( - second_dataset_config.dataset["collections"] - == Dataset.from_orm(ctl_dataset).collections - ) first_dataset_config.delete(db) second_dataset_config.delete(db) @@ -623,11 +660,11 @@ def test_patch_update_dataset_configs( assert dataset_config.updated_at != updated assert response_body["succeeded"][0]["fides_key"] == "new_ctl_dataset" assert response_body["succeeded"][0]["description"] == "updated description" - assert dataset_config.dataset["description"] == "updated description" - assert len(dataset_config.dataset["collections"]) == 1 + assert dataset_config.ctl_dataset.description == "updated description" + assert len(dataset_config.ctl_dataset.collections) == 1 @pytest.mark.unit_saas - def test_patch_datasets_missing_saas_config( + def test_patch_dataset_configs_missing_saas_config( self, saas_example_connection_config_without_saas_config, saas_ctl_dataset, @@ -663,7 +700,7 @@ def test_patch_datasets_missing_saas_config( ) @pytest.mark.unit_saas - def test_patch_datasets_extra_reference( + def test_patch_dataset_configs_extra_reference( self, saas_example_connection_config, saas_ctl_dataset, @@ -711,7 +748,7 @@ def test_patch_datasets_extra_reference( ) @pytest.mark.unit_saas - def test_patch_datasets_extra_identity( + def test_patch_dataset_configs_extra_identity( self, saas_example_connection_config, saas_ctl_dataset, @@ -753,7 +790,7 @@ def test_patch_datasets_extra_identity( ), "Validation is done when attaching dataset to Saas Config" @pytest.mark.unit_saas - def test_patch_datasets_fides_key_mismatch( + def test_patch_dataset_configs_fides_key_mismatch( self, saas_example_connection_config, saas_ctl_dataset, @@ -795,7 +832,7 @@ def test_patch_datasets_fides_key_mismatch( ) @mock.patch("fides.api.ops.models.datasetconfig.DatasetConfig.create_or_update") - def test_patch_datasets_failed_response( + def test_patch_dataset_configs_failed_response( self, mock_create: Mock, request_body, @@ -820,6 +857,28 @@ def test_patch_datasets_failed_response( for index, failed in enumerate(response_body["failed"]): assert failed["data"]["fides_key"] == request_body[0]["fides_key"] + def test_patch_dataset_configs_failed_ctl_dataset_validation( + self, + ctl_dataset, + generate_auth_header, + api_client, + datasets_url, + db, + request_body, + ): + ctl_dataset.organization_fides_key = None + db.add(ctl_dataset) + db.commit() + db.refresh(ctl_dataset) + + auth_header = generate_auth_header(scopes=[DATASET_CREATE_OR_UPDATE]) + response = api_client.patch( + datasets_url, + headers=auth_header, + json=request_body, + ) + assert response.status_code == 422 + class TestPutDatasets: @pytest.fixture @@ -1485,6 +1544,53 @@ def test_get_datasets( assert response_body["size"] == Params().size +class TestGetDatasetConfigs: + @pytest.fixture + def datasets_url(self, connection_config) -> str: + path = V1_URL_PREFIX + DATASET_CONFIGS + path_params = {"connection_key": connection_config.key} + return path.format(**path_params) + + def test_get_dataset_configs_not_authenticated( + self, datasets_url, api_client: TestClient + ) -> None: + response = api_client.get(datasets_url, headers={}) + assert response.status_code == 401 + + def test_get_dataset_configs_invalid_connection_key( + self, datasets_url, api_client: TestClient, generate_auth_header + ) -> None: + auth_header = generate_auth_header(scopes=[DATASET_READ]) + path = V1_URL_PREFIX + DATASET_CONFIGS + path_params = {"connection_key": "nonexistent_key"} + datasets_url = path.format(**path_params) + + response = api_client.get(datasets_url, headers=auth_header) + assert response.status_code == 404 + + def test_get_dataset_configs( + self, dataset_config, datasets_url, api_client: TestClient, generate_auth_header + ) -> None: + auth_header = generate_auth_header(scopes=[DATASET_READ]) + response = api_client.get(datasets_url, headers=auth_header) + assert response.status_code == 200 + + response_body = json.loads(response.text) + assert len(response_body["items"]) == 1 + dataset_response = response_body["items"][0] + assert dataset_response["fides_key"] == "postgres_example_subscriptions_dataset" + + assert ( + dataset_response["ctl_dataset"]["fides_key"] + == "postgres_example_subscriptions_dataset" + ) + assert len(dataset_response["ctl_dataset"]["collections"]) == 1 + + assert response_body["total"] == 1 + assert response_body["page"] == 1 + assert response_body["size"] == Params().size + + def get_dataset_url( connection_config: Optional[ConnectionConfig] = None, dataset_config: Optional[DatasetConfig] = None, @@ -1563,6 +1669,88 @@ def test_get_dataset( assert len(response_body["collections"]) == 1 +def get_dataset_config_url( + connection_config: Optional[ConnectionConfig] = None, + dataset_config: Optional[DatasetConfig] = None, +) -> str: + """Helper to construct the DATASETCONFIG_BY_KEY URL, substituting valid/invalid keys in the path""" + path = V1_URL_PREFIX + DATASETCONFIG_BY_KEY + connection_key = "nonexistent_key" + if connection_config: + connection_key = connection_config.key + fides_key = "nonexistent_key" + if dataset_config: + fides_key = dataset_config.fides_key + path_params = {"connection_key": connection_key, "fides_key": fides_key} + return path.format(**path_params) + + +class TestGetDatasetConfig: + def test_get_dataset_config_not_authenticated( + self, dataset_config, connection_config, api_client + ) -> None: + dataset_url = get_dataset_config_url(connection_config, dataset_config) + response = api_client.get(dataset_url, headers={}) + assert response.status_code == 401 + + def test_get_dataset_config_wrong_scope( + self, + dataset_config, + connection_config, + api_client: TestClient, + generate_auth_header, + ) -> None: + dataset_url = get_dataset_config_url(connection_config, dataset_config) + auth_header = generate_auth_header(scopes=[DATASET_CREATE_OR_UPDATE]) + response = api_client.get(dataset_url, headers=auth_header) + assert response.status_code == 403 + + def test_get_dataset_config_does_not_exist( + self, + dataset_config, + connection_config, + api_client: TestClient, + generate_auth_header, + ) -> None: + dataset_url = get_dataset_config_url(connection_config, None) + auth_header = generate_auth_header(scopes=[DATASET_READ]) + response = api_client.get(dataset_url, headers=auth_header) + assert response.status_code == 404 + + def test_get_dataset_config_invalid_connection_key( + self, + dataset_config, + connection_config, + api_client: TestClient, + generate_auth_header, + ) -> None: + dataset_url = get_dataset_config_url(None, dataset_config) + dataset_url.replace(connection_config.key, "nonexistent_key") + auth_header = generate_auth_header(scopes=[DATASET_READ]) + response = api_client.get(dataset_url, headers=auth_header) + assert response.status_code == 404 + + def test_get_dataset_config( + self, + dataset_config, + connection_config, + api_client: TestClient, + generate_auth_header, + ): + dataset_url = get_dataset_config_url(connection_config, dataset_config) + auth_header = generate_auth_header(scopes=[DATASET_READ]) + response = api_client.get(dataset_url, headers=auth_header) + assert response.status_code == 200 + + response_body = json.loads(response.text) + assert response_body["fides_key"] == dataset_config.fides_key + assert ( + response_body["ctl_dataset"]["fides_key"] + == dataset_config.ctl_dataset.fides_key + ) + assert len(response_body["ctl_dataset"]["collections"]) == 1 + + class TestDeleteDataset: def test_delete_dataset_not_authenticated( self, dataset_config, connection_config, api_client @@ -1622,9 +1810,6 @@ def test_delete_dataset( data={ "connection_config_id": connection_config.id, "fides_key": "postgres_example_subscriptions", - "dataset": Dataset.from_orm( - ctl_dataset - ).dict(), # Temporary, soon remove writing to this field. "ctl_dataset_id": ctl_dataset.id, }, ) diff --git a/tests/ops/fixtures/application_fixtures.py b/tests/ops/fixtures/application_fixtures.py index c4818c8c72..0c45f16efe 100644 --- a/tests/ops/fixtures/application_fixtures.py +++ b/tests/ops/fixtures/application_fixtures.py @@ -1260,31 +1260,6 @@ def dataset_config( "connection_config_id": connection_config.id, "fides_key": "postgres_example_subscriptions_dataset", "ctl_dataset_id": ctl_dataset.id, - "dataset": { - "fides_key": "postgres_example_subscriptions_dataset", - "name": "Postgres Example Subscribers Dataset", - "description": "Example Postgres dataset created in test fixtures", - "dataset_type": "PostgreSQL", - "location": "postgres_example.test", - "collections": [ - { - "name": "subscriptions", - "fields": [ - { - "name": "id", - "data_categories": ["system.operations"], - }, - { - "name": "email", - "data_categories": ["user.contact.email"], - "fidesops_meta": { - "identity": "email", - }, - }, - ], - }, - ], - }, }, ) yield dataset_config @@ -1304,31 +1279,6 @@ def dataset_config_preview( "connection_config_id": connection_config.id, "fides_key": "postgres", "ctl_dataset_id": ctl_dataset.id, - "dataset": { - "fides_key": "postgres", - "name": "Postgres Example Subscribers Dataset", - "description": "Example Postgres dataset created in test fixtures", - "dataset_type": "PostgreSQL", - "location": "postgres_example.test", - "collections": [ - { - "name": "subscriptions", - "fields": [ - { - "name": "id", - "data_categories": ["system.operations"], - }, - { - "name": "email", - "data_categories": ["user.contact.email"], - "fidesops_meta": { - "identity": "email", - }, - }, - ], - }, - ], - }, }, ) yield dataset_config diff --git a/tests/ops/fixtures/bigquery_fixtures.py b/tests/ops/fixtures/bigquery_fixtures.py index b47ad7e58b..6197961774 100644 --- a/tests/ops/fixtures/bigquery_fixtures.py +++ b/tests/ops/fixtures/bigquery_fixtures.py @@ -83,7 +83,6 @@ def bigquery_example_test_dataset_config( data={ "connection_config_id": bigquery_connection_config.id, "fides_key": fides_key, - "dataset": bigquery_dataset, "ctl_dataset_id": ctl_dataset.id, }, ) diff --git a/tests/ops/fixtures/email_fixtures.py b/tests/ops/fixtures/email_fixtures.py index 56f4730c50..e7ba1c72df 100644 --- a/tests/ops/fixtures/email_fixtures.py +++ b/tests/ops/fixtures/email_fixtures.py @@ -49,7 +49,6 @@ def email_dataset_config( data={ "connection_config_id": email_connection_config.id, "fides_key": fides_key, - "dataset": email_dataset, "ctl_dataset_id": ctl_dataset.id, }, ) diff --git a/tests/ops/fixtures/fides_connector_example_fixtures.py b/tests/ops/fixtures/fides_connector_example_fixtures.py index facc731387..a5b11bca71 100644 --- a/tests/ops/fixtures/fides_connector_example_fixtures.py +++ b/tests/ops/fixtures/fides_connector_example_fixtures.py @@ -87,7 +87,6 @@ def fides_connector_example_test_dataset_config( data={ "connection_config_id": fides_connector_connection_config.id, "fides_key": fides_key, - "dataset": fides_connector_dataset, "ctl_dataset_id": ctl_dataset.id, }, ) diff --git a/tests/ops/fixtures/manual_fixtures.py b/tests/ops/fixtures/manual_fixtures.py index df5c9f26c8..f70523ae3b 100644 --- a/tests/ops/fixtures/manual_fixtures.py +++ b/tests/ops/fixtures/manual_fixtures.py @@ -47,7 +47,6 @@ def manual_dataset_config( data={ "connection_config_id": integration_manual_config.id, "fides_key": fides_key, - "dataset": manual_dataset, "ctl_dataset_id": ctl_dataset.id, }, ) diff --git a/tests/ops/fixtures/mariadb_fixtures.py b/tests/ops/fixtures/mariadb_fixtures.py index 64d100f36a..500c5539f6 100644 --- a/tests/ops/fixtures/mariadb_fixtures.py +++ b/tests/ops/fixtures/mariadb_fixtures.py @@ -80,7 +80,6 @@ def mariadb_example_test_dataset_config( data={ "connection_config_id": connection_config_mariadb.id, "fides_key": fides_key, - "dataset": mariadb_dataset, "ctl_dataset_id": ctl_dataset.id, }, ) diff --git a/tests/ops/fixtures/mssql_fixtures.py b/tests/ops/fixtures/mssql_fixtures.py index 81fe82beb6..19a01e6d14 100644 --- a/tests/ops/fixtures/mssql_fixtures.py +++ b/tests/ops/fixtures/mssql_fixtures.py @@ -41,7 +41,6 @@ def mssql_example_test_dataset_config( data={ "connection_config_id": connection_config_mssql.id, "fides_key": fides_key, - "dataset": mssql_dataset, "ctl_dataset_id": ctl_dataset.id, }, ) diff --git a/tests/ops/fixtures/mysql_fixtures.py b/tests/ops/fixtures/mysql_fixtures.py index 0277861e84..56e1725e46 100644 --- a/tests/ops/fixtures/mysql_fixtures.py +++ b/tests/ops/fixtures/mysql_fixtures.py @@ -59,7 +59,6 @@ def dataset_config_mysql( data={ "connection_config_id": connection_config.id, "fides_key": "mysql_example_subscriptions_dataset", - "dataset": dataset, "ctl_dataset_id": ctl_dataset.id, }, ) @@ -88,7 +87,6 @@ def mysql_example_test_dataset_config( data={ "connection_config_id": connection_config_mysql.id, "fides_key": fides_key, - "dataset": mysql_dataset, "ctl_dataset_id": ctl_dataset.id, }, ) diff --git a/tests/ops/fixtures/postgres_fixtures.py b/tests/ops/fixtures/postgres_fixtures.py index 4276960da2..2e7a4dbb5a 100644 --- a/tests/ops/fixtures/postgres_fixtures.py +++ b/tests/ops/fixtures/postgres_fixtures.py @@ -49,7 +49,6 @@ def postgres_example_test_dataset_config( data={ "connection_config_id": connection_config.id, "fides_key": fides_key, - "dataset": postgres_dataset, "ctl_dataset_id": ctl_dataset.id, }, ) @@ -74,7 +73,6 @@ def postgres_example_test_dataset_config_read_access( data={ "connection_config_id": read_connection_config.id, "fides_key": fides_key, - "dataset": postgres_dataset, "ctl_dataset_id": ctl_dataset.id, }, ) diff --git a/tests/ops/fixtures/redshift_fixtures.py b/tests/ops/fixtures/redshift_fixtures.py index bbe173e082..5ca6d68b3e 100644 --- a/tests/ops/fixtures/redshift_fixtures.py +++ b/tests/ops/fixtures/redshift_fixtures.py @@ -61,7 +61,6 @@ def redshift_example_test_dataset_config( data={ "connection_config_id": redshift_connection_config.id, "fides_key": fides_key, - "dataset": dataset, "ctl_dataset_id": ctl_dataset.id, }, ) diff --git a/tests/ops/fixtures/saas/adobe_campaign_fixtures.py b/tests/ops/fixtures/saas/adobe_campaign_fixtures.py index 0b9acfaa9b..4ff3a8c2d3 100644 --- a/tests/ops/fixtures/saas/adobe_campaign_fixtures.py +++ b/tests/ops/fixtures/saas/adobe_campaign_fixtures.py @@ -115,7 +115,6 @@ def adobe_campaign_dataset_config( data={ "connection_config_id": adobe_campaign_connection_config.id, "fides_key": fides_key, - "dataset": adobe_campaign_dataset, "ctl_dataset_id": ctl_dataset.id, }, ) diff --git a/tests/ops/fixtures/saas/auth0_fixtures.py b/tests/ops/fixtures/saas/auth0_fixtures.py index 3677f5b509..2ef2c429c8 100644 --- a/tests/ops/fixtures/saas/auth0_fixtures.py +++ b/tests/ops/fixtures/saas/auth0_fixtures.py @@ -94,7 +94,6 @@ def auth0_dataset_config( data={ "connection_config_id": auth0_connection_config.id, "fides_key": fides_key, - "dataset": auth0_dataset, "ctl_dataset_id": ctl_dataset.id, }, ) diff --git a/tests/ops/fixtures/saas/braze_fixtures.py b/tests/ops/fixtures/saas/braze_fixtures.py index 0a497eec86..0941dd5941 100644 --- a/tests/ops/fixtures/saas/braze_fixtures.py +++ b/tests/ops/fixtures/saas/braze_fixtures.py @@ -107,7 +107,6 @@ def braze_dataset_config( data={ "connection_config_id": braze_connection_config.id, "fides_key": fides_key, - "dataset": braze_dataset, "ctl_dataset_id": ctl_dataset.id, }, ) diff --git a/tests/ops/fixtures/saas/datadog_fixtures.py b/tests/ops/fixtures/saas/datadog_fixtures.py index e0781e6683..6befdf0c27 100644 --- a/tests/ops/fixtures/saas/datadog_fixtures.py +++ b/tests/ops/fixtures/saas/datadog_fixtures.py @@ -104,7 +104,6 @@ def datadog_dataset_config( data={ "connection_config_id": datadog_connection_config.id, "fides_key": fides_key, - "dataset": datadog_dataset, "ctl_dataset_id": ctl_dataset.id, }, ) diff --git a/tests/ops/fixtures/saas/domo_fixtures.py b/tests/ops/fixtures/saas/domo_fixtures.py index 97f0d4c0c9..fbf63b41d8 100644 --- a/tests/ops/fixtures/saas/domo_fixtures.py +++ b/tests/ops/fixtures/saas/domo_fixtures.py @@ -116,7 +116,6 @@ def domo_dataset_config( data={ "connection_config_id": domo_connection_config.id, "fides_key": fides_key, - "dataset": domo_dataset, "ctl_dataset_id": ctl_dataset.id, }, ) diff --git a/tests/ops/fixtures/saas/doordash_fixtures.py b/tests/ops/fixtures/saas/doordash_fixtures.py index d0a4505b79..3499d1c3c5 100644 --- a/tests/ops/fixtures/saas/doordash_fixtures.py +++ b/tests/ops/fixtures/saas/doordash_fixtures.py @@ -110,7 +110,6 @@ def doordash_dataset_config( data={ "connection_config_id": doordash_connection_config.id, "fides_key": fides_key, - "dataset": doordash_dataset, "ctl_dataset_id": ctl_dataset.id, }, ) @@ -162,7 +161,6 @@ def doordash_postgres_dataset_config( data={ "connection_config_id": connection_config.id, "fides_key": fides_key, - "dataset": doordash_postgres_dataset, "ctl_dataset_id": ctl_dataset.id, }, ) diff --git a/tests/ops/fixtures/saas/friendbuy_fixtures.py b/tests/ops/fixtures/saas/friendbuy_fixtures.py index e5278714fb..e21da1f563 100644 --- a/tests/ops/fixtures/saas/friendbuy_fixtures.py +++ b/tests/ops/fixtures/saas/friendbuy_fixtures.py @@ -7,6 +7,7 @@ from sqlalchemy.orm import Session from sqlalchemy_utils.functions import create_database, database_exists, drop_database +from fides.api.ctl.sql_models import Dataset as CtlDataset from fides.api.ops.models.connectionconfig import ( AccessLevel, ConnectionConfig, @@ -104,16 +105,20 @@ def friendbuy_dataset_config( friendbuy_connection_config.name = fides_key friendbuy_connection_config.key = fides_key friendbuy_connection_config.save(db=db) + + ctl_dataset = CtlDataset.create_from_dataset_dict(db, friendbuy_dataset) + dataset = DatasetConfig.create( db=db, data={ "connection_config_id": friendbuy_connection_config.id, "fides_key": fides_key, - "dataset": friendbuy_dataset, + "ctl_dataset_id": ctl_dataset.id, }, ) yield dataset dataset.delete(db=db) + ctl_dataset.delete(db=db) @pytest.fixture() @@ -151,16 +156,20 @@ def friendbuy_postgres_dataset_config( connection_config.name = fides_key connection_config.key = fides_key connection_config.save(db=db) + + ctl_dataset = CtlDataset.create_from_dataset_dict(db, friendbuy_postgres_dataset) + dataset = DatasetConfig.create( db=db, data={ "connection_config_id": connection_config.id, "fides_key": fides_key, - "dataset": friendbuy_postgres_dataset, + "ctl_dataset_id": ctl_dataset.id, }, ) yield dataset dataset.delete(db=db) + ctl_dataset.delete(db=db) @pytest.fixture(scope="function") diff --git a/tests/ops/fixtures/saas/fullstory_fixtures.py b/tests/ops/fixtures/saas/fullstory_fixtures.py index 0d6e93a26e..9879849c01 100644 --- a/tests/ops/fixtures/saas/fullstory_fixtures.py +++ b/tests/ops/fixtures/saas/fullstory_fixtures.py @@ -116,7 +116,6 @@ def fullstory_dataset_config( data={ "connection_config_id": fullstory_connection_config.id, "fides_key": fides_key, - "dataset": fullstory_dataset, "ctl_dataset_id": ctl_dataset.id, }, ) @@ -168,7 +167,6 @@ def fullstory_postgres_dataset_config( data={ "connection_config_id": connection_config.id, "fides_key": fides_key, - "dataset": fullstory_postgres_dataset, "ctl_dataset_id": ctl_dataset.id, }, ) diff --git a/tests/ops/fixtures/saas/hubspot_fixtures.py b/tests/ops/fixtures/saas/hubspot_fixtures.py index f9ea6290d0..2b0a0ffe34 100644 --- a/tests/ops/fixtures/saas/hubspot_fixtures.py +++ b/tests/ops/fixtures/saas/hubspot_fixtures.py @@ -103,7 +103,6 @@ def dataset_config_hubspot( data={ "connection_config_id": connection_config_hubspot.id, "fides_key": fides_key, - "dataset": hubspot_dataset, "ctl_dataset_id": ctl_dataset.id, }, ) diff --git a/tests/ops/fixtures/saas/mailchimp_fixtures.py b/tests/ops/fixtures/saas/mailchimp_fixtures.py index 1333894a35..dc47882c93 100644 --- a/tests/ops/fixtures/saas/mailchimp_fixtures.py +++ b/tests/ops/fixtures/saas/mailchimp_fixtures.py @@ -98,7 +98,6 @@ def mailchimp_dataset_config( data={ "connection_config_id": mailchimp_connection_config.id, "fides_key": fides_key, - "dataset": mailchimp_dataset, "ctl_dataset_id": ctl_dataset.id, }, ) diff --git a/tests/ops/fixtures/saas/outreach_fixtures.py b/tests/ops/fixtures/saas/outreach_fixtures.py index b84e27deeb..69e59260d3 100644 --- a/tests/ops/fixtures/saas/outreach_fixtures.py +++ b/tests/ops/fixtures/saas/outreach_fixtures.py @@ -108,7 +108,6 @@ def outreach_dataset_config( data={ "connection_config_id": outreach_connection_config.id, "fides_key": fides_key, - "dataset": outreach_dataset, "ctl_dataset_id": ctl_dataset.id, }, ) diff --git a/tests/ops/fixtures/saas/request_override/firebase_auth_fixtures.py b/tests/ops/fixtures/saas/request_override/firebase_auth_fixtures.py index 123e80af25..47ee385adf 100644 --- a/tests/ops/fixtures/saas/request_override/firebase_auth_fixtures.py +++ b/tests/ops/fixtures/saas/request_override/firebase_auth_fixtures.py @@ -167,7 +167,6 @@ def firebase_auth_dataset_config( data={ "connection_config_id": firebase_auth_connection_config.id, "fides_key": fides_key, - "dataset": firebase_auth_dataset, "ctl_dataset_id": ctl_dataset.id, }, ) diff --git a/tests/ops/fixtures/saas/request_override/mailchimp_override_fixtures.py b/tests/ops/fixtures/saas/request_override/mailchimp_override_fixtures.py index cbe6b5c17b..057d17b981 100644 --- a/tests/ops/fixtures/saas/request_override/mailchimp_override_fixtures.py +++ b/tests/ops/fixtures/saas/request_override/mailchimp_override_fixtures.py @@ -71,7 +71,6 @@ def mailchimp_override_dataset_config( data={ "connection_config_id": mailchimp_override_connection_config.id, "fides_key": fides_key, - "dataset": mailchimp_override_dataset, "ctl_dataset_id": ctl_dataset.id, }, ) diff --git a/tests/ops/fixtures/saas/rollbar_fixtures.py b/tests/ops/fixtures/saas/rollbar_fixtures.py index 3052ea37cd..586e23a9ec 100644 --- a/tests/ops/fixtures/saas/rollbar_fixtures.py +++ b/tests/ops/fixtures/saas/rollbar_fixtures.py @@ -108,7 +108,6 @@ def rollbar_dataset_config( data={ "connection_config_id": rollbar_connection_config.id, "fides_key": fides_key, - "dataset": rollbar_dataset, "ctl_dataset_id": ctl_dataset.id, }, ) diff --git a/tests/ops/fixtures/saas/salesforce_fixtures.py b/tests/ops/fixtures/saas/salesforce_fixtures.py index bd73487453..fdf5468ef1 100644 --- a/tests/ops/fixtures/saas/salesforce_fixtures.py +++ b/tests/ops/fixtures/saas/salesforce_fixtures.py @@ -120,7 +120,6 @@ def salesforce_dataset_config( data={ "connection_config_id": salesforce_connection_config.id, "fides_key": fides_key, - "dataset": salesforce_dataset, "ctl_dataset_id": ctl_dataset.id, }, ) diff --git a/tests/ops/fixtures/saas/segment_fixtures.py b/tests/ops/fixtures/saas/segment_fixtures.py index 387b84e82f..c3eeb72adf 100644 --- a/tests/ops/fixtures/saas/segment_fixtures.py +++ b/tests/ops/fixtures/saas/segment_fixtures.py @@ -110,7 +110,6 @@ def segment_dataset_config( data={ "connection_config_id": segment_connection_config.id, "fides_key": fides_key, - "dataset": segment_dataset, "ctl_dataset_id": ctl_dataset.id, }, ) diff --git a/tests/ops/fixtures/saas/sendgrid_fixtures.py b/tests/ops/fixtures/saas/sendgrid_fixtures.py index 45382f873e..a1a6120ec2 100644 --- a/tests/ops/fixtures/saas/sendgrid_fixtures.py +++ b/tests/ops/fixtures/saas/sendgrid_fixtures.py @@ -103,7 +103,6 @@ def sendgrid_dataset_config( data={ "connection_config_id": sendgrid_connection_config.id, "fides_key": fides_key, - "dataset": sendgrid_dataset, "ctl_dataset_id": ctl_dataset.id, }, ) diff --git a/tests/ops/fixtures/saas/sentry_fixtures.py b/tests/ops/fixtures/saas/sentry_fixtures.py index ee3b28c3fc..8b133b50a9 100644 --- a/tests/ops/fixtures/saas/sentry_fixtures.py +++ b/tests/ops/fixtures/saas/sentry_fixtures.py @@ -95,7 +95,6 @@ def sentry_dataset_config( data={ "connection_config_id": sentry_connection_config.id, "fides_key": fides_key, - "dataset": sentry_dataset, "ctl_dataset_id": ctl_dataset.id, }, ) diff --git a/tests/ops/fixtures/saas/shopify_fixtures.py b/tests/ops/fixtures/saas/shopify_fixtures.py index 629532e042..3d4b5e30af 100644 --- a/tests/ops/fixtures/saas/shopify_fixtures.py +++ b/tests/ops/fixtures/saas/shopify_fixtures.py @@ -104,7 +104,6 @@ def shopify_dataset_config( data={ "connection_config_id": shopify_connection_config.id, "fides_key": fides_key, - "dataset": shopify_dataset, "ctl_dataset_id": ctl_dataset.id, }, ) diff --git a/tests/ops/fixtures/saas/slack_enterprise_fixtures.py b/tests/ops/fixtures/saas/slack_enterprise_fixtures.py index ceb2ba3a17..94ca659318 100644 --- a/tests/ops/fixtures/saas/slack_enterprise_fixtures.py +++ b/tests/ops/fixtures/saas/slack_enterprise_fixtures.py @@ -98,7 +98,6 @@ def slack_enterprise_dataset_config( data={ "connection_config_id": slack_enterprise_connection_config.id, "fides_key": fides_key, - "dataset": slack_enterprise_dataset, "ctl_dataset_id": ctl_dataset.id, }, ) diff --git a/tests/ops/fixtures/saas/square_fixtures.py b/tests/ops/fixtures/saas/square_fixtures.py index dab36623a0..ca6679438b 100644 --- a/tests/ops/fixtures/saas/square_fixtures.py +++ b/tests/ops/fixtures/saas/square_fixtures.py @@ -113,7 +113,6 @@ def square_dataset_config( data={ "connection_config_id": square_connection_config.id, "fides_key": fides_key, - "dataset": square_dataset, "ctl_dataset_id": ctl_dataset.id, }, ) diff --git a/tests/ops/fixtures/saas/stripe_fixtures.py b/tests/ops/fixtures/saas/stripe_fixtures.py index d70c124039..3e775da57e 100644 --- a/tests/ops/fixtures/saas/stripe_fixtures.py +++ b/tests/ops/fixtures/saas/stripe_fixtures.py @@ -106,7 +106,6 @@ def stripe_dataset_config( data={ "connection_config_id": stripe_connection_config.id, "fides_key": fides_key, - "dataset": stripe_dataset, "ctl_dataset_id": ctl_dataset.id, }, ) diff --git a/tests/ops/fixtures/saas/twilio_conversations_fixtures.py b/tests/ops/fixtures/saas/twilio_conversations_fixtures.py index 9236e8a4d7..f55015bb5f 100644 --- a/tests/ops/fixtures/saas/twilio_conversations_fixtures.py +++ b/tests/ops/fixtures/saas/twilio_conversations_fixtures.py @@ -117,7 +117,6 @@ def twilio_conversations_dataset_config( data={ "connection_config_id": twilio_conversations_connection_config.id, "fides_key": fides_key, - "dataset": twilio_conversations_dataset, "ctl_dataset_id": ctl_dataset.id, }, ) @@ -169,7 +168,6 @@ def twilio_postgres_dataset_config( data={ "connection_config_id": connection_config.id, "fides_key": fides_key, - "dataset": twilio_postgres_dataset, "ctl_dataset_id": ctl_dataset.id, }, ) diff --git a/tests/ops/fixtures/saas/zendesk_fixtures.py b/tests/ops/fixtures/saas/zendesk_fixtures.py index 4cfebe0cf0..b813dd736b 100644 --- a/tests/ops/fixtures/saas/zendesk_fixtures.py +++ b/tests/ops/fixtures/saas/zendesk_fixtures.py @@ -102,7 +102,6 @@ def zendesk_dataset_config( data={ "connection_config_id": zendesk_connection_config.id, "fides_key": fides_key, - "dataset": zendesk_dataset, "ctl_dataset_id": ctl_dataset.id, }, ) diff --git a/tests/ops/fixtures/saas_example_fixtures.py b/tests/ops/fixtures/saas_example_fixtures.py index 1c1d4c793d..ce543a78c1 100644 --- a/tests/ops/fixtures/saas_example_fixtures.py +++ b/tests/ops/fixtures/saas_example_fixtures.py @@ -142,7 +142,6 @@ def saas_example_dataset_config( data={ "connection_config_id": saas_example_connection_config.id, "fides_key": fides_key, - "dataset": saas_example_dataset, "ctl_dataset_id": ctl_dataset.id, }, ) @@ -169,7 +168,6 @@ def saas_external_example_dataset_config( data={ "connection_config_id": saas_external_example_connection_config.id, "fides_key": fides_key, - "dataset": saas_external_example_dataset, "ctl_dataset_id": ctl_dataset.id, }, ) diff --git a/tests/ops/fixtures/snowflake_fixtures.py b/tests/ops/fixtures/snowflake_fixtures.py index 2b979cd1c6..b0be5526ca 100644 --- a/tests/ops/fixtures/snowflake_fixtures.py +++ b/tests/ops/fixtures/snowflake_fixtures.py @@ -75,7 +75,6 @@ def snowflake_example_test_dataset_config( data={ "connection_config_id": snowflake_connection_config.id, "fides_key": fides_key, - "dataset": dataset, "ctl_dataset_id": ctl_dataset.id, }, ) diff --git a/tests/ops/models/test_datasetconfig.py b/tests/ops/models/test_datasetconfig.py index 4c84a6824f..de4d9a7a85 100644 --- a/tests/ops/models/test_datasetconfig.py +++ b/tests/ops/models/test_datasetconfig.py @@ -22,7 +22,6 @@ def test_create_dataset( data={ "connection_config_id": connection_config.id, "fides_key": postgres_dataset["fides_key"], - "dataset": postgres_dataset, "ctl_dataset_id": ctl_dataset.id, }, ) @@ -32,13 +31,13 @@ def test_create_dataset( assert dataset_config.connection_config_id == connection_config.id assert dataset_config.fides_key == postgres_dataset["fides_key"] - assert dataset_config.dataset["fides_key"] == postgres_dataset["fides_key"] - assert len(dataset_config.dataset["collections"]) == 11 + assert dataset_config.ctl_dataset.fides_key == ctl_dataset.fides_key + assert len(dataset_config.ctl_dataset.collections) == 1 assert dataset_config.created_at is not None orig_updated = dataset_config.updated_at assert orig_updated is not None - dataset_config.dataset["description"] = "Updated description" + dataset_config.fides_key = "new fides key" dataset_config.save(db=db) assert dataset_config.updated_at is not None assert dataset_config.updated_at > orig_updated @@ -175,8 +174,8 @@ def test_validate_dataset_reference_invalid(db: Session, dataset_config: Dataset Test that various types of invalid references to datasets raise expected errors """ dataset_key = "fake_dataset" - collection_name = dataset_config.dataset["collections"][0]["name"] - field_name = dataset_config.dataset["collections"][0]["fields"][0]["name"] + collection_name = dataset_config.ctl_dataset.collections[0]["name"] + field_name = dataset_config.ctl_dataset.collections[0]["fields"][0]["name"] dsr = FidesDatasetReference( dataset=dataset_key, field=f"{collection_name}.{field_name}" ) @@ -273,13 +272,6 @@ def test_no_existing_dataset_config_or_ctl_dataset( assert ctl_dataset.data_categories == postgres_dataset.get("data_categories") assert ctl_dataset.collections is not None - dataset = dataset_config.dataset - assert dataset["description"] == postgres_dataset["description"] - assert ( - dataset.get("organization_fides_key") is None - ), "Existing behavior, DatasetConfig.dataset not validated by Fideslang Dataset first" - assert dataset["collections"] is not None - dataset_config.delete(db) ctl_dataset.delete(db) @@ -346,13 +338,6 @@ def test_no_existing_dataset_config_but_ctl_dataset_exists( assert ctl_dataset.data_categories is None assert ctl_dataset.collections is not None - dataset = dataset_config.dataset - assert dataset["description"] == "New Dataset Description" - assert ( - dataset.get("organization_fides_key") is None - ), "Existing behavior, DatasetConfig.dataset not validated by Fideslang Dataset first" - assert dataset["collections"] is not None - dataset_config.delete(db) ctl_dataset.delete(db) @@ -408,11 +393,3 @@ def test_existing_dataset_config_and_ctl_dataset(self, dataset_config, db): assert updated_ctl_dataset.name == "New Dataset Name", "Updated name" assert updated_ctl_dataset.collections is not None - - dataset = updated_dataset_config.dataset - - assert dataset["description"] == "New Dataset Description" - assert ( - dataset.get("organization_fides_key") is None - ), "Existing behavior, DatasetConfig.dataset not validated by Fideslang Dataset first" - assert dataset["collections"] is not None