Unified Fides Resources Feature

.fides/db_dataset.yml

@@ -430,7 +430,7 @@ dataset:
       data_categories:
       - system.operations
       data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
-    - name: fidesctl_meta
+    - name: fides_meta
       data_categories:
       - system.operations
       data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
@@ -1025,10 +1025,10 @@ dataset:
     - name: connection_config_id
       data_categories: [system.operations]
       data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
-    - name: created_at
-      data_categories: [system.operations]
+    - name: ctl_dataset_id
+      data_categories: [ system.operations ]
       data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
-    - name: dataset
+    - name: created_at
       data_categories: [system.operations]
       data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
     - name: fides_key
@@ -1514,4 +1514,4 @@ dataset:
       description: 'The name of the organization this Fides deployment belongs to'
       data_categories:
       - user.workplace
-      data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+      data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified

.fides/fides.toml

@@ -57,4 +57,4 @@ task_default_queue = "fides"
 task_always_eager = true
 
 [notifications]
-notification_service_type = "mailgun"
+notification_service_type = "mailgun"

.fides/redis_dataset.yml

@@ -7,15 +7,11 @@ dataset:
         fields:
           - name: EN_ACCESS_GRAPH__<privacy_request_id>
             description: This graph is summarized and sent to Fideslog to create high level insight into how graphs change between retries to inform future features.
-            data_categories:
-            - system.operations
             data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
             fidesops_meta:
               data_type: object # Stores an encrypted representation of the fidesops graph that executes the privacy requests.
             fields:
               - name: <dataset_name>:<collection_name>  # The current collection
-                data_categories:
-                - system.operations
                 data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
                 fidesops_meta:
                   data_type: object

CHANGELOG.md

@@ -35,6 +35,7 @@ The types of changes are:
 * Adds support for Twilio email service (Sendgrid) [#2154](https://github.com/ethyca/fides/pull/2154)
 * Access and erasure support for Recharge [#1709](https://github.com/ethyca/fides/pull/1709)
 * Access and erasure support for Friendbuy Nextgen [#2085](https://github.com/ethyca/fides/pull/2085)
+* Unified Fides Resources: Added a dataset dropdown selector when configuring a connector to link an existing dataset to the connector configuration. [#2162](https://github.com/ethyca/fides/pull/2162)
 
 ### Changed
 
@@ -45,6 +46,11 @@ The types of changes are:
 * Made privacy declarations optional when adding systems manually - [#2173](https://github.com/ethyca/fides/pull/2173)
 * Dynamic imports of custom overrides and SaaS test fixtures [#2169](https://github.com/ethyca/fides/pull/2169)
 * Added `AuthenticatedClient` to custom request override interface [#2171](https://github.com/ethyca/fides/pull/2171)
+* Unified Fides Resources: Remove several fidesops schemas for DSR's in favor of updated Fideslang schemas [#2009](https://github.com/ethyca/fides/pull/2009)
+* Unified Fides Resources: New datasetconfig.ctl_dataset_id field to unify fides dataset resources [#2046](https://github.com/ethyca/fides/pull/2046)
+* Unified Fides Resources: Remove DatasetConfig.dataset field [#2096](https://github.com/ethyca/fides/pull/2096)
+* Unified Fides Resources: Update UI dataset config routes to use new unified routes [#2113](https://github.com/ethyca/fides/pull/2113)
+* Unified Fides Resources: Validate request body on crud endpoints on upsert. Validate dataset data categories before save. [#2134](https://github.com/ethyca/fides/pull/2134/)
 
 ### Removed
 
@@ -64,6 +70,7 @@ The types of changes are:
 
 * Remove "Create New System" button when viewing systems. All systems can now be created via the "Add systems" button on the home page. [#2132](https://github.com/ethyca/fides/pull/2132)
 
+
 ## [2.4.0](https://github.com/ethyca/fides/compare/2.3.1...2.4.0)
 
 ### Developer Experience
@@ -750,4 +757,4 @@ The types of changes are:
 * Potential Duplication on Data Map
 * Exceptions are no longer raised when sending `AnalyticsEvent`s on Windows
 * Running `fidesctl init` now generates a `server_host` and `server_protocol`
-  rather than `server_url`
+  rather than `server_url`

Dockerfile

@@ -26,6 +26,7 @@ RUN apt-get update && \
     g++ \
     gnupg \
     gcc \
+    git \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 

clients/admin-ui/cypress/e2e/connectors.cy.ts

@@ -0,0 +1,131 @@
+describe("Connectors", () => {
+  beforeEach(() => {
+    cy.login();
+  });
+  describe("Configuring connectors", () => {
+    beforeEach(() => {
+      cy.intercept("GET", "/api/v1/connection*", {
+        fixture: "connectors/list.json",
+      }).as("getConnectors");
+      cy.intercept("GET", "/api/v1/connection_type*", {
+        fixture: "connectors/connection_types.json",
+      }).as("getConnectionTypes");
+      cy.intercept("GET", "/api/v1/connection/postgres_connector", {
+        fixture: "connectors/postgres_connector.json",
+      }).as("getPostgresConnector");
+      cy.intercept("GET", "/api/v1/connection_type/postgres/secret", {
+        fixture: "connectors/postgres_secret.json",
+      }).as("getPostgresConnectorSecret");
+      cy.intercept(
+        "GET",
+        "/api/v1/connection/postgres_connector/datasetconfig",
+        {
+          fixture: "connectors/datasetconfig.json",
+        }
+      ).as("getPostgresConnectorDatasetconfig");
+
+      cy.intercept("POST", "/api/v1/dataset/upsert", { body: {} }).as(
+        "upsertDataset"
+      );
+      cy.intercept(
+        "PATCH",
+        "/api/v1/connection/postgres_connector/datasetconfig",
+        { body: {} }
+      ).as("patchDatasetconfig");
+      cy.intercept("GET", "/api/v1/dataset", { fixture: "datasets.json" }).as(
+        "getDatasets"
+      );
+    });
+
+    it("Should show data store connections and view configuration", () => {
+      cy.visit("/datastore-connection");
+      cy.getByTestId("connection-grid-item-mongodb_connector");
+      cy.getByTestId("connection-grid-item-postgres_connector").within(() => {
+        cy.getByTestId("connection-menu-btn").click();
+      });
+      cy.getByTestId("connection-menu-postgres_connector").within(() => {
+        cy.getByTestId("configure-btn").click();
+      });
+      cy.getByTestId("input-name").should("have.value", "postgres_connector");
+    });
+
+    it("Should allow saving a dataset configuration via dropdown", () => {
+      cy.visit("/datastore-connection/postgres_connector");
+      cy.getByTestId("tab-Dataset configuration").click();
+      cy.wait("@getPostgresConnectorDatasetconfig");
+
+      // The yaml editor will start off disabled
+      cy.getByTestId("save-yaml-btn").should("be.disabled");
+      // The dataset dropdown selector should have the value of the existing connected dataset
+      cy.getByTestId("save-dataset-link-btn").should("be.enabled");
+      cy.getByTestId("dataset-selector").should(
+        "have.value",
+        "postgres_example_test_dataset"
+      );
+
+      // Change the linked dataset
+      cy.getByTestId("dataset-selector").select("demo_users_dataset_2");
+      cy.getByTestId("save-dataset-link-btn").click();
+
+      cy.wait("@patchDatasetconfig").then((interception) => {
+        expect(interception.request.body).to.eql([
+          {
+            fides_key: "postgres_example_test_dataset",
+            ctl_dataset_fides_key: "demo_users_dataset_2",
+          },
+        ]);
+      });
+    });
+
+    it("Should allow saving a dataset configuration via yaml", () => {
+      cy.visit("/datastore-connection/postgres_connector");
+      cy.getByTestId("tab-Dataset configuration").click();
+      cy.wait("@getPostgresConnectorDatasetconfig");
+
+      // Unset the linked dataset, which should switch the save button enable-ness
+      cy.getByTestId("dataset-selector").select("Select");
+      cy.getByTestId("save-dataset-link-btn").should("be.disabled");
+      // The monaco yaml editor takes a bit to load
+      // eslint-disable-next-line cypress/no-unnecessary-waiting
+      cy.wait(1000);
+      cy.getByTestId("save-yaml-btn").click();
+
+      // Click past the confirmation modal
+      cy.getByTestId("confirmation-modal");
+      cy.getByTestId("continue-btn").click();
+
+      cy.wait("@upsertDataset").then((interception) => {
+        expect(interception.request.body.length).to.eql(1);
+        expect(interception.request.body[0].fides_key).to.eql(
+          "postgres_example_test_dataset"
+        );
+      });
+      cy.wait("@patchDatasetconfig").then((interception) => {
+        expect(interception.request.body).to.eql([
+          {
+            fides_key: "postgres_example_test_dataset",
+            ctl_dataset_fides_key: "postgres_example_test_dataset",
+          },
+        ]);
+      });
+    });
+
+    it("Should not show the dataset selector if no datasets exist", () => {
+      cy.intercept("GET", "/api/v1/dataset", { body: [] }).as("getDatasets");
+      cy.intercept(
+        "GET",
+        "/api/v1/connection/postgres_connector/datasetconfig",
+        {
+          body: {
+            items: [],
+          },
+        }
+      ).as("getEmptyPostgresConnectorDatasetconfig");
+
+      cy.visit("/datastore-connection/postgres_connector");
+      cy.getByTestId("tab-Dataset configuration").click();
+      cy.wait("@getEmptyPostgresConnectorDatasetconfig");
+      cy.getByTestId("dataset-selector-section").should("not.exist");
+    });
+  });
+});

clients/admin-ui/cypress/fixtures/connectors/connection_types.json

@@ -0,0 +1,67 @@
+{
+  "items": [
+    {
+      "identifier": "bigquery",
+      "type": "database",
+      "human_readable": "BigQuery",
+      "encoded_icon": null
+    },
+    {
+      "identifier": "mariadb",
+      "type": "database",
+      "human_readable": "MariaDB",
+      "encoded_icon": null
+    },
+    {
+      "identifier": "mongodb",
+      "type": "database",
+      "human_readable": "MongoDB",
+      "encoded_icon": null
+    },
+    {
+      "identifier": "mssql",
+      "type": "database",
+      "human_readable": "Microsoft SQL Server",
+      "encoded_icon": null
+    },
+    {
+      "identifier": "mysql",
+      "type": "database",
+      "human_readable": "MySQL",
+      "encoded_icon": null
+    },
+    {
+      "identifier": "postgres",
+      "type": "database",
+      "human_readable": "PostgreSQL",
+      "encoded_icon": null
+    },
+    {
+      "identifier": "redshift",
+      "type": "database",
+      "human_readable": "Amazon Redshift",
+      "encoded_icon": null
+    },
+    {
+      "identifier": "snowflake",
+      "type": "database",
+      "human_readable": "Snowflake",
+      "encoded_icon": null
+    },
+    {
+      "identifier": "timescale",
+      "type": "database",
+      "human_readable": "TimescaleDB",
+      "encoded_icon": null
+    },
+    {
+      "identifier": "manual_webhook",
+      "type": "manual",
+      "human_readable": "Manual Webhook",
+      "encoded_icon": null
+    }
+  ],
+  "total": 10,
+  "page": 1,
+  "size": 50
+}