diff --git a/.fides/db_dataset.yml b/.fides/db_dataset.yml index 001fabf84a..352e7a8f74 100644 --- a/.fides/db_dataset.yml +++ b/.fides/db_dataset.yml @@ -376,6 +376,10 @@ dataset: data_categories: - system.operations data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: active + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified - name: ctl_data_qualifiers data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified fields: @@ -419,6 +423,10 @@ dataset: data_categories: - system.operations data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: active + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified - name: ctl_data_subjects data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified fields: @@ -470,6 +478,10 @@ dataset: data_categories: - system.operations data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: active + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified - name: ctl_data_uses data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified fields: @@ -536,6 +548,10 @@ dataset: data_categories: - system.operations data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: active + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified - name: ctl_datasets data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified fields: @@ -2204,6 +2220,10 @@ dataset: data_categories: - system.operations data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: served_notice_history_id + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified - name: userregistration description: 'Records the registration status of this Fides deployment' data_categories: null @@ -2240,4 +2260,167 @@ dataset: description: 'The name of the organization this Fides deployment belongs to' data_categories: - user.workplace + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: cookies + description: 'Fides Generated Description for Table: cookies' + data_categories: [] + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + fields: + - name: created_at + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: domain + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: id + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: name + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: path + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: privacy_declaration_id + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: system_id + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: updated_at + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: lastservednotice + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + fields: + - name: created_at + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: fides_user_device_provided_identity_id + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: id + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: privacy_notice_history_id + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: privacy_notice_id + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: provided_identity_id + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: served_notice_history_id + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: updated_at + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: servednoticehistory + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + fields: + - name: acknowledge_mode + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: anonymized_ip_address + data_categories: + - user + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: created_at + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: updated_at + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: email + data_categories: + - user + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: fides_user_device + data_categories: + - user + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: fides_user_device_provided_identity_id + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: hashed_email + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: hashed_fides_user_device + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: hashed_phone_number + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: id + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: phone_number + data_categories: + - user + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: privacy_experience_config_history_id + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: privacy_experience_id + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: privacy_notice_history_id + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: provided_identity_id + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: request_origin + data_categories: + - user + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: serving_component + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: updated_at + description: 'Fides Generated Description for Column: updated_at' + data_categories: [ ] + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: url_recorded + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: user_agent + data_categories: + - user + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: user_geography + data_categories: + - user data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified \ No newline at end of file diff --git a/.fides/redis_dataset.yml b/.fides/redis_dataset.yml index 56f66e7bf5..c2c3831df3 100644 --- a/.fides/redis_dataset.yml +++ b/.fides/redis_dataset.yml @@ -20,6 +20,17 @@ dataset: data_categories: [system.operations] fidesops_meta: data_type: string[] # List of edges between the upstream collection and the current collection + - name: EN_DATA_USE_MAP__ + description: This map of traversed `Collection`s to associated `DataUse`s is stored and retrieved to be included in access request output packages. + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + fidesops_meta: + data_type: object # Dict mapping `Collection` addresses -> set of associated `DataUse`s + fields: + - name: : # `Collection` address + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + data_categories: [system.operations] + fidesops_meta: + data_type: string[] # set of `DataUse`s associated with this `Collection` - name: EN_EMAIL_INFORMATION________ # Usage: For building emails associated with email-connector datasets at the end of the privacy request. This encrypted raw information is retrieved from each relevant email-based collection and used to build a single email per email connector, with instructions on how to mask data on the given dataset. fidesops_meta: data_type: object # Stores how to locate and mask records for a given "email" collection. diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 2c6cfad0c6..24ea93ab79 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,5 +1,10 @@ Closes # +### Description Of Changes + +_Write some things here about the changes and any potential caveats_ + + ### Code Changes * [ ] _list your code changes here_ @@ -18,7 +23,3 @@ Closes # * [ ] Relevant Follow-Up Issues Created * [ ] Update `CHANGELOG.md` * [ ] For API changes, the [Postman collection](https://github.com/ethyca/fides/blob/main/docs/fides/docs/development/postman/Fides.postman_collection.json) has been updated - -### Description Of Changes - -_Write some things here about the changes and any potential caveats_ diff --git a/.github/workflows/backend_checks.yml b/.github/workflows/backend_checks.yml index 500e5a11f2..8a8ff4e7f9 100644 --- a/.github/workflows/backend_checks.yml +++ b/.github/workflows/backend_checks.yml @@ -12,7 +12,7 @@ on: env: IMAGE: ethyca/fides:local - DEFAULT_PYTHON_VERSION: "3.10.11" + DEFAULT_PYTHON_VERSION: "3.10.12" jobs: ############### @@ -41,7 +41,7 @@ jobs: strategy: matrix: # NOTE: These are the currently supported/tested Python Versions - python_version: ["3.8.16", "3.9.16", "3.10.11"] + python_version: ["3.8.17", "3.9.17", "3.10.12"] runs-on: ubuntu-latest steps: - name: Checkout @@ -183,7 +183,7 @@ jobs: needs: Check-Container-Startup strategy: matrix: - python_version: ["3.8.16", "3.9.16", "3.10.11"] + python_version: ["3.8.17", "3.9.17", "3.10.12"] test_selection: - "ctl-not-external" - "ops-unit" @@ -234,7 +234,7 @@ jobs: strategy: max-parallel: 1 # This prevents collisions in shared external resources matrix: - python_version: ["3.8.16", "3.9.16", "3.10.11"] + python_version: ["3.8.17", "3.9.17", "3.10.12"] runs-on: ubuntu-latest timeout-minutes: 20 # In PRs run with the "unsafe" label, or run on a "push" event to main @@ -260,8 +260,8 @@ jobs: env: SNOWFLAKE_FIDESCTL_PASSWORD: ${{ secrets.SNOWFLAKE_FIDESCTL_PASSWORD }} REDSHIFT_FIDESCTL_PASSWORD: ${{ secrets.REDSHIFT_FIDESCTL_PASSWORD }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_FIDESCTL_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_FIDESCTL_ACCESS_KEY }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_CTL_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_CTL_SECRET_ACCESS_KEY }} OKTA_CLIENT_TOKEN: ${{ secrets.OKTA_FIDESCTL_CLIENT_TOKEN }} AWS_DEFAULT_REGION: us-east-1 BIGQUERY_CONFIG: ${{ secrets.BIGQUERY_CONFIG }} @@ -274,7 +274,7 @@ jobs: strategy: max-parallel: 1 # This prevents collisions in shared external resources matrix: - python_version: ["3.8.16", "3.9.16", "3.10.11"] + python_version: ["3.8.17", "3.9.17", "3.10.12"] runs-on: ubuntu-latest timeout-minutes: 20 # In PRs run with the "unsafe" label, or run on a "push" event to main @@ -320,7 +320,7 @@ jobs: strategy: max-parallel: 1 # This prevents collisions in shared external resources matrix: - python_version: ["3.8.16", "3.9.16", "3.10.11"] + python_version: ["3.8.17", "3.9.17", "3.10.12"] steps: - name: Download container uses: actions/download-artifact@v3 diff --git a/.github/workflows/cli_checks.yml b/.github/workflows/cli_checks.yml index 2b39411fc2..161648fd8d 100644 --- a/.github/workflows/cli_checks.yml +++ b/.github/workflows/cli_checks.yml @@ -14,7 +14,7 @@ on: - "main" env: - DEFAULT_PYTHON_VERSION: "3.10.11" + DEFAULT_PYTHON_VERSION: "3.10.12" jobs: # Basic smoke test of a local install of the fides Python CLI diff --git a/CHANGELOG.md b/CHANGELOG.md index a7be3ffb29..05541a565c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,33 +15,102 @@ The types of changes are: - `Fixed` for any bug fixes. - `Security` in case of vulnerabilities. -## [Unreleased](https://github.com/ethyca/fides/compare/2.15.0...main) +## [Unreleased](https://github.com/ethyca/fides/compare/2.16.0...main) ### Added -- Included optional env vars to have postgres or Redshift connected via bastion host [#3374](https://github.com/ethyca/fides/pull/3374/) -- Support for acknowledge button for notice-only Privacy Notices and to disable toggling them off [#3546](https://github.com/ethyca/fides/pull/3546) -- HTML format for privacy request storage destinations [#3427](https://github.com/ethyca/fides/pull/3427) +- Tab component for `fides-js` [#3782](https://github.com/ethyca/fides/pull/3782) +- Prefetches API calls as part of Fides.js [#3698](https://github.com/ethyca/fides/pull/3698) + +### Developer Experience + +- Changed where db-dependent routers were imported to avoid dependency issues [#3741](https://github.com/ethyca/fides/pull/3741) ### Changed -- Removed `pyodbc` in favor of `pymssql` for handling SQL Server connections [#3435](https://github.com/ethyca/fides/pull/3435) +- Bumped supported Python versions to `3.10.12`, `3.9.17`, and `3.8.17` [#3733](https://github.com/ethyca/fides/pull/3733) +- Logging Updates [#3758](https://github.com/ethyca/fides/pull/3758) +- Add polyfill service to fides-js route [#3759](https://github.com/ethyca/fides/pull/3759) +- Show/hide integration values [#3775](https://github.com/ethyca/fides/pull/3775) +- Sort system cards alphabetically by name on "View systems" page [#3781](https://github.com/ethyca/fides/pull/3781) +- Update admin ui to use new integration delete route [#3785](https://github.com/ethyca/fides/pull/3785) + +### Removed + +- Removed "Custom field(s) successfully saved" toast [#3779](https://github.com/ethyca/fides/pull/3779) + +### Added + +- Record when consent is served [#3777](https://github.com/ethyca/fides/pull/3777) +- Add an `active` property to taxonomy elements [#3784](https://github.com/ethyca/fides/pull/3784) ### Fixed +- Privacy notice UI's list of possible regions now matches the backend's list [#3787](https://github.com/ethyca/fides/pull/3787) +- Admin UI "property does not existing" build issue [#3831](https://github.com/ethyca/fides/pull/3831) -- Fix race condition with consent modal link rendering [#3521](https://github.com/ethyca/fides/pull/3521) -- Hide custom fields section when there are no custom fields created [#3554](https://github.com/ethyca/fides/pull/3554) -- Disable connector dropdown in integration tab on save [#3552](https://github.com/ethyca/fides/pull/3552) -- Handles an edge case for non-existent identities with the Kustomer API [#3513](https://github.com/ethyca/fides/pull/3513) -- remove the configure privacy request tile from the home screen [#3555](https://github.com/ethyca/fides/pull/3555) +## [2.16.0](https://github.com/ethyca/fides/compare/2.15.1...2.16.0) + +### Added + +- Empty state for when there are no relevant privacy notices in the privacy center [#3640](https://github.com/ethyca/fides/pull/3640) +- GPC indicators in fides-js banner and modal [#3673](https://github.com/ethyca/fides/pull/3673) +- Include `data_use` and `data_category` metadata in `upload` of access results [#3674](https://github.com/ethyca/fides/pull/3674) +- Add enable/disable toggle to integration tab [#3593] (https://github.com/ethyca/fides/pull/3593) + +### Fixed + +- Render linebreaks in the Fides.js overlay descriptions, etc. [#3665](https://github.com/ethyca/fides/pull/3665) +- Broken link to Fides docs site on the About Fides page in Admin UI [#3643](https://github.com/ethyca/fides/pull/3643) +- Add Systems Applicable Filter to Privacy Experience List [#3654](https://github.com/ethyca/fides/pull/3654) +- Privacy center and fides-js now pass in `Unescape-Safestr` as a header so that special characters can be rendered properly [#3706](https://github.com/ethyca/fides/pull/3706) +- Fixed ValidationError for saving PrivacyPreferences [#3719](https://github.com/ethyca/fides/pull/3719) +- Fixed issue preventing ConnectionConfigs with duplicate names from saving [#3770](https://github.com/ethyca/fides/pull/3770) +- Fixed creating and editing manual integrations [#3772](https://github.com/ethyca/fides/pull/3772) +- Fix lingering integration artifacts by cascading deletes from System [#3771](https://github.com/ethyca/fides/pull/3771) ### Developer Experience -- Optimize GitHub workflows used for docker image publishing [#3526](https://github.com/ethyca/fides/pull/3526) +- Reorganized some `api.api.v1` code to avoid circular dependencies on `quickstart` [#3692](https://github.com/ethyca/fides/pull/3692) +- Treat underscores as special characters in user passwords [#3717](https://github.com/ethyca/fides/pull/3717) +- Allow Privacy Notices banner and modal to scroll as needed [#3713](https://github.com/ethyca/fides/pull/3713) +- Make malicious url test more robust to environmental differences [#3748](https://github.com/ethyca/fides/pull/3748) +- Ignore type checker on click decorators to bypass known issue with `click` version `8.1.4` [#3746](https://github.com/ethyca/fides/pull/3746) + +### Changed + +- Moved GPC preferences slightly earlier in Fides.js lifecycle [#3561](https://github.com/ethyca/fides/pull/3561) +- Changed results from clicking "Test connection" to be a toast instead of statically displayed on the page [#3700](https://github.com/ethyca/fides/pull/3700) +- Moved "management" tab from nav into settings icon in top right [#3701](https://github.com/ethyca/fides/pull/3701) +- Remove name and description fields from integration form [#3684](https://github.com/ethyca/fides/pull/3684) +- Update EU PrivacyNoticeRegion codes and allow experience filtering to drop back to country filtering if region not found [#3630](https://github.com/ethyca/fides/pull/3630) +- Fields with default fields are now flagged as required in the front-end [#3694](https://github.com/ethyca/fides/pull/3694) +- In "view systems", system cards can now be clicked and link to that system's `configure/[id]` page [#3734](https://github.com/ethyca/fides/pull/3734) +- Enable privacy notice and privacy experience feature flags by default [#3773](https://github.com/ethyca/fides/pull/3773) + +### Security +- Resolve Zip bomb file upload vulnerability [CVE-2023-37480](https://github.com/ethyca/fides/security/advisories/GHSA-g95c-2jgm-hqc6) +- Resolve SVG bomb (billion laughs) file upload vulnerability [CVE-2023-37481](https://github.com/ethyca/fides/security/advisories/GHSA-3rw2-wfc8-wmj5) + +## [2.15.1](https://github.com/ethyca/fides/compare/2.15.0...2.15.1) + +### Added +- Set `sslmode` to `prefer` if connecting to Redshift via ssh [#3685](https://github.com/ethyca/fides/pull/3685) + +### Changed +- Privacy center action cards are now able to expand to accommodate longer text [#3669](https://github.com/ethyca/fides/pull/3669) +- Update integration endpoint permissions [#3707](https://github.com/ethyca/fides/pull/3707) + +### Fixed +- Handle names with a double underscore when processing access and erasure requests [#3688](https://github.com/ethyca/fides/pull/3688) +- Allow Privacy Notices banner and modal to scroll as needed [#3713](https://github.com/ethyca/fides/pull/3713) + +### Security +- Resolve path traversal vulnerability in webserver API [CVE-2023-36827](https://github.com/ethyca/fides/security/advisories/GHSA-r25m-cr6v-p9hq) ## [2.15.0](https://github.com/ethyca/fides/compare/2.14.1...2.15.0) ### Added + - Privacy center can now render its consent values based on Privacy Notices and Privacy Experiences [#3411](https://github.com/ethyca/fides/pull/3411) - Add Google Tag Manager and Privacy Center ENV vars to sample app [#2949](https://github.com/ethyca/fides/pull/2949) - Add `notice_key` field to Privacy Notice UI form [#3403](https://github.com/ethyca/fides/pull/3403) @@ -61,6 +130,14 @@ The types of changes are: - Add new dataset route that has additional filters [#3558](https://github.com/ethyca/fides/pull/3558) - Update dataset dropdown to use new api filter [#3565](https://github.com/ethyca/fides/pull/3565) - Filter out saas datasets from the rest of the UI [#3568](https://github.com/ethyca/fides/pull/3568) +- Included optional env vars to have postgres or Redshift connected via bastion host [#3374](https://github.com/ethyca/fides/pull/3374/) +- Support for acknowledge button for notice-only Privacy Notices and to disable toggling them off [#3546](https://github.com/ethyca/fides/pull/3546) +- HTML format for privacy request storage destinations [#3427](https://github.com/ethyca/fides/pull/3427) +- Persistent message showing result and timestamp of last integration test to "Integrations" tab in system view [#3628](https://github.com/ethyca/fides/pull/3628) +- Access and erasure support for SurveyMonkey [#3590](https://github.com/ethyca/fides/pull/3590) +- New Cookies Table for storing cookies associated with systems and privacy declarations [#3572](https://github.com/ethyca/fides/pull/3572) +- `fides-js` and privacy center now delete cookies associated with notices that were opted out of [#3569](https://github.com/ethyca/fides/pull/3569) +- Cookie input field on system data use tab [#3571](https://github.com/ethyca/fides/pull/3571) ### Fixed @@ -73,6 +150,20 @@ The types of changes are: - Fix bug where `fides-js` toggles were not reflecting changes from rejecting or accepting all notices [#3522](https://github.com/ethyca/fides/pull/3522) - Remove the `fides-js` banner from tab order when it is hidden and move the overlay components to the top of the tab order. [#3510](https://github.com/ethyca/fides/pull/3510) - Fix bug where `fides-js` toggle states did not always initialize properly [#3597](https://github.com/ethyca/fides/pull/3597) +- Fix race condition with consent modal link rendering [#3521](https://github.com/ethyca/fides/pull/3521) +- Hide custom fields section when there are no custom fields created [#3554](https://github.com/ethyca/fides/pull/3554) +- Disable connector dropdown in integration tab on save [#3552](https://github.com/ethyca/fides/pull/3552) +- Handles an edge case for non-existent identities with the Kustomer API [#3513](https://github.com/ethyca/fides/pull/3513) +- remove the configure privacy request tile from the home screen [#3555](https://github.com/ethyca/fides/pull/3555) +- Updated Privacy Experience Safe Strings Serialization [#3600](https://github.com/ethyca/fides/pull/3600/) +- Only create default experience configs on startup, not update [#3605](https://github.com/ethyca/fides/pull/3605) +- Update to latest asyncpg dependency to avoid build error [#3614](https://github.com/ethyca/fides/pull/3614) +- Fix bug where editing a data use on a system could delete existing data uses [#3627](https://github.com/ethyca/fides/pull/3627) +- Restrict Privacy Center debug logging to development-only [#3638](https://github.com/ethyca/fides/pull/3638) +- Fix bug where linking an integration would not update the tab when creating a new system [#3662](https://github.com/ethyca/fides/pull/3662) +- Fix dataset yaml not properly reflecting the dataset in the dropdown of system integrations tab [#3666](https://github.com/ethyca/fides/pull/3666) +- Fix privacy notices not being able to be edited via the UI after the addition of the `cookies` field [#3670](https://github.com/ethyca/fides/pull/3670) +- Add a transform in the case of `null` name fields in privacy declarations for the data use forms [#3683](https://github.com/ethyca/fides/pull/3683) ### Changed @@ -89,17 +180,25 @@ The types of changes are: - Update `fideslang` to `1.4.1` to allow arbitrary nested metadata on `System`s and `Dataset`s `meta` property [#3463](https://github.com/ethyca/fides/pull/3463) - Remove form validation to allow both email & phone inputs for consent requests [#3529](https://github.com/ethyca/fides/pull/3529) - Removed dataset dropdown from saas connector configuration [#3563](https://github.com/ethyca/fides/pull/3563) +- Removed `pyodbc` in favor of `pymssql` for handling SQL Server connections [#3435](https://github.com/ethyca/fides/pull/3435) +- Only create a PrivacyRequest when saving consent if at least one notice has system-wide enforcement [#3626](https://github.com/ethyca/fides/pull/3626) +- Increased the character limit for the `SafeStr` type from 500 to 32000 [#3647](https://github.com/ethyca/fides/pull/3647) +- Changed "connection" to "integration" on system view and edit pages [#3659](https://github.com/ethyca/fides/pull/3659) ### Developer Experience - Add ability to pass ENV vars to both privacy center and sample app during `fides deploy` via `.env` [#2949](https://github.com/ethyca/fides/pull/2949) - Handle an edge case when generating tags that finds them out of sequence [#3405](https://github.com/ethyca/fides/pull/3405) - Add support for pushing `prerelease` and `rc` tagged images to Dockerhub [#3474](https://github.com/ethyca/fides/pull/3474) +- Optimize GitHub workflows used for docker image publishing [#3526](https://github.com/ethyca/fides/pull/3526) ### Removed - Removed the deprecated `system_dependencies` from `System` resources, migrating to `egress` [#3285](https://github.com/ethyca/fides/pull/3285) +### Docs + +- Updated developer docs for ARM platform users related to `pymssql` [#3615](https://github.com/ethyca/fides/pull/3615) ## [2.14.1](https://github.com/ethyca/fides/compare/2.14.0...2.14.1) @@ -113,7 +212,6 @@ The types of changes are: - Update privacy centre email and phone validation to allow for both to be blank [#3432](https://github.com/ethyca/fides/pull/3432) - ## [2.14.0](https://github.com/ethyca/fides/compare/2.13.0...2.14.0) ### Added @@ -165,7 +263,6 @@ The types of changes are: - Remove `fides export` command and backing code [#3256](https://github.com/ethyca/fides/pull/3256) - ## [2.13.0](https://github.com/ethyca/fides/compare/2.12.1...2.13.0) ### Added @@ -198,7 +295,7 @@ The types of changes are: ### Developer Experience -- Use prettier to format *all* source files in client packages [#3240](https://github.com/ethyca/fides/pull/3240) +- Use prettier to format _all_ source files in client packages [#3240](https://github.com/ethyca/fides/pull/3240) ### Deprecated @@ -263,7 +360,6 @@ The types of changes are: - Fixed unit tests for saas connector type endpoints now that we have >50 [#3101](https://github.com/ethyca/fides/pull/3101) - Fixed nox docs link [#3121](https://github.com/ethyca/fides/pull/3121/files) - ### Developer Experience - Update fides deploy to use a new database.load_samples setting to initialize sample Systems, Datasets, and Connections for testing [#3102](https://github.com/ethyca/fides/pull/3102) @@ -271,7 +367,6 @@ The types of changes are: - Add smoke tests for consent management [#3158](https://github.com/ethyca/fides/pull/3158) - Added nox command that opens dev docs [#3082](https://github.com/ethyca/fides/pull/3082) - ## [2.11.0](https://github.com/ethyca/fides/compare/2.10.0...2.11.0) ### Added @@ -546,6 +641,7 @@ The types of changes are: - This PR contains a migration that deletes duplicate users and keeps the oldest original account. - Update Logos for shipped connectors [#2464](https://github.com/ethyca/fides/pull/2587) - Search field on privacy request page isn't working [#2270](https://github.com/ethyca/fides/pull/2595) +- Fix connection dropdown in integration table to not be disabled add system creation [#3589](https://github.com/ethyca/fides/pull/3589) ### Developer Experience diff --git a/Dockerfile b/Dockerfile index e880aa91df..47905e6676 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # If you update this, also update `DEFAULT_PYTHON_VERSION` in the GitHub workflow files -ARG PYTHON_VERSION="3.10.11" +ARG PYTHON_VERSION="3.10.12" ######################### ## Compile Python Deps ## ######################### diff --git a/MANIFEST.in b/MANIFEST.in index ee1575c366..c419da707b 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,7 +4,7 @@ include requirements.txt include dev-requirements.txt include dangerous-requirements.txt include versioneer.py -include src/fides/api/alembic.ini +include src/fides/api/alembic/alembic.ini include src/fides/_version.py include src/fides/py.typed diff --git a/README.md b/README.md index 2057257c23..2eeeab1516 100644 --- a/README.md +++ b/README.md @@ -22,11 +22,26 @@ Fides (pronounced */fee-dhez/*, from Latin: Fidēs) is an open-source privacy en In order to get started quickly with Fides, a sample project is bundled within the Fides CLI that will set up a server, privacy center, and a sample application for you to experiment with. -#### Minimum requirements +#### Minimum requirements (for all platforms) * [Docker](https://www.docker.com/products/docker-desktop) (version 20.10.11 or later) * [Python](https://www.python.org/downloads/) (version 3.8 through 3.10) +#### Additional requirements (for ARM Mac Users) + +Due to platform differences, the following dependencies and steps are also required: + +```bash +brew install freetds openssl +``` + +**Add the following to your run commands (i.e. `.zshrc`), updating any path/versions to match yours** + +```bash +export LDFLAGS="-L/opt/homebrew/Cellar/freetds/1.3.18/lib -L/opt/homebrew/Cellar/openssl@1.1/1.1.1u/lib"` +export CFLAGS="-I/opt/homebrew/Cellar/freetds/1.3.18/include" +``` + #### Download and install Fides You can easily download and install Fides using `pip`. Run the following command to get started: diff --git a/clients/admin-ui/cypress/e2e/nav-bar.cy.ts b/clients/admin-ui/cypress/e2e/nav-bar.cy.ts index b9ec2ae7ec..890983ccfb 100644 --- a/clients/admin-ui/cypress/e2e/nav-bar.cy.ts +++ b/clients/admin-ui/cypress/e2e/nav-bar.cy.ts @@ -6,11 +6,10 @@ describe("Nav Bar", () => { it("renders all navigation links", () => { cy.visit("/"); - cy.get("nav a").should("have.length", 4); + cy.get("nav a").should("have.length", 3); cy.contains("nav a", "Home"); - cy.contains("nav a", "Privacy requests"); cy.contains("nav a", "Data map"); - cy.contains("nav a", "Management"); + cy.contains("nav a", "Privacy requests"); }); it("styles the active navigation link based on the current route", () => { @@ -22,18 +21,18 @@ describe("Nav Bar", () => { cy.contains("nav a", "Home") .should("have.css", "background-color") .should("eql", ACTIVE_COLOR); - cy.contains("nav a", "Management") + cy.contains("nav a", "Data map") .should("have.css", "background-color") .should("not.eql", ACTIVE_COLOR); // Navigate by clicking a nav link. - cy.contains("nav a", "Management").click(); + cy.contains("nav a", "Data map").click(); // The nav should update which page is active. cy.contains("nav a", "Home") .should("have.css", "background-color") .should("not.eql", ACTIVE_COLOR); - cy.contains("nav a", "Management") + cy.contains("nav a", "Data map") .should("have.css", "background-color") .should("eql", ACTIVE_COLOR); }); diff --git a/clients/admin-ui/cypress/e2e/organization.cy.ts b/clients/admin-ui/cypress/e2e/organization.cy.ts index 4cbfa0492e..2f38247c07 100644 --- a/clients/admin-ui/cypress/e2e/organization.cy.ts +++ b/clients/admin-ui/cypress/e2e/organization.cy.ts @@ -10,7 +10,7 @@ describe("Organization page", () => { it("can navigate to the Organization page", () => { cy.visit("/"); - cy.contains("nav a", "Management").click(); + cy.getByTestId("management-btn").click(); cy.contains("nav a", "Organization").click(); cy.getByTestId("organization-management"); }); diff --git a/clients/admin-ui/cypress/e2e/privacy-notices.cy.ts b/clients/admin-ui/cypress/e2e/privacy-notices.cy.ts index 15af445a26..8f3123e844 100644 --- a/clients/admin-ui/cypress/e2e/privacy-notices.cy.ts +++ b/clients/admin-ui/cypress/e2e/privacy-notices.cy.ts @@ -227,23 +227,41 @@ describe("Privacy notices", () => { }); it("can make an edit", () => { - cy.visit(`${PRIVACY_NOTICES_ROUTE}/${ESSENTIAL_NOTICE_ID}`); - cy.wait("@getNoticeDetail"); - const newName = "new name"; - cy.getByTestId("input-name").clear().type(newName); - // should not reflect the new name since this is the edit form - cy.getByTestId("input-notice_key").should("have.value", "essential"); - // but we can still update it - const newKey = "custom_key"; - cy.getByTestId("input-notice_key").clear().type(newKey); - - cy.getByTestId("save-btn").click(); - cy.wait("@patchNotices").then((interception) => { - const { body } = interception.request; - expect(body[0].name).to.eql(newName); - expect(body[0].notice_key).to.eql(newKey); + cy.fixture("privacy-notices/notice.json").then((notice) => { + cy.visit(`${PRIVACY_NOTICES_ROUTE}/${ESSENTIAL_NOTICE_ID}`); + cy.wait("@getNoticeDetail"); + const newName = "new name"; + cy.getByTestId("input-name").clear().type(newName); + // should not reflect the new name since this is the edit form + cy.getByTestId("input-notice_key").should("have.value", "essential"); + // but we can still update it + const newKey = "custom_key"; + cy.getByTestId("input-notice_key").clear().type(newKey); + + cy.getByTestId("save-btn").click(); + cy.wait("@patchNotices").then((interception) => { + const { body } = interception.request; + const expected = { + name: newName, + notice_key: newKey, + consent_mechanism: notice.consent_mechanism, + data_uses: notice.data_uses, + description: notice.description, + disabled: notice.disabled, + displayed_in_api: notice.displayed_in_api, + displayed_in_overlay: notice.displayed_in_overlay, + displayed_in_privacy_center: notice.displayed_in_privacy_center, + enforcement_level: notice.enforcement_level, + has_gpc_flag: notice.has_gpc_flag, + id: notice.id, + internal_description: notice.internal_description, + origin: notice.origin, + regions: notice.regions, + }; + expect(body[0]).to.eql(expected); + }); + cy.wait("@getNoticeDetail"); }); - cy.wait("@getNoticeDetail"); }); }); diff --git a/clients/admin-ui/cypress/e2e/systems.cy.ts b/clients/admin-ui/cypress/e2e/systems.cy.ts index ec7538c3dc..94a83b4121 100644 --- a/clients/admin-ui/cypress/e2e/systems.cy.ts +++ b/clients/admin-ui/cypress/e2e/systems.cy.ts @@ -111,6 +111,9 @@ describe("System management page", () => { it("Can step through the flow", () => { cy.fixture("systems/system.json").then((system) => { + cy.intercept("GET", "/api/v1/system/*", { + body: { ...system, privacy_declarations: [] }, + }).as("getDemoSystem"); // Fill in the describe form based on fixture data cy.visit(ADD_SYSTEMS_ROUTE); cy.getByTestId("manual-btn").click(); @@ -145,6 +148,7 @@ describe("System management page", () => { "@getDataSubjects", "@getDataUses", "@getFilteredDatasets", + "@getDemoSystem", ]); cy.getByTestId("new-declaration-form"); const declaration = system.privacy_declarations[0]; @@ -172,38 +176,49 @@ describe("System management page", () => { data_categories: declaration.data_categories, data_subjects: declaration.data_subjects, dataset_references: ["demo_users_dataset_2"], + cookies: [], + id: "", }); }); }); }); it("can render a warning when there is unsaved data", () => { - cy.visit(ADD_SYSTEMS_MANUAL_ROUTE); - cy.wait("@getSystems"); - cy.wait("@getConnectionTypes"); - cy.getByTestId("create-system-btn").click(); - cy.getByTestId("input-name").type("test"); - cy.getByTestId("input-fides_key").type("test"); - cy.getByTestId("save-btn").click(); - cy.wait("@postSystem"); + cy.fixture("systems/system.json").then((system) => { + cy.intercept("GET", "/api/v1/system/*", { + body: { ...system, privacy_declarations: [] }, + }).as("getDemoSystem"); + cy.visit(ADD_SYSTEMS_MANUAL_ROUTE); + cy.wait("@getSystems"); + cy.wait("@getConnectionTypes"); + cy.getByTestId("create-system-btn").click(); + cy.getByTestId("input-name").type(system.name); + cy.getByTestId("input-fides_key").type(system.fides_key); + cy.getByTestId("input-description").type(system.description); + cy.getByTestId("save-btn").click(); + cy.wait("@postSystem"); - // start typing a description - const description = "half formed thought"; - cy.getByTestId("input-description").type(description); - // then try navigating to the privacy declarations tab - cy.getByTestId("tab-Data uses").click(); - cy.getByTestId("confirmation-modal"); - // make sure canceling works - cy.getByTestId("cancel-btn").click(); - cy.getByTestId("input-description").should("have.value", description); - // now actually discard - cy.getByTestId("tab-Data uses").click(); - cy.getByTestId("continue-btn").click(); - // should load the privacy declarations page - cy.getByTestId("privacy-declaration-step"); - // navigate back - cy.getByTestId("tab-System information").click(); - cy.getByTestId("input-description").should("have.value", ""); + // start typing a description + const description = "half formed thought"; + cy.getByTestId("input-description").clear().type(description); + // then try navigating to the privacy declarations tab + cy.getByTestId("tab-Data uses").click(); + cy.getByTestId("confirmation-modal"); + // make sure canceling works + cy.getByTestId("cancel-btn").click(); + cy.getByTestId("input-description").should("have.value", description); + // now actually discard + cy.getByTestId("tab-Data uses").click(); + cy.getByTestId("continue-btn").click(); + // should load the privacy declarations page + cy.getByTestId("privacy-declaration-step"); + // navigate back and make sure description has the original description + cy.getByTestId("tab-System information").click(); + cy.getByTestId("input-description").should( + "have.value", + system.description + ); + }); }); }); }); @@ -277,6 +292,14 @@ describe("System management page", () => { cy.getByTestId("system-not-found"); }); + it("Can go to a system's edit page by clicking its card", () => { + cy.visit(SYSTEM_ROUTE); + cy.getByTestId("system-fidesctl_system").within(() => { + cy.getByTestId("system-box").click(); + }); + cy.url().should("contain", "/systems/configure/fidesctl_system"); + }); + it("Can go through the edit flow", () => { cy.getByTestId("system-fidesctl_system").within(() => { cy.getByTestId("more-btn").click(); @@ -309,6 +332,7 @@ describe("System management page", () => { const { body } = interception.request; expect(body.joint_controller.name).to.eql(controllerName); }); + cy.wait("@getFidesctlSystem"); // Switch to the Data Uses tab cy.getByTestId("tab-Data uses").click(); @@ -334,15 +358,15 @@ describe("System management page", () => { // edit the existing declaration cy.getByTestId("accordion-header-improve.system").click(); cy.getByTestId("improve.system-form").within(() => { - cy.getByTestId("input-data_subjects").type(`anonymous{enter}`); + cy.getByTestId("input-data_subjects").type(`customer{enter}`); cy.getByTestId("save-btn").click(); }); cy.wait("@putSystem").then((interception) => { const { body } = interception.request; expect(body.privacy_declarations.length).to.eql(1); expect(body.privacy_declarations[0].data_subjects).to.eql([ - "customer", "anonymous_user", + "customer", ]); }); cy.getByTestId("saved-indicator"); @@ -518,6 +542,72 @@ describe("System management page", () => { cy.getByTestId("toast-success-msg"); }); + it("can edit an accordion data use while persisting a newly added data use", () => { + cy.visit(`${SYSTEM_ROUTE}/configure/fidesctl_system`); + cy.getByTestId("tab-Data uses").click(); + cy.getByTestId("add-btn").click(); + cy.wait(["@getDataCategories", "@getDataSubjects", "@getDataUses"]); + + const newDeclaration = { + id: "pri_bf701ddb-1d05-48f9-913f-b5ff05b8f987", + name: "Second data use", + data_categories: ["user.biometric"], + data_use: "collect", + data_qualifier: + "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", + data_subjects: ["anonymous"], + dataset_references: [], + }; + // We need to update both the PUT and GET fixtures to make sure they return + // the data use we are adding. This is how we can get the form into a state + // where there are both "accordion" declarations and the one declaration + // in the new form + cy.fixture("systems/system.json").then((system) => { + const { privacy_declarations: declarations } = system; + const updatedSystem = { + ...system, + fides_key: "fidesctl_system", + privacy_declarations: [...declarations, newDeclaration], + }; + cy.intercept("PUT", "/api/v1/system*", { + body: updatedSystem, + }).as("putSystemWithAddedDataUse"); + cy.intercept("GET", "/api/v1/system/*", { + body: updatedSystem, + }).as("getSystemWithAddedDataUse"); + }); + + // Add one data use (one already exists) + cy.getByTestId("new-declaration-form").within(() => { + cy.getByTestId("input-data_use").type( + `${newDeclaration.data_use}{enter}` + ); + cy.getByTestId("input-name").type(newDeclaration.name); + cy.getByTestId("input-data_categories").type( + `${newDeclaration.data_categories[0]}{enter}` + ); + cy.getByTestId("input-data_subjects").type( + `${newDeclaration.data_subjects[0]}{enter}` + ); + cy.getByTestId("save-btn").click(); + cy.wait("@putSystemWithAddedDataUse") + .its("request.body.privacy_declarations") + .should("have.length", 2); + cy.wait("@getSystemWithAddedDataUse"); + }); + + // Edit the existing data use + cy.getByTestId("privacy-declaration-accordion").within(() => { + cy.getByTestId("accordion-header-improve.system").click(); + // Add a data subject + cy.getByTestId("input-data_subjects").type(`citizen{enter}`); + cy.getByTestId("save-btn").click(); + cy.wait("@putSystemWithAddedDataUse") + .its("request.body.privacy_declarations") + .should("have.length", 2); + }); + }); + describe("delete privacy declaration", () => { beforeEach(() => { cy.fixture("systems/systems_with_data_uses.json").then((systems) => { diff --git a/clients/admin-ui/cypress/e2e/taxonomy.cy.ts b/clients/admin-ui/cypress/e2e/taxonomy.cy.ts index 70921c53f1..5861fd56f7 100644 --- a/clients/admin-ui/cypress/e2e/taxonomy.cy.ts +++ b/clients/admin-ui/cypress/e2e/taxonomy.cy.ts @@ -8,7 +8,7 @@ describe("Taxonomy management page", () => { it("Can navigate to the taxonomy page", () => { cy.visit("/"); - cy.contains("nav a", "Management").click(); + cy.getByTestId("management-btn").click(); cy.contains("nav a", "Taxonomy").click(); cy.getByTestId("taxonomy-tabs"); cy.getByTestId("tab-Data Categories"); diff --git a/clients/admin-ui/cypress/fixtures/privacy-notices/list.json b/clients/admin-ui/cypress/fixtures/privacy-notices/list.json index 65c3c24ada..934666fa55 100644 --- a/clients/admin-ui/cypress/fixtures/privacy-notices/list.json +++ b/clients/admin-ui/cypress/fixtures/privacy-notices/list.json @@ -27,7 +27,7 @@ "description": "Ensures you are correctly notifying the user about your advertising practices and for appropriate locations, collecting the users consent preferences.", "internal_description": null, "origin": null, - "regions": ["eu_fr", "eu_ie"], + "regions": ["fr", "ie"], "consent_mechanism": "opt_in", "data_uses": ["advertising.first_party.contextual"], "enforcement_level": "system_wide", @@ -48,7 +48,7 @@ "description": "Ensures you are correctly notifying the user about your advertising practices and for appropriate locations, collecting the users consent preferences.", "internal_description": null, "origin": null, - "regions": ["eu_fr", "eu_ie"], + "regions": ["fr", "ie"], "consent_mechanism": "opt_in", "data_uses": ["collect"], "enforcement_level": "system_wide", @@ -69,7 +69,7 @@ "description": "This is for data processing activities that enhance the capability or features of your site but may not be strictly necessary.", "internal_description": null, "origin": null, - "regions": ["eu_fr", "eu_ie"], + "regions": ["fr", "ie"], "consent_mechanism": "opt_in", "data_uses": ["improve.system"], "enforcement_level": "system_wide", @@ -90,7 +90,7 @@ "description": "Notify the user about data processing activities that are essential to your services functionality. Typically consent is not required for this.", "internal_description": null, "origin": null, - "regions": ["eu_fr", "eu_ie"], + "regions": ["fr", "ie"], "consent_mechanism": "notice_only", "data_uses": ["provide.service"], "enforcement_level": "system_wide", diff --git a/clients/admin-ui/cypress/fixtures/privacy-notices/notice.json b/clients/admin-ui/cypress/fixtures/privacy-notices/notice.json index 4824ec44ae..b2e4fd85bc 100644 --- a/clients/admin-ui/cypress/fixtures/privacy-notices/notice.json +++ b/clients/admin-ui/cypress/fixtures/privacy-notices/notice.json @@ -4,7 +4,7 @@ "description": "Notify the user about data processing activities that are essential to your services functionality. Typically consent is not required for this.", "internal_description": null, "origin": null, - "regions": ["eu_fr", "eu_ie"], + "regions": ["fr", "ie"], "consent_mechanism": "notice_only", "data_uses": ["provide.service"], "enforcement_level": "not_applicable", diff --git a/clients/admin-ui/cypress/fixtures/systems/system.json b/clients/admin-ui/cypress/fixtures/systems/system.json index 5929eb18dd..09e930e287 100644 --- a/clients/admin-ui/cypress/fixtures/systems/system.json +++ b/clients/admin-ui/cypress/fixtures/systems/system.json @@ -16,7 +16,9 @@ "data_use": "improve.system", "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", "data_subjects": ["customer"], - "dataset_references": ["demo_users_dataset"] + "dataset_references": ["demo_users_dataset"], + "cookies": [], + "id": "pri_ac9d4dfb-d033-4b06-bc7f-968df8d125ff" } ], "joint_controller": { "name": "Sally Controller" }, diff --git a/clients/admin-ui/cypress/fixtures/systems/systems.json b/clients/admin-ui/cypress/fixtures/systems/systems.json index a73c0d419f..6e9e6aa264 100644 --- a/clients/admin-ui/cypress/fixtures/systems/systems.json +++ b/clients/admin-ui/cypress/fixtures/systems/systems.json @@ -17,7 +17,9 @@ "data_use": "improve.system", "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", "data_subjects": ["anonymous_user"], - "dataset_references": ["public"] + "dataset_references": ["public"], + "cookies": [], + "id": "pri_ac9d4dfb-d033-4b06-bc7f-968df8d125ff" } ], "joint_controller": null, @@ -59,7 +61,9 @@ "data_subjects": ["customer"], "dataset_references": ["demo_users_dataset"], "egress": null, - "ingress": null + "ingress": null, + "cookies": [], + "id": "pri_ac9d4dfb-d033-4b06-bc7f-968df8d125ff" } ], "joint_controller": null, @@ -99,7 +103,9 @@ "data_subjects": ["customer"], "dataset_references": null, "egress": null, - "ingress": null + "ingress": null, + "cookies": [], + "id": "pri_06430a1c-1365-422e-90a7-d444ddb32181" } ], "joint_controller": null, diff --git a/clients/admin-ui/cypress/fixtures/systems/systems_with_data_uses.json b/clients/admin-ui/cypress/fixtures/systems/systems_with_data_uses.json index 533a20f7d9..37c89489c3 100644 --- a/clients/admin-ui/cypress/fixtures/systems/systems_with_data_uses.json +++ b/clients/admin-ui/cypress/fixtures/systems/systems_with_data_uses.json @@ -17,7 +17,9 @@ "data_use": "improve.system", "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", "data_subjects": ["anonymous_user"], - "dataset_references": ["public"] + "dataset_references": ["public"], + "cookies": [], + "id": "pri_ac9d4dfb-d033-4b06-bc7f-968df8d125ff" }, { "name": "Collect data for marketing", @@ -27,7 +29,9 @@ "data_subjects": ["customer"], "dataset_references": null, "egress": null, - "ingress": null + "ingress": null, + "cookies": [], + "id": "pri_bc6e6efe-f122-3e33-ac9a-732ae8b437bb" } ], "joint_controller": null, diff --git a/clients/admin-ui/src/features/common/Header.tsx b/clients/admin-ui/src/features/common/Header.tsx index 2c17dbd745..75b06810ff 100644 --- a/clients/admin-ui/src/features/common/Header.tsx +++ b/clients/admin-ui/src/features/common/Header.tsx @@ -1,6 +1,7 @@ import { Button, Flex, + IconButton, Link, Menu, MenuButton, @@ -8,6 +9,7 @@ import { MenuItem, MenuList, QuestionIcon, + SettingsIcon, Stack, Text, UserIcon, @@ -21,6 +23,8 @@ import { INDEX_ROUTE } from "~/constants"; import { logout, selectUser, useLogoutMutation } from "~/features/auth"; import Image from "~/features/common/Image"; +import { USER_MANAGEMENT_ROUTE } from "./nav/v2/routes"; + const useHeader = () => { const { username } = useAppSelector(selectUser) ?? { username: "" }; return { username }; @@ -59,6 +63,15 @@ const Header: React.FC = () => { + + } + data-testid="management-btn" + /> + {username && ( { - const { errorAlert, successAlert } = useAlert(); + const { errorAlert } = useAlert(); const { plus: isEnabled } = useFeatures(); // This keeps track of the fides key that was initially passed in. If that key started out blank, @@ -149,7 +149,10 @@ export const useCustomFields = ({ // This will be undefined if the form never rendered a `CustomFieldList` that would assign // form values. - if (!customFieldValuesFromForm) { + if ( + !customFieldValuesFromForm || + Object.keys(customFieldValuesFromForm).length === 0 + ) { return; } @@ -184,10 +187,6 @@ export const useCustomFields = ({ return upsertCustomFieldMutationTrigger(body); }) ); - - successAlert( - `Custom field(s) successfully saved and added to this ${resourceType} form.` - ); } catch (e) { errorAlert( `One or more custom fields have failed to save, please try again.` @@ -202,9 +201,7 @@ export const useCustomFields = ({ deleteCustomFieldMutationTrigger, errorAlert, resourceFidesKey, - resourceType, sortedCustomFieldDefinitionIds, - successAlert, upsertCustomFieldMutationTrigger, ] ); diff --git a/clients/admin-ui/src/features/common/form/inputs.tsx b/clients/admin-ui/src/features/common/form/inputs.tsx index d242550648..7f6de443f1 100644 --- a/clients/admin-ui/src/features/common/form/inputs.tsx +++ b/clients/admin-ui/src/features/common/form/inputs.tsx @@ -316,21 +316,26 @@ const CreatableSelectInput = ({ size={size} classNamePrefix="custom-creatable-select" chakraStyles={{ - container: (provided) => ({ ...provided, flexGrow: 1 }), + container: (provided) => ({ + ...provided, + flexGrow: 1, + backgroundColor: "white", + }), dropdownIndicator: (provided) => ({ ...provided, - background: "white", + bg: "transparent", + px: 2, + cursor: "inherit", + }), + indicatorSeparator: (provided) => ({ + ...provided, + display: "none", }), multiValue: (provided) => ({ ...provided, background: "primary.400", color: "white", }), - multiValueRemove: (provided) => ({ - ...provided, - display: "none", - visibility: "hidden", - }), }} components={components} isSearchable={isSearchable} diff --git a/clients/admin-ui/src/features/common/form/validation.ts b/clients/admin-ui/src/features/common/form/validation.ts index 5f691a6961..c0c78be891 100644 --- a/clients/admin-ui/src/features/common/form/validation.ts +++ b/clients/admin-ui/src/features/common/form/validation.ts @@ -6,4 +6,4 @@ export const passwordValidation = Yup.string() .matches(/[0-9]/, "Password must have at least one number.") .matches(/[A-Z]/, "Password must have at least one capital letter.") .matches(/[a-z]/, "Password must have at least one lowercase letter.") - .matches(/[\W]/, "Password must have at least one symbol."); + .matches(/[\W_]/, "Password must have at least one symbol."); diff --git a/clients/admin-ui/src/features/common/nav/v2/NavTopBar.tsx b/clients/admin-ui/src/features/common/nav/v2/NavTopBar.tsx index 7dc04bde9a..493e5ce0a3 100644 --- a/clients/admin-ui/src/features/common/nav/v2/NavTopBar.tsx +++ b/clients/admin-ui/src/features/common/nav/v2/NavTopBar.tsx @@ -21,6 +21,10 @@ export const NavTopBar = () => { borderColor="gray.100" > {nav.groups.map((group) => { + // "Management" is navigated to via the gear icon, so don't display it in the nav + if (group.title === "Management") { + return null; + } // The group links to its first child's path. const { path } = group.children[0]!; diff --git a/clients/admin-ui/src/features/common/nav/v2/nav-config.test.ts b/clients/admin-ui/src/features/common/nav/v2/nav-config.test.ts index fb04f77367..4af3cf0713 100644 --- a/clients/admin-ui/src/features/common/nav/v2/nav-config.test.ts +++ b/clients/admin-ui/src/features/common/nav/v2/nav-config.test.ts @@ -35,19 +35,8 @@ describe("configureNavGroups", () => { children: [{ title: "Home", path: "/" }], }); - expect(navGroups[1]).toMatchObject({ - title: "Privacy requests", - children: [ - { title: "Request manager", path: routes.PRIVACY_REQUESTS_ROUTE }, - { - title: "Connection manager", - path: routes.DATASTORE_CONNECTION_ROUTE, - }, - ], - }); - // NOTE: the data map should _not_ include the Plus routes (/plus/datamap, /classify-systems, etc.) - expect(navGroups[2]).toMatchObject({ + expect(navGroups[1]).toMatchObject({ title: "Data map", children: [ { title: "View systems", path: routes.SYSTEM_ROUTE }, @@ -56,12 +45,14 @@ describe("configureNavGroups", () => { ], }); - expect(navGroups[3]).toMatchObject({ - title: "Management", + expect(navGroups[2]).toMatchObject({ + title: "Privacy requests", children: [ - { title: "Users", path: routes.USER_MANAGEMENT_ROUTE }, - { title: "Taxonomy", path: routes.TAXONOMY_ROUTE }, - { title: "About Fides", path: routes.ABOUT_ROUTE }, + { title: "Request manager", path: routes.PRIVACY_REQUESTS_ROUTE }, + { + title: "Connection manager", + path: routes.DATASTORE_CONNECTION_ROUTE, + }, ], }); }); @@ -79,7 +70,7 @@ describe("configureNavGroups", () => { }); // The data map _should_ include the actual "/plus/datamap". - expect(navGroups[2]).toMatchObject({ + expect(navGroups[1]).toMatchObject({ title: "Data map", children: [ { title: "View map", path: routes.DATAMAP_ROUTE }, @@ -147,7 +138,7 @@ describe("configureNavGroups", () => { }); // The data map should _not_ include the actual "/plus/datamap". - expect(navGroups[2]).toMatchObject({ + expect(navGroups[1]).toMatchObject({ title: "Data map", children: [ { title: "View systems", path: routes.SYSTEM_ROUTE }, diff --git a/clients/admin-ui/src/features/common/nav/v2/nav-config.ts b/clients/admin-ui/src/features/common/nav/v2/nav-config.ts index 8c9fca5ed0..ff8e1730ba 100644 --- a/clients/admin-ui/src/features/common/nav/v2/nav-config.ts +++ b/clients/admin-ui/src/features/common/nav/v2/nav-config.ts @@ -32,6 +32,41 @@ export const NAV_CONFIG: NavConfigGroup[] = [ }, ], }, + { + title: "Data map", + routes: [ + { + title: "View map", + path: routes.DATAMAP_ROUTE, + requiresPlus: true, + scopes: [ScopeRegistryEnum.DATAMAP_READ], + }, + { + title: "View systems", + path: routes.SYSTEM_ROUTE, + scopes: [ScopeRegistryEnum.SYSTEM_READ], + }, + { + title: "Add systems", + path: routes.ADD_SYSTEMS_ROUTE, + scopes: [ScopeRegistryEnum.SYSTEM_CREATE], + }, + { + title: "Manage datasets", + path: routes.DATASET_ROUTE, + scopes: [ + ScopeRegistryEnum.CTL_DATASET_CREATE, + ScopeRegistryEnum.CTL_DATASET_UPDATE, + ], + }, + { + title: "Classify systems", + path: routes.CLASSIFY_SYSTEMS_ROUTE, + requiresPlus: true, + scopes: [ScopeRegistryEnum.SYSTEM_UPDATE], // temporary scope until we decide what to do here + }, + ], + }, { title: "Privacy requests", routes: [ @@ -79,41 +114,6 @@ export const NAV_CONFIG: NavConfigGroup[] = [ }, ], }, - { - title: "Data map", - routes: [ - { - title: "View map", - path: routes.DATAMAP_ROUTE, - requiresPlus: true, - scopes: [ScopeRegistryEnum.DATAMAP_READ], - }, - { - title: "View systems", - path: routes.SYSTEM_ROUTE, - scopes: [ScopeRegistryEnum.SYSTEM_READ], - }, - { - title: "Add systems", - path: routes.ADD_SYSTEMS_ROUTE, - scopes: [ScopeRegistryEnum.SYSTEM_CREATE], - }, - { - title: "Manage datasets", - path: routes.DATASET_ROUTE, - scopes: [ - ScopeRegistryEnum.CTL_DATASET_CREATE, - ScopeRegistryEnum.CTL_DATASET_UPDATE, - ], - }, - { - title: "Classify systems", - path: routes.CLASSIFY_SYSTEMS_ROUTE, - requiresPlus: true, - scopes: [ScopeRegistryEnum.SYSTEM_UPDATE], // temporary scope until we decide what to do here - }, - ], - }, { title: "Management", routes: [ diff --git a/clients/admin-ui/src/features/connector-templates/ConnectorTemplateUploadModal.tsx b/clients/admin-ui/src/features/connector-templates/ConnectorTemplateUploadModal.tsx index 6d1c8413e5..58d7e1c235 100644 --- a/clients/admin-ui/src/features/connector-templates/ConnectorTemplateUploadModal.tsx +++ b/clients/admin-ui/src/features/connector-templates/ConnectorTemplateUploadModal.tsx @@ -68,7 +68,9 @@ const ConnectorTemplateUploadModal: React.FC = ({ if (uploadedFile) { try { await registerConnectorTemplate(uploadedFile).unwrap(); - toast(successToastParams("Connector template uploaded successfully.")); + toast( + successToastParams("Integration template uploaded successfully.") + ); // refresh the connection types const { data } = await refetchConnectionTypes(); @@ -96,10 +98,10 @@ const ConnectorTemplateUploadModal: React.FC = ({ - Upload connector template + Upload integration template - Drag and drop your connector template zip file here, or click to + Drag and drop your integration template zip file here, or click to browse your files. = ({ {renderFileText()} - A connector template zip file must include a SaaS config and + An integration template zip file must include a SaaS config and dataset, but may also contain an icon (.svg) and custom functions (.py) as optional files. diff --git a/clients/admin-ui/src/features/datastore-connections/ConnectionGridItem.tsx b/clients/admin-ui/src/features/datastore-connections/ConnectionGridItem.tsx index 7cbc85ee81..f21bd57bae 100644 --- a/clients/admin-ui/src/features/datastore-connections/ConnectionGridItem.tsx +++ b/clients/admin-ui/src/features/datastore-connections/ConnectionGridItem.tsx @@ -3,7 +3,6 @@ import { formatDate } from "common/utils"; import React, { useMemo } from "react"; import { useAppSelector } from "~/app/hooks"; -import ConnectedCircle from "~/features/common/ConnectedCircle"; import { selectConnectionTypeState } from "~/features/connection-type"; import { ConnectionConfigurationResponse } from "~/types/api"; @@ -11,33 +10,7 @@ import ConnectionMenu from "./ConnectionMenu"; import ConnectionStatusBadge from "./ConnectionStatusBadge"; import ConnectionTypeLogo from "./ConnectionTypeLogo"; import { useLazyGetDatastoreConnectionStatusQuery } from "./datastore-connection.slice"; - -type TestDataProps = { - succeeded?: boolean; - timestamp: string; -}; - -const TestData: React.FC = ({ succeeded, timestamp }) => { - const date = timestamp ? formatDate(timestamp) : ""; - const testText = timestamp - ? `Last tested on ${date}` - : "This connection has not been tested yet"; - - return ( - <> - - - {testText} - - - ); -}; +import TestData from "./TestData"; type ConnectionGridItemProps = { connectionData: ConnectionConfigurationResponse; @@ -85,7 +58,7 @@ const ConnectionGridItem: React.FC = ({ diff --git a/clients/admin-ui/src/features/datastore-connections/ConnectionMenu.tsx b/clients/admin-ui/src/features/datastore-connections/ConnectionMenu.tsx index 930946600e..239da56ac9 100644 --- a/clients/admin-ui/src/features/datastore-connections/ConnectionMenu.tsx +++ b/clients/admin-ui/src/features/datastore-connections/ConnectionMenu.tsx @@ -65,6 +65,7 @@ const ConnectionMenu: React.FC = ({ connection_type={connection_type} access_type={access_type} name={name} + isSwitch={false} /> diff --git a/clients/admin-ui/src/features/datastore-connections/ConnectionTypeLogo.tsx b/clients/admin-ui/src/features/datastore-connections/ConnectionTypeLogo.tsx index be7c752110..66cf2696d5 100644 --- a/clients/admin-ui/src/features/datastore-connections/ConnectionTypeLogo.tsx +++ b/clients/admin-ui/src/features/datastore-connections/ConnectionTypeLogo.tsx @@ -48,7 +48,7 @@ const ConnectionTypeLogo: React.FC = ({ }; const getAltValue = (): string => { if (isDatastoreConnection(data)) { - return data.name; + return data.name ?? data.key; } if (isConnectionSystemTypeMap(data)) { return data.human_readable; diff --git a/clients/admin-ui/src/features/datastore-connections/DisableConnectionModal.tsx b/clients/admin-ui/src/features/datastore-connections/DisableConnectionModal.tsx index 5648c03a4b..19f5dffd92 100644 --- a/clients/admin-ui/src/features/datastore-connections/DisableConnectionModal.tsx +++ b/clients/admin-ui/src/features/datastore-connections/DisableConnectionModal.tsx @@ -1,5 +1,6 @@ import { Button, + Flex, MenuItem, Modal, ModalBody, @@ -9,6 +10,7 @@ import { ModalHeader, ModalOverlay, Stack, + Switch, Text, useDisclosure, } from "@fidesui/react"; @@ -25,6 +27,7 @@ type DataConnectionProps = { name: string; access_type: AccessLevel; connection_type: ConnectionType; + isSwitch: boolean; }; const DisableConnectionModal: React.FC = ({ @@ -33,6 +36,7 @@ const DisableConnectionModal: React.FC = ({ name, access_type, connection_type, + isSwitch, }) => { const { isOpen, onOpen, onClose } = useDisclosure(); const [patchConnection, patchConnectionResult] = @@ -40,13 +44,14 @@ const DisableConnectionModal: React.FC = ({ const handleDisableConnection = async () => { const shouldDisable = !disabled; - patchConnection({ + await patchConnection({ key: connection_key, name, disabled: shouldDisable, access: access_type, connection_type, }); + onClose(); }; const closeIfComplete = () => { @@ -57,12 +62,24 @@ const DisableConnectionModal: React.FC = ({ return ( <> - - {disabled ? "Enable" : "Disable"} - + {isSwitch ? ( + + Enable integration + + + ) : ( + + {disabled ? "Enable" : "Disable"} + + )} diff --git a/clients/admin-ui/src/features/datastore-connections/TestData.tsx b/clients/admin-ui/src/features/datastore-connections/TestData.tsx new file mode 100644 index 0000000000..d0f854074f --- /dev/null +++ b/clients/admin-ui/src/features/datastore-connections/TestData.tsx @@ -0,0 +1,33 @@ +import { Text } from "@fidesui/react"; + +import ConnectedCircle from "../common/ConnectedCircle"; +import { formatDate } from "../common/utils"; + +type TestDataProps = { + succeeded?: boolean; + timestamp: string | number; +}; + +const TestData: React.FC = ({ succeeded, timestamp }) => { + const date = timestamp ? formatDate(timestamp) : ""; + const testText = timestamp + ? `Last tested on ${date}` + : "This connection has not been tested yet"; + + return ( + <> + + + {testText} + + + ); +}; + +export default TestData; diff --git a/clients/admin-ui/src/features/datastore-connections/add-connection/forms/ConnectorParametersForm.tsx b/clients/admin-ui/src/features/datastore-connections/add-connection/forms/ConnectorParametersForm.tsx index b47559edc5..ebaafbff06 100644 --- a/clients/admin-ui/src/features/datastore-connections/add-connection/forms/ConnectorParametersForm.tsx +++ b/clients/admin-ui/src/features/datastore-connections/add-connection/forms/ConnectorParametersForm.tsx @@ -195,7 +195,7 @@ const ConnectorParametersForm: React.FC = ({ const getInitialValues = () => { const initialValues = { ...defaultValues }; if (connection?.key) { - initialValues.name = connection.name; + initialValues.name = connection.name ?? ""; initialValues.description = connection.description as string; initialValues.instance_key = connection.connection_type === ConnectionType.SAAS diff --git a/clients/admin-ui/src/features/datastore-connections/add-connection/manual/ConnectorParametersForm.tsx b/clients/admin-ui/src/features/datastore-connections/add-connection/manual/ConnectorParametersForm.tsx index 160086262d..d33ec9c540 100644 --- a/clients/admin-ui/src/features/datastore-connections/add-connection/manual/ConnectorParametersForm.tsx +++ b/clients/admin-ui/src/features/datastore-connections/add-connection/manual/ConnectorParametersForm.tsx @@ -30,7 +30,7 @@ const ConnectorParametersForm: React.FC = ({ ); const getInitialValues = () => { if (connection?.key) { - defaultValues.name = connection.name; + defaultValues.name = connection.name ?? ""; defaultValues.description = connection.description as string; } return defaultValues; diff --git a/clients/admin-ui/src/features/datastore-connections/datastore-connection.slice.ts b/clients/admin-ui/src/features/datastore-connections/datastore-connection.slice.ts index 905d62720d..6402580630 100644 --- a/clients/admin-ui/src/features/datastore-connections/datastore-connection.slice.ts +++ b/clients/admin-ui/src/features/datastore-connections/datastore-connection.slice.ts @@ -89,7 +89,7 @@ const initialState: DatastoreConnectionParams = { }; export type CreateSaasConnectionConfig = { - connectionConfig: CreateSaasConnectionConfigRequest; + connectionConfig: Omit; systemFidesKey: string; }; @@ -351,7 +351,7 @@ export const datastoreConnectionApi = baseApi.injectEndpoints({ method: "PATCH", body: [{ key, name, disabled, connection_type, access }], }), - invalidatesTags: () => ["Datastore Connection", "Datasets"], + invalidatesTags: () => ["Datastore Connection", "Datasets", "System"], }), updateDatastoreConnectionSecrets: build.mutation< DatastoreConnectionSecretsResponse, diff --git a/clients/admin-ui/src/features/datastore-connections/system_portal_config/ConnectionForm.tsx b/clients/admin-ui/src/features/datastore-connections/system_portal_config/ConnectionForm.tsx index 697a102a02..143b6850fb 100644 --- a/clients/admin-ui/src/features/datastore-connections/system_portal_config/ConnectionForm.tsx +++ b/clients/admin-ui/src/features/datastore-connections/system_portal_config/ConnectionForm.tsx @@ -65,10 +65,10 @@ const ConnectionForm = ({ connectionConfig, systemFidesKey }: Props) => { {!connectionConfig && orphanedConnectionConfigs.length > 0 ? ( @@ -79,7 +79,7 @@ const ConnectionForm = ({ connectionConfig, systemFidesKey }: Props) => { ) : null} void; + onDelete: () => void; deleteResult: any; }; const DeleteConnectionModal: React.FC = ({ - connectionKey, onDelete, deleteResult, }) => { const { isOpen, onOpen, onClose } = useDisclosure(); const handleDeleteConnection = () => { - onDelete(connectionKey); + onDelete(); }; const closeIfComplete = () => { @@ -44,26 +46,24 @@ const DeleteConnectionModal: React.FC = ({ <> <> - + + Delete Integration + } + isDisabled={deleteResult.isLoading} + onClick={onOpen} + size="sm" + /> + - Delete Connection + Delete Integration @@ -73,7 +73,7 @@ const DeleteConnectionModal: React.FC = ({ fontWeight="sm" lineHeight="20px" > - Deleting a connection may impact any privacy request that is + Deleting an integration may impact any privacy request that is currently in progress. Do you wish to proceed? @@ -108,7 +108,7 @@ const DeleteConnectionModal: React.FC = ({ color: "gray.600", }} > - Delete connection + Delete integration diff --git a/clients/admin-ui/src/features/datastore-connections/system_portal_config/OrphanedConnectionModal.tsx b/clients/admin-ui/src/features/datastore-connections/system_portal_config/OrphanedConnectionModal.tsx index 7c54bb7021..8a099f9a4e 100644 --- a/clients/admin-ui/src/features/datastore-connections/system_portal_config/OrphanedConnectionModal.tsx +++ b/clients/admin-ui/src/features/datastore-connections/system_portal_config/OrphanedConnectionModal.tsx @@ -106,17 +106,15 @@ const OrphanedConnectionModal: React.FC = ({ loadingText="Deleting" onClick={onOpen} size="sm" - variant="solid" - color="white" - colorScheme="primary" + variant="outline" > - Link connector + Link integration - Unlinked Connections + Unlinked Integrations @@ -126,8 +124,8 @@ const OrphanedConnectionModal: React.FC = ({ fontWeight="sm" lineHeight="20px" > - These are all the connections that are not linked to a system. - Please select a connection to link to a system. + These are all the integrations that are not linked to a system. + Please select an integration to link to a system. {connectionConfigs.map((connectionConfig) => ( @@ -192,7 +190,7 @@ const OrphanedConnectionModal: React.FC = ({ color: "gray.600", }} > - Link connector + Link integration diff --git a/clients/admin-ui/src/features/datastore-connections/system_portal_config/TestConnection.tsx b/clients/admin-ui/src/features/datastore-connections/system_portal_config/TestConnection.tsx deleted file mode 100644 index ee3a503015..0000000000 --- a/clients/admin-ui/src/features/datastore-connections/system_portal_config/TestConnection.tsx +++ /dev/null @@ -1,122 +0,0 @@ -import { - Box, - Divider, - ErrorWarningIcon, - GreenCheckCircleIcon, - Heading, - HStack, - Tag, - Text, - VStack, -} from "@fidesui/react"; -import { formatDate } from "common/utils"; -import React from "react"; - -import { ConnectionSystemTypeMap } from "~/types/api"; - -type TestConnectionProps = { - response: any; - connectionOption: ConnectionSystemTypeMap; -}; - -const TestConnection: React.FC = ({ - response, - connectionOption, -}) => ( - <> - - - {response.data?.test_status === "succeeded" && ( - <> - - - Successfully connected to {connectionOption!.human_readable} - - - Success - - - - {formatDate(response.fulfilledTimeStamp)} - - - - - - - Success message: - - - {response.data.msg} - - - - - - )} - {response.data?.test_status === "failed" && ( - <> - - - Output error to {connectionOption!.human_readable} - - - Error - - - - {formatDate(response.fulfilledTimeStamp)} - - - - - - - Error message: - - - {response.data.failure_reason} - - - {response.data.msg} - - - - - - )} - - -); - -export default TestConnection; diff --git a/clients/admin-ui/src/features/datastore-connections/system_portal_config/TestConnectionMessage.tsx b/clients/admin-ui/src/features/datastore-connections/system_portal_config/TestConnectionMessage.tsx new file mode 100644 index 0000000000..2e4d62f81a --- /dev/null +++ b/clients/admin-ui/src/features/datastore-connections/system_portal_config/TestConnectionMessage.tsx @@ -0,0 +1,19 @@ +import { Text } from "@fidesui/react"; +import React from "react"; + +type TestConnectionMessageProps = { + status: "success" | "error"; +}; + +const TestConnectionMessage: React.FC = ({ + status, +}) => ( + <> + {status === "success" && Connection test was successful} + {status === "error" && ( + Test failed: please check your connection info + )} + +); + +export default TestConnectionMessage; diff --git a/clients/admin-ui/src/features/datastore-connections/system_portal_config/forms/ConnectorParameters.tsx b/clients/admin-ui/src/features/datastore-connections/system_portal_config/forms/ConnectorParameters.tsx index 0ce3cf6d3c..d866afc89f 100644 --- a/clients/admin-ui/src/features/datastore-connections/system_portal_config/forms/ConnectorParameters.tsx +++ b/clients/admin-ui/src/features/datastore-connections/system_portal_config/forms/ConnectorParameters.tsx @@ -1,31 +1,33 @@ -import { Box, SlideFade } from "@fidesui/react"; +import { Box, Flex, Spacer, useToast, UseToastOptions } from "@fidesui/react"; import { useAPIHelper } from "common/hooks"; import { useAlert } from "common/hooks/useAlert"; import { ConnectionTypeSecretSchemaReponse } from "connection-type/types"; import { CreateSaasConnectionConfig, useCreateSassConnectionConfigMutation, - useDeleteDatastoreConnectionMutation, useGetConnectionConfigDatasetConfigsQuery, - useUpdateDatastoreConnectionSecretsMutation, } from "datastore-connections/datastore-connection.slice"; import { useDatasetConfigField } from "datastore-connections/system_portal_config/forms/fields/DatasetConfigField/DatasetConfigField"; import { CreateSaasConnectionConfigRequest, CreateSaasConnectionConfigResponse, - DatastoreConnectionSecretsRequest, DatastoreConnectionSecretsResponse, } from "datastore-connections/types"; -import { useState } from "react"; +import { useMemo, useState } from "react"; import { useAppDispatch, useAppSelector } from "~/app/hooks"; +import { DEFAULT_TOAST_PARAMS } from "~/features/common/toast"; import { useGetConnectionTypeSecretSchemaQuery } from "~/features/connection-type"; import { formatKey } from "~/features/datastore-connections/system_portal_config/helpers"; -import TestConnection from "~/features/datastore-connections/system_portal_config/TestConnection"; +import TestConnectionMessage from "~/features/datastore-connections/system_portal_config/TestConnectionMessage"; +import TestData from "~/features/datastore-connections/TestData"; import { + ConnectionConfigSecretsRequest, selectActiveSystem, setActiveSystem, + useDeleteSystemConnectionConfigMutation, usePatchSystemConnectionConfigsMutation, + usePatchSystemConnectionSecretsMutation, } from "~/features/system/system.slice"; import { AccessLevel, @@ -38,7 +40,9 @@ import { } from "~/types/api"; import { ConnectionConfigFormValues } from "../types"; -import ConnectorParametersForm from "./ConnectorParametersForm"; +import ConnectorParametersForm, { + TestConnectionResponse, +} from "./ConnectorParametersForm"; /** * Only handles creating saas connectors. The BE handler automatically @@ -52,9 +56,8 @@ const createSaasConnector = async ( systemFidesKey: string, createSaasConnectorFunc: any ) => { - const connectionConfig: CreateSaasConnectionConfigRequest = { + const connectionConfig: Omit = { description: values.description, - name: values.name, instance_key: formatKey(values.instance_key as string), saas_connector_type: connectionOption.identifier, secrets: {}, @@ -66,7 +69,7 @@ const createSaasConnector = async ( }; Object.entries(secretsSchema!.properties).forEach((key) => { - params.connectionConfig.secrets[key[0]] = values[key[0]]; + params.connectionConfig.secrets[key[0]] = values.secrets[key[0]]; }); return (await createSaasConnectorFunc( params @@ -86,20 +89,22 @@ export const patchConnectionConfig = async ( patchFunc: any ) => { const key = - [SystemType.DATABASE, SystemType.EMAIL].indexOf(connectionOption.type) > -1 + [SystemType.DATABASE, SystemType.EMAIL, SystemType.MANUAL].indexOf( + connectionOption.type + ) > -1 ? formatKey(values.instance_key as string) : connectionConfig?.key; - const params1: Omit = { - access: AccessLevel.WRITE, - connection_type: (connectionOption.type === SystemType.SAAS - ? connectionOption.type - : connectionOption.identifier) as ConnectionType, - description: values.description, - disabled: false, - key, - name: values.name, - }; + const params1: Omit = + { + access: AccessLevel.WRITE, + connection_type: (connectionOption.type === SystemType.SAAS + ? connectionOption.type + : connectionOption.identifier) as ConnectionType, + description: values.description, + disabled: false, + key, + }; const payload = await patchFunc({ systemFidesKey, connectionConfigs: [params1], @@ -119,18 +124,32 @@ export const patchConnectionConfig = async ( const upsertConnectionConfigSecrets = async ( values: ConnectionConfigFormValues, secretsSchema: ConnectionTypeSecretSchemaReponse, - connectionConfigFidesKey: string, - upsertFunc: any + systemFidesKey: string, + originalSecrets: Record, + patchFunc: any ) => { - const params2: DatastoreConnectionSecretsRequest = { - connection_key: connectionConfigFidesKey, + const params2: ConnectionConfigSecretsRequest = { + systemFidesKey, secrets: {}, }; Object.entries(secretsSchema!.properties).forEach((key) => { - params2.secrets[key[0]] = values[key[0]]; + /* + * Only patch secrets that have changed. Otherwise, sensitive secrets + * would get overwritten with "**********" strings + */ + if ( + !(key[0] in originalSecrets) || + values.secrets[key[0]] !== originalSecrets[key[0]] + ) { + params2.secrets[key[0]] = values.secrets[key[0]]; + } }); - return (await upsertFunc( + if (Object.keys(params2.secrets).length === 0) { + return Promise.resolve(); + } + + return (await patchFunc( params2 ).unwrap()) as DatastoreConnectionSecretsResponse; }; @@ -175,24 +194,28 @@ export const useConnectorForm = ({ }); const [createSassConnectionConfig] = useCreateSassConnectionConfigMutation(); - const [updateDatastoreConnectionSecrets] = - useUpdateDatastoreConnectionSecretsMutation(); + const [updateSystemConnectionSecrets] = + usePatchSystemConnectionSecretsMutation(); const [patchDatastoreConnection] = usePatchSystemConnectionConfigsMutation(); const [deleteDatastoreConnection, deleteDatastoreConnectionResult] = - useDeleteDatastoreConnectionMutation(); + useDeleteSystemConnectionConfigMutation(); const { data: allDatasetConfigs } = useGetConnectionConfigDatasetConfigsQuery( connectionConfig?.key || "" ); + const originalSecrets = useMemo( + () => (connectionConfig ? { ...connectionConfig.secrets } : {}), + [connectionConfig] + ); const activeSystem = useAppSelector(selectActiveSystem) as SystemResponse; - const handleDelete = async (id: string) => { + const handleDelete = async () => { try { - await deleteDatastoreConnection(id); + await deleteDatastoreConnection(systemFidesKey); // @ts-ignore connection_configs isn't on the type yet but will be in the future dispatch(setActiveSystem({ ...activeSystem, connection_configs: null })); setSelectedConnectionOption(undefined); - successAlert(`Connector successfully deleted!`); + successAlert(`Integration successfully deleted!`); } catch (e) { handleError(e); } @@ -243,8 +266,9 @@ export const useConnectorForm = ({ await upsertConnectionConfigSecrets( secretsPayload, secretsSchema!, - payload.succeeded[0].key, - updateDatastoreConnectionSecrets + systemFidesKey, + originalSecrets, + updateSystemConnectionSecrets ); } } @@ -269,7 +293,7 @@ export const useConnectorForm = ({ } successAlert( - `Connector successfully ${ + `Integration successfully ${ isCreatingConnectionConfig ? "added" : "updated" }!` ); @@ -296,11 +320,22 @@ export const ConnectorParameters: React.FC = ({ connectionConfig, setSelectedConnectionOption, }) => { - const [response, setResponse] = useState(); + const [response, setResponse] = useState(); - const handleTestConnectionClick = (value: any) => { + const toast = useToast(); + + const handleTestConnectionClick = (value: TestConnectionResponse) => { setResponse(value); + const status: UseToastOptions["status"] = + value.data?.test_status === "succeeded" ? "success" : "error"; + const toastParams = { + ...DEFAULT_TOAST_PARAMS, + status, + description: , + }; + toast(toastParams); }; + const skip = connectionOption.type === SystemType.MANUAL; const { data: secretsSchema } = useGetConnectionTypeSecretSchemaQuery( connectionOption!.identifier, @@ -338,10 +373,18 @@ export const ConnectorParameters: React.FC = ({ return ( <> - + Connect to your {connectionOption!.human_readable} environment by providing credential information below. Once you have saved your - connector credentials, you can review what data is included when + integration credentials, you can review what data is included when processing a privacy request in your Dataset configuration. = ({ deleteResult={deleteDatastoreConnectionResult} /> - {response && ( - - - + {response && + response.data && + response.fulfilledTimeStamp !== undefined ? ( + + ) : ( + - - - )} + )} + + + ) : null} ); }; diff --git a/clients/admin-ui/src/features/datastore-connections/system_portal_config/forms/ConnectorParametersForm.tsx b/clients/admin-ui/src/features/datastore-connections/system_portal_config/forms/ConnectorParametersForm.tsx index db35b8d5be..c603a5fe40 100644 --- a/clients/admin-ui/src/features/datastore-connections/system_portal_config/forms/ConnectorParametersForm.tsx +++ b/clients/admin-ui/src/features/datastore-connections/system_portal_config/forms/ConnectorParametersForm.tsx @@ -13,12 +13,11 @@ import { NumberInput, NumberInputField, NumberInputStepper, - Textarea, + Spacer, Tooltip, VStack, } from "@fidesui/react"; import { Option } from "common/form/inputs"; -import { useAPIHelper } from "common/hooks"; import { ConnectionTypeSecretSchemaProperty, ConnectionTypeSecretSchemaReponse, @@ -26,8 +25,10 @@ import { import { useLazyGetDatastoreConnectionStatusQuery } from "datastore-connections/datastore-connection.slice"; import DSRCustomizationModal from "datastore-connections/system_portal_config/forms/DSRCustomizationForm/DSRCustomizationModal"; import { Field, FieldInputProps, Form, Formik, FormikProps } from "formik"; -import React, { useEffect, useRef } from "react"; +import React from "react"; +import { DatastoreConnectionStatus } from "src/features/datastore-connections/types"; +import DisableConnectionModal from "~/features/datastore-connections/DisableConnectionModal"; import DatasetConfigField from "~/features/datastore-connections/system_portal_config/forms/fields/DatasetConfigField/DatasetConfigField"; import { ConnectionConfigurationResponse, @@ -42,6 +43,11 @@ import { fillInDefaults } from "./helpers"; const FIDES_DATASET_REFERENCE = "#/definitions/FidesDatasetReference"; +export interface TestConnectionResponse { + data?: DatastoreConnectionStatus; + fulfilledTimeStamp?: number; +} + type ConnectorParametersFormProps = { secretsSchema?: ConnectionTypeSecretSchemaReponse; defaultValues: ConnectionConfigFormValues; @@ -53,7 +59,7 @@ type ConnectorParametersFormProps = { /** * Parent callback when Test Connection is clicked */ - onTestConnectionClick: (value: any) => void; + onTestConnectionClick: (value: TestConnectionResponse) => void; /** * Text for the test button. Defaults to "Test connection" */ @@ -62,7 +68,7 @@ type ConnectorParametersFormProps = { connectionOption: ConnectionSystemTypeMap; isCreatingConnectionConfig: boolean; datasetDropdownOptions: Option[]; - onDelete: (id: string) => void; + onDelete: () => void; deleteResult: any; }; @@ -72,7 +78,7 @@ const ConnectorParametersForm: React.FC = ({ isSubmitting = false, onSaveClick, onTestConnectionClick, - testButtonLabel = "Test connection", + testButtonLabel = "Test integration", connectionOption, connectionConfig, datasetDropdownOptions, @@ -80,25 +86,23 @@ const ConnectorParametersForm: React.FC = ({ onDelete, deleteResult, }) => { - const mounted = useRef(false); - const { handleError } = useAPIHelper(); - - const [trigger, result] = useLazyGetDatastoreConnectionStatusQuery(); + const [trigger, { isLoading, isFetching }] = + useLazyGetDatastoreConnectionStatusQuery(); const validateConnectionIdentifier = (value: string) => { let error; if (typeof value === "undefined" || value === "") { - error = "Connection Identifier is required"; + error = "Integration Identifier is required"; } if (value && isNumeric(value)) { - error = "Connection Identifier must be an alphanumeric value"; + error = "Integration Identifier must be an alphanumeric value"; } return error; }; const validateField = (label: string, value: string, type?: string) => { let error; - if (typeof value === "undefined" || value === "") { + if (typeof value === "undefined" || value === "" || value === undefined) { error = `${label} is required`; } if (type === FIDES_DATASET_REFERENCE) { @@ -133,84 +137,103 @@ const ConnectorParametersForm: React.FC = ({ return undefined; }; + const isRequiredSecretValue = (key: string): boolean => + secretsSchema?.required?.includes(key) || + (secretsSchema?.properties?.[key] !== undefined && + "default" in secretsSchema.properties[key]); + const getFormField = ( key: string, item: ConnectionTypeSecretSchemaProperty ): JSX.Element => ( validateField(item.title, value, item.allOf?.[0].$ref) : false } > - {({ field, form }: { field: FieldInputProps; form: any }) => ( - - {getFormLabel(key, item.title)} - - {item.type !== "integer" && ( - - )} - {item.type === "integer" && ( - - - - - - - - )} - {form.errors[key]} - - ; form: any }) => { + const error = form.errors.secrets && form.errors.secrets[key]; + const touch = form.touched.secrets ? form.touched.secrets[key] : false; + + return ( + - + {item.type !== "integer" && ( + + )} + {item.type === "integer" && ( + { + form.setFieldValue(field.name, value); + }} + defaultValue={field.value ?? 0} + min={0} + size="sm" + > + + + + + + + )} + {error} + + - - - - - )} + + + + + + ); + }} ); const getInitialValues = () => { const initialValues = { ...defaultValues }; if (connectionConfig?.key) { - initialValues.name = connectionConfig.name; + initialValues.name = connectionConfig.name ?? ""; initialValues.description = connectionConfig.description as string; initialValues.instance_key = connectionConfig.connection_type === ConnectionType.SAAS ? (connectionConfig.saas_config?.fides_key as string) : connectionConfig.key; + // @ts-ignore + initialValues.secrets = connectionConfig.secrets; + return initialValues; } return fillInDefaults(initialValues, secretsSchema); }; @@ -238,22 +261,11 @@ const ConnectorParametersForm: React.FC = ({ }; const handleTestConnectionClick = async () => { - try { - await trigger(connectionConfig!.key).unwrap(); - } catch (error) { - handleError(error); - } + const result = await trigger(connectionConfig!.key); + onTestConnectionClick(result); }; - useEffect(() => { - mounted.current = true; - if (result.isSuccess) { - onTestConnectionClick(result); - } - return () => { - mounted.current = false; - }; - }, [onTestConnectionClick, result]); + const isDisabledConnection = connectionConfig?.disabled || false; return ( = ({ {(props: FormikProps) => (
+ + {connectionConfig ? ( + + ) : null} + {connectionConfig ? ( + + ) : null} + + {/* Connection Identifier */} validateField("Name", value)} + id="instance_key" + name="instance_key" + validate={validateConnectionIdentifier} > {({ field }: { field: FieldInputProps }) => ( - {getFormLabel("name", "Name")} + {getFormLabel("instance_key", "Integration Identifier")} - {props.errors.name} + + {props.errors.instance_key} + - - - - - )} - - {/* Description */} - - {({ field }: { field: FieldInputProps }) => ( - - {getFormLabel("description", "Description")} -