Merge branch 'main' into 3518_ajackson_remove_dataset_from_saas

.github/workflows/backend_checks.yml

@@ -109,8 +109,37 @@ jobs:
   #################
   ## Misc Checks ##
   #################
-  Misc-Tests:
+  Check-Container-Startup:
     needs: Build
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Download container
+        uses: actions/download-artifact@v3
+        with:
+          name: python-${{ env.DEFAULT_PYTHON_VERSION }}
+          path: /tmp/
+
+      - name: Load image
+        run: docker load --input /tmp/python-${{ env.DEFAULT_PYTHON_VERSION }}.tar
+
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Set Up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
+          cache: "pip"
+
+      - name: Install Nox
+        run: pip install nox>=2022
+
+      - name: Run test suite
+        run: nox -s check_container_startup
+
+  Misc-Tests:
+    needs: Check-Container-Startup
     strategy:
       matrix:
         test_selection:
@@ -151,7 +180,7 @@ jobs:
   ## Safe Tests ##
   ################
   Safe-Tests:
-    needs: Build
+    needs: Check-Container-Startup
     strategy:
       matrix:
         python_version: ["3.8.16", "3.9.16", "3.10.11"]
@@ -201,7 +230,7 @@ jobs:
   # NOTE: Matrixes aren't used here due to the danger of race conditions for external resources
 
   Pytest-Ctl-External:
-    needs: Build
+    needs: Check-Container-Startup
     strategy:
       max-parallel: 1 # This prevents collisions in shared external resources
       matrix:
@@ -241,7 +270,7 @@ jobs:
           DYNAMODB_ACCESS_KEY: ${{ secrets.DYNAMODB_ACCESS_KEY }}
 
   External-Datastores:
-    needs: Build
+    needs: Check-Container-Startup
     strategy:
       max-parallel: 1 # This prevents collisions in shared external resources
       matrix:
@@ -280,7 +309,7 @@ jobs:
         run: nox -s "pytest(ops-external-datastores)"
 
   External-SaaS-Connectors:
-    needs: Build
+    needs: Check-Container-Startup
     runs-on: ubuntu-latest
     timeout-minutes: 30
     # In PRs run with the "unsafe" label, or run on a "push" event to main

.github/workflows/publish_docker.yaml

@@ -33,33 +33,33 @@ jobs:
         id: check-prod-tag
         run: |
           if [[ ${{ github.event.ref }} =~ ^refs/tags/[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
-              echo ::set-output name=match::true
+            echo "match=true" >> $GITHUB_OUTPUT
           else
-              echo ::set-output name=match::false
+            echo "match=false" >> $GITHUB_OUTPUT
           fi
       - name: Check RC Tag
         id: check-rc-tag
         run: |
           if [[ ${{ github.event.ref }} =~ ^refs/tags/[0-9]+\.[0-9]+\.[0-9]+rc[0-9]+$ ]]; then
-              echo ::set-output name=match::true
+            echo "match=true" >> $GITHUB_OUTPUT
           else
-              echo ::set-output name=match::false
+            echo "match=false" >> $GITHUB_OUTPUT
           fi
       - name: Check alpha Tag
         id: check-alpha-tag
         run: |
           if [[ ${{ github.event.ref }} =~ ^refs/tags/[0-9]+\.[0-9]+\.[0-9]+a[0-9]+$ ]]; then
-              echo ::set-output name=match::true
+            echo "match=true" >> $GITHUB_OUTPUT
           else
-              echo ::set-output name=match::false
+            echo "match=false" >> $GITHUB_OUTPUT
           fi
       - name: Check beta Tag
         id: check-beta-tag
         run: |
           if [[ ${{ github.event.ref }} =~ ^refs/tags/[0-9]+\.[0-9]+\.[0-9]+b[0-9]+$ ]]; then
-              echo ::set-output name=match::true
+            echo "match=true" >> $GITHUB_OUTPUT
           else
-              echo ::set-output name=match::false
+            echo "match=false" >> $GITHUB_OUTPUT
           fi
 
       # if neither prod, rc, beta or alpha git tag, then push images with the ":dev" tag

.github/workflows/publish_package.yaml

@@ -53,33 +53,33 @@ jobs:
         id: check-prod-tag
         run: |
           if [[ ${{ github.event.ref }} =~ ^refs/tags/[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
-              echo ::set-output name=match::true
+            echo "match=true" >> $GITHUB_OUTPUT
           else
-              echo ::set-output name=match::false
+            echo "match=false" >> $GITHUB_OUTPUT
           fi
       - name: Check RC Tag
         id: check-rc-tag
         run: |
           if [[ ${{ github.event.ref }} =~ ^refs/tags/[0-9]+\.[0-9]+\.[0-9]+rc[0-9]+$ ]]; then
-              echo ::set-output name=match::true
+            echo "match=true" >> $GITHUB_OUTPUT
           else
-              echo ::set-output name=match::false
+            echo "match=false" >> $GITHUB_OUTPUT
           fi
       - name: Check alpha Tag
         id: check-alpha-tag
         run: |
           if [[ ${{ github.event.ref }} =~ ^refs/tags/[0-9]+\.[0-9]+\.[0-9]+a[0-9]+$ ]]; then
-              echo ::set-output name=match::true
+            echo "match=true" >> $GITHUB_OUTPUT
           else
-              echo ::set-output name=match::false
+            echo "match=false" >> $GITHUB_OUTPUT
           fi
       - name: Check beta Tag
         id: check-beta-tag
         run: |
           if [[ ${{ github.event.ref }} =~ ^refs/tags/[0-9]+\.[0-9]+\.[0-9]+b[0-9]+$ ]]; then
-              echo ::set-output name=match::true
+            echo "match=true" >> $GITHUB_OUTPUT
           else
-              echo ::set-output name=match::false
+            echo "match=false" >> $GITHUB_OUTPUT
           fi
 
       # Prod and 'beta' tags go to PyPI; 'rc', 'alpha', all other tags and untagged commits go to TestPyPI

CHANGELOG.md

@@ -15,26 +15,30 @@ The types of changes are:
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
-
 ## [Unreleased](https://github.com/ethyca/fides/compare/2.15.0...main)
 
 ### Added
 
 - Included optional env vars to have postgres or Redshift connected via bastion host [#3374](https://github.com/ethyca/fides/pull/3374/)
 - Support for acknowledge button for notice-only Privacy Notices and to disable toggling them off [#3546](https://github.com/ethyca/fides/pull/3546)
+- HTML format for privacy request storage destinations [#3427](https://github.com/ethyca/fides/pull/3427)
+
+### Changed
+
+- Removed `pyodbc` in favor of `pymssql` for handling SQL Server connections [#3435](https://github.com/ethyca/fides/pull/3435)
 
 ### Fixed
 
 - Fix race condition with consent modal link rendering [#3521](https://github.com/ethyca/fides/pull/3521)
-- Remove the `fides-js` banner from tab order when it is hidden and move the overlay components to the top of the tab order. [#3510](https://github.com/ethyca/fides/pull/3510)
+- Hide custom fields section when there are no custom fields created [#3554](https://github.com/ethyca/fides/pull/3554)
 - Disable connector dropdown in integration tab on save [#3552](https://github.com/ethyca/fides/pull/3552)
 - Handles an edge case for non-existent identities with the Kustomer API [#3513](https://github.com/ethyca/fides/pull/3513)
+- remove the configure privacy request tile from the home screen [#3555](https://github.com/ethyca/fides/pull/3555)
 
 ### Developer Experience
 
 - Optimize GitHub workflows used for docker image publishing [#3526](https://github.com/ethyca/fides/pull/3526)
 
-
 ## [2.15.0](https://github.com/ethyca/fides/compare/2.14.1...2.15.0)
 
 ### Added
@@ -54,6 +58,7 @@ The types of changes are:
 - Add ability to close `fides-js` banner and modal via a button or ESC [#3411](https://github.com/ethyca/fides/pull/3411)
 - Add ability to open the `fides-js` modal from a link on the host site [#3411](https://github.com/ethyca/fides/pull/3411)
 - GPC preferences are automatically applied via `fides-js` [#3411](https://github.com/ethyca/fides/pull/3411)
+- Add new dataset route that has additinonal filters [#3558](https://github.com/ethyca/fides/pull/3558)
 
 ### Fixed
 
@@ -64,6 +69,7 @@ The types of changes are:
 - Flaky custom field Cypress test on systems page [#3408](https://github.com/ethyca/fides/pull/3408)
 - Fix NextJS errors & warnings for Cookie House sample app [#3411](https://github.com/ethyca/fides/pull/3411)
 - Fix bug where `fides-js` toggles were not reflecting changes from rejecting or accepting all notices [#3522](https://github.com/ethyca/fides/pull/3522)
+- Remove the `fides-js` banner from tab order when it is hidden and move the overlay components to the top of the tab order. [#3510](https://github.com/ethyca/fides/pull/3510)
 
 ### Changed
 

Dockerfile

@@ -1,11 +1,9 @@
 # If you update this, also update `DEFAULT_PYTHON_VERSION` in the GitHub workflow files
 ARG PYTHON_VERSION="3.10.11"
-
 #########################
 ## Compile Python Deps ##
 #########################
 FROM python:${PYTHON_VERSION}-slim-bullseye as compile_image
-ARG TARGETPLATFORM
 
 # Install auxiliary software
 RUN apt-get update && \
@@ -17,16 +15,33 @@ RUN apt-get update && \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
+
+# Install FreeTDS (used for PyMSSQL)
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+    libssl-dev \
+    libffi-dev \
+    libxslt-dev \
+    libkrb5-dev \
+    unixodbc \
+    unixodbc-dev \
+    freetds-dev \
+    freetds-bin \
+    python-dev \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install Python Dependencies
+COPY dev-requirements.txt .
+RUN pip install --user -U pip --no-cache-dir install -r dev-requirements.txt
+
 # Activate a Python venv
 RUN python3 -m venv /opt/fides
 ENV PATH="/opt/fides/bin:${PATH}"
 
 # Install Python Dependencies
 RUN pip --no-cache-dir --disable-pip-version-check install --upgrade pip setuptools wheel
 
-COPY dangerous-requirements.txt .
-RUN if [ $TARGETPLATFORM != linux/arm64 ] ; then pip install --no-cache-dir install -r dangerous-requirements.txt ; fi
-
 COPY requirements.txt .
 RUN pip install --no-cache-dir install -r requirements.txt
 
@@ -37,38 +52,20 @@ RUN pip install --no-cache-dir install -r dev-requirements.txt
 ## Backend Base ##
 ##################
 FROM python:${PYTHON_VERSION}-slim-bullseye as backend
-ARG TARGETPLATFORM
 
-# Loads compiled requirements and adds the to the path
-COPY --from=compile_image /opt/fides /opt/fides
-ENV PATH=/opt/fides/bin:$PATH
-
-# These are all required for MSSQL
-RUN : \
-    && apt-get update \
-    && apt-get install \
-    -y --no-install-recommends \
-    apt-transport-https \
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
     curl \
     git \
-    gnupg \
-    unixodbc-dev \
+    freetds-dev \
+    freetds-bin \
+    python-dev \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
-# SQL Server (MS SQL)
-# https://docs.microsoft.com/en-us/sql/connect/odbc/linux-mac/installing-the-microsoft-odbc-driver-for-sql-server?view=sql-server-ver15
-RUN curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add -
-RUN curl https://packages.microsoft.com/config/debian/11/prod.list | tee /etc/apt/sources.list.d/msprod.list
-ENV ACCEPT_EULA=y DEBIAN_FRONTEND=noninteractive
-RUN if [ "$TARGETPLATFORM" != "linux/arm64" ] ; \
-    then apt-get update \
-    && apt-get install \
-    -y --no-install-recommends \
-    mssql-tools \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/* ; \
-    fi
+# Loads compiled requirements and adds the to the path
+COPY --from=compile_image /opt/fides /opt/fides
+ENV PATH=/opt/fides/bin:$PATH
 
 # General Application Setup ##
 COPY . /fides

MANIFEST.in

@@ -4,13 +4,15 @@ include requirements.txt
 include dev-requirements.txt
 include dangerous-requirements.txt
 include versioneer.py
-include src/fides/api/ctl/alembic.ini
+include src/fides/api/alembic.ini
 include src/fides/_version.py
 include src/fides/py.typed
 
 recursive-include src/fides/data *
 recursive-include src/fides/ui-build *
 
 graft src/fides/api/email_templates/templates
+graft src/fides/api/service/privacy_request/dsr_package/templates
+graft src/fides/api/service/privacy_request/dsr_package/assets
 
 global-exclude *.pyc

clients/admin-ui/cypress/e2e/home.cy.ts

@@ -6,7 +6,6 @@ const ALL_TILES = [
   "View data map",
   "Add systems",
   "View systems",
-  "Configure privacy requests",
   "Review privacy requests",
 ];