Merge branch 'main' of github.com:ethyca/fides into SteveDMurphy-824-…

…fideslang-updates

CHANGELOG.md

@@ -20,24 +20,28 @@ The types of changes are:
 ### Added
 
 * Add datasets via YAML in the UI [#813](https://github.com/ethyca/fides/pull/813)
-* Add datasets via database connection (UI only) [#834](https://github.com/ethyca/fides/pull/834)
+* Add datasets via database connection [#834](https://github.com/ethyca/fides/pull/834) [#889](https://github.com/ethyca/fides/pull/889)
 * Add delete confirmation when deleting a field or collection from a dataset [#809](https://github.com/ethyca/fides/pull/809)
 * Add ability to delete datasets from the UI
 * Initial configuration wizard UI view
   * System scanning step: AWS credentials form and initial `generate` API usage.
   * System scanning results: AWS systems are stored and can be selected for review
 * Added Cypress for testing [713](https://github.com/ethyca/fides/pull/833)
 * CustomInput type "password" with show/hide icon.
+* Sync CLI command now checks for untracked/unstaged files in the manifests dir [#869](https://github.com/ethyca/fides/pull/869)
 * Add Okta support to the `/generate` endpoint [#842](https://github.com/ethyca/fides/pull/842)
 * Add db support to `/generate` endpoint [849](https://github.com/ethyca/fides/pull/849)
 * Added OpenAPI TypeScript client generation for the UI app. See the [README](/clients/admin-ui/src/types/api/README.md) for more details.
 
 ### Changed
 
 * Updated the `datamap` endpoint to return human-readable column names as the first response item [#779](https://github.com/ethyca/fides/pull/779)
+* Moved `scan` and `generate` to the list of commands that can be run in local mode [#841](https://github.com/ethyca/fides/pull/841)
 * Webserver dependencies now come as a standard part of the package [#881](https://github.com/ethyca/fides/pull/881)
 * Initial configuration wizard UI view
   * Refactored step & form results management to use Redux Toolkit slice.
+* Remove the `obscure` requirement from the `generate` endpoint [#819](https://github.com/ethyca/fides/pull/819)
+* Remove the `obscure` requirement from the `generate` endpoint [#819](https://github.com/ethyca/fides/pull/819)
 
 ### Docs
 
@@ -48,10 +52,8 @@ The types of changes are:
 * CustomSelect input tooltips appear next to selector instead of wrapping to a new row.
 * Datasets without the `third_country_transfer` will not cause the editing dataset form to not render.
 * Fixed a build issue causing an `unknown` version of `fidesctl` to be installed in published Docker images [#836](https://github.com/ethyca/fides/pull/836)
-
-### Changed
-
-* Remove the `obscure` requirement from the `generate` endpoint [#819](https://github.com/ethyca/fides/pull/819)
+* Fixed an M1-related SQLAlchemy bug [#816](https://github.com/ethyca/fides/pull/891)
+* Endpoints now work with or without a trailing slash. [#886](https://github.com/ethyca/fides/pull/886)
 
 ## [1.7.0](https://github.com/ethyca/fides/compare/1.6.1...1.7.0) - 2022-06-23
 

clients/admin-ui/src/features/common/helpers.ts

@@ -78,25 +78,14 @@ export function isYamlException(error: unknown): error is YAMLException {
   );
 }
 
-type RTKReturnType =
-  | {
-      data: any;
-    }
-  | {
-      error: FetchBaseQueryError | SerializedError;
-    };
-export function getErrorFromResult(result: RTKReturnType) {
-  if ("error" in result) {
-    const { error } = result;
-    let errorMsg = "An unexpected error occurred. Please try again.";
-    if (isErrorWithDetail(error)) {
-      errorMsg = error.data.detail;
-    } else if (isErrorWithDetailArray(error)) {
-      errorMsg = `${error.data.detail[0].msg}: ${error.data.detail[0].loc}`;
-    } else if (isConflictError(error)) {
-      errorMsg = `${error.data.detail.error} (${error.data.detail.fides_key})`;
-    }
-    return errorMsg;
+export function getErrorMessage(error: FetchBaseQueryError | SerializedError) {
+  let errorMsg = "An unexpected error occurred. Please try again.";
+  if (isErrorWithDetail(error)) {
+    errorMsg = error.data.detail;
+  } else if (isErrorWithDetailArray(error)) {
+    errorMsg = `${error.data.detail[0].msg}: ${error.data.detail[0].loc}`;
+  } else if (isConflictError(error)) {
+    errorMsg = `${error.data.detail.error} (${error.data.detail.fides_key})`;
   }
-  return null;
+  return errorMsg;
 }

clients/admin-ui/src/features/dataset/DatabaseConnectForm.tsx

@@ -1,25 +1,87 @@
-import { Box, Button, Text } from "@fidesui/react";
-import { Form, Formik } from "formik";
+import { Box, Button, Spinner, Text, useToast } from "@fidesui/react";
+import { SerializedError } from "@reduxjs/toolkit";
+import { FetchBaseQueryError } from "@reduxjs/toolkit/dist/query";
+import { Form, Formik, FormikHelpers } from "formik";
+import { useRouter } from "next/router";
 import * as Yup from "yup";
 
+import { GenerateResponse, GenerateTypes, ValidTargets } from "~/types/api";
+
 import { CustomTextInput } from "../common/form/inputs";
+import { getErrorMessage } from "../common/helpers";
+import { successToastParams } from "../common/toast";
+import {
+  setActiveDataset,
+  useCreateDatasetMutation,
+  useGenerateDatasetMutation,
+} from "./dataset.slice";
+import { Dataset } from "./types";
 
 const initialValues = { url: "" };
 
+type FormValues = typeof initialValues;
+
 const ValidationSchema = Yup.object().shape({
   url: Yup.string().required().label("Database URL"),
 });
 
 const DatabaseConnectForm = () => {
-  const handleCreate = () => {
-    //   TODO when backend supports this (829)
+  // TODO: where should this come from?
+  const organizationKey = "default_organization";
+  const [generate, { isLoading: isGenerating }] = useGenerateDatasetMutation();
+  const [createDataset, { isLoading: isCreating }] = useCreateDatasetMutation();
+  const toast = useToast();
+  const router = useRouter();
+
+  const handleSubmit = async (
+    values: FormValues,
+    formikHelpers: FormikHelpers<FormValues>
+  ) => {
+    const { setErrors } = formikHelpers;
+
+    const handleError = (error: FetchBaseQueryError | SerializedError) => {
+      const errorMessage = getErrorMessage(error);
+      setErrors({ url: errorMessage });
+    };
+
+    const handleGenerateResults = async (
+      results: GenerateResponse["generate_results"]
+    ) => {
+      if (results && results.length) {
+        const newDataset = results[0] as Dataset;
+        const createResult = await createDataset(newDataset);
+        if ("error" in createResult) {
+          handleError(createResult.error);
+        } else {
+          toast(successToastParams("Successfully loaded new dataset"));
+          setActiveDataset(createResult.data);
+          router.push(`/dataset/${createResult.data.fides_key}`);
+        }
+      }
+    };
+
+    const response = await generate({
+      organization_key: organizationKey,
+      generate: {
+        config: { connection_string: values.url },
+        target: ValidTargets.DB,
+        type: GenerateTypes.DATASETS,
+      },
+    });
+
+    if ("error" in response) {
+      const errorMessage = getErrorMessage(response.error);
+      setErrors({ url: errorMessage });
+    } else {
+      handleGenerateResults(response.data.generate_results);
+    }
   };
 
   return (
     <Formik
       initialValues={initialValues}
       validationSchema={ValidationSchema}
-      onSubmit={handleCreate}
+      onSubmit={handleSubmit}
       validateOnChange={false}
       validateOnBlur={false}
     >
@@ -33,14 +95,18 @@ const DatabaseConnectForm = () => {
           <Box mb={8}>
             <CustomTextInput name="url" label="Database URL" />
           </Box>
-          <Button
-            size="sm"
-            colorScheme="primary"
-            type="submit"
-            disabled={isSubmitting}
-          >
-            Create dataset
-          </Button>
+          <Box display="flex" alignItems="center">
+            <Button
+              size="sm"
+              colorScheme="primary"
+              type="submit"
+              disabled={isSubmitting}
+              mr="2"
+            >
+              Create dataset
+            </Button>
+            {isGenerating || isCreating ? <Spinner /> : null}
+          </Box>
         </Form>
       )}
     </Formik>

clients/admin-ui/src/features/dataset/DatasetYamlForm.tsx

@@ -4,7 +4,7 @@ import yaml from "js-yaml";
 import { useRouter } from "next/router";
 
 import { CustomTextArea } from "../common/form/inputs";
-import { getErrorFromResult, isYamlException } from "../common/helpers";
+import { getErrorMessage, isYamlException } from "../common/helpers";
 import { successToastParams } from "../common/toast";
 import { setActiveDataset, useCreateDatasetMutation } from "./dataset.slice";
 import { Dataset } from "./types";
@@ -44,9 +44,9 @@ const DatasetYamlForm = () => {
     }
 
     const result = await createDataset(dataset);
-    const error = getErrorFromResult(result);
-    if (error) {
-      setErrors({ datasetYaml: error });
+    if ("error" in result) {
+      const errorMessage = getErrorMessage(result.error);
+      setErrors({ datasetYaml: errorMessage });
     } else if ("data" in result) {
       toast(successToastParams("Successfully loaded new dataset YAML"));
       setActiveDataset(result.data);

clients/admin-ui/src/features/dataset/dataset.slice.ts

@@ -3,6 +3,7 @@ import { createApi, fetchBaseQuery } from "@reduxjs/toolkit/query/react";
 import { HYDRATE } from "next-redux-wrapper";
 
 import type { AppState } from "~/app/store";
+import { GenerateRequestPayload, GenerateResponse } from "~/types/api";
 
 import { FidesKey } from "../common/fides-types";
 import { Dataset } from "./types";
@@ -72,6 +73,13 @@ export const datasetApi = createApi({
       }),
       invalidatesTags: ["Datasets"],
     }),
+    generateDataset: build.mutation<GenerateResponse, GenerateRequestPayload>({
+      query: (payload) => ({
+        url: `generate/`,
+        method: "POST",
+        body: payload,
+      }),
+    }),
   }),
 });
 
@@ -81,6 +89,7 @@ export const {
   useUpdateDatasetMutation,
   useCreateDatasetMutation,
   useDeleteDatasetMutation,
+  useGenerateDatasetMutation,
 } = datasetApi;
 
 export const datasetSlice = createSlice({

clients/admin-ui/src/types/api/index.ts

@@ -4,6 +4,7 @@
 
 export type { AWSConfig } from './models/AWSConfig';
 export type { ContactDetails } from './models/ContactDetails';
+export type { DatabaseConfig } from './models/DatabaseConfig';
 export type { DataCategory } from './models/DataCategory';
 export type { DataProtectionImpactAssessment } from './models/DataProtectionImpactAssessment';
 export type { DataQualifier } from './models/DataQualifier';

clients/admin-ui/src/types/api/models/DatabaseConfig.ts

@@ -0,0 +1,10 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+/**
+ * The model for the connection config for databases
+ */
+export type DatabaseConfig = {
+  connection_string: string;
+};

clients/admin-ui/src/types/api/models/Generate.ts

@@ -3,6 +3,7 @@
 /* eslint-disable */
 
 import type { AWSConfig } from './AWSConfig';
+import type { DatabaseConfig } from './DatabaseConfig';
 import type { GenerateTypes } from './GenerateTypes';
 import type { OktaConfig } from './OktaConfig';
 import type { ValidTargets } from './ValidTargets';
@@ -11,7 +12,7 @@ import type { ValidTargets } from './ValidTargets';
  * Defines attributes for generating resources included in a request.
  */
 export type Generate = {
-  config: (AWSConfig | OktaConfig);
+  config: (AWSConfig | OktaConfig | DatabaseConfig);
   target: ValidTargets;
   type: GenerateTypes;
 };

clients/admin-ui/src/types/api/models/ValidTargets.ts

@@ -7,5 +7,6 @@
  */
 export enum ValidTargets {
   AWS = 'aws',
+  DB = 'db',
   OKTA = 'okta',
 }

docker-compose.yml

@@ -29,6 +29,7 @@ services:
       FIDESCTL_TEST_MODE: "True"
       FIDESCTL__CLI__SERVER_HOST: "fidesctl"
       FIDESCTL__CLI__SERVER_PORT: "8080"
+      FIDESCTL__CLI__ANALYTICS_ID: ${FIDESCTL__CLI__ANALYTICS_ID}
       FIDESCTL__API__DATABASE_HOST: "fidesctl-db"
 
   fidesctl-ui:

docs/fides/docs/cli.md

@@ -6,5 +6,5 @@ These docs reflect the latest PyPI release.
 
 ::: mkdocs-click
     :module: fidesctl.cli
-    :command: cli_docs
+    :command: cli
     :depth: 1

requirements.txt

@@ -8,6 +8,7 @@ fastapi==0.77.1
 fideslang==1.1.0
 fideslib==2.2.2
 fideslog==1.1.5
+GitPython==3.1
 loguru>=0.5,<0.6
 openpyxl==3.0.9
 pandas==1.4
@@ -17,7 +18,7 @@ pydantic>=1.8.1,<2.0.0 # Required by fastapi
 PyJWT==2.4.0
 pyyaml>=5,<6
 requests>=2,<3
-sqlalchemy==1.4.36
+sqlalchemy[asyncio]==1.4.36
 SQLAlchemy-Utils==0.38.2
 toml>=0.10.1
 uvicorn==0.17.6

src/fidesapi/database/database.py

@@ -133,3 +133,13 @@ def get_db_health(database_url: str) -> str:
         error_type = get_full_exception_name(error)
         log.error(f"Unable to reach the database: {error_type}: {error}")
         return "unhealthy"
+
+
+async def configure_db(database_url: str) -> None:
+    "Set up the db to be used by the app."
+    try:
+        create_db_if_not_exists(database_url)
+        await init_db(database_url)
+    except Exception as error:  # pylint: disable=broad-except
+        error_type = get_full_exception_name(error)
+        log.error(f"Unable to configure database: {error_type}: {error}")