From de1e4f16b7f8cca65eb3002728fe7f74a4be803d Mon Sep 17 00:00:00 2001 From: Steve Murphy Date: Mon, 25 Jul 2022 14:48:40 -0400 Subject: [PATCH] Add Exception Handling to Generate Route (#911) * validate credentials for connectors * bigquery handling, failure tests * changelog * add missing ctl path to imports --- CHANGELOG.md | 1 + src/fidesctl/api/ctl/routes/generate.py | 39 ++++++++++++++- src/fidesctl/ctl/core/utils.py | 15 ++++++ tests/ctl/api/test_generate.py | 66 +++++++++++++++++++++++++ tests/ctl/cli/test_cli.py | 4 +- 5 files changed, 122 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e30d591012..102ad029ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,7 @@ The types of changes are: * Update `fideslang` to `1.1.0`, simplifying the default taxonomy and adding `tags` for resources [#865](https://github.com/ethyca/fides/pull/865) * Remove the `obscure` requirement from the `generate` endpoint [#819](https://github.com/ethyca/fides/pull/819) * Merge existing configurations with `fideslib` library [#913](https://github.com/ethyca/fides/pull/913) +* Replicated the error response handling from the `/validate` endpoint to the `/generate` endpoint [#911](https://github.com/ethyca/fides/pull/911) * Moved frontend static files to `src/ui-build/static` [#934](https://github.com/ethyca/fides/pull/934) ### Developer Experience diff --git a/src/fidesctl/api/ctl/routes/generate.py b/src/fidesctl/api/ctl/routes/generate.py index caea288e8f..27fd07141e 100644 --- a/src/fidesctl/api/ctl/routes/generate.py +++ b/src/fidesctl/api/ctl/routes/generate.py @@ -22,11 +22,13 @@ AWSConfig, BigQueryConfig, ConnectorAuthFailureException, + ConnectorFailureException, DatabaseConfig, OktaConfig, ) from fidesctl.ctl.core.dataset import generate_bigquery_datasets, generate_db_datasets from fidesctl.ctl.core.system import generate_aws_systems, generate_okta_systems +from fidesctl.ctl.core.utils import validate_db_engine class ValidTargets(str, Enum): @@ -168,6 +170,17 @@ def generate_aws( """ Returns a list of Systems found in AWS. """ + from fidesctl.ctl.connectors.aws import validate_credentials + + log.info("Validating AWS credentials") + try: + validate_credentials(aws_config) + except ConnectorAuthFailureException as error: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail=str(error), + ) + log.info("Generating systems from AWS") aws_systems = generate_aws_systems(organization=organization, aws_config=aws_config) @@ -181,6 +194,16 @@ async def generate_okta( """ Returns a list of Systems found in Okta. """ + from fidesctl.ctl.connectors.okta import validate_credentials + + log.info("Validating Okta credentials") + try: + validate_credentials(okta_config) + except ConnectorAuthFailureException as error: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail=str(error), + ) log.info("Generating systems from Okta") okta_systems = await generate_okta_systems( organization=organization, okta_config=okta_config @@ -192,6 +215,14 @@ def generate_db(db_config: DatabaseConfig) -> List[Dict[str, str]]: """ Returns a list of datasets found in a database. """ + log.info("Validating database credentials") + try: + validate_db_engine(db_config.connection_string) + except ConnectorAuthFailureException as error: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail=str(error), + ) log.info("Generating datasets from database") db_datasets = generate_db_datasets(connection_string=db_config.connection_string) @@ -204,5 +235,11 @@ def generate_bigquery(bigquery_config: BigQueryConfig) -> List[Dict[str, str]]: Returns a list of datasets found in a BigQuery dataset """ log.info("Generating datasets from BigQuery") - bigquery_datasets = generate_bigquery_datasets(bigquery_config) + try: + bigquery_datasets = generate_bigquery_datasets(bigquery_config) + except ConnectorFailureException as error: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail=str(error), + ) return [i.dict(exclude_none=True) for i in bigquery_datasets] diff --git a/src/fidesctl/ctl/core/utils.py b/src/fidesctl/ctl/core/utils.py index 33c6b4d546..961691aadd 100644 --- a/src/fidesctl/ctl/core/utils.py +++ b/src/fidesctl/ctl/core/utils.py @@ -14,6 +14,9 @@ from fideslang.validation import FidesValidationError from git.repo import Repo from sqlalchemy.engine import Engine +from sqlalchemy.exc import SQLAlchemyError + +from fidesctl.ctl.connectors.models import ConnectorAuthFailureException logger = logging.getLogger("server_api") @@ -36,6 +39,18 @@ def check_response(response: requests.Response) -> requests.Response: return response +def validate_db_engine(connection_string: str) -> None: + """ + Use SQLAlchemy to create a DB engine. + """ + try: + engine = sqlalchemy.create_engine(connection_string) + with engine.begin() as connection: + connection.execute("SELECT 1") + except SQLAlchemyError as error: + raise ConnectorAuthFailureException(error) + + def get_db_engine(connection_string: str) -> Engine: """ Use SQLAlchemy to create a DB engine. diff --git a/tests/ctl/api/test_generate.py b/tests/ctl/api/test_generate.py index dca6cd4152..f8c108a3bf 100644 --- a/tests/ctl/api/test_generate.py +++ b/tests/ctl/api/test_generate.py @@ -31,6 +31,37 @@ }, } +EXTERNAL_FAILURE_CONFIG_BODY = { + "aws": { + "region_name": getenv("AWS_DEFAULT_REGION", ""), + "aws_access_key_id": "ILLEGAL_ACCESS_KEY_ID", + "aws_secret_access_key": "ILLEGAL_SECRET_ACCESS_KEY_ID", + }, + "bigquery": { + "dataset": "fidesopstest", + "keyfile_creds": loads( + b64decode(getenv("BIGQUERY_CONFIG", "e30=").encode("utf-8")).decode("utf-8") + ), + }, + "db": { + "connection_string": "postgresql+psycopg2://postgres:postgres@postgres-test:5432/INVALID_DB" + }, + "okta": { + "orgUrl": "https://dev-78908748.okta.com", + "token": "INVALID_TOKEN", + }, +} +EXTERNAL_FAILURE_CONFIG_BODY["bigquery"]["keyfile_creds"][ + "project_id" +] = "INVALID_PROJECT_ID" + +EXPECTED_FAILURE_MESSAGES = { + "aws": "The security token included in the request is invalid.", + "okta": "Invalid token provided", + "db": '(psycopg2.OperationalError) FATAL: database "INVALID_DB" does not exist\n\n(Background on this error at: https://sqlalche.me/e/14/e3q8)', + "bigquery": "Invalid project ID 'INVALID_PROJECT_ID'. Project IDs must contain 6-63 lowercase letters, digits, or dashes. Some project IDs also include domain name separated by a colon. IDs must start with a letter and may not end with a dash.", +} + @pytest.mark.external @pytest.mark.parametrize( @@ -66,3 +97,38 @@ def test_generate( generate_response = GenerateResponse.parse_raw(response.text) assert len(generate_response.generate_results) > 0 assert response.status_code == 200 + + +@pytest.mark.external +@pytest.mark.parametrize( + "generate_type, generate_target", + [ + ("systems", "aws"), + ("systems", "okta"), + ("datasets", "db"), + ("datasets", "bigquery"), + ], +) +def test_generate_failure( + test_config: FidesctlConfig, + generate_type: str, + generate_target: str, + test_client: TestClient, +) -> None: + + data = { + "organization_key": "default_organization", + "generate": { + "config": EXTERNAL_FAILURE_CONFIG_BODY[generate_target], + "target": generate_target, + "type": generate_type, + }, + } + + response = test_client.post( + test_config.cli.server_url + API_PREFIX + "/generate/", + headers=test_config.user.request_headers, + data=dumps(data), + ) + + assert loads(response.text)["detail"] == EXPECTED_FAILURE_MESSAGES[generate_target] diff --git a/tests/ctl/cli/test_cli.py b/tests/ctl/cli/test_cli.py index 7903d7ab90..b6fd104c80 100644 --- a/tests/ctl/cli/test_cli.py +++ b/tests/ctl/cli/test_cli.py @@ -764,7 +764,7 @@ def test_generate_dataset_bigquery_credentials_id( ) -> None: tmp_output_file = tmpdir.join("dataset.yml") - config_data = os.getenv("BIGQUERY_CONFIG", "") + config_data = os.getenv("BIGQUERY_CONFIG", "e30=") config_data_decoded = loads(b64decode(config_data.encode("utf-8")).decode("utf-8")) os.environ["FIDESCTL__CREDENTIALS__BIGQUERY_1__PROJECT_ID"] = config_data_decoded[ "project_id" @@ -813,7 +813,7 @@ def test_generate_dataset_bigquery_keyfile_path( tmp_output_file = tmpdir.join("dataset.yml") tmp_keyfile = tmpdir.join("bigquery.json") - config_data = os.getenv("BIGQUERY_CONFIG", "") + config_data = os.getenv("BIGQUERY_CONFIG", "e30=") config_data_decoded = loads(b64decode(config_data.encode("utf-8")).decode("utf-8")) with open(tmp_keyfile, "w", encoding="utf-8") as keyfile: dump(config_data_decoded, keyfile)