Add Okta support to generate endpoint

CHANGELOG.md

@@ -25,6 +25,7 @@ The types of changes are:
   * System scanning step: AWS credentials form and initial `generate` API usage.
 * Added Cypress for testing [713](https://github.com/ethyca/fides/pull/833)
 * CustomInput type "password" with show/hide icon.
+* Add Okta support to the `/generate` endpoint [#842](https://github.com/ethyca/fides/pull/842)
 
 ### Changed
 * Updated the `datamap` endpoint to return human-readable column names as the first response item [#779](https://github.com/ethyca/fides/pull/779)

src/fidesapi/routes/generate.py

@@ -10,14 +10,18 @@
 from pydantic import BaseModel, root_validator
 
 from fidesapi.routes.crud import get_resource
-from fidesapi.routes.util import API_PREFIX, route_requires_aws_connector
+from fidesapi.routes.util import (
+    API_PREFIX,
+    route_requires_aws_connector,
+    route_requires_okta_connector,
+)
 from fidesapi.sql_models import sql_model_map
 from fidesctl.connectors.models import (
     AWSConfig,
     ConnectorAuthFailureException,
     OktaConfig,
 )
-from fidesctl.core.system import generate_aws_systems
+from fidesctl.core.system import generate_aws_systems, generate_okta_systems
 
 
 class ValidTargets(str, Enum):
@@ -26,6 +30,7 @@ class ValidTargets(str, Enum):
     """
 
     AWS = "aws"
+    OKTA = "okta"
 
 
 class GenerateTypes(str, Enum):
@@ -55,7 +60,7 @@ def target_matches_type(cls, values: Dict) -> Dict:
         pair (returning an error on an ('aws', 'dataset') as an example).
         """
         target_type = (values.get("target"), values.get("type"))
-        valid_target_types = [("aws", "systems")]
+        valid_target_types = [("aws", "systems"), ("okta", "systems")]
         if target_type not in valid_target_types:
             raise ValueError("Target and Type are not a valid match")
         return values
@@ -104,23 +109,34 @@ async def generate(
 
     Currently generates Fides resources for the following:
     * AWS: Systems
+    * Okta: Systems
 
     In the future, this will include options for other Systems & Datasets,
     examples include:
-    * Okta: Systems
     * Snowflake: Datasets
 
     All production deployments should implement HTTPS for security purposes
     """
     organization = await get_resource(
         sql_model_map["organization"], generate_request_payload.organization_key
     )
-    if generate_request_payload.generate.target.lower() == "aws":
-        generated_systems = generate_aws(
-            aws_config=AWSConfig(**generate_request_payload.generate.config.dict()),
-            organization=organization,
+    try:
+        if generate_request_payload.generate.target.lower() == "aws":
+            generate_results = generate_aws(
+                aws_config=generate_request_payload.generate.config,
+                organization=organization,
+            )
+        elif generate_request_payload.generate.target.lower() == "okta":
+            generate_results = await generate_okta(
+                okta_config=generate_request_payload.generate.config,
+                organization=organization,
+            )
+    except ConnectorAuthFailureException as error:
+        raise HTTPException(
+            status_code=status.HTTP_403_FORBIDDEN,
+            detail=str(error),
         )
-    return GenerateResponse(generate_results=generated_systems)
+    return GenerateResponse(generate_results=generate_results)
 
 
 @route_requires_aws_connector
@@ -130,16 +146,20 @@ def generate_aws(
     """
     Returns a list of Systems found in AWS.
     """
-    log.info("Setting config for AWS")
-    try:
-        log.info("Generating systems from AWS")
-        aws_systems = generate_aws_systems(
-            organization=organization, aws_config=aws_config
-        )
-    except ConnectorAuthFailureException as error:
-        raise HTTPException(
-            status_code=status.HTTP_403_FORBIDDEN,
-            detail=str(error),
-        )
-
+    log.info("Generating systems from AWS")
+    aws_systems = generate_aws_systems(organization=organization, aws_config=aws_config)
     return [i.dict(exclude_none=True) for i in aws_systems]
+
+
+@route_requires_okta_connector
+async def generate_okta(
+    okta_config: OktaConfig, organization: Organization
+) -> List[Dict[str, str]]:
+    """
+    Returns a list of Systems found in Okta.
+    """
+    log.info("Generating systems from Okta")
+    okta_systems = await generate_okta_systems(
+        organization=organization, okta_config=okta_config
+    )
+    return [i.dict(exclude_none=True) for i in okta_systems]

src/fidesctl/cli/commands/generate.py

@@ -11,6 +11,7 @@
     include_null_flag,
     okta_org_url_option,
     okta_token_option,
+    organization_fides_key_option,
 )
 from fidesctl.cli.utils import (
     handle_aws_credentials_options,
@@ -89,6 +90,7 @@ def generate_system(ctx: click.Context) -> None:
 @okta_org_url_option
 @okta_token_option
 @include_null_flag
+@organization_fides_key_option
 @with_analytics
 def generate_system_okta(
     ctx: click.Context,
@@ -97,6 +99,7 @@ def generate_system_okta(
     token: str,
     org_url: str,
     include_null: bool,
+    org_key: str,
 ) -> None:
     """
     Generates systems for your Okta applications. Connect to an Okta admin
@@ -107,8 +110,9 @@ def generate_system_okta(
     This is a one-time operation that does not track the state of the okta resources.
     It will need to be run again if the tracked resources change.
     """
+    config = ctx.obj["CONFIG"]
     okta_config = handle_okta_credentials_options(
-        fides_config=ctx.obj["CONFIG"],
+        fides_config=config,
         token=token,
         org_url=org_url,
         credentials_id=credentials_id,
@@ -118,6 +122,9 @@ def generate_system_okta(
         okta_config=okta_config,
         file_name=output_filename,
         include_null=include_null,
+        organization_key=org_key,
+        url=config.cli.server_url,
+        headers=config.user.request_headers,
     )
 
 
@@ -129,13 +136,13 @@ def generate_system_okta(
 @aws_secret_access_key_option
 @aws_region_option
 @include_null_flag
-@click.option("-o", "--organization", type=str, default="default_organization")
+@organization_fides_key_option
 @with_analytics
 def generate_system_aws(
     ctx: click.Context,
     output_filename: str,
     include_null: bool,
-    organization: str,
+    org_key: str,
     credentials_id: str,
     access_key_id: str,
     secret_access_key: str,
@@ -163,7 +170,7 @@ def generate_system_aws(
     _system.generate_system_aws(
         file_name=output_filename,
         include_null=include_null,
-        organization_key=organization,
+        organization_key=org_key,
         aws_config=aws_config,
         url=config.cli.server_url,
         headers=config.user.request_headers,

src/fidesctl/cli/commands/scan.py

@@ -12,6 +12,7 @@
     manifests_dir_argument,
     okta_org_url_option,
     okta_token_option,
+    organization_fides_key_option,
 )
 from fidesctl.cli.utils import (
     handle_aws_credentials_options,
@@ -94,6 +95,7 @@ def scan_system(ctx: click.Context) -> None:
 @credentials_id_option
 @okta_org_url_option
 @okta_token_option
+@organization_fides_key_option
 @coverage_threshold_option
 @with_analytics
 def scan_system_okta(
@@ -102,6 +104,7 @@ def scan_system_okta(
     credentials_id: str,
     org_url: str,
     token: str,
+    org_key: str,
     coverage_threshold: int,
 ) -> None:
     """
@@ -123,6 +126,7 @@ def scan_system_okta(
     _system.scan_system_okta(
         okta_config=okta_config,
         coverage_threshold=coverage_threshold,
+        organization_key=org_key,
         manifest_dir=manifests_dir,
         url=config.cli.server_url,
         headers=config.user.request_headers,
@@ -136,7 +140,7 @@ def scan_system_okta(
 @aws_access_key_id_option
 @aws_secret_access_key_option
 @aws_region_option
-@click.option("-o", "--organization", type=str, default="default_organization")
+@organization_fides_key_option
 @coverage_threshold_option
 @with_analytics
 def scan_system_aws(
@@ -146,7 +150,7 @@ def scan_system_aws(
     access_key_id: str,
     secret_access_key: str,
     region: str,
-    organization: str,
+    org_key: str,
     coverage_threshold: int,
 ) -> None:
     """
@@ -169,7 +173,7 @@ def scan_system_aws(
 
     _system.scan_system_aws(
         manifest_dir=manifests_dir,
-        organization_key=organization,
+        organization_key=org_key,
         aws_config=aws_config,
         coverage_threshold=coverage_threshold,
         url=config.cli.server_url,

src/fidesctl/connectors/okta.py

@@ -77,7 +77,9 @@ async def list_okta_applications(okta_client: OktaClient) -> List[OktaApplicatio
     return applications
 
 
-def create_okta_systems(okta_applications: List[OktaApplication]) -> List[System]:
+def create_okta_systems(
+    okta_applications: List[OktaApplication], organization_key: str
+) -> List[System]:
     """
     Returns a list of system objects given a list of Okta applications
     """
@@ -90,6 +92,7 @@ def create_okta_systems(okta_applications: List[OktaApplication]) -> List[System
             ),
             description=f"Fides Generated Description for Okta Application: {application.label}",
             system_type="okta_application",
+            organization_fides_key=organization_key,
             privacy_declarations=[],
         )
         for application in okta_applications

src/fidesctl/core/system.py

@@ -154,25 +154,51 @@ def generate_system_aws(
     return file_name
 
 
-def generate_system_okta(
-    okta_config: Optional[OktaConfig], file_name: str, include_null: bool
-) -> str:
+async def generate_okta_systems(
+    organization: Organization, okta_config: Optional[OktaConfig]
+) -> List[System]:
     """
-    Given an organization url, generates a system manifest from existing Okta
-    applications.
+    Given an okta configuration, calls okta for a list of
+    applications and returns the corresponding systems.
     """
 
-    _check_okta_connector_import()
-
     import fidesctl.connectors.okta as okta_connector
 
     okta_client = okta_connector.get_okta_client(okta_config)
-    okta_applications = asyncio.run(
-        okta_connector.list_okta_applications(okta_client=okta_client)
+    okta_applications = await okta_connector.list_okta_applications(
+        okta_client=okta_client
     )
     okta_systems = okta_connector.create_okta_systems(
-        okta_applications=okta_applications
+        okta_applications=okta_applications, organization_key=organization.fides_key
     )
+    return okta_systems
+
+
+def generate_system_okta(
+    organization_key: str,
+    okta_config: Optional[OktaConfig],
+    file_name: str,
+    include_null: bool,
+    url: AnyHttpUrl,
+    headers: Dict[str, str],
+) -> str:
+    """
+    Generates a system manifest from existing Okta applications.
+    """
+
+    _check_okta_connector_import()
+
+    organization = get_organization(
+        organization_key=organization_key,
+        manifest_organizations=[],
+        url=url,
+        headers=headers,
+    )
+
+    okta_systems = asyncio.run(
+        generate_okta_systems(organization=organization, okta_config=okta_config)
+    )
+
     write_system_manifest(
         file_name=file_name, include_null=include_null, systems=okta_systems
     )
@@ -357,14 +383,15 @@ def scan_system_aws(
 
 def scan_system_okta(
     manifest_dir: str,
+    organization_key: str,
     okta_config: Optional[OktaConfig],
     coverage_threshold: int,
     url: AnyHttpUrl,
     headers: Dict[str, str],
 ) -> None:
     """
-    Given an organization url, fetches Okta applications and compares them
-    against existing systems in the server and manifest supplied.
+    Fetches Okta applications and compares them against existing
+    systems in the server and manifest supplied.
     """
 
     _check_okta_connector_import()
@@ -375,15 +402,19 @@ def scan_system_okta(
         url=url, headers=headers, exclude_systems=manifest_systems
     )
 
-    import fidesctl.connectors.okta as okta_connector
-
-    okta_client = okta_connector.get_okta_client(okta_config)
-    okta_applications = asyncio.run(
-        okta_connector.list_okta_applications(okta_client=okta_client)
+    organization = get_organization(
+        organization_key=organization_key,
+        manifest_organizations=manifest_taxonomy.organization
+        if manifest_taxonomy
+        else [],
+        url=url,
+        headers=headers,
     )
-    okta_systems = okta_connector.create_okta_systems(
-        okta_applications=okta_applications
+
+    okta_systems = asyncio.run(
+        generate_okta_systems(organization=organization, okta_config=okta_config)
     )
+
     if len(okta_systems) < 1:
         echo_red("Okta did not return any applications to scan systems")
         raise SystemExit(1)

tests/api/test_generate.py

@@ -14,12 +14,18 @@
         "region_name": getenv("AWS_DEFAULT_REGION", ""),
         "aws_access_key_id": getenv("AWS_ACCESS_KEY_ID", ""),
         "aws_secret_access_key": getenv("AWS_SECRET_ACCESS_KEY", ""),
-    }
+    },
+    "okta": {
+        "orgUrl": "https://dev-78908748.okta.com",
+        "token": getenv("OKTA_CLIENT_TOKEN", ""),
+    },
 }
 
 
 @pytest.mark.external
-@pytest.mark.parametrize("generate_type, generate_target", [("systems", "aws")])
+@pytest.mark.parametrize(
+    "generate_type, generate_target", [("systems", "aws"), ("systems", "okta")]
+)
 def test_generate(
     test_config: FidesctlConfig,
     generate_type: str,