Scan/Generate Endpoint - Core Updates (#645)

* add aws config option, return the systems as json

This is still an iterative state, with the focus being on avoiding any impact to the current cli functionality. One suggestion has been to pass the client around instead of the configuration.

Further work will still be required to separate the cli commands from the core files. This will be done in separate issues.

* pulling changes after testing against new endpoint

After testing this PR against the new endpoint, some issues were surfaced with how some core functionality is used. It seemed the best next option was to utilize the specific Core commands required and only update the config option.

Further work in here will look at replacing the config option with generating a session and passing that instead.

* changelog update

* reorganization to embed filter_aws_systems

CHANGELOG.md

@@ -27,6 +27,7 @@ The types of changes are:
 
 * Comparing server and CLI versions ignores `.dirty` only differences, and is quiet on success when running general CLI commands
 * Migrate all endpoints to be prefixed by `/api/v1` [#623](https://github.com/ethyca/fides/issues/623)
+* Allow credentials to be passed to the generate systems from aws functionality via the API [#645](https://github.com/ethyca/fides/pull/645)
 
 ### Developer Experience
 

src/fidesctl/core/system.py

@@ -15,14 +15,15 @@
 from .utils import echo_green, echo_red
 
 
-def describe_redshift_clusters() -> Dict[str, List[Dict]]:
+def describe_redshift_clusters(aws_config: Dict[str, str]) -> Dict[str, List[Dict]]:
     """
     Creates boto3 redshift client and returns describe_clusters response.
     """
     import boto3
 
     redshift_client = boto3.client(
         "redshift",
+        **aws_config,
     )
     describe_clusters = redshift_client.describe_clusters()
     return describe_clusters
@@ -58,38 +59,42 @@ def transform_redshift_systems(
     return redshift_systems
 
 
-def generate_redshift_systems(organization_key: str) -> List[System]:
+def generate_redshift_systems(
+    organization_key: str, aws_config: Dict[str, str]
+) -> List[System]:
     """
     Fetches Redshift clusters from AWS and returns the transformed Sytem representations.
     """
-    describe_clusters = describe_redshift_clusters()
+    describe_clusters = describe_redshift_clusters(aws_config)
     redshift_systems = transform_redshift_systems(
         describe_clusters=describe_clusters, organization_key=organization_key
     )
     return redshift_systems
 
 
-def describe_rds_clusters() -> Dict[str, List[Dict]]:
+def describe_rds_clusters(aws_config: Dict[str, str]) -> Dict[str, List[Dict]]:
     """
     Creates boto3 rds client and returns describe_db_clusters response.
     """
     import boto3
 
     rds_client = boto3.client(
         "rds",
+        **aws_config,
     )
     describe_clusters = rds_client.describe_db_clusters()
     return describe_clusters
 
 
-def describe_rds_instances() -> Dict[str, List[Dict]]:
+def describe_rds_instances(aws_config: Dict[str, str]) -> Dict[str, List[Dict]]:
     """
     Creates boto3 rds client and returns describe_db_instances response.
     """
     import boto3
 
     rds_client = boto3.client(
         "rds",
+        **aws_config,
     )
     describe_instances = rds_client.describe_db_instances()
     return describe_instances
@@ -147,12 +152,14 @@ def transform_rds_systems(
     return rds_cluster_systems + rds_instances_systems
 
 
-def generate_rds_systems(organization_key: str) -> List[System]:
+def generate_rds_systems(
+    organization_key: str, aws_config: Dict[str, str]
+) -> List[System]:
     """
     Fetches RDS clusters and instances from AWS and returns the transformed Sytem representations.
     """
-    describe_clusters = describe_rds_clusters()
-    describe_instances = describe_rds_instances()
+    describe_clusters = describe_rds_clusters(aws_config)
+    describe_instances = describe_rds_instances(aws_config)
     rds_systems = transform_rds_systems(
         describe_clusters=describe_clusters,
         describe_instances=describe_instances,
@@ -196,17 +203,26 @@ def get_organization(
     return server_organization
 
 
-def generate_aws_systems(organization_key: str) -> List[System]:
+def generate_aws_systems(
+    organization: Organization, aws_config: Dict[str, str]
+) -> List[System]:
     """
     Calls each generate system function for aws resources
+
+    Returns a list of systems with any filters applied
     """
     generate_system_functions = [generate_redshift_systems, generate_rds_systems]
+
     aws_systems = [
         found_system
         for generate_function in generate_system_functions
-        for found_system in generate_function(organization_key)
+        for found_system in generate_function(organization.fides_key, aws_config)
     ]
-    return aws_systems
+
+    filtered_aws_systems = filter_aws_systems(
+        systems=aws_systems, organization=organization
+    )
+    return filtered_aws_systems
 
 
 def generate_system_aws(
@@ -221,22 +237,22 @@ def generate_system_aws(
     configuration and extract tracked resource to write a System manifest with.
     Tracked resources: [Redshift, RDS]
     """
-
     _check_boto3_import()
 
-    aws_systems = generate_aws_systems(organization_key=organization_key)
+    empty_aws_config_dict: Dict = {}  # not used via CLI today
+
     organization = get_organization(
         organization_key=organization_key,
         manifest_organizations=[],
         url=url,
         headers=headers,
     )
-    filtered_aws_systems = filter_aws_systems(
-        systems=aws_systems, organization=organization
-    )
+    aws_systems = generate_aws_systems(organization, aws_config=empty_aws_config_dict)
+
+    output_list_of_dicts = [i.dict(exclude_none=not include_null) for i in aws_systems]
     manifests.write_manifest(
         file_name,
-        [i.dict(exclude_none=not include_null) for i in filtered_aws_systems],
+        output_list_of_dicts,
         "system",
     )
     echo_green(f"Generated system manifest written to {file_name}")
@@ -374,7 +390,6 @@ def scan_system_aws(
     )
     existing_system_arns = get_system_arns(systems=manifest_systems + server_systems)
 
-    aws_systems = generate_aws_systems(organization_key=organization_key)
     organization = get_organization(
         organization_key=organization_key,
         manifest_organizations=manifest_taxonomy.organization
@@ -383,16 +398,15 @@ def scan_system_aws(
         url=url,
         headers=headers,
     )
-    filtered_aws_systems = filter_aws_systems(
-        systems=aws_systems, organization=organization
-    )
+
+    aws_systems = generate_aws_systems(organization=organization, aws_config={})
 
     (
         scan_text_output,
         scanned_resource_count,
         missing_resource_count,
     ) = scan_aws_systems(
-        aws_systems=filtered_aws_systems,
+        aws_systems=aws_systems,
         existing_system_arns=existing_system_arns,
     )
 

tests/core/test_system.py

@@ -44,8 +44,12 @@ def create_test_server_systems(
 @pytest.fixture(scope="function")
 def create_external_server_systems(test_config: FidesctlConfig) -> Generator:
     systems = _system.generate_redshift_systems(
-        organization_key="default_organization"
-    ) + _system.generate_rds_systems(organization_key="default_organization")
+        organization_key="default_organization",
+        aws_config={},
+    ) + _system.generate_rds_systems(
+        organization_key="default_organization",
+        aws_config={},
+    )
     delete_server_systems(test_config, systems)
     create_server_systems(test_config, systems)
     yield