From 66364e828ae123c33cab8627c4d72948893dd15a Mon Sep 17 00:00:00 2001 From: Oleksandr Moskalenko Date: Wed, 29 Dec 2021 14:15:34 +0100 Subject: [PATCH 01/12] Rework QuickSight module, implement logging --- cid/cli.py | 29 +++- cid/common.py | 29 ++-- cid/helpers/athena.py | 11 +- cid/helpers/quicksight.py | 292 ++++++++++++++++++++++++-------------- 4 files changed, 228 insertions(+), 133 deletions(-) diff --git a/cid/cli.py b/cid/cli.py index b9f7bc0a..4244e4a1 100644 --- a/cid/cli.py +++ b/cid/cli.py @@ -4,13 +4,27 @@ from cid import Cid -logger = logging.getLogger(__name__) +logger = logging.getLogger('cid') +# create formatter +formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(filename)s:%(funcName)s:%(lineno)d - %(message)s') +# File handler logs everything down to DEBUG level +fh = logging.FileHandler('cid.log') +fh.setLevel(logging.DEBUG) +fh.setFormatter(formatter) +# Console handler logs everything down to ERROR level +ch = logging.StreamHandler() +ch.setLevel(logging.ERROR) +# create formatter and add it to the handlers +ch.setFormatter(formatter) +# add the handlers to logger +logger.addHandler(ch) +logger.addHandler(fh) version = '1.0 Beta' prog_name="CLOUD INTELLIGENCE DASHBOARDS (CID) CLI" print(f'{prog_name} {version}\n') -App = Cid() +App = None @click.group() @click.option('--profile_name', help='AWS Profile name to use', default=env.get('AWS_PROFILE')) @@ -21,7 +35,16 @@ @click.option('-v', '--verbose', count=True) @click.pass_context def main(ctx, **kwargs): - logger.setLevel(logger.getEffectiveLevel()-10*kwargs.pop('verbose')) + global App + _verbose = kwargs.pop('verbose') + if _verbose: + # Limit Logging level to DEBUG, base level is WARNING + _verbose = 2 if _verbose > 2 else _verbose + logger.setLevel(logger.getEffectiveLevel()-10*_verbose) + # Logging application start here due to logging configuration + logger.info(f'{prog_name} {version} starting') + print(f'Logging level set to: {logging.getLevelName(logger.getEffectiveLevel())}') + App = Cid() App.run(**kwargs) diff --git a/cid/common.py b/cid/common.py index 7e11f87a..359afde6 100644 --- a/cid/common.py +++ b/cid/common.py @@ -111,9 +111,11 @@ def __loadPlugins(self) -> dict: plugins = dict() _entry_points = entry_points().get('cid.plugins') print('Loading plugins...') + logger.info('Loading plugins...') for ep in _entry_points: plugin = Plugin(ep.value) print(f"\t{ep.name} loaded") + logger.info(f'Plugin "{ep.name}" loaded') plugins.update({ep.value: plugin}) try: self.resources = always_merger.merge( @@ -121,6 +123,7 @@ def __loadPlugins(self) -> dict: except AttributeError: pass print('done\n') + logger.info('Finished loading plugins') return plugins def getPlugin(self, plugin) -> dict: @@ -133,16 +136,21 @@ def run(self, **kwargs): if kwargs.get('profile_name'): print('\tprofile name: {name}'.format( name=kwargs.get('profile_name'))) + logger.info(f'AWS profile name: {kwargs.get("profile_name")}') sts = utils.get_boto_client(service_name='sts', **kwargs) self.awsIdentity = sts.get_caller_identity() except NoCredentialsError: print('Error: Not authenticated, please check AWS credentials') + logger.info('Not authenticated, exiting') exit() print('\taccountId: {}\n\tAWS userId: {}'.format( self.awsIdentity.get('Account'), self.awsIdentity.get('Arn').split(':')[5] )) + logger.info(f'AWS accountId: {self.awsIdentity.get("Account")}') + logger.info(f'AWS userId: {self.awsIdentity.get("Arn").split(":")[5]}') print('\tRegion: {}'.format(kwargs.get('region_name'))) + logger.info(f'AWS region: {kwargs.get("region_name")}') print('done\n') @@ -159,7 +167,7 @@ def deploy(self, **kwargs): ) try: selected_dashboard = questionary.select( - "\nPlease select dashboard to install", + "Please select dashboard to install", choices=selection ).ask() except: @@ -274,11 +282,7 @@ def status(self, dashboard_id, **kwargs): dashboard = self.qs.dashboards.get(dashboard_id) else: # Describe dashboard by the ID given, no discovery - result = self.qs.describe_dashboard(DashboardId=dashboard_id) - if result: - dashboard = Dashboard(result.get('Dashboard')) - else: - dashboard = None + dashboard = self.qs.describe_dashboard(DashboardId=dashboard_id) click.echo('Getting dashboard status...', nl=False) if dashboard is not None: @@ -547,15 +551,18 @@ def create_dataset(self, dataset_definition) -> bool: # Read dataset definition from template dataset_file = dataset_definition.get('File') if dataset_file: - athena_datasources = { - k: v for (k, v) in self.qs.datasources.items() if v.get('Type') == 'ATHENA'} - if not len(athena_datasources): - return False + + if not len(self.qs.athena_datasources): + logger.info('No Athena datasources found, attempting to create one') + self.qs.create_data_source() + if not len(self.qs.athena_datasources): + logger.error('No Athena datasources available, failing') + return False # Load TPL file columns_tpl = dict() columns_tpl.update({ 'cur_table_name': self.cur.tableName if dataset_definition.get('dependsOn').get('cur') else None, - 'athena_datasource_arn': next(iter(athena_datasources)), + 'athena_datasource_arn': next(iter(self.qs.athena_datasources)), 'athena_database_name': self.athena.DatabaseName, 'user_arn': self.qs.user.get('Arn') }) diff --git a/cid/helpers/athena.py b/cid/helpers/athena.py index 7b394bd8..6d5821f1 100644 --- a/cid/helpers/athena.py +++ b/cid/helpers/athena.py @@ -53,7 +53,7 @@ def CatalogName(self) -> str: "Select AWS DataCatalog to use", choices=glue_data_catalogs ).ask() - logging.debug(f'Using datacatalog: {self._CatalogName}') + logger.info(f'Using datacatalog: {self._CatalogName}') return self._CatalogName @property @@ -85,7 +85,7 @@ def DatabaseName(self) -> str: "Select AWS Athena database to use", choices=[d['Name'] for d in athena_databases] ).ask() - logging.debug(f'Using Athena database: {self._DatabaseName}') + logger.info(f'Using Athena database: {self._DatabaseName}') return self._DatabaseName def list_data_catalogs(self) -> list: @@ -168,14 +168,7 @@ def execute_query(self, sql_query, sleep_duration=1) -> str: # Return result, either positive or negative if (current_status == "SUCCEEDED"): return query_id - elif (current_status == "FAILED") or (current_status == "CANCELLED"): - failure_reason = response['QueryExecution']['Status']['StateChangeReason'] - logger.error('Athena query failed: {}'.format(failure_reason)) - logger.error('Full query: {}'.format(sql_query)) - - raise Exception(failure_reason) else: - failure_reason = response['QueryExecution']['Status']['StateChangeReason'] failure_reason = response['QueryExecution']['Status']['StateChangeReason'] logger.error('Athena query failed: {}'.format(failure_reason)) logger.error('Full query: {}'.format(sql_query)) diff --git a/cid/helpers/quicksight.py b/cid/helpers/quicksight.py index 0003f451..8a32f42a 100644 --- a/cid/helpers/quicksight.py +++ b/cid/helpers/quicksight.py @@ -13,20 +13,19 @@ logger = logging.getLogger(__name__) class Dashboard(): + # Initialize properties + datasets = dict() + _status = None + status_detail = None + # Source template in origin account + sourceTemplate = None + # Dashboard definition + definition = None + # Locally saved deployment + localConfig = None + def __init__(self, dashboard) -> None: self.dashboard = dashboard - self.status = None - self.health = True - self.latest = False - self.status_detail = None - self.datasets = dict() - # Source template in origin account - self.sourceTemplate = None - # Dashboard definition - self.template = None - self.dashboard_id = None - # Locally saved deployment - self.localConfig = None @property def id(self) -> str: @@ -44,6 +43,10 @@ def arn(self) -> str: def version(self) -> dict: return self.get_property('Version') + @property + def latest(self) -> bool: + return self.latest_version == self.deployed_version + @property def latest_version(self) -> int: return int(self.sourceTemplate.get('Version').get('VersionNumber')) @@ -58,16 +61,44 @@ def deployed_version(self) -> int: return int(self.deployed_arn.split('/')[-1]) except: return 0 - + + @property + def health(self) -> bool: + return self.status not in ['broken', 'unsupported'] + + @property + def status(self) -> str: + if not self._status: + # Unsupported + if not self.definition: + self._status = 'unsupported' + # Missing dataset + if not self.datasets or (len(self.datasets) < len(self.definition.get('dependsOn').get('datasets'))): + self.status_detail = 'missing dataset(s)' + self._status = 'broken' + logger.info(f"Found datasets: {self.datasets}") + logger.info(f"Required datasets: {self.definition.get('dependsOn').get('datasets')}") + # Source Template has changed + if not self.deployed_arn.startswith(self.sourceTemplate.get('Arn')): + self._status = 'legacy' + else: + if self.latest_version > self.deployed_version: + self._status = f'update available {self.deployed_version}->{self.latest_version}' + elif self.latest: + self._status = 'up to date' + return self._status + @property def templateId(self) -> str: return str(self.version.get('SourceEntityArn').split('/')[1]) - def get_property(self, property) -> str: + def get_property(self, property: str) -> str: return self.dashboard.get(property) def find_local_config(self, account_id): + if self.localConfig: + return self.localConfig # Set base paths abs_path = Path().absolute() @@ -79,12 +110,13 @@ def find_local_config(self, account_id): f'work/{account_id}/{self.id.lower()}-update-dashboard-{account_id}.json', f'work/{account_id}/{self.id.lower()}-create-dashboard-{account_id}.json', ] - for localConfig in self.template.get('localConfigs', list()): + for localConfig in self.definition.get('localConfigs', list()): files_to_find.append(f'work/{account_id}/{localConfig.lower()}') for file in files_to_find: - logger.debug(f'Checking local config file {file}') + logger.info(f'Checking local config file {file}') if os.path.isfile(os.path.join(abs_path, file)): file_path = os.path.join(abs_path, file) + logger.info(f'Found local config file {file}, using it') break if file_path: @@ -93,10 +125,11 @@ def find_local_config(self, account_id): self.localConfig = json.loads(f.read()) if self.localConfig: for dataset in self.localConfig.get('SourceEntity').get('SourceTemplate').get('DataSetReferences'): - # self.datasets.update({dataset.get('DataSetPlaceholder'): dataset.get('DataSetArn').split(':')[-1].split('/')[-1]}) - self.datasets.update({dataset.get('DataSetPlaceholder'): dataset.get('DataSetArn')}) + if not self.datasets.get(dataset.get('DataSetPlaceholder')): + logger.info(f"Using dataset {dataset.get('DataSetPlaceholder')} ({dataset.get('DataSetId')})") + self.datasets.update({dataset.get('DataSetPlaceholder'): dataset.get('DataSetArn')}) except: - raise + logger.info(f'Failed to load local config file {file_path}') class QuickSight(): @@ -136,43 +169,124 @@ def user(self): ) ) try: - self._user = questionary.select("\nPlease select QuickSight to use", choices=selection).ask() + self._user = questionary.select("Please select QuickSight to use", choices=selection).ask() except: return None - + logger.info(f"Using QuickSight user {self._user.get('UserName')}") return self._user + @property + def supported_dashboards(self) -> list: + return self._resources.get('dashboards') + @property def dashboards(self) -> dict: """Returns a list of deployed dashboards""" if not self._dashboards: - self.discover_dashboards(self._resources.get('dashboards')) + self.discover_dashboards() return self._dashboards - # @property - # def datasources(self, _type: str='All') -> dict: - # """Returns a list of existing datasources""" - # if _type not in ['All', 'Athena']: - # _type = 'Athena' - # if not self._datasources: - # self.discover_data_sources() - # if _type is not 'All': - # return {v.get('DataSourceId'):v for v in self._datasources if v.get('Type') == _type} - # else: - # return self._datasources + @property + def athena_datasources(self) -> dict: + """Returns a list of existing athena datasources""" + return {k: v for (k, v) in self.datasources.items() if v.get('Type') == 'ATHENA'} @property def datasources(self) -> dict: """Returns a list of existing datasources""" if not self._datasources: + logger.info(f"Discovering datasources for account {self.account_id}") self.discover_data_sources() return self._datasources - def discover_dashboard(self, dashboard_id) -> Dashboard: + def discover_dashboard(self, dashboardId: str): """Discover single dashboard""" - - return self.describe_dashboard(DashboardId=dashboard_id) + + dashboard = self.describe_dashboard(DashboardId=dashboardId) + # Look for dashboard definition by DashboardId + _definition = next((v for v in self.supported_dashboards.values() if v['dashboardId'] == dashboard.id), None) + if not _definition: + # Look for dashboard definition by templateId + _definition = next((v for v in self.supported_dashboards.values() if v['templateId'] == dashboard.templateId), None) + if not _definition: + logger.info(f'Unsupported dashboard "{dashboard.name}" ({dashboard.deployed_arn})') + else: + logger.info(f'Supported dashboard "{dashboard.name}" ({dashboard.deployed_arn})') + dashboard.definition = _definition + logger.info(f'Found definition for "{dashboard.name}" ({dashboard.deployed_arn})') + for dataset in dashboard.version.get('DataSetArns'): + dataset_id = dataset.split('/')[-1] + try: + _dataset = self.describe_dataset(id=dataset_id) + if not _dataset: + dashboard.datasets.update({dataset_id: 'missing'}) + logger.info(f'Dataset "{dataset_id}" is missing') + else: + logger.info('Using dataset "{name}" ({id})'.format(name=_dataset.get('Name'), id=_dataset.get('DataSetId'))) + dashboard.datasets.update({_dataset.get('Name'): _dataset.get('Arn')}) + self._datasets.update({_dataset.get('Name'): _dataset}) + except self.client.exceptions.AccessDeniedException: + logger.info(f'Looking local config for {dashboardId}') + dashboard.find_local_config(self.account_id) + except self.client.exceptions.InvalidParameterValueException: + logger.info(f'Invalid dataset {dataset_id}') + templateAccountId = _definition.get('sourceAccountId') + try: + template = self.describe_template(dashboard.templateId, account_id=templateAccountId) + dashboard.sourceTemplate = template + except: + logger.info(f'Unable to describe template {dashboard.templateId} in {templateAccountId}') + self._dashboards.update({dashboardId: dashboard}) + logger.info(f'"{dashboard.name}" ({dashboardId}) discover complete') + + def create_data_source(self) -> bool: + """Create a new data source""" + logger.info('Creating Athena data source') + params = { + "AwsAccountId": self.account_id, + "DataSourceId": "95aa6f18-abb4-436f-855f-182b199a961f", + "Name": "Athena", + "Type": "ATHENA", + "DataSourceParameters": { + "AthenaParameters": { + "WorkGroup": "primary" + } + }, + "Permissions": [ + { + "Principal": self.user.get('Arn'), + "Actions": [ + "quicksight:UpdateDataSourcePermissions", + "quicksight:DescribeDataSource", + "quicksight:DescribeDataSourcePermissions", + "quicksight:PassDataSource", + "quicksight:UpdateDataSource", + "quicksight:DeleteDataSource" + ] + } + ] + } + try: + logger.info(f'Creating data source {params}') + create_status = self.client.create_data_source(**params) + logger.info(f'Data source createtion status {create_status}') + current_status = create_status['CreationStatus'] + logger.info(f'Data source creation status {current_status}') + # Poll for the current status of query as long as its not finished + while current_status in ['CREATION_IN_PROGRESS', 'UPDATE_IN_PROGRESS']: + response = self.describe_data_source(create_status['DataSourceId']) + current_status = response.get('Status') + + if (current_status != "CREATION_SUCCESSFUL"): + failure_reason = response.get('Errors') + raise Exception(failure_reason) + return True + except self.client.exceptions.ResourceExistsException: + logger.error('Data source already exists') + except self.client.exceptions.AccessDeniedException: + logger.info('Access denied creating data source') + return False def discover_data_sources(self) -> None: """ Discover existing datasources""" @@ -184,75 +298,26 @@ def discover_data_sources(self) -> None: for _,map in v.get('PhysicalTableMap').items(): self.describe_data_source(map.get('RelationalTable').get('DataSourceArn').split('/')[-1]) - def discover_dashboards(self, supported_dashboards, display: bool=False) -> None: + def discover_dashboards(self, display: bool=False) -> None: """ Discover deployed dashboards """ + logger.info('Discovering deployed dashboards') deployed_dashboards=self.list_dashboards() + logger.info(f'Found {len(deployed_dashboards)} deployed dashboards') + logger.debug(deployed_dashboards) with click.progressbar( deployed_dashboards, label='Discovering deployed dashboards...', item_show_func=lambda a: a ) as bar: - for index, item in enumerate(deployed_dashboards, start=1): - # Iterate through loaded list of dashboards by dashboardId - dashboardName = item.get('Name') - dashboardId = item.get('DashboardId') + for index, dashboard in enumerate(deployed_dashboards, start=1): + # Discover found dashboards + dashboardName = dashboard.get('Name') + dashboardId = dashboard.get('DashboardId') # Update progress bar bar.update(index, f'"{dashboardName}" ({dashboardId})') - dashboard = self.describe_dashboard(DashboardId=dashboardId) - # Iterate dashboards by DashboardId - res = next((sub for sub in supported_dashboards.values() if sub['dashboardId'] == dashboard.id), None) - if not res: - # Iterate dashboards by templateId - res = next((sub for sub in supported_dashboards.values() if sub['templateId'] == dashboard.templateId), None) - if not res: - logging.info(f'\nNot a supported dashboard "{dashboard.name}" ({dashboard.deployed_arn})\n') - else: - dashboard.status = 'healthy' - dashboard.template = res - for dataset in dashboard.version.get('DataSetArns'): - dataset_id = dataset.split('/')[-1] - try: - _dataset = self.describe_dataset(id=dataset_id) - if not _dataset: - dashboard.status = 'broken' - dashboard.health = False - dashboard.status_detail = 'missing dataset' - dashboard.datasets.update({dataset_id: 'missing'}) - else: - logging.info('Found dataset "{name}" ({arn})'.format(name=_dataset.get('Name'), arn=_dataset.get('Arn'))) - dashboard.datasets.update({_dataset.get('Name'): _dataset.get('Arn')}) - self._datasets.update({_dataset.get('Name'): _dataset}) - except self.client.exceptions.AccessDeniedException: - dashboard.status = 'broken' - dashboard.health = False - dashboard.status_detail = 'missing dataset' - logger.debug(f'Looking local config for {dashboardId}') - dashboard.find_local_config(self.account_id) - except self.client.exceptions.InvalidParameterValueException: - logger.debug(f'Invalid dataset {dataset_id}') - continue - # if not dashboard.status.startswith('broken'): - templateAccountId = res.get('sourceAccountId') - try: - template = self.describe_template(dashboard.templateId, account_id=templateAccountId) - except: - logging.info(f'Unable to describe template {dashboard.templateId} in {templateAccountId}') - template = None - if not template: - continue - dashboard.sourceTemplate = template - if not dashboard.deployed_arn.startswith(template.get('Arn')): - dashboard.status = 'legacy' - else: - if dashboard.latest_version > dashboard.deployed_version: - dashboard.status = f'update available {dashboard.deployed_version}->{dashboard.latest_version}' - elif dashboard.latest_version == dashboard.deployed_version: - dashboard.latest = True - dashboard.status = 'up to date' - self._dashboards.update({dashboardId: dashboard}) - # Update progress bar - bar.update(index, '') - logging.info(f'\t"{dashboardName}" ({dashboardId})') + logger.info(f'Discovering dashboard "{dashboardName}" ({dashboardId})') + self.discover_dashboard(dashboardId) + # Update progress bar bar.update(index, 'Complete') # print('Discovered dashboards:') if not display: @@ -275,6 +340,7 @@ def list_dashboards(self) -> list: print(f'Error, {result}') exit() else: + logger.debug(result) return result.get('DashboardSummaryList') except: return list() @@ -314,7 +380,7 @@ def select_dashboard(self, force=False): ) try: dashboard_id = questionary.select( - "\nPlease select installation(s) from the list", + "Please select installation(s) from the list", choices=selection ).ask() except: @@ -364,28 +430,31 @@ def delete_dashboard(self, dashboard_id): def describe_dataset(self, id) -> dict: """ Describes an AWS QuickSight dataset """ try: - result = self.client.describe_data_set(AwsAccountId=self.account_id,DataSetId=id) - if not self._datasets.get(result.get('DataSet').get('Name')): - self._datasets.update({result.get('DataSet').get('Name'): result.get('DataSet')}) + _dataset = self.client.describe_data_set(AwsAccountId=self.account_id,DataSetId=id).get('DataSet') + if not self._datasets.get(_dataset.get('Name')): + logger.info(f'Saving dataset details "{_dataset.get("Name")}" ({_dataset.get("DataSetId")})') + self._datasets.update({_dataset.get('Name'): _dataset}) except self.client.exceptions.ResourceNotFoundException: - logging.info(f'Warning: DataSetId {id} do not exist') + logger.info(f'DataSetId {id} do not exist') raise except self.client.exceptions.AccessDeniedException: - logging.info(f'No quicksight:DescribeDataSet permission or missing DataSetId {id}') + logger.debug(f'No quicksight:DescribeDataSet permission or missing DataSetId {id}') raise - return result.get('DataSet') + return _dataset def describe_data_source(self, id): """ Describes an AWS QuickSight data source """ try: result = self.client.describe_data_source(AwsAccountId=self.account_id,DataSourceId=id) - if not self._datasources.get(result.get('DataSource').get('Arn')): + logger.debug(result) + _described_data_source = self._datasources.get(result.get('DataSource').get('Arn')) + if not _described_data_source or _described_data_source.get('Status') in ['CREATION_IN_PROGRESS', 'UPDATE_IN_PROGRESS']: self._datasources.update({result.get('DataSource').get('Arn'): result.get('DataSource')}) except self.client.exceptions.ResourceNotFoundException: - logging.info(f'Warning: DataSource {id} do not exist') + logger.info(f'DataSource {id} do not exist') raise except self.client.exceptions.AccessDeniedException: - logging.info(f'No quicksight:DescribeDataSource permission or missing DataSetId {id}') + logger.info(f'No quicksight:DescribeDataSource permission or missing DataSetId {id}') raise return result.get('DataSource') @@ -396,13 +465,12 @@ def describe_template(self, template_id: str, account_id: str=None ): account_id=self.cidAccountId try: result = self.use1Client.describe_template(AwsAccountId=account_id,TemplateId=template_id) - logging.debug(result) + logger.debug(result) except: print(f'Error: Template {template_id} is not available in account {account_id}') exit(1) - # return None return result.get('Template') - + def describe_user(self, username: str) -> dict: """ Describes an AWS QuickSight template """ parameters = { @@ -424,7 +492,11 @@ def describe_user(self, username: str) -> dict: def create_dataset(self, dataset: dict) -> dict: """ Creates an AWS QuickSight dataset """ dataset.update({'AwsAccountId': self.account_id}) - response = self.client.create_data_set(**dataset) + try: + response = self.client.create_data_set(**dataset) + logger.info(f'Created dataset {dataset.get("Name")} ({response.get("DataSetId")})') + except self.client.exceptions.ResourceExistsException: + logger.info(f'Dataset {dataset.get("Name")} already exists') self.describe_dataset(response.get('DataSetId')) def create_dashboard(self, dashboard, sleep_duration=1, **kwargs) -> Dashboard: From a2648f5bec115bfd4443315f92003dfe314334bf Mon Sep 17 00:00:00 2001 From: Oleksandr Moskalenko Date: Wed, 29 Dec 2021 16:10:53 +0100 Subject: [PATCH 02/12] Exclude log files --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index a02f5963..8f171c0f 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ *~ *# *.swp +*.log # Python specific build/ From fed3dce2182083f50fe9493709d40b9d12e52294 Mon Sep 17 00:00:00 2001 From: Oleksandr Moskalenko Date: Wed, 29 Dec 2021 16:13:57 +0100 Subject: [PATCH 03/12] Refactor displaying status --- cid/common.py | 43 ++++++++------------------------------- cid/helpers/quicksight.py | 29 +++++++++++++++++++++++++- 2 files changed, 37 insertions(+), 35 deletions(-) diff --git a/cid/common.py b/cid/common.py index 359afde6..2c68c440 100644 --- a/cid/common.py +++ b/cid/common.py @@ -139,6 +139,10 @@ def run(self, **kwargs): logger.info(f'AWS profile name: {kwargs.get("profile_name")}') sts = utils.get_boto_client(service_name='sts', **kwargs) self.awsIdentity = sts.get_caller_identity() + self.qs_url_params = { + 'account_id': self.awsIdentity.get('Account'), + 'region': self.session.region_name + } except NoCredentialsError: print('Error: Not authenticated, please check AWS credentials') logger.info('Not authenticated, exiting') @@ -217,9 +221,7 @@ def deploy(self, **kwargs): f"Latest template: {latest_template.get('Arn')}/version/{latest_template.get('Version').get('VersionNumber')}") click.echo('\nDeploying...', nl=False) _url = self.qs_url.format( - account_id=self.awsIdentity.get('Account'), - region=self.session.region_name, - dashboard_id=dashboard_definition.get('dashboardId') + dashboard_id=dashboard_definition.get('dashboardId'), **self.qs_url_params ) try: self.qs.create_dashboard(dashboard_definition, **kwargs) @@ -259,13 +261,8 @@ def open(self, dashboard_id): indent=4, sort_keys=True, default=str)) click.echo( f'\nDashboard is unhealthy, please check errors above.') - _url = self.qs_url.format( - account_id=self.awsIdentity.get('Account'), - region=self.session.region_name, - dashboard_id=dashboard_id - ) click.echo('healthy, opening...') - click.launch(_url) + click.launch(self.qs_url.format(dashboard_id=dashboard_id, **self.qs_url_params)) else: click.echo('not deployed.') @@ -284,24 +281,9 @@ def status(self, dashboard_id, **kwargs): # Describe dashboard by the ID given, no discovery dashboard = self.qs.describe_dashboard(DashboardId=dashboard_id) - click.echo('Getting dashboard status...', nl=False) if dashboard is not None: - print('\n'+json.dumps(dashboard.dashboard, - indent=4, sort_keys=True, default=str)) - if dashboard.version.get('Status') in ['CREATION_SUCCESSFUL']: - click.echo(f'\nDashboard is healthy.') - else: - print(json.dumps(dashboard.version.get('Errors'), - indent=4, sort_keys=True, default=str)) - click.echo( - f'\nDashboard is unhealthy, please check errors above.') - _url = self.qs_url.format( - account_id=self.awsIdentity.get('Account'), - region=self.session.region_name, - dashboard_id=dashboard_id - ) - click.echo( - f"#######\n####### {dashboard.name} is available at: " + _url + "\n#######") + dashboard.display_status() + dashboard.display_url(self.qs_url, **self.qs_url_params) else: click.echo('not deployed.') @@ -379,14 +361,7 @@ def update(self, dashboard_id, **kwargs): try: self.qs.update_dashboard(dashboard, **kwargs) click.echo('completed') - _url = self.qs_url.format( - account_id=self.awsIdentity.get('Account'), - region=self.session.region_name, - dashboard_id=dashboard_id - ) - print(f"#######\n####### {dashboard.name} is available at: {_url}\n#######") - if click.confirm('Do you wish to open it in your browser?'): - click.launch(_url) + dashboard.display_url(self.qs_url, launch=True, **self.qs_url_params) except Exception as e: # Catch exception and dump a reason click.echo('failed, dumping error message') diff --git a/cid/helpers/quicksight.py b/cid/helpers/quicksight.py index 8a32f42a..2feb36a5 100644 --- a/cid/helpers/quicksight.py +++ b/cid/helpers/quicksight.py @@ -9,7 +9,6 @@ import logging -from questionary.constants import NO logger = logging.getLogger(__name__) class Dashboard(): @@ -78,6 +77,10 @@ def status(self) -> str: self._status = 'broken' logger.info(f"Found datasets: {self.datasets}") logger.info(f"Required datasets: {self.definition.get('dependsOn').get('datasets')}") + # Deployment failed + if self.version.get('Status') not in ['CREATION_SUCCESSFUL']: + self._status = 'broken' + self.status_detail = f"{self.version.get('Status')}: {self.version.get('Errors')}" # Source Template has changed if not self.deployed_arn.startswith(self.sourceTemplate.get('Arn')): self._status = 'legacy' @@ -131,6 +134,30 @@ def find_local_config(self, account_id): except: logger.info(f'Failed to load local config file {file_path}') + def display_status(self): + print('\nDashboard status:') + print(f" Name (id): {self.name} ({self.id})") + print(f" Status: {self.status}") + print(f" Health: {'healthy' if self.health else 'unhealthy'}") + if self.status_detail: + print(f" Status detail: {self.status_detail}") + if self.latest: + print(f" Latest version: {self.latest_version}") + if self.deployed_version: + print(f" Deployed version: {self.deployed_version}") + if self.localConfig: + print(f" Local config: {self.localConfig.get('SourceEntity').get('SourceTemplate').get('Name')}") + if self.datasets: + print(f" Datasets: {', '.join(sorted(self.datasets.keys()))}") + print('\n') + if click.confirm('Display dashboard raw data?'): + print(json.dumps(self.dashboard, indent=4, sort_keys=True, default=str)) + + def display_url(self, url_template: str, launch: bool = False, **kwargs): + url = url_template.format(dashboard_id=self.id, **kwargs) + print(f"#######\n####### {self.name} is available at: " + url + "\n#######") + if launch and click.confirm('Do you wish to open it in your browser?'): + click.launch(url) class QuickSight(): # Define defaults From 9d03b2c27ab31d0862313775c4a437687824370a Mon Sep 17 00:00:00 2001 From: Oleksandr Moskalenko Date: Thu, 30 Dec 2021 17:22:09 +0100 Subject: [PATCH 04/12] More logging, refactor plugin class --- cid/builtin/core/__init__.py | 9 +-------- cid/common.py | 6 +++--- cid/helpers/quicksight.py | 2 +- cid/plugin.py | 36 ++++++++++++++++++++++++++---------- 4 files changed, 31 insertions(+), 22 deletions(-) diff --git a/cid/builtin/core/__init__.py b/cid/builtin/core/__init__.py index 216d1293..fdacb785 100644 --- a/cid/builtin/core/__init__.py +++ b/cid/builtin/core/__init__.py @@ -1,8 +1 @@ -# This plugin implements TAO dashboard - -def onInit(cxt): - # STS context, extract user session -> detect Bubblewand - cxt.defaultAthenaDataSource = 'defaultDS' - cxt.defaultAthenaDatabase = 'defaultDS' - cxt.defaultAthenaTable = 'defaultDS' - pass +# This plugin implements Core dashboards diff --git a/cid/common.py b/cid/common.py index 2c68c440..73869fae 100644 --- a/cid/common.py +++ b/cid/common.py @@ -2,7 +2,7 @@ import questionary from cid import utils -from cid.helpers import Athena, CUR, Glue, QuickSight, Dashboard +from cid.helpers import Athena, CUR, Glue, QuickSight from cid.helpers.account_map import AccountMap from cid.plugin import Plugin @@ -111,11 +111,11 @@ def __loadPlugins(self) -> dict: plugins = dict() _entry_points = entry_points().get('cid.plugins') print('Loading plugins...') - logger.info('Loading plugins...') + logger.info(f'Located {len(_entry_points)} plugin(s)') for ep in _entry_points: + logger.info(f'Loading plugin: {ep.name} ({ep.value})') plugin = Plugin(ep.value) print(f"\t{ep.name} loaded") - logger.info(f'Plugin "{ep.name}" loaded') plugins.update({ep.value: plugin}) try: self.resources = always_merger.merge( diff --git a/cid/helpers/quicksight.py b/cid/helpers/quicksight.py index 2feb36a5..af5087df 100644 --- a/cid/helpers/quicksight.py +++ b/cid/helpers/quicksight.py @@ -297,7 +297,7 @@ def create_data_source(self) -> bool: try: logger.info(f'Creating data source {params}') create_status = self.client.create_data_source(**params) - logger.info(f'Data source createtion status {create_status}') + logger.debug(f'Data source creation result {create_status}') current_status = create_status['CreationStatus'] logger.info(f'Data source creation status {current_status}') # Poll for the current status of query as long as its not finished diff --git a/cid/plugin.py b/cid/plugin.py index 789b4ef3..09e43b5d 100644 --- a/cid/plugin.py +++ b/cid/plugin.py @@ -8,44 +8,60 @@ resource_isdir, resource_stream ) +import logging + +logger = logging.getLogger(__name__) class Plugin(): def __init__(self, name): + logger.info(f'Initializing plugin {name}') self.resources = {} self.name = name pkg_resources_db_directory = 'data' for pkg_resource in resource_listdir(self.name,pkg_resources_db_directory): if not resource_isdir(self.name, f'data/{pkg_resource}'): + logger.info(f'Located data file: {pkg_resource}') ext = pkg_resource.rsplit('.', -1)[-1].lower() if ext == 'json': content = json.loads(resource_string(self.name, f'data/{pkg_resource}')) + logger.info(f'Loaded {pkg_resource} as JSON') elif ext in ['yaml', 'yml']: with resource_stream(self.name, f'data/{pkg_resource}') as yaml_stream: content = yaml.load(yaml_stream, Loader=yaml.SafeLoader) + logger.info(f'Loaded {pkg_resource} as YAML') if content is None: + logger.info(f'Unsupported file type: {pkg_resource}') continue + # If plugin has resources defined in different files, + # they will be merged into one dict resource_kind = pkg_resource.rsplit('.', -1)[0] supported_resource_kinds = ['dashboards', 'views', 'datasets'] if resource_kind in supported_resource_kinds: - for item in content.values(): - if item is not None: - item.update({'providedBy': self.name}) self.resources.update({ resource_kind: content }) + logger.info(f'Loaded {resource_kind} from {pkg_resource}') + # If plugin has resources defined in one file, + # simply add it to resources dict else: - for v in content.values(): - for item in v.values(): - if item is not None: - item.update({'providedBy': self.name}) self.resources.update(content) - + # Add plugin name to every resource + for v in self.resources.values(): + for item in v.values(): + if item is not None: + item.update({'providedBy': self.name}) + logger.info(f'Plugin {self.name} initialized') + def provides(self) -> dict: + logger.debug(f'Provides: {self.resources}') return self.resources - + def get_resource(self, resource_name) -> str: _resource = f'data/{resource_name}' if resource_exists(self.name, _resource): - return resource_string(self.name, _resource).decode("utf-8") + logger.info(f'Resource {resource_name} found') + _content = resource_string(self.name, _resource).decode("utf-8") + logger.debug(f'Resource {resource_name} content: {_content}') + return _content return None From 0b1ad1eef970cf27b90ae93a0c2953ca36492d0c Mon Sep 17 00:00:00 2001 From: Oleksandr Moskalenko Date: Thu, 30 Dec 2021 19:46:32 +0100 Subject: [PATCH 05/12] Move logging setup to the main class --- cid/cli.py | 30 ++++-------------------------- cid/common.py | 26 ++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 26 deletions(-) diff --git a/cid/cli.py b/cid/cli.py index 4244e4a1..7db4f0d8 100644 --- a/cid/cli.py +++ b/cid/cli.py @@ -1,25 +1,8 @@ -import logging import click from os import environ as env from cid import Cid -logger = logging.getLogger('cid') -# create formatter -formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(filename)s:%(funcName)s:%(lineno)d - %(message)s') -# File handler logs everything down to DEBUG level -fh = logging.FileHandler('cid.log') -fh.setLevel(logging.DEBUG) -fh.setFormatter(formatter) -# Console handler logs everything down to ERROR level -ch = logging.StreamHandler() -ch.setLevel(logging.ERROR) -# create formatter and add it to the handlers -ch.setFormatter(formatter) -# add the handlers to logger -logger.addHandler(ch) -logger.addHandler(fh) - version = '1.0 Beta' prog_name="CLOUD INTELLIGENCE DASHBOARDS (CID) CLI" print(f'{prog_name} {version}\n') @@ -36,15 +19,10 @@ @click.pass_context def main(ctx, **kwargs): global App - _verbose = kwargs.pop('verbose') - if _verbose: - # Limit Logging level to DEBUG, base level is WARNING - _verbose = 2 if _verbose > 2 else _verbose - logger.setLevel(logger.getEffectiveLevel()-10*_verbose) - # Logging application start here due to logging configuration - logger.info(f'{prog_name} {version} starting') - print(f'Logging level set to: {logging.getLevelName(logger.getEffectiveLevel())}') - App = Cid() + params = { + 'verbose': kwargs.pop('verbose'), + } + App = Cid(**params) App.run(**kwargs) diff --git a/cid/common.py b/cid/common.py index 73869fae..6cca8507 100644 --- a/cid/common.py +++ b/cid/common.py @@ -29,6 +29,8 @@ class Cid: } def __init__(self, **kwargs) -> None: + self.__setupLogging(verbosity=kwargs.pop('verbose')) + logger.info('Initializing CID') # Defined resources self.resources = dict() self.dashboards = dict() @@ -158,6 +160,30 @@ def run(self, **kwargs): print('done\n') + def __setupLogging(self, verbosity: int=0, log_filename: str='cid.log') -> None: + _logger = logging.getLogger('cid') + # create formatter + formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(filename)s:%(funcName)s:%(lineno)d - %(message)s') + # File handler logs everything down to DEBUG level + fh = logging.FileHandler(log_filename) + fh.setLevel(logging.DEBUG) + fh.setFormatter(formatter) + # Console handler logs everything down to ERROR level + ch = logging.StreamHandler() + ch.setLevel(logging.ERROR) + # create formatter and add it to the handlers + ch.setFormatter(formatter) + # add the handlers to logger + _logger.addHandler(ch) + _logger.addHandler(fh) + if verbosity: + # Limit Logging level to DEBUG, base level is WARNING + verbosity = 2 if verbosity > 2 else verbosity + _logger.setLevel(logger.getEffectiveLevel()-10*verbosity) + # Logging application start here due to logging configuration + print(f'Logging level set to: {logging.getLevelName(logger.getEffectiveLevel())}') + + def deploy(self, **kwargs): """ Deploy Dashboard """ From 7f7d0660649323648e5f4b7f74d5a18b951fc327 Mon Sep 17 00:00:00 2001 From: Oleksandr Moskalenko Date: Sun, 2 Jan 2022 13:24:32 +0100 Subject: [PATCH 06/12] Refactor session creation, rely on boto fully --- cid/cli.py | 31 ++++++++++++++++++------------- cid/common.py | 13 ++++++------- cid/helpers/quicksight.py | 3 ++- cid/utils.py | 13 +++++++++++-- 4 files changed, 37 insertions(+), 23 deletions(-) diff --git a/cid/cli.py b/cid/cli.py index 7db4f0d8..c201065e 100644 --- a/cid/cli.py +++ b/cid/cli.py @@ -7,33 +7,34 @@ prog_name="CLOUD INTELLIGENCE DASHBOARDS (CID) CLI" print(f'{prog_name} {version}\n') -App = None @click.group() -@click.option('--profile_name', help='AWS Profile name to use', default=env.get('AWS_PROFILE')) -@click.option('--region_name', help="AWS Region (default:'us-east-1')", default=env.get('AWS_REGION', env.get('AWS_DEFAULT_REGION', 'us-east-1'))) -@click.option('--aws_access_key_id', help='', default=env.get('AWS_ACCESS_KEY_ID')) -@click.option('--aws_secret_access_key', help='', default=env.get('AWS_SECRET_ACCESS_KEY')) -@click.option('--aws_session_token', help='', default=env.get('AWS_SESSION_TOKEN')) +@click.option('--profile_name', help='AWS Profile name to use', default=None) +@click.option('--region_name', help="AWS Region (default:'us-east-1')", default=None) +@click.option('--aws_access_key_id', help='', default=None) +@click.option('--aws_secret_access_key', help='', default=None) +@click.option('--aws_session_token', help='', default=None) @click.option('-v', '--verbose', count=True) @click.pass_context def main(ctx, **kwargs): - global App params = { 'verbose': kwargs.pop('verbose'), } App = Cid(**params) App.run(**kwargs) + ctx.obj = App @main.command() -def map(): +@click.pass_obj +def map(App): """Create account mapping""" App.map() @main.command() -def deploy(): +@click.pass_obj +def deploy(App): """Deploy Dashboard""" App.deploy() @@ -41,14 +42,16 @@ def deploy(): @main.command() @click.option('--dashboard-id', help='QuickSight dashboard id', default=None) -def status(**kwargs): +@click.pass_obj +def status(App, **kwargs): """Show Dashboard status""" App.status(dashboard_id=kwargs.get('dashboard_id')) @main.command() @click.option('--dashboard-id', help='QuickSight dashboard id', default=None) -def delete(**kwargs): +@click.pass_obj +def delete(App, **kwargs): """Delete Dashboard""" App.delete(dashboard_id=kwargs.get('dashboard_id')) @@ -56,7 +59,8 @@ def delete(**kwargs): @main.command() @click.option('--dashboard-id', help='QuickSight dashboard id', default=None) @click.option('--force', help='Allow force update', is_flag=True) -def update(**kwargs): +@click.pass_obj +def update(App, **kwargs): """Update Dashboard""" App.update(dashboard_id=kwargs.get('dashboard_id'), force=kwargs.get('force')) @@ -64,7 +68,8 @@ def update(**kwargs): @main.command() @click.option('--dashboard-id', help='QuickSight dashboard id', default=None) -def open(**kwargs): +@click.pass_obj +def open(App, **kwargs): """Open Dashboard in browser""" App.open(dashboard_id=kwargs.get('dashboard_id')) diff --git a/cid/common.py b/cid/common.py index 6cca8507..4ff34b50 100644 --- a/cid/common.py +++ b/cid/common.py @@ -135,11 +135,10 @@ def run(self, **kwargs): print('Checking AWS environment...') try: self.session = utils.get_boto_session(**kwargs) - if kwargs.get('profile_name'): - print('\tprofile name: {name}'.format( - name=kwargs.get('profile_name'))) - logger.info(f'AWS profile name: {kwargs.get("profile_name")}') - sts = utils.get_boto_client(service_name='sts', **kwargs) + if self.session.profile_name: + print(f'\tprofile name: {self.session.profile_name}') + logger.info(f'AWS profile name: {self.session.profile_name}') + sts = self.session.client('sts') self.awsIdentity = sts.get_caller_identity() self.qs_url_params = { 'account_id': self.awsIdentity.get('Account'), @@ -155,8 +154,8 @@ def run(self, **kwargs): )) logger.info(f'AWS accountId: {self.awsIdentity.get("Account")}') logger.info(f'AWS userId: {self.awsIdentity.get("Arn").split(":")[5]}') - print('\tRegion: {}'.format(kwargs.get('region_name'))) - logger.info(f'AWS region: {kwargs.get("region_name")}') + print('\tRegion: {}'.format(self.session.region_name)) + logger.info(f'AWS region: {self.session.region_name}') print('done\n') diff --git a/cid/helpers/quicksight.py b/cid/helpers/quicksight.py index af5087df..19f91490 100644 --- a/cid/helpers/quicksight.py +++ b/cid/helpers/quicksight.py @@ -173,7 +173,8 @@ def __init__(self, session, awsIdentity, resources=None): self._resources = resources # QuickSight client - self.client = session.client('quicksight', region_name=self.region) + logger.info(f'Creating QuickSight client') + self.client = session.client('quicksight') self.use1Client = session.client('quicksight', region_name='us-east-1') @property diff --git a/cid/utils.py b/cid/utils.py index 62ccfc6a..3f9d8405 100644 --- a/cid/utils.py +++ b/cid/utils.py @@ -1,6 +1,6 @@ import boto3 -from botocore.exceptions import NoCredentialsError, CredentialRetrievalError +from botocore.exceptions import NoCredentialsError, CredentialRetrievalError, NoRegionError import logging logger = logging.getLogger(__name__) @@ -16,10 +16,19 @@ def get_boto_session(**kwargs): :return: Boto3 Client """ try: - return boto3.session.Session(**kwargs) + session = boto3.session.Session(**kwargs) + logger.info('Boto3 session created') + logger.debug(session) + if not session.region_name: + raise NoRegionError + return session except (NoCredentialsError, CredentialRetrievalError): print('Error: unable to initialize session, please check your AWS credentials, exiting') exit(1) + except NoRegionError: + logger.info('No AWS region set, defaulting to us-east-1') + kwargs.update({'region_name': 'us-east-1'}) + return get_boto_session(**kwargs) except: raise From 1e41697007af6e4765def2bdd301f5fbd6e053f3 Mon Sep 17 00:00:00 2001 From: Oleksandr Moskalenko Date: Mon, 3 Jan 2022 09:44:57 +0100 Subject: [PATCH 07/12] Use Dashboard object properties --- cid/common.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/cid/common.py b/cid/common.py index 4ff34b50..73a835c5 100644 --- a/cid/common.py +++ b/cid/common.py @@ -348,18 +348,13 @@ def update(self, dashboard_id, **kwargs): return print(f'\nChecking for updates...') - try: - deployed_template = dashboard.version.get('SourceEntityArn') - except AttributeError: - click.echo(f'not deployed') - return click.echo(f'Deployed template: {dashboard.deployed_arn}') click.echo( f"Latest template: {dashboard.sourceTemplate.get('Arn')}/version/{dashboard.latest_version}") - if not deployed_template.startswith(dashboard.sourceTemplate.get('Arn')): + if dashboard.status == 'legacy': click.confirm( "\nDashboard template changed, update it anyway?", abort=True) - elif not (dashboard.deployed_version < dashboard.latest_version): + elif dashboard.latest: click.confirm( "\nNo updates available, should I update it anyway?", abort=True) From 1ec53844ef3d9bc131f13922f04848d8f5174d6f Mon Sep 17 00:00:00 2001 From: Oleksandr Moskalenko Date: Mon, 3 Jan 2022 11:43:29 +0100 Subject: [PATCH 08/12] Update log formatting --- cid/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cid/common.py b/cid/common.py index 73a835c5..1a037fbd 100644 --- a/cid/common.py +++ b/cid/common.py @@ -162,7 +162,7 @@ def run(self, **kwargs): def __setupLogging(self, verbosity: int=0, log_filename: str='cid.log') -> None: _logger = logging.getLogger('cid') # create formatter - formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(filename)s:%(funcName)s:%(lineno)d - %(message)s') + formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(name)s:%(funcName)s:%(lineno)d - %(message)s') # File handler logs everything down to DEBUG level fh = logging.FileHandler(log_filename) fh.setLevel(logging.DEBUG) From 692d41048fb35bc33d335f49a4dda0f1e2b2ab6f Mon Sep 17 00:00:00 2001 From: Oleksandr Moskalenko Date: Mon, 3 Jan 2022 15:34:49 +0100 Subject: [PATCH 09/12] Handle CredentialRetrievalError --- cid/common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cid/common.py b/cid/common.py index 1a037fbd..1e317c99 100644 --- a/cid/common.py +++ b/cid/common.py @@ -15,7 +15,7 @@ import json from pathlib import Path -from botocore.exceptions import NoCredentialsError +from botocore.exceptions import NoCredentialsError, CredentialRetrievalError from deepmerge import always_merger @@ -144,7 +144,7 @@ def run(self, **kwargs): 'account_id': self.awsIdentity.get('Account'), 'region': self.session.region_name } - except NoCredentialsError: + except (NoCredentialsError, CredentialRetrievalError): print('Error: Not authenticated, please check AWS credentials') logger.info('Not authenticated, exiting') exit() From 419cf610df3df40ecd8ec999fd3b7dcc83093007 Mon Sep 17 00:00:00 2001 From: Oleksandr Moskalenko Date: Mon, 3 Jan 2022 18:08:38 +0100 Subject: [PATCH 10/12] Refactor QuickSight module --- cid/common.py | 12 ++- cid/helpers/quicksight.py | 174 +++++++++++++++++++++----------------- 2 files changed, 100 insertions(+), 86 deletions(-) diff --git a/cid/common.py b/cid/common.py index 1e317c99..07a4a674 100644 --- a/cid/common.py +++ b/cid/common.py @@ -77,15 +77,12 @@ def cur(self) -> CUR: "Error: please ensure CUR is enabled, if yes allow it some time to propagate") exit(1) print(f'\tAthena table: {_cur.tableName}') - print('\tResource IDs: {}'.format( - 'yes' if _cur.hasResourceIDs else 'no')) + print(f"\tResource IDs: {'yes' if _cur.hasResourceIDs else 'no'}") if not _cur.hasResourceIDs: print("Error: CUR has to be created with Resource IDs") exit(1) - print('\tSavingsPlans: {}'.format( - 'yes' if _cur.hasSavingsPlans else 'no')) - print('\tReserved Instances: {}'.format( - 'yes' if _cur.hasReservations else 'no')) + print(f"\tSavingsPlans: {'yes' if _cur.hasSavingsPlans else 'no'}") + print(f"\tReserved Instances: {'yes' if _cur.hasReservations else 'no'}") print('done') self._clients.update({ 'cur': _cur @@ -215,8 +212,9 @@ def deploy(self, **kwargs): dashboard_definition.update({'datasets': {}}) dashboard_datasets = dashboard_definition.get('datasets') for dataset_name in required_datasets: + arn = next(v.get('Arn') for v in self.qs._datasets.values() if v.get('Name') == dataset_name) dashboard_datasets.update( - {dataset_name: self.qs._datasets.get(dataset_name).get('Arn')}) + {dataset_name: arn}) kwargs = dict() local_overrides = f'work/{self.awsIdentity.get("Account")}/{dashboard_definition.get("dashboardId")}.json' diff --git a/cid/helpers/quicksight.py b/cid/helpers/quicksight.py index 19f91490..59a46c04 100644 --- a/cid/helpers/quicksight.py +++ b/cid/helpers/quicksight.py @@ -12,19 +12,18 @@ logger = logging.getLogger(__name__) class Dashboard(): - # Initialize properties - datasets = dict() - _status = None - status_detail = None - # Source template in origin account - sourceTemplate = None - # Dashboard definition - definition = None - # Locally saved deployment - localConfig = None - - def __init__(self, dashboard) -> None: - self.dashboard = dashboard + def __init__(self, dashboard: dict) -> None: + self.dashboard: dict = dashboard + # Initialize properties + self.datasets = dict() + self._status = str() + self.status_detail = str() + # Source template in origin account + self.sourceTemplate = dict() + # Dashboard definition + self.definition = dict() + # Locally saved deployment + self.localConfig = dict() @property def id(self) -> str: @@ -38,6 +37,10 @@ def name(self) -> str: def arn(self) -> str: return self.get_property('Arn') + @property + def account_id(self) -> str: + return self.get_property('Arn').split(':')[4] + @property def version(self) -> dict: return self.get_property('Version') @@ -48,7 +51,7 @@ def latest(self) -> bool: @property def latest_version(self) -> int: - return int(self.sourceTemplate.get('Version').get('VersionNumber')) + return int(self.sourceTemplate.get('Version', dict()).get('VersionNumber', -1)) @property def deployed_arn(self) -> str: @@ -63,26 +66,26 @@ def deployed_version(self) -> int: @property def health(self) -> bool: - return self.status not in ['broken', 'unsupported'] + return self.status not in ['broken'] @property def status(self) -> str: if not self._status: - # Unsupported + # Not dicovered yet if not self.definition: - self._status = 'unsupported' + self._status = 'undiscovered' # Missing dataset - if not self.datasets or (len(self.datasets) < len(self.definition.get('dependsOn').get('datasets'))): + elif not self.datasets or (len(self.datasets) < len(self.definition.get('dependsOn').get('datasets'))): self.status_detail = 'missing dataset(s)' self._status = 'broken' logger.info(f"Found datasets: {self.datasets}") logger.info(f"Required datasets: {self.definition.get('dependsOn').get('datasets')}") # Deployment failed - if self.version.get('Status') not in ['CREATION_SUCCESSFUL']: + elif self.version.get('Status') not in ['CREATION_SUCCESSFUL']: self._status = 'broken' self.status_detail = f"{self.version.get('Status')}: {self.version.get('Errors')}" # Source Template has changed - if not self.deployed_arn.startswith(self.sourceTemplate.get('Arn')): + elif self.deployed_arn and self.sourceTemplate.get('Arn') and not self.deployed_arn.startswith(self.sourceTemplate.get('Arn')): self._status = 'legacy' else: if self.latest_version > self.deployed_version: @@ -98,7 +101,7 @@ def templateId(self) -> str: def get_property(self, property: str) -> str: return self.dashboard.get(property) - def find_local_config(self, account_id): + def find_local_config(self) -> Union[dict, None]: if self.localConfig: return self.localConfig @@ -108,13 +111,12 @@ def find_local_config(self, account_id): # Load TPL file file_path = None files_to_find = [ - f'work/{account_id}/{self.id.lower()}-update-dashboard.json', - f'work/{account_id}/{self.id.lower()}-create-dashboard.json', - f'work/{account_id}/{self.id.lower()}-update-dashboard-{account_id}.json', - f'work/{account_id}/{self.id.lower()}-create-dashboard-{account_id}.json', + f'work/{self.account_id}/{self.id.lower()}-update-dashboard.json', + f'work/{self.account_id}/{self.id.lower()}-create-dashboard.json', + f'work/{self.account_id}/{self.id.lower()}-update-dashboard-{self.account_id}.json', + f'work/{self.account_id}/{self.id.lower()}-create-dashboard-{self.account_id}.json', ] - for localConfig in self.definition.get('localConfigs', list()): - files_to_find.append(f'work/{account_id}/{localConfig.lower()}') + files_to_find += [f'work/{self.account_id}/{f.lower()}' for f in self.definition.get('localConfigs', list())] for file in files_to_find: logger.info(f'Checking local config file {file}') if os.path.isfile(os.path.join(abs_path, file)): @@ -142,9 +144,9 @@ def display_status(self): if self.status_detail: print(f" Status detail: {self.status_detail}") if self.latest: - print(f" Latest version: {self.latest_version}") - if self.deployed_version: - print(f" Deployed version: {self.deployed_version}") + print(f" Version: {self.deployed_version}") + else: + print(f" Version (deployed, latest): {self.deployed_version}, {self.latest_version}") if self.localConfig: print(f" Local config: {self.localConfig.get('SourceEntity').get('SourceTemplate').get('Name')}") if self.datasets: @@ -162,10 +164,10 @@ def display_url(self, url_template: str, launch: bool = False, **kwargs): class QuickSight(): # Define defaults cidAccountId = '223485597511' - _dashboards = dict() + _dashboards: dict[Dashboard] = {} _datasets = dict() - _datasources = dict() - _user = None + _datasources: dict() = {} + _user: dict = None def __init__(self, session, awsIdentity, resources=None): self.region = session.region_name @@ -182,7 +184,7 @@ def account_id(self) -> str: return self.awsIdentity.get('Account') @property - def user(self): + def user(self) -> dict: if not self._user: self._user = self.describe_user('/'.join(self.awsIdentity.get('Arn').split('/')[-2:])) if not self._user: @@ -208,7 +210,7 @@ def supported_dashboards(self) -> list: return self._resources.get('dashboards') @property - def dashboards(self) -> dict: + def dashboards(self) -> dict[Dashboard]: """Returns a list of deployed dashboards""" if not self._dashboards: self.discover_dashboards() @@ -251,12 +253,11 @@ def discover_dashboard(self, dashboardId: str): dashboard.datasets.update({dataset_id: 'missing'}) logger.info(f'Dataset "{dataset_id}" is missing') else: - logger.info('Using dataset "{name}" ({id})'.format(name=_dataset.get('Name'), id=_dataset.get('DataSetId'))) + logger.info(f"Using dataset \"{_dataset.get('Name')}\" ({_dataset.get('DataSetId')} for {dashboard.name})") dashboard.datasets.update({_dataset.get('Name'): _dataset.get('Arn')}) - self._datasets.update({_dataset.get('Name'): _dataset}) except self.client.exceptions.AccessDeniedException: logger.info(f'Looking local config for {dashboardId}') - dashboard.find_local_config(self.account_id) + dashboard.find_local_config() except self.client.exceptions.InvalidParameterValueException: logger.info(f'Invalid dataset {dataset_id}') templateAccountId = _definition.get('sourceAccountId') @@ -266,6 +267,7 @@ def discover_dashboard(self, dashboardId: str): except: logger.info(f'Unable to describe template {dashboard.templateId} in {templateAccountId}') self._dashboards.update({dashboardId: dashboard}) + logger.info(f"{dashboard.name} has {len(dashboard.datasets)} datasets") logger.info(f'"{dashboard.name}" ({dashboardId}) discover complete') def create_data_source(self) -> bool: @@ -322,7 +324,7 @@ def discover_data_sources(self) -> None: for v in self.list_data_sources(): self.describe_data_source(v.get('DataSourceId')) except: - for _,v in self._datasets.items(): + for _,v in self.datasets.items(): for _,map in v.get('PhysicalTableMap').items(): self.describe_data_source(map.get('RelationalTable').get('DataSourceArn').split('/')[-1]) @@ -389,7 +391,7 @@ def list_data_sources(self) -> list: except: return list() - def select_dashboard(self, force=False): + def select_dashboard(self, force=False) -> str: """ Select from a list of discovered dashboards """ selection = list() dashboard_id = None @@ -432,20 +434,40 @@ def list_data_sets(self): except: return None - def describe_dashboard(self, **kwargs) -> Union[None, Dashboard]: + def describe_dashboard(self, poll: bool=False, **kwargs) -> Union[None, Dashboard]: """ Describes an AWS QuickSight dashboard Keyword arguments: DashboardId """ - + poll_interval = kwargs.get('poll_interval', 1) try: - return Dashboard(self.client.describe_dashboard(AwsAccountId=self.account_id, **kwargs).get('Dashboard')) + dashboard: Dashboard = None + current_status = None + # Poll for the current status of query as long as its not finished + while current_status in [None, 'CREATION_IN_PROGRESS', 'UPDATE_IN_PROGRESS']: + if current_status: + logger.info(f'Dashboard {dashboard.name} status is {current_status}, waiting for {poll_interval} seconds') + # Sleep before polling again + time.sleep(poll_interval) + elif poll: + logger.info(f'Polling for dashboard {kwargs.get("DashboardId")}') + response = self.client.describe_dashboard(AwsAccountId=self.account_id, **kwargs).get('Dashboard') + logger.debug(response) + dashboard = Dashboard(response) + current_status = dashboard.version.get('Status') + if not poll: + break + logger.info(f'Dashboard {dashboard.name} status is {current_status}') + return dashboard except self.client.exceptions.ResourceNotFoundException: return None except self.client.exceptions.UnsupportedUserEditionException: print('Error: AWS QuickSight Enterprise Edition is required') exit(1) + except Exception as e: + print(f'Error: {e}') + raise def delete_dashboard(self, dashboard_id): """ Deletes an AWS QuickSight dashboard """ @@ -457,18 +479,19 @@ def delete_dashboard(self, dashboard_id): def describe_dataset(self, id) -> dict: """ Describes an AWS QuickSight dataset """ - try: - _dataset = self.client.describe_data_set(AwsAccountId=self.account_id,DataSetId=id).get('DataSet') - if not self._datasets.get(_dataset.get('Name')): + if not self._datasets.get(id): + logger.info(f'Describing dataset {id}') + try: + _dataset = self.client.describe_data_set(AwsAccountId=self.account_id,DataSetId=id).get('DataSet') logger.info(f'Saving dataset details "{_dataset.get("Name")}" ({_dataset.get("DataSetId")})') - self._datasets.update({_dataset.get('Name'): _dataset}) - except self.client.exceptions.ResourceNotFoundException: - logger.info(f'DataSetId {id} do not exist') - raise - except self.client.exceptions.AccessDeniedException: - logger.debug(f'No quicksight:DescribeDataSet permission or missing DataSetId {id}') - raise - return _dataset + self._datasets.update({_dataset.get('DataSetId'): _dataset}) + except self.client.exceptions.ResourceNotFoundException: + logger.info(f'DataSetId {id} do not exist') + raise + except self.client.exceptions.AccessDeniedException: + logger.debug(f'No quicksight:DescribeDataSet permission or missing DataSetId {id}') + raise + return self._datasets.get(id) def describe_data_source(self, id): """ Describes an AWS QuickSight data source """ @@ -527,10 +550,11 @@ def create_dataset(self, dataset: dict) -> dict: logger.info(f'Dataset {dataset.get("Name")} already exists') self.describe_dataset(response.get('DataSetId')) - def create_dashboard(self, dashboard, sleep_duration=1, **kwargs) -> Dashboard: + + def create_dashboard(self, definition: dict, **kwargs) -> Dashboard: """ Creates an AWS QuickSight dashboard """ DataSetReferences = list() - for k, v in dashboard.get('datasets', dict()).items(): + for k, v in definition.get('datasets', dict()).items(): DataSetReferences.append({ 'DataSetPlaceholder': k, 'DataSetArn': v @@ -538,8 +562,8 @@ def create_dashboard(self, dashboard, sleep_duration=1, **kwargs) -> Dashboard: create_parameters = { 'AwsAccountId': self.account_id, - 'DashboardId': dashboard.get('dashboardId'), - 'Name': dashboard.get('name'), + 'DashboardId': definition.get('dashboardId'), + 'Name': definition.get('name'), 'Permissions': [ { "Principal": self.user.get('Arn'), @@ -557,7 +581,7 @@ def create_dashboard(self, dashboard, sleep_duration=1, **kwargs) -> Dashboard: ], 'SourceEntity': { 'SourceTemplate': { - 'Arn': f"{dashboard.get('sourceTemplate').get('Arn')}/version/{dashboard.get('sourceTemplate').get('Version').get('VersionNumber')}", + 'Arn': f"{definition.get('sourceTemplate').get('Arn')}/version/{definition.get('sourceTemplate').get('Version').get('VersionNumber')}", 'DataSetReferences': DataSetReferences } } @@ -566,6 +590,7 @@ def create_dashboard(self, dashboard, sleep_duration=1, **kwargs) -> Dashboard: create_parameters = always_merger.merge(create_parameters, kwargs) try: create_status = self.client.create_dashboard(**create_parameters) + logger.debug(create_status) except self.client.exceptions.ResourceExistsException: raise created_version = int(create_status['VersionArn'].split('/')[-1]) @@ -573,21 +598,18 @@ def create_dashboard(self, dashboard, sleep_duration=1, **kwargs) -> Dashboard: # Poll for the current status of query as long as its not finished describe_parameters = { - 'DashboardId': dashboard.get('dashboardId'), + 'DashboardId': definition.get('dashboardId'), 'VersionNumber': created_version } - while current_status in ['CREATION_IN_PROGRESS', 'UPDATE_IN_PROGRESS']: - response = self.describe_dashboard(**describe_parameters) - current_status = response.version.get('Status') - - if (current_status != "CREATION_SUCCESSFUL"): - failure_reason = response.version.get('Errors') + dashboard = self.describe_dashboard(poll=True, **describe_parameters) + if not dashboard.health: + failure_reason = dashboard.version.get('Errors') raise Exception(failure_reason) - return response.dashboard + return dashboard - def update_dashboard(self, dashboard, sleep_duration=1, **kwargs): + def update_dashboard(self, dashboard: Dashboard, **kwargs): """ Updates an AWS QuickSight dashboard """ DataSetReferences = list() for k, v in dashboard.datasets.items(): @@ -602,26 +624,20 @@ def update_dashboard(self, dashboard, sleep_duration=1, **kwargs): 'Name': dashboard.name, 'SourceEntity': { 'SourceTemplate': { - 'Arn': dashboard.sourceTemplate.get('Arn'), + 'Arn': f"{dashboard.sourceTemplate.get('Arn')}/version/{dashboard.latest_version}", 'DataSetReferences': DataSetReferences } } } update_parameters = always_merger.merge(update_parameters, kwargs) + logger.debug(f"Update parameters: {update_parameters}") update_status = self.client.update_dashboard(**update_parameters) + logger.debug(update_status) updated_version = int(update_status['VersionArn'].split('/')[-1]) - current_status = update_status['CreationStatus'] - - # Poll for the current status of query as long as its not finished - while current_status in ['CREATION_IN_PROGRESS', 'UPDATE_IN_PROGRESS']: - # Sleep before polling again - time.sleep(sleep_duration) - dashboard = self.describe_dashboard(DashboardId=dashboard.id, VersionNumber=updated_version) - current_status = dashboard.version.get('Status') - - if (current_status != "CREATION_SUCCESSFUL"): + dashboard = self.describe_dashboard(poll=True, DashboardId=dashboard.id, VersionNumber=updated_version) + if not dashboard.health: failure_reason = dashboard.version.get('Errors') raise Exception(failure_reason) From a1bbf6715292227cda72c2962c321f96862c8015 Mon Sep 17 00:00:00 2001 From: Oleksandr Moskalenko Date: Tue, 4 Jan 2022 01:38:56 +0100 Subject: [PATCH 11/12] Refactor account mapping creation --- .../core/data/queries/shared/account_map.sql | 4 +- .../core/data/queries/shared/aws_accounts.sql | 7 +- cid/common.py | 9 +- cid/helpers/account_map.py | 219 +++++++++++------- cid/helpers/athena.py | 3 + 5 files changed, 157 insertions(+), 85 deletions(-) diff --git a/cid/builtin/core/data/queries/shared/account_map.sql b/cid/builtin/core/data/queries/shared/account_map.sql index b5b43218..025e8e91 100644 --- a/cid/builtin/core/data/queries/shared/account_map.sql +++ b/cid/builtin/core/data/queries/shared/account_map.sql @@ -1,4 +1,4 @@ CREATE OR REPLACE VIEW account_map AS -SELECT account_id, - concat(account_name, ': ', account_id) account_name +SELECT ${account_id} as account_id, + concat(${account_name}, ': ', ${account_id}) account_name FROM ${metadata_table_name} diff --git a/cid/builtin/core/data/queries/shared/aws_accounts.sql b/cid/builtin/core/data/queries/shared/aws_accounts.sql index dc59b0a0..06b79350 100644 --- a/cid/builtin/core/data/queries/shared/aws_accounts.sql +++ b/cid/builtin/core/data/queries/shared/aws_accounts.sql @@ -1,6 +1,7 @@ -CREATE OR REPLACE VIEW aws_accounts AS WITH m AS ( - SELECT account_id, - account_name, +CREATE OR REPLACE VIEW aws_accounts AS WITH + m AS ( + SELECT ${account_id} as account_id, + ${account_name} as account_name, email account_email_id FROM ${metadata_table_name} ), diff --git a/cid/common.py b/cid/common.py index 07a4a674..2da85231 100644 --- a/cid/common.py +++ b/cid/common.py @@ -633,16 +633,21 @@ def create_view(self, view_name: str) -> None: return # Create a view print(f'\nCreating view: {view_name}') + logger.info(f'Creating view: {view_name}') + logger.info(f'Getting view definition') view_definition = self.resources.get('views').get(view_name, dict()) + logger.debug(f'View definition: {view_definition}') # Discover dependency views (may not be discovered earlier) dependency_views = view_definition.get( 'dependsOn', dict()).get('views', list()) + logger.info(f"Dependency views: {', '.join(dependency_views)}" if dependency_views else 'No dependency views') self.athena.discover_views(dependency_views) while dependency_views: dep = dependency_views.copy().pop() # for dep in dependency_views: if dep not in self.athena._metadata.keys(): print(f'Missing dependency view: {dep}, trying to create') + logger.info(f'Missing dependency view: {dep}, trying to create') self.create_view(dep) dependency_views.remove(dep) view_query = self.get_view_query(view_name=view_name) @@ -702,6 +707,6 @@ def get_view_query(self, view_name: str) -> str: def map(self): """Create account mapping Athena views""" + for v in ['account_map', 'aws_accounts']: + self.accountMap.create(v) - self.accountMap.create('account_map') - self.accountMap.create('aws_accounts') diff --git a/cid/helpers/account_map.py b/cid/helpers/account_map.py index f7ae2f5b..5a041921 100644 --- a/cid/helpers/account_map.py +++ b/cid/helpers/account_map.py @@ -1,4 +1,5 @@ import csv +import logging from pathlib import Path import questionary import click @@ -6,21 +7,28 @@ from string import Template from cid.helpers import Athena, CUR +logger = logging.getLogger(__name__) + class AccountMap(): defaults = { - 'AthenaTableName': 'acc_metadata_details' - } + 'MetadataTableNames': ['acc_metadata_details', 'organisation_data'] + } _clients = dict() _accounts = list() + _metadata_source: str = None + _AthenaTableName: str = None mappings = { 'account_map': { - 'metadata_fields': 'account_id, account_name' + 'acc_metadata_details': {'account_id': 'account_id', 'account_name': 'account_name'}, + 'organisation_data': {'account_id': 'id', 'account_name': 'name', 'email': 'email'} }, 'aws_accounts': { - 'metadata_fields': 'account_id, account_name, email', - 'cur_fields': 'bill_payer_account_id' + 'acc_metadata_details': {'account_id': 'account_id', 'account_name': 'account_name', 'email': 'email'}, + 'organisation_data': { 'account_id': 'id', 'account_name': 'name', 'email': 'email', 'status': 'status'}, + 'cur_fields': ['bill_payer_account_id'] } } + session = None def __init__(self, session=None) -> None: self.session = session @@ -55,45 +63,7 @@ def cur(self, client) -> CUR: @property def accounts(self) -> dict: - if not self._accounts: - # Ask user which method to use to retreive account list - selection = list() - account_map_sources = { - 'csv': 'CSV file (relative path required)', - 'organization': 'AWS Organizations (one time account listing)', - 'dummy': 'Dummy (creates dummy account mapping)' - } - for k,v in account_map_sources.items(): - selection.append( - questionary.Choice( - title=f'{v}', - value=k - ) - ) - selected_source=questionary.select( - "\nWhich method would you like to use for collecting an account list ?", - choices=selection - ).ask() - - # Collect account list from different sources of user choice - if selected_source == 'csv': - finished = False - while not finished: - mapping_file = click.prompt("Enter file path", type=str) - finished = self.check_file_exists(mapping_file) - if not finished: - click.echo('File not found, ', nl=False) - click.echo('\nCollecting account info...', nl=False) - self._accounts = self.get_csv_accounts(mapping_file) - elif selected_source == 'organization': - click.echo('\nCollecting account info...', nl=False) - self._accounts = self.get_organization_accounts() - elif selected_source == 'dummy': - click.echo('Notice: Dummy account mapping will be created') - else: - raise Exception('Unsupported selection') - - click.echo(f'collected accounts: {len(self._accounts)}') + if self._accounts: # Required keys mapping, used in renaming below key_mapping = { 'accountid': 'account_id', @@ -113,21 +83,42 @@ def accounts(self) -> dict: }) return self._accounts - def create(self, map_name= str) -> bool: + def create(self, name) -> bool: """Create account map""" - print(f'\nCreating {map_name}...') + print(f'\nCreating {name}...') + logger.info(f'Creating account mapping "{name}"...') try: - # Autodiscover - print('\tautodiscovering...', end='') - accounts = self.athena.get_table_metadata(self.defaults.get('AthenaTableName')).get('Columns') - field_found = [v.get('Name') for v in accounts] - field_required = self.mappings.get(map_name).get('metadata_fields').replace(" ", "").split(',') - # Check if we have all the required fields - if all(v in field_found for v in field_required): - self._AthenaTableName = self.defaults.get('AthenaTableName') + if self.accounts: + logger.info('Account information found, skipping autodiscovery') + raise Exception + if not self._AthenaTableName: + # Autodiscover + print('\tautodiscovering...', end='') + logger.info('Autodiscovering metadata table...') + tables = self.athena.list_table_metadata() + tables = [t for t in tables if t.get('TableType') == 'EXTERNAL_TABLE'] + tables = [t for t in tables if t.get('Name') in self.defaults.get('MetadataTableNames')] + if not len(tables): + logger.info('Metadata table not found') + print('account metadata not detected') + raise Exception + table = next(iter(tables)) + logger.info(f"Detected metadata table {table.get('Name')}") + accounts = table.get('Columns') + field_found = [v.get('Name') for v in accounts] + field_required = list(self.mappings.get(name).get(table.get('Name')).values()) + logger.info(f"Detected fields: {field_found}") + logger.info(f"Required fields: {field_required}") + # Check if we have all the required fields + if all(v in field_found for v in field_required): + logger.info('All required fields found') + self._AthenaTableName = table.get('Name') + else: + logger.info('Missing required fields') + if self._AthenaTableName: # Query path - view_definition = self.athena._resources.get('views').get(map_name, dict()) + view_definition = self.athena._resources.get('views').get(name, dict()) view_file = view_definition.get('File') template = Template(resource_string(view_definition.get('providedBy'), f'data/queries/{view_file}').decode('utf-8')) @@ -135,18 +126,21 @@ def create(self, map_name= str) -> bool: # Fill in TPLs columns_tpl = dict() parameters = { - 'metadata_table_name': self.defaults.get('AthenaTableName'), + 'metadata_table_name': self._AthenaTableName, 'cur_table_name': self.cur.tableName } columns_tpl.update(**parameters) + for k,v in self.mappings.get(name).get(self._AthenaTableName).items(): + logger.info(f'Mapping field {k} to {v}') + columns_tpl.update({k: v}) compiled_query = template.safe_substitute(columns_tpl) print('compiled view.') else: - print('failed, continuing..') - compiled_query = self.create_account_mapping_sql(map_name) + logger.info('Metadata table not found') + print('account metadata not detected') + raise Exception except: - print('failed, continuing..') - compiled_query = self.create_account_mapping_sql(map_name) + compiled_query = self.create_account_mapping_sql(name) finally: # Execute query click.echo('\tcreating view...', nl=False) @@ -155,6 +149,27 @@ def create(self, map_name= str) -> bool: response = self.athena.get_query_results(query_id) click.echo('done') + def get_dummy_account_mapping_sql(self, name) -> list: + """Create dummy account mapping""" + logger.info(f'Creating dummy account mapping for {name}') + template_str = '''CREATE OR REPLACE VIEW ${athena_view_name} AS SELECT DISTINCT + line_item_usage_account_id account_id, bill_payer_account_id parent_account_id, + line_item_usage_account_id account_name, line_item_usage_account_id account_email_id + FROM + ${cur_table_name} + ''' + template = Template(template_str) + # Fill in TPLs + columns_tpl = dict() + parameters = { + 'athena_view_name': name, + 'cur_table_name': self.cur.tableName + } + columns_tpl.update(**parameters) + compiled_query = template.safe_substitute(columns_tpl) + + return compiled_query + def get_organization_accounts(self) -> list: """ Retreive AWS Organization account """ @@ -197,9 +212,53 @@ def get_csv_accounts(self, file_path) -> list: return accounts - def create_account_mapping_sql(self, mapping_name) -> str: + def select_metadata_collection_method(self) -> str: + """ Selects the method to collect metadata """ + logger.info('Metadata source selection') + # Ask user which method to use to retreive account list + selection = list() + account_map_sources = { + 'csv': 'CSV file (relative path required)', + 'organization': 'AWS Organizations (one time account listing)', + 'dummy': 'Dummy (CUR account data, no names)' + } + for k,v in account_map_sources.items(): + selection.append( + questionary.Choice(title=f'{v}', value=k) + ) + selected_source=questionary.select( + "Please select account metadata collection method", + choices=selection + ).ask() + if selected_source in account_map_sources.keys(): + logger.info(f'Selected {selected_source}') + self._metadata_source = selected_source + + # Collect account list from different sources of user choice + if self._metadata_source == 'csv': + finished = False + while not finished: + mapping_file = click.prompt("Enter file path", type=str) + finished = self.check_file_exists(mapping_file) + if not finished: + click.echo('File not found, ', nl=False) + click.echo('\nCollecting account info...', nl=False) + self._accounts = self.get_csv_accounts(mapping_file) + logger.info(f'Found {len(self._accounts)} accounts') + click.echo(f' {len(self.accounts)} collected') + elif self._metadata_source == 'organization': + click.echo('\nCollecting account info...', nl=False) + self._accounts = self.get_organization_accounts() + logger.info(f'Found {len(self._accounts)} accounts') + click.echo(f' {len(self.accounts)} collected') + elif self._metadata_source == 'dummy': + click.echo('Notice: Dummy account mapping will be created') + else: + print('Unsupported selection') + return False + + def create_account_mapping_sql(self, name) -> str: """ Returns account mapping Athena query """ - template_str = '''CREATE OR REPLACE VIEW ${athena_view_name} AS SELECT * @@ -207,22 +266,26 @@ def create_account_mapping_sql(self, mapping_name) -> str: ( VALUES ${rows} ) ignored_table_name (account_id, account_name, parent_account_id, account_status, account_email) ''' - # Wait for account list - while not self.accounts: - pass - template = Template(template_str) - accounts_sql = list() - for account in self.accounts: - accounts_sql.append( - "ROW ('{account_id}', '{account_name}:{account_id}', '{parent_account_id}', '{account_status}', '{account_email}')".format(**account)) - # Fill in TPLs - columns_tpl = dict() - parameters = { - 'athena_view_name': mapping_name, - 'rows': ','.join(accounts_sql) - } - columns_tpl.update(**parameters) - compiled_query = template.safe_substitute(columns_tpl) + while not self.accounts and self._metadata_source != 'dummy': + self.select_metadata_collection_method() + + if self._metadata_source == 'dummy': + compiled_query = self.get_dummy_account_mapping_sql(name) + else: + template = Template(template_str) + accounts_sql = list() + for account in self.accounts: + accounts_sql.append( + "ROW ('{account_id}', '{account_name}:{account_id}', '{parent_account_id}', '{account_status}', '{account_email}')".format(**account)) + + # Fill in TPLs + columns_tpl = dict() + parameters = { + 'athena_view_name': name, + 'rows': ','.join(accounts_sql) + } + columns_tpl.update(**parameters) + compiled_query = template.safe_substitute(columns_tpl) return compiled_query diff --git a/cid/helpers/athena.py b/cid/helpers/athena.py index 6d5821f1..5d4091b1 100644 --- a/cid/helpers/athena.py +++ b/cid/helpers/athena.py @@ -22,6 +22,9 @@ class Athena(): _DatabaseName = None ahq_queries = None _metadata = dict() + _resources = dict() + _client = None + region: str = None def __init__(self, session, resources: dict=None): self.region = session.region_name From 86a0f2f2518de151098f1a4a1906e011e39eaa10 Mon Sep 17 00:00:00 2001 From: Oleksandr Moskalenko Date: Tue, 4 Jan 2022 17:59:26 +0100 Subject: [PATCH 12/12] Bump version --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index d1ede1b5..d1e4f888 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,7 +4,7 @@ universal = 1 [metadata] name = cid-cmd # version = attr: VERSION -version = 0.1.5 +version = 0.1.6 keywords = aws, cmd, cli, cost intelligence dashboards description = Cloud Intelligence Dashboards deployment helper tool long_description = file: README.md