From 4b1c5a7cb927aaf8cb16cb07b9cb2436371a703b Mon Sep 17 00:00:00 2001 From: Oleksandr Moskalenko Date: Tue, 21 Dec 2021 18:30:24 +0100 Subject: [PATCH 1/3] IMprove CUR detection --- cid/helpers/cur.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/cid/helpers/cur.py b/cid/helpers/cur.py index e106d212..7b9a0afc 100644 --- a/cid/helpers/cur.py +++ b/cid/helpers/cur.py @@ -5,6 +5,23 @@ class CUR: defaults = { 'TableName': 'customer_all' } + requiredColumns = [ + 'identity_line_item_id', + 'identity_time_interval', + 'bill_invoice_id', + 'bill_billing_entity', + 'bill_bill_type', + 'bill_payer_account_id', + 'bill_billing_period_start_date', + 'bill_billing_period_end_date', + 'line_item_usage_account_id', + 'line_item_line_item_type', + 'line_item_usage_start_date', + 'line_item_usage_end_date', + 'line_item_product_code', + 'line_item_usage_type', + 'line_item_operation', + ] _tableName = None _metadata = None _clients = dict() @@ -89,12 +106,11 @@ def metadata(self) -> dict: tables = self.athena.list_table_metadata() tables = [v for v in tables if v.get('TableType') == 'EXTERNAL_TABLE'] for table in tables: - if table.get('Name') in self.athena.DatabaseName: - self._metadata = self.athena.get_table_metadata(table.get('Name')) + _metadata = self.athena.get_table_metadata(table.get('Name')) + if all(v in self.requiredColumns for v in _metadata.get('Columns')): + self._metadata = _metadata self._tableName = table.get('Name') break - # self._metadata = self.athena.get_table_metadata(self.athena._DatabaseName.rpartition('_')[2]) - # self._tableName = self.athena._DatabaseName.rpartition('_')[2] except self.athena.client.exceptions.MetadataException: # TODO: ask user print('Error: CUR metadata not found') From b282bada1be21d9a9510ff932d2cb9124a7b7738 Mon Sep 17 00:00:00 2001 From: Oleksandr Moskalenko Date: Wed, 22 Dec 2021 13:54:47 +0100 Subject: [PATCH 2/3] Select from multiple CUR tables --- cid/helpers/cur.py | 47 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/cid/helpers/cur.py b/cid/helpers/cur.py index 7b9a0afc..49446834 100644 --- a/cid/helpers/cur.py +++ b/cid/helpers/cur.py @@ -1,10 +1,8 @@ import json +import questionary from cid.helpers import Athena class CUR: - defaults = { - 'TableName': 'customer_all' - } requiredColumns = [ 'identity_line_item_id', 'identity_time_interval', @@ -63,20 +61,20 @@ def configured(self) -> bool: return self._configured @property - def tableName(self): + def tableName(self) -> str: if self.metadata is None: print('Error: CUR not detected') exit(1) return self.metadata.get('Name') @property - def hasResourceIDs(self): + def hasResourceIDs(self) -> bool: if self._configured and self._hasResourceIDs is None: self._hasResourceIDs = 'line_item_resource_id' in self.fields return self._hasResourceIDs @property - def hasReservations(self): + def hasReservations(self) -> bool: if self._configured and self._hasReservations is None: kwargs = { 'cur_table_name': self._tableName @@ -85,7 +83,7 @@ def hasReservations(self): return self._hasReservations @property - def hasSavingsPlans(self): + def hasSavingsPlans(self) -> bool: if self._configured and self._hasSavingsPlans is None: kwargs = { 'cur_table_name': self._tableName @@ -97,23 +95,24 @@ def hasSavingsPlans(self): def metadata(self) -> dict: if not self._metadata: try: - # Look for default CUR table - self._metadata = self.athena.get_table_metadata(self.defaults.get('TableName')) - self._tableName = self.defaults.get('TableName') - except self.athena.client.exceptions.MetadataException: - # Look based on CUR Athena database name - try: - tables = self.athena.list_table_metadata() - tables = [v for v in tables if v.get('TableType') == 'EXTERNAL_TABLE'] - for table in tables: - _metadata = self.athena.get_table_metadata(table.get('Name')) - if all(v in self.requiredColumns for v in _metadata.get('Columns')): - self._metadata = _metadata - self._tableName = table.get('Name') - break - except self.athena.client.exceptions.MetadataException: - # TODO: ask user - print('Error: CUR metadata not found') + # Look other tables + tables = self.athena.list_table_metadata() + # Filter tables with type = 'EXTERNAL_TABLE' + tables = [v for v in tables if v.get('TableType') == 'EXTERNAL_TABLE'] + # Filter tables having CUR structure + for table in tables.copy(): + columns = [c.get('Name') for c in table.get('Columns')] + if not all([c in columns for c in self.requiredColumns]): + tables.remove(table) + if len(tables) == 1: + self._metadata = tables[0] + self._tableName = self._metadata.get('Name') + elif len(tables) > 1: + self._tableName=questionary.select( + "Multiple CUR tables found, please select one", + choices=[v.get('Name') for v in tables] + ).ask() + self._metadata = self.athena.get_table_metadata(self._tableName) except Exception as e: # For other errors dump the message print(json.dumps(e, indent=4, sort_keys=True, default=str)) From e3b2fc5fcffa46c7066b74a2e5b0a21df0c3d094 Mon Sep 17 00:00:00 2001 From: Oleksandr Moskalenko Date: Wed, 22 Dec 2021 13:59:57 +0100 Subject: [PATCH 3/3] Bump version --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 9c8a7177..a8e904c8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,7 +4,7 @@ universal = 1 [metadata] name = cid-cmd # version = attr: VERSION -version = 0.1.3 +version = 0.1.4 keywords = aws, cmd, cli, cost intelligence dashboards description = Cloud Intelligence Dashboards deployment helper tool long_description = file: README.md