Autodetect identity region and other fixes

cfn-templates/cid-cfn.tests.bats

@@ -76,7 +76,6 @@ setup_file() {
       CURTableName=""\
       CidVersion="$cid_version"\
       QuickSightDataSetRefreshSchedule="cron(0 4 * * ? *)"\
-      QuicksightIdentityRegion="us-east-1"\
       LambdaLayerBucketPrefix="aws-managed-cost-intelligence-dashboards"\
       Suffix=""\
     --stack-name "$stackname"

cfn-templates/cid-cfn.yml

@@ -32,7 +32,6 @@ Metadata:
           - CURTableName
           - CidVersion
           - Suffix
-          - QuicksightIdentityRegion
           - QuickSightDataSetRefreshSchedule
           - LambdaLayerBucketPrefix
   cfn-lint:
@@ -65,11 +64,6 @@ Parameters:
     Description: >
         REQUIRED - User name of QuickSight user from default namespace (as displayed in QuickSight admin panel).
         Dashboard created by this template with be owned by this user. See https://quicksight.aws.amazon.com/sn/admin#users
-  QuicksightIdentityRegion: #CID can detect QS Idenity region, but we need in permissions
-    Type: String
-    MinLength: 8
-    Description: Quicksight Identity region(can be different than the region you deploy the dashboard, typically us-east-1)
-    Default: us-east-1
   QuickSightDataSetRefreshSchedule:
     Type: String
     MinLength: 3
@@ -325,6 +319,8 @@ Resources:
           import botocore
           import urllib3
 
+          from cid.helpers import QuickSight # from layer
+
           BUCKET = os.environ['BUCKET']
           WORKGROUP = os.environ['WORKGROUP']
           CRAWLER = os.environ['CRAWLER']
@@ -333,10 +329,12 @@ Resources:
               print(event)
               type_ = event.get('RequestType', 'Undef')
               res = (True, f"Un error on {type_}. Check logs")
+              identity_region = ''
               try:
                   if type_ == 'Create': res = on_create()
                   elif type_ == 'Delete': res = on_delete()
                   else: res = (True, f"Not supported operation: {type_}")
+                  identity_region = get_identity_region()
               finally:
                   url = event.get('ResponseURL')
                   body = {}
@@ -347,7 +345,7 @@ Resources:
                   body['RequestId'] = event.get('RequestId')
                   body['LogicalResourceId'] = event.get('LogicalResourceId')
                   body['NoEcho'] = False
-                  body['Data'] =  {'Reason': res[1], 'uuid': str(uuid.uuid1()) }
+                  body['Data'] =  {'Reason': res[1], 'uuid': str(uuid.uuid1()), 'IdentityRegion': identity_region}
                   print(body)
                   if not url: return
                   json_body=json.dumps(body)
@@ -358,6 +356,10 @@ Resources:
                   except Exception as exc:
                       print("Failed sending PUT to CFN: " + str(exc))
 
+          def get_identity_region():
+              qs = QuickSight(boto3.session.Session())
+              return qs.identityRegion
+
           def on_create():
               if CRAWLER:
                   # FIXME: this can be replaced with AWS::Glue::Trigger
@@ -411,6 +413,8 @@ Resources:
                       log.append(f'ERROR: WorkGroup {WORKGROUP} Error: {exc}')
               print('\n'.join(log))
               return (True, '\n'.join(log))
+      Layers:
+        - !Ref CidResourceLambdaLayer
       Environment:
         Variables:
           BUCKET: !If [NeedAthenaQueryResultsBucket, !Ref MyAthenaQueryResultsBucket, '']
@@ -452,6 +456,10 @@ Resources:
                 Action:
                   - glue:StartCrawler
                 Resource: '*' # FIXME: use MyGlueCURCrawler
+              - Effect: Allow
+                Action:
+                  - quicksight:DescribeUser
+                Resource: '*' #FIXME: use !Sub 'arn:${AWS::Partition}:quicksight:*:${AWS::AccountId}:user/default/${QuickSightUser}'
       ManagedPolicyArns:
         - !Sub arn:${AWS::Partition}:iam::aws:policy/AWSLambdaExecute
   InitialSetup:
@@ -747,7 +755,7 @@ Resources:
             - 'quicksight:UpdateDataSource'
             - 'quicksight:DeleteDataSource'
             - 'quicksight:UpdateDataSourcePermissions'
-          Principal: !Sub 'arn:${AWS::Partition}:quicksight:${QuicksightIdentityRegion}:${AWS::AccountId}:user/default/${QuickSightUser}'
+          Principal: !Sub 'arn:${AWS::Partition}:quicksight:${InitialSetup.IdentityRegion}:${AWS::AccountId}:user/default/${QuickSightUser}'
 
   CidExecRole:
     Type: AWS::IAM::Role

cid/helpers/quicksight/__init__.py

@@ -14,7 +14,7 @@
 from cid.helpers.quicksight.dashboard import Dashboard
 from cid.helpers.quicksight.dataset import Dataset
 from cid.helpers.quicksight.datasource import Datasource
-from cid.utils import get_parameter
+from cid.utils import get_parameter, get_parameters
 
 logger = logging.getLogger(__name__)
 
@@ -50,8 +50,9 @@ def AthenaWorkGroup(self, value):
     @property
     def user(self) -> dict:
         if not self._user:
+            username = get_parameters().get('quicksight-user', self.username)
             try:
-                self._user =  self.describe_user(self.username)
+                self._user =  self.describe_user(username)
             except Exception as exc:
                 logger.debug(exc, stack_info=True)
                 logger.error(f'Failed to find your QuickSight username ({exc}). Is QuickSight activated?')
@@ -69,9 +70,10 @@ def identityRegion(self) -> str:
         if not self._identityRegion:
             try:
                 logger.info(f'Detecting QuickSight identity region, trying {self.region}')
+                username = get_parameters().get('quicksight-user', self.username)
                 parameters = {
                     'AwsAccountId': self.account_id,
-                    'UserName': '/'.join(self.username),
+                    'UserName': username,
                     'Namespace': 'default'
                 }
                 self.client.describe_user(**parameters)
@@ -410,14 +412,11 @@ def list_data_sources(self) -> list:
         parameters = {
             'AwsAccountId': self.account_id
         }
+        data_sources = []
         try:
-            result = self.client.list_data_sources(**parameters)
-            logger.debug(result)
-            if result.get('Status') != 200:
-                print(f'Error, {result}')
-                exit()
-            else:
-                return result.get('DataSources')
+            for page in self.client.get_paginator('list_data_sources').paginate(**parameters):
+                data_sources += page.get('DataSources',[])
+            return data_sources
         except self.client.exceptions.AccessDeniedException:
             logger.info('Access denied listing data sources')
             raise

cid/test/bats/10-deploy-update-delete/compute-optimizer-dashboard.bats

@@ -5,6 +5,7 @@ account_id=$(aws sts get-caller-identity --query "Account" --output text )
 @test "Install" {
   run cid-cmd -vv deploy  \
     --dashboard-id compute-optimizer-dashboard \
+    --share-with-account yes \
     --athena-database 'optimization_data' \
     --view-compute-optimizer-lambda-lines-s3FolderPath       's3://costoptimizationdata{account_id}/Compute_Optimizer/Compute_Optimizer_ec2_lambda' \
     --view-compute-optimizer-ebs-volume-lines-s3FolderPath   's3://costoptimizationdata{account_id}/Compute_Optimizer/Compute_Optimizer_ebs_volume' \

cid/test/bats/10-deploy-update-delete/cost_intelligence_dashboard.bats

@@ -9,6 +9,7 @@ database_name="${database_name:-athenacurcfn_cur1}" # If variable not set or nul
     --dashboard-id cost_intelligence_dashboard \
     --athena-database $database_name\
     --account-map-source dummy \
+    --share-with-account yes \
 
   [ "$status" -eq 0 ]
 }

cid/test/bats/10-deploy-update-delete/cudos.bats

@@ -11,6 +11,7 @@ quicksight_user="${quicksight_user:-cicd-staging}" # If variable not set or null
     --athena-database $database_name\
     --account-map-source dummy \
     --quicksight-user $quicksight_user \
+    --share-with-account yes \
 
   [ "$status" -eq 0 ]
 }

cid/test/bats/10-deploy-update-delete/kpi_dashboard.bats

@@ -9,6 +9,7 @@ database_name="${database_name:-athenacurcfn_cur1}" # If variable not set or nul
     --dashboard-id kpi_dashboard \
     --athena-database $database_name\
     --account-map-source dummy \
+    --share-with-account yes \
 
   [ "$status" -eq 0 ]
 }

cid/test/bats/10-deploy-update-delete/ta-organizational-view.bats

@@ -6,6 +6,8 @@ account_id=$(aws sts get-caller-identity --query "Account" --output text )
   run cid-cmd -vv deploy  \
     --dashboard-id ta-organizational-view \
     --athena-database 'optimization_data' \
+    --share-with-account yes \
+
     --view-ta-organizational-view-reports-s3FolderPath "s3://costoptimizationdata$account_id/optics-data-collector/ta-data'"
 
   [ "$status" -eq 0 ]

cid/test/bats/10-deploy-update-delete/trends-dashboard.bats

@@ -9,6 +9,7 @@ database_name="${database_name:-athenacurcfn_cur1}" # If variable not set or nul
     --dashboard-id trends-dashboard \
     --athena-database $database_name\
     --account-map-source dummy \
+    --share-with-account yes \
 
   [ "$status" -eq 0 ]
 }