local dev scripting updates (#2632)

scripts/e2e-secrets.yml

@@ -14,9 +14,79 @@ labels:
   app: koku
   template: koku-secret
 objects:
+# postgresql-exporter
 - apiVersion: v1
   stringData:
     data-source-name: postgresql://${DB_MONITOR_USER}:${DB_MONITOR_USER_PW}@${DATABASE_HOST}:5432/dbname?sslmode=disable
+    query-yaml: |-
+      pg_stat_statements:
+        query: "SELECT queryid, query, calls, total_time, min_time, max_time, mean_time, stddev_time, rows, shared_blks_hit, shared_blks_read, shared_blks_dirtied, shared_blks_written, local_blks_hit, local_blks_read, local_blks_dirtied, local_blks_written, temp_blks_read, temp_blks_written, blk_read_time, blk_write_time FROM public.pg_stat_statements() WHERE userid in (SELECT usesysid FROM pg_user WHERE usename != any(array['postgres', 'kokumon']) and usename !~ 'test') AND mean_time > 500 AND query !~ 'INSERT INTO' AND query !~ 'CREATE'"
+        master: true
+        cache_seconds: 60
+        metrics:
+          - queryid:
+              usage: "LABEL"
+              description: "Query identifier"
+          - query:
+              usage: "LABEL"
+              description: "Parameterized SQL query"
+          - calls:
+              usage: "COUNTER"
+              description: "Number of times executed"
+          - total_time:
+              usage: "COUNTER"
+              description: "Total time spent in the statement, in milliseconds"
+          - min_time:
+              usage: "GAUGE"
+              description: "Minimum time spent in the statement, in milliseconds"
+          - max_time:
+              usage: "GAUGE"
+              description: "Maximum time spent in the statement, in milliseconds"
+          - mean_time:
+              usage: "GAUGE"
+              description: "Mean time spent in the statement, in milliseconds"
+          - stddev_time:
+              usage: "GAUGE"
+              description: "Population standard deviation of time spent in the statement, in milliseconds"
+          - rows:
+              usage: "COUNTER"
+              description: "Total number of rows retrieved or affected by the statement"
+          - shared_blks_hit:
+              usage: "COUNTER"
+              description: "Total number of shared block cache hits by the statement"
+          - shared_blks_read:
+              usage: "COUNTER"
+              description: "Total number of shared blocks read by the statement"
+          - shared_blks_dirtied:
+              usage: "COUNTER"
+              description: "Total number of shared blocks dirtied by the statement"
+          - shared_blks_written:
+              usage: "COUNTER"
+              description: "Total number of shared blocks written by the statement"
+          - local_blks_hit:
+              usage: "COUNTER"
+              description: "Total number of local block cache hits by the statement"
+          - local_blks_read:
+              usage: "COUNTER"
+              description: "Total number of local blocks read by the statement"
+          - local_blks_dirtied:
+              usage: "COUNTER"
+              description: "Total number of local blocks dirtied by the statement"
+          - local_blks_written:
+              usage: "COUNTER"
+              description: "Total number of local blocks written by the statement"
+          - temp_blks_read:
+              usage: "COUNTER"
+              description: "Total number of temp blocks read by the statement"
+          - temp_blks_written:
+              usage: "COUNTER"
+              description: "Total number of temp blocks written by the statement"
+          - blk_read_time:
+              usage: "COUNTER"
+              description: "Total time the statement spent reading blocks, in milliseconds (if track_io_timing is enabled, otherwise zero)"
+          - blk_write_time:
+              usage: "COUNTER"
+              description: "Total time the statement spent writing blocks, in milliseconds (if track_io_timing is enabled, otherwise zero)"
   kind: Secret
   metadata:
     labels:
@@ -25,6 +95,8 @@ objects:
     name: postgresql-exporter
     namespace: secrets
   type: Opaque
+
+# rdsexporter
 - apiVersion: v1
   stringData:
     aws-access-key-id: ${AWS_ACCESS_KEY_ID}
@@ -40,6 +112,8 @@ objects:
     name: rdsexporter
     namespace: secrets
   type: Opaque
+
+# koku-aws
 - apiVersion: v1
   stringData:
     aws-access-key-id: ${AWS_ACCESS_KEY_ID}
@@ -55,6 +129,8 @@ objects:
     name: koku-aws
     namespace: secrets
   type: Opaque
+
+# cloudwatch
 - apiVersion: v1
   stringData:
     aws_access_key_id: ${CLOUDWATCH_AWS_ACCESS_KEY_ID}
@@ -77,6 +153,8 @@ objects:
     name: cloudwatch
     namespace: secrets
   type: Opaque
+
+# koku-gcp
 - apiVersion: v1
   data:
     gcp-credentials: ${GCP_CREDENTIALS}
@@ -90,6 +168,8 @@ objects:
     name: koku-gcp
     namespace: secrets
   type: Opaque
+
+# koku-db
 - apiVersion: v1
   stringData:
     db.name: ${DATABASE_NAME}
@@ -112,6 +192,23 @@ objects:
     name: koku-db
     namespace: secrets
   type: Opaque
+
+# hive-db
+- apiVersion: v1
+  stringData:
+    database_password: ${HIVE_DATABASE_PASSWORD}
+    database_user: ${HIVE_DATABASE_USER}
+    database_name: ${HIVE_DATABASE_NAME}
+  kind: Secret
+  metadata:
+    labels:
+      app: koku
+      template: koku-secret
+    name: hive-db
+    namespace: secrets
+  type: Opaque
+
+# koku-flower
 - apiVersion: v1
   stringData:
     flower-auth-scheme: ${FLOWER_AUTH_SCHEME}
@@ -129,6 +226,8 @@ objects:
     name: koku-flower
     namespace: secrets
   type: Opaque
+
+# koku-secret
 - apiVersion: v1
   stringData:
     django-secret-key: ${DJANGO_SECRET_KEY}
@@ -145,6 +244,8 @@ objects:
     name: koku-secret
     namespace: secrets
   type: Opaque
+
+# koku-daily-secret
 - apiVersion: v1
   stringData:
     django-secret-key: ${DJANGO_SECRET_KEY}
@@ -161,6 +262,8 @@ objects:
     name: koku-daily-secret
     namespace: secrets
   type: Opaque
+
+# koku-sentry
 - apiVersion: v1
   kind: Secret
   metadata:
@@ -173,6 +276,7 @@ objects:
     template: koku-secret
   name: koku-sentry
   namespace: secrets
+
 # aws s3 secrets
 - apiVersion: v1
   kind: Secret
@@ -190,6 +294,8 @@ objects:
     aws_secret_access_key: ${S3_SECRET}
     bucket: ${S3_BUCKET}
     endpoint: ${S3_ENDPOINT}
+
+# nise-populator-secret
 - apiVersion: v1
   kind: Secret
   metadata:
@@ -278,13 +384,24 @@ objects:
 # rds-client-ca
 - apiVersion: v1
   data:
-    rds-client-ca: null
+    rds-cacert: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM4RENDQWRpZ0F3SUJBZ0lVZjJ3YnY3K1dDd0E4cXV2S05ZOGZLU2hwRHVJd0RRWUpLb1pJaHZjTkFRRUwKQlFBd0ZERVNNQkFHQTFVRUF3d0piRzlqWVd4b2IzTjBNQjRYRFRJeE1ERXlNakUzTlRBeU5Wb1hEVEl4TURJeQpNVEUzTlRBeU5Wb3dGREVTTUJBR0ExVUVBd3dKYkc5allXeG9iM04wTUlJQklqQU5CZ2txaGtpRzl3MEJBUUVGCkFBT0NBUThBTUlJQkNnS0NBUUVBeFV0OWxCZzFsOXBaMzRsV09BbGNnM0lidjE4UU9ZZkRmcmZaOHVlQ3ptK1UKM1o5QmE5R3ZhNzY2WUxwSHhxL0x5QTBIdlh1VjFPWUtNcmUzUlpMbmJ4TGgvcEpjWFhDUDhnUExlaTRXK2dPVQpXTkc5QXl0eG1hdVpHa2RCcjA4L1VHNDE1SFNKU3RkMW9ZZ1RsSTlJandlNWp6RUZZbHhDNDJreG44a0cxL3pWCmRyVkNVTlM2eE9xRTdteTdjQkJQVm5YWXRFcWdkeWZyMExSYnEwNldPTVdaM0ZEVE5Oc1hDUGtxMkZJSjlmRGwKTVFUei9qS0VVbG51WG5nZjFkcThSOXFHYURUeDhFaDQ3Mys3U2ZWdzlrUlJ1OVRoY0ZNT2VqaTAzRnZKV25PSQpBYW5OMUpqcXE2TGwrUzk0NFp0OTRrTHh6d0FkRHg2bU1Ea2NxOHJWNndJREFRQUJvem93T0RBVUJnTlZIUkVFCkRUQUxnZ2xzYjJOaGJHaHZjM1F3Q3dZRFZSMFBCQVFEQWdlQU1CTUdBMVVkSlFRTU1Bb0dDQ3NHQVFVRkJ3TUIKTUEwR0NTcUdTSWIzRFFFQkN3VUFBNElCQVFDK2VlNmZQOUFCN2FZQW82KzZ0eCt6L3hNa1ZtU2FsWDZMWnFDcQpmRnI3blV3MHJtYUNFRGhFM1pzNHVwdGJwUXV6M3diWjhtUTB4dXlBRDI5NjZXc3JidDZFeWljVHNzUmlBUkZpCjFjc0JkM05Cc3ArMDk1dXhLcTU4OFFEQWVGeHRUKzlsYnltOXArcmtSdzZvM3VSdmIwcG1uQi84cERlV3lzZTYKL3FCNEhuSUV1Q3AxZWZCczZvb2YxdEJIcFdjZlgwWXFnTHpVVDF2by9rWHRlM3pnaFRjdGpJYW9ZdmsxdzgyMApUaHpoQVo4Q0lodXhKakorWVdNcklxOWJrMW1aRk01S3BsWUI5QnM1U1hkdmZJdE4vR2Z1R05EZFJGOTl5WGxXCjF4cXg0QVNVZ3hITDJQVk01TGc5U05iV3c2RnhVd215dmlFWWFzcXNvK0RNaEhoRAotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg==
   kind: Secret
   metadata:
     name: rds-client-ca
     namespace: secrets
   type: Opaque
 
+# elasticache-redis
+- apiVersion: v1
+  stringData:
+    db.endpoint: 'koku-redis.hccm.svc.cluster.local'
+  kind: Secret
+  metadata:
+    name: elasticache-redis
+    namespace: secrets
+  type: Opaque
+
+
 parameters:
 # database params
 - displayName: Database Name
@@ -333,6 +450,22 @@ parameters:
   from: '[a-zA-Z0-9]{16}'
   generate: expression
 
+# hive db params
+- displayName: Hive Database Name
+  name: HIVE_DATABASE_NAME
+  required: true
+  value: hive
+- displayName: Hive Database Username
+  from: 'hive_[a-zA-Z0-9]{12}'
+  generate: expression
+  name: HIVE_DATABASE_USER
+  required: true
+- displayName: Hive Database Password
+  from: '[a-zA-Z0-9]{16}'
+  generate: expression
+  name: HIVE_DATABASE_PASSWORD
+  required: true
+
 # application secret keys
 - description: Set this to a long random string.
   displayName: Django Secret Key
@@ -408,7 +541,6 @@ parameters:
   name: KOKU_CELERY_SENTRY_DSN
   required: false
 
-
 # AWS S3 account creds
 - displayName: S3 Bucket Access Key
   name: S3_ACCESS_KEY

scripts/show_test_graph.py

@@ -18,6 +18,8 @@
 """draw a graph from cost & forecast data."""
 import datetime
 import json
+import sys
+from pprint import pformat
 
 import matplotlib as mlt
 import matplotlib.pyplot as plt
@@ -35,7 +37,19 @@ def _get_data(url):
     except HTTPError as err:
         print("Error: %s" % err)
 
-    resp = json.loads(response.content)
+    try:
+        resp = json.loads(response.content)
+    except json.decoder.JSONDecodeError as exc:
+        print("Error decoding API response: %s" % exc)
+
+        from bs4 import BeautifulSoup
+
+        errdoc = BeautifulSoup(response.content, "html.parser")
+        print(errdoc.find(id="traceback_area").get_text())
+
+        # print(response.text)
+        sys.exit(1)
+
     # print(resp)
     return resp.get("data")
 
@@ -79,7 +93,13 @@ def _get_values(key_name, data, fields=["total"]):
 ax.scatter(x, y, label="aws cost")
 
 forecast_data = _get_data(FORECAST_URL)
-x, y, upper, lower = _get_values("cost", forecast_data, fields=["total", "upper_conf_y", "lower_conf_y"])
+
+try:
+    x, y, upper, lower = _get_values("cost", forecast_data, fields=["total", "upper_conf_y", "lower_conf_y"])
+except ValueError:
+    print("Error: unexpected API response")
+    print("Response: %s" % pformat(forecast_data))
+    sys.exit(-1)
 
 print("Forecast X: %s" % x)
 print("Forecast Y: %s" % y)