From 6e70521a43f691ff5e6c9d036e1fc65deaa1b492 Mon Sep 17 00:00:00 2001 From: Brett Lentz Date: Wed, 3 Feb 2021 12:37:07 -0500 Subject: [PATCH] local dev scripting updates --- scripts/e2e-secrets.yml | 136 ++++++++++++++++++++++++++++++++++++- scripts/show_test_graph.py | 24 ++++++- 2 files changed, 156 insertions(+), 4 deletions(-) diff --git a/scripts/e2e-secrets.yml b/scripts/e2e-secrets.yml index 1ef3d36034..407823dac3 100644 --- a/scripts/e2e-secrets.yml +++ b/scripts/e2e-secrets.yml @@ -14,9 +14,79 @@ labels: app: koku template: koku-secret objects: +# postgresql-exporter - apiVersion: v1 stringData: data-source-name: postgresql://${DB_MONITOR_USER}:${DB_MONITOR_USER_PW}@${DATABASE_HOST}:5432/dbname?sslmode=disable + query-yaml: |- + pg_stat_statements: + query: "SELECT queryid, query, calls, total_time, min_time, max_time, mean_time, stddev_time, rows, shared_blks_hit, shared_blks_read, shared_blks_dirtied, shared_blks_written, local_blks_hit, local_blks_read, local_blks_dirtied, local_blks_written, temp_blks_read, temp_blks_written, blk_read_time, blk_write_time FROM public.pg_stat_statements() WHERE userid in (SELECT usesysid FROM pg_user WHERE usename != any(array['postgres', 'kokumon']) and usename !~ 'test') AND mean_time > 500 AND query !~ 'INSERT INTO' AND query !~ 'CREATE'" + master: true + cache_seconds: 60 + metrics: + - queryid: + usage: "LABEL" + description: "Query identifier" + - query: + usage: "LABEL" + description: "Parameterized SQL query" + - calls: + usage: "COUNTER" + description: "Number of times executed" + - total_time: + usage: "COUNTER" + description: "Total time spent in the statement, in milliseconds" + - min_time: + usage: "GAUGE" + description: "Minimum time spent in the statement, in milliseconds" + - max_time: + usage: "GAUGE" + description: "Maximum time spent in the statement, in milliseconds" + - mean_time: + usage: "GAUGE" + description: "Mean time spent in the statement, in milliseconds" + - stddev_time: + usage: "GAUGE" + description: "Population standard deviation of time spent in the statement, in milliseconds" + - rows: + usage: "COUNTER" + description: "Total number of rows retrieved or affected by the statement" + - shared_blks_hit: + usage: "COUNTER" + description: "Total number of shared block cache hits by the statement" + - shared_blks_read: + usage: "COUNTER" + description: "Total number of shared blocks read by the statement" + - shared_blks_dirtied: + usage: "COUNTER" + description: "Total number of shared blocks dirtied by the statement" + - shared_blks_written: + usage: "COUNTER" + description: "Total number of shared blocks written by the statement" + - local_blks_hit: + usage: "COUNTER" + description: "Total number of local block cache hits by the statement" + - local_blks_read: + usage: "COUNTER" + description: "Total number of local blocks read by the statement" + - local_blks_dirtied: + usage: "COUNTER" + description: "Total number of local blocks dirtied by the statement" + - local_blks_written: + usage: "COUNTER" + description: "Total number of local blocks written by the statement" + - temp_blks_read: + usage: "COUNTER" + description: "Total number of temp blocks read by the statement" + - temp_blks_written: + usage: "COUNTER" + description: "Total number of temp blocks written by the statement" + - blk_read_time: + usage: "COUNTER" + description: "Total time the statement spent reading blocks, in milliseconds (if track_io_timing is enabled, otherwise zero)" + - blk_write_time: + usage: "COUNTER" + description: "Total time the statement spent writing blocks, in milliseconds (if track_io_timing is enabled, otherwise zero)" kind: Secret metadata: labels: @@ -25,6 +95,8 @@ objects: name: postgresql-exporter namespace: secrets type: Opaque + +# rdsexporter - apiVersion: v1 stringData: aws-access-key-id: ${AWS_ACCESS_KEY_ID} @@ -40,6 +112,8 @@ objects: name: rdsexporter namespace: secrets type: Opaque + +# koku-aws - apiVersion: v1 stringData: aws-access-key-id: ${AWS_ACCESS_KEY_ID} @@ -55,6 +129,8 @@ objects: name: koku-aws namespace: secrets type: Opaque + +# cloudwatch - apiVersion: v1 stringData: aws_access_key_id: ${CLOUDWATCH_AWS_ACCESS_KEY_ID} @@ -77,6 +153,8 @@ objects: name: cloudwatch namespace: secrets type: Opaque + +# koku-gcp - apiVersion: v1 data: gcp-credentials: ${GCP_CREDENTIALS} @@ -90,6 +168,8 @@ objects: name: koku-gcp namespace: secrets type: Opaque + +# koku-db - apiVersion: v1 stringData: db.name: ${DATABASE_NAME} @@ -112,6 +192,23 @@ objects: name: koku-db namespace: secrets type: Opaque + +# hive-db +- apiVersion: v1 + stringData: + database_password: ${HIVE_DATABASE_PASSWORD} + database_user: ${HIVE_DATABASE_USER} + database_name: ${HIVE_DATABASE_NAME} + kind: Secret + metadata: + labels: + app: koku + template: koku-secret + name: hive-db + namespace: secrets + type: Opaque + +# koku-flower - apiVersion: v1 stringData: flower-auth-scheme: ${FLOWER_AUTH_SCHEME} @@ -129,6 +226,8 @@ objects: name: koku-flower namespace: secrets type: Opaque + +# koku-secret - apiVersion: v1 stringData: django-secret-key: ${DJANGO_SECRET_KEY} @@ -145,6 +244,8 @@ objects: name: koku-secret namespace: secrets type: Opaque + +# koku-daily-secret - apiVersion: v1 stringData: django-secret-key: ${DJANGO_SECRET_KEY} @@ -161,6 +262,8 @@ objects: name: koku-daily-secret namespace: secrets type: Opaque + +# koku-sentry - apiVersion: v1 kind: Secret metadata: @@ -173,6 +276,7 @@ objects: template: koku-secret name: koku-sentry namespace: secrets + # aws s3 secrets - apiVersion: v1 kind: Secret @@ -190,6 +294,8 @@ objects: aws_secret_access_key: ${S3_SECRET} bucket: ${S3_BUCKET} endpoint: ${S3_ENDPOINT} + +# nise-populator-secret - apiVersion: v1 kind: Secret metadata: @@ -278,13 +384,24 @@ objects: # rds-client-ca - apiVersion: v1 data: - rds-client-ca: null + rds-cacert: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM4RENDQWRpZ0F3SUJBZ0lVZjJ3YnY3K1dDd0E4cXV2S05ZOGZLU2hwRHVJd0RRWUpLb1pJaHZjTkFRRUwKQlFBd0ZERVNNQkFHQTFVRUF3d0piRzlqWVd4b2IzTjBNQjRYRFRJeE1ERXlNakUzTlRBeU5Wb1hEVEl4TURJeQpNVEUzTlRBeU5Wb3dGREVTTUJBR0ExVUVBd3dKYkc5allXeG9iM04wTUlJQklqQU5CZ2txaGtpRzl3MEJBUUVGCkFBT0NBUThBTUlJQkNnS0NBUUVBeFV0OWxCZzFsOXBaMzRsV09BbGNnM0lidjE4UU9ZZkRmcmZaOHVlQ3ptK1UKM1o5QmE5R3ZhNzY2WUxwSHhxL0x5QTBIdlh1VjFPWUtNcmUzUlpMbmJ4TGgvcEpjWFhDUDhnUExlaTRXK2dPVQpXTkc5QXl0eG1hdVpHa2RCcjA4L1VHNDE1SFNKU3RkMW9ZZ1RsSTlJandlNWp6RUZZbHhDNDJreG44a0cxL3pWCmRyVkNVTlM2eE9xRTdteTdjQkJQVm5YWXRFcWdkeWZyMExSYnEwNldPTVdaM0ZEVE5Oc1hDUGtxMkZJSjlmRGwKTVFUei9qS0VVbG51WG5nZjFkcThSOXFHYURUeDhFaDQ3Mys3U2ZWdzlrUlJ1OVRoY0ZNT2VqaTAzRnZKV25PSQpBYW5OMUpqcXE2TGwrUzk0NFp0OTRrTHh6d0FkRHg2bU1Ea2NxOHJWNndJREFRQUJvem93T0RBVUJnTlZIUkVFCkRUQUxnZ2xzYjJOaGJHaHZjM1F3Q3dZRFZSMFBCQVFEQWdlQU1CTUdBMVVkSlFRTU1Bb0dDQ3NHQVFVRkJ3TUIKTUEwR0NTcUdTSWIzRFFFQkN3VUFBNElCQVFDK2VlNmZQOUFCN2FZQW82KzZ0eCt6L3hNa1ZtU2FsWDZMWnFDcQpmRnI3blV3MHJtYUNFRGhFM1pzNHVwdGJwUXV6M3diWjhtUTB4dXlBRDI5NjZXc3JidDZFeWljVHNzUmlBUkZpCjFjc0JkM05Cc3ArMDk1dXhLcTU4OFFEQWVGeHRUKzlsYnltOXArcmtSdzZvM3VSdmIwcG1uQi84cERlV3lzZTYKL3FCNEhuSUV1Q3AxZWZCczZvb2YxdEJIcFdjZlgwWXFnTHpVVDF2by9rWHRlM3pnaFRjdGpJYW9ZdmsxdzgyMApUaHpoQVo4Q0lodXhKakorWVdNcklxOWJrMW1aRk01S3BsWUI5QnM1U1hkdmZJdE4vR2Z1R05EZFJGOTl5WGxXCjF4cXg0QVNVZ3hITDJQVk01TGc5U05iV3c2RnhVd215dmlFWWFzcXNvK0RNaEhoRAotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== kind: Secret metadata: name: rds-client-ca namespace: secrets type: Opaque +# elasticache-redis +- apiVersion: v1 + stringData: + db.endpoint: 'koku-redis.hccm.svc.cluster.local' + kind: Secret + metadata: + name: elasticache-redis + namespace: secrets + type: Opaque + + parameters: # database params - displayName: Database Name @@ -333,6 +450,22 @@ parameters: from: '[a-zA-Z0-9]{16}' generate: expression +# hive db params +- displayName: Hive Database Name + name: HIVE_DATABASE_NAME + required: true + value: hive +- displayName: Hive Database Username + from: 'hive_[a-zA-Z0-9]{12}' + generate: expression + name: HIVE_DATABASE_USER + required: true +- displayName: Hive Database Password + from: '[a-zA-Z0-9]{16}' + generate: expression + name: HIVE_DATABASE_PASSWORD + required: true + # application secret keys - description: Set this to a long random string. displayName: Django Secret Key @@ -408,7 +541,6 @@ parameters: name: KOKU_CELERY_SENTRY_DSN required: false - # AWS S3 account creds - displayName: S3 Bucket Access Key name: S3_ACCESS_KEY diff --git a/scripts/show_test_graph.py b/scripts/show_test_graph.py index df61a92279..0bf00b934a 100755 --- a/scripts/show_test_graph.py +++ b/scripts/show_test_graph.py @@ -18,6 +18,8 @@ """draw a graph from cost & forecast data.""" import datetime import json +import sys +from pprint import pformat import matplotlib as mlt import matplotlib.pyplot as plt @@ -35,7 +37,19 @@ def _get_data(url): except HTTPError as err: print("Error: %s" % err) - resp = json.loads(response.content) + try: + resp = json.loads(response.content) + except json.decoder.JSONDecodeError as exc: + print("Error decoding API response: %s" % exc) + + from bs4 import BeautifulSoup + + errdoc = BeautifulSoup(response.content, "html.parser") + print(errdoc.find(id="traceback_area").get_text()) + + # print(response.text) + sys.exit(1) + # print(resp) return resp.get("data") @@ -79,7 +93,13 @@ def _get_values(key_name, data, fields=["total"]): ax.scatter(x, y, label="aws cost") forecast_data = _get_data(FORECAST_URL) -x, y, upper, lower = _get_values("cost", forecast_data, fields=["total", "upper_conf_y", "lower_conf_y"]) + +try: + x, y, upper, lower = _get_values("cost", forecast_data, fields=["total", "upper_conf_y", "lower_conf_y"]) +except ValueError: + print("Error: unexpected API response") + print("Response: %s" % pformat(forecast_data)) + sys.exit(-1) print("Forecast X: %s" % x) print("Forecast Y: %s" % y)