diff --git a/artifacts/grafana-dashboards/eks/opensearch/opensearch-logs.json b/artifacts/grafana-dashboards/eks/opensearch/opensearch-logs.json new file mode 100644 index 0000000..bed8370 --- /dev/null +++ b/artifacts/grafana-dashboards/eks/opensearch/opensearch-logs.json @@ -0,0 +1,1107 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 12, + "title": "summary", + "type": "row" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${opensearch_datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 10, + "options": { + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": false, + "values": [ + "value" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "kubernetes.namespace_name.keyword", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + }, + { + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${opensearch_datasource}" + }, + "format": "table", + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "", + "queryType": "lucene", + "refId": "A", + "timeField": "@timestamp" + } + ], + "title": "Log Count per Namespace", + "type": "piechart" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${opensearch_datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "log" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 4, + "y": 1 + }, + "id": 5, + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "kubernetes.namespace_name.keyword", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + }, + { + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${opensearch_datasource}" + }, + "format": "table", + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "", + "queryType": "lucene", + "refId": "A", + "timeField": "@timestamp" + } + ], + "title": "Log Count per Namespace", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${opensearch_datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 48, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 8, + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "kubernetes.pod_name.keyword", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + }, + { + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${opensearch_datasource}" + }, + "format": "table", + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "kubernetes.namespace_name:\"$namespace\"", + "queryType": "lucene", + "refId": "A", + "timeField": "@timestamp" + } + ], + "title": "Log Count per Pod", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${opensearch_datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 9, + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "kubernetes.container_name.keyword", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + }, + { + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${opensearch_datasource}" + }, + "format": "table", + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "kubernetes.namespace_name:\"$namespace\" AND kubernetes.pod_name:\"$pod\"", + "queryType": "lucene", + "refId": "A", + "timeField": "@timestamp" + } + ], + "title": "Log Count per Container", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 6, + "panels": [], + "title": "$namespace / $pod / $container", + "type": "row" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${opensearch_datasource}" + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 4, + "options": { + "dedupStrategy": "numbers", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${opensearch_datasource}" + }, + "format": "table", + "metrics": [ + { + "id": "1", + "type": "logs" + } + ], + "query": "kubernetes.namespace_name:\"$namespace\"\nAND kubernetes.pod_name:\"$pod\"\nAND kubernetes.container_name:\"$container\"", + "queryType": "lucene", + "refId": "A", + "timeField": "@timestamp" + } + ], + "title": "Logs", + "type": "logs" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${opensearch_datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "Count" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Count" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 11, + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${opensearch_datasource}" + }, + "format": "table", + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "kubernetes.namespace_name:\"$namespace\"\nAND kubernetes.pod_name:\"$pod\"\nAND kubernetes.container_name:\"$container\"", + "queryType": "lucene", + "refId": "A", + "timeField": "@timestamp" + } + ], + "title": "Log Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${amp_datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "CPU usage" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 13 + }, + "id": 1, + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${amp_datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=\"$pod\", container=\"$container\"}[5m])) by (container)", + "format": "table", + "instant": false, + "legendFormat": "{{container}}", + "range": true, + "refId": "A" + } + ], + "title": "CPU Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${amp_datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Memory Usage" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "sum(container_memory_usage_bytes{namespace=\"default\", pod=\"my-otel-demo-productcatalogservice-7c476d6cfc-w8qrs\", container=\"productcatalogservice\"})" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 13 + }, + "id": 2, + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${amp_datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(container) (container_memory_usage_bytes{namespace=\"$namespace\", pod=\"$pod\", container=\"$container\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{container}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Memory Usage", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false + }, + "hide": 0, + "includeAll": false, + "label": "AMP", + "multi": false, + "name": "amp_datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "Amazon OpenSearch", + "value": "bdu7z6cnnfitcf" + }, + "hide": 0, + "includeAll": false, + "label": "OpenSearch", + "multi": false, + "name": "opensearch_datasource", + "options": [], + "query": "grafana-opensearch-datasource", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${amp_datasource}" + }, + "definition": "label_values(namespace)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "namespace", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(namespace)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${amp_datasource}" + }, + "definition": "label_values({namespace=~\"$namespace\"},pod)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "pod", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({namespace=~\"$namespace\"},pod)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${amp_datasource}" + }, + "definition": "label_values({namespace=~\"$namespace\", pod=~\"$pod\"},container)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "container", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({namespace=~\"$namespace\", pod=~\"$pod\"},container)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Kubernetes / Logs and Metrics", + "uid": "ae4f3fayxjvnkf", + "version": 6, + "weekStart": "" +} \ No newline at end of file diff --git a/artifacts/grafana-operator-manifests/eks/opensearch/amg_grafana-dashboards.yaml b/artifacts/grafana-operator-manifests/eks/opensearch/amg_grafana-dashboards.yaml new file mode 100644 index 0000000..4d1373a --- /dev/null +++ b/artifacts/grafana-operator-manifests/eks/opensearch/amg_grafana-dashboards.yaml @@ -0,0 +1,12 @@ +apiVersion: grafana.integreatly.org/v1beta1 +kind: GrafanaDashboard +metadata: + name: opensearch-logs-grafanadashboard + namespace: grafana-operator +spec: + folder: "Observability Accelerator Dashboards" + instanceSelector: + matchLabels: + dashboards: "external-grafana" + url: ${GRAFANA_OPENSEARCH_LOGS_DASH_URL} + diff --git a/artifacts/grafana-operator-manifests/eks/opensearch/amg_grafana-opensearch-datasource.yaml b/artifacts/grafana-operator-manifests/eks/opensearch/amg_grafana-opensearch-datasource.yaml new file mode 100644 index 0000000..2e07c52 --- /dev/null +++ b/artifacts/grafana-operator-manifests/eks/opensearch/amg_grafana-opensearch-datasource.yaml @@ -0,0 +1,29 @@ +apiVersion: grafana.integreatly.org/v1beta1 +kind: GrafanaDatasource +metadata: + name: grafanadatasource-opensearch + namespace: grafana-operator +spec: + instanceSelector: + matchLabels: + dashboards: "external-grafana" + datasource: + name: aws-observability-accelerator-opensearch + type: grafana-opensearch-datasource + access: proxy + url: ${OPENSEARCH_DOMAIN_URL} + isDefault: false + jsonData: + 'flavor': "opensearch" + 'logLevelField': "severityText" + 'logMessageField': "log" + 'maxConcurrentShardRequests': "20" + 'pplEnabled': true + 'serverless': false + 'sigV4Auth': true + 'sigV4AuthType': "ec2_iam_role" + 'sigV4Region': ${AMG_AWS_REGION} + 'timeField': "@timestamp" + 'version': "2.15.0" + 'versionLabel': "OpenSearch 2.15.0" + editable: true diff --git a/artifacts/grafana-operator-manifests/eks/opensearch/kustomization.yaml b/artifacts/grafana-operator-manifests/eks/opensearch/kustomization.yaml new file mode 100644 index 0000000..0d22ede --- /dev/null +++ b/artifacts/grafana-operator-manifests/eks/opensearch/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - amg_grafana-opensearch-datasource.yaml + - amg_grafana-dashboards.yaml