Merge pull request #1 from Howlla/fargateOSS

Fargate OSS blog logs from ADOT
Howlla · Jan 31, 2024 · d74aa38 · d74aa38
2 parents dc2ba96 + e26039c
commit d74aa38
Show file tree

Hide file tree

Showing 5 changed files with 263 additions and 29 deletions.
diff --git a/cdk.json b/cdk.json
@@ -30,24 +30,31 @@
         "GRAFANA_CLUSTER_DASH_URL" : "https://raw.githubusercontent.com/aws-observability/aws-observability-accelerator/main/artifacts/grafana-dashboards/eks/infrastructure/cluster.json",
         "GRAFANA_KUBELET_DASH_URL" : "https://raw.githubusercontent.com/aws-observability/aws-observability-accelerator/main/artifacts/grafana-dashboards/eks/infrastructure/kubelet.json",
         "GRAFANA_NSWRKLDS_DASH_URL" : "https://raw.githubusercontent.com/aws-observability/aws-observability-accelerator/main/artifacts/grafana-dashboards/eks/infrastructure/namespace-workloads.json",
-        "GRAFANA_NODEEXP_DASH_URL" : "https://raw.githubusercontent.com/aws-observability/aws-observability-accelerator/main/artifacts/grafana-dashboards/eks/infrastructure/nodeexporter-nodes.json",
         "GRAFANA_NODES_DASH_URL" : "https://raw.githubusercontent.com/aws-observability/aws-observability-accelerator/main/artifacts/grafana-dashboards/eks/infrastructure/nodes.json",
-        "GRAFANA_WORKLOADS_DASH_URL" : "https://raw.githubusercontent.com/aws-observability/aws-observability-accelerator/main/artifacts/grafana-dashboards/eks/infrastructure/workloads.json"
+        "GRAFANA_WORKLOADS_DASH_URL" : "https://raw.githubusercontent.com/aws-observability/aws-observability-accelerator/main/artifacts/grafana-dashboards/eks/infrastructure/workloads.json",
+        "GRAFANA_NGINX_DASH_URL" : "https://raw.githubusercontent.com/aws-observability/aws-observability-accelerator/main/artifacts/grafana-dashboards/eks/nginx/nginx.json",
+        "GRAFANA_JAVA_JMX_DASH_URL" : "https://raw.githubusercontent.com/aws-observability/aws-observability-accelerator/main/artifacts/grafana-dashboards/eks/java/default.json",
+        "GRAFANA_APISERVER_BASIC_DASH_URL" : "https://raw.githubusercontent.com/aws-observability/aws-observability-accelerator/main/artifacts/grafana-dashboards/eks/apiserver/apiserver-basic.json"
       },
       "kustomizations": [
         {
           "kustomizationPath": "./artifacts/grafana-operator-manifests/eks/infrastructure"
+        },
+        {
+          "kustomizationPath": "./artifacts/grafana-operator-manifests/eks/nginx"
+        },
+        {
+          "kustomizationPath": "./artifacts/grafana-operator-manifests/eks/java"
+        },
+        {
+          "kustomizationPath": "./artifacts/grafana-operator-manifests/eks/apiserver"
         }
       ]
     },
-    "gpuNodeGroup": {
-      "instanceType": "g4dn.xlarge",
-      "desiredSize": 2, 
-      "minSize": 2, 
-      "maxSize": 3,
-      "ebsSize": 50
-    },
-    "existing.cluster.name": "single-new-eks-observability-accelerator",
-    "existing.kubectl.rolename": "YOUR_KUBECTL_ROLE"
+    "nginx.pattern.enabled": true,
+    "java.pattern.enabled": true,
+    "apiserver.pattern.enabled": true,
+    "adotcontainerlogs.pattern.enabled": true,
+    "adotcollectormetrics.pattern.enabled": true
   }
 }
diff --git a/lib/common/resources/otel-collector-config.yml b/lib/common/resources/otel-collector-config.yml
@@ -68,7 +68,7 @@ spec:
                 regex: (.+)
                 target_label: __metrics_path__
                 replacement: /api/v1/nodes/$${1}/proxy/metrics/cadvisor
-            {{ if enableAPIserverJob }}  
+            {{ start enableAPIserverJob }}  
             - job_name: 'apiserver'
               scheme: https
               tls_config:
@@ -94,7 +94,7 @@ spec:
                 regex: apiserver_request_duration_seconds_bucket;(0.15|0.2|0.3|0.35|0.4|0.45|0.6|0.7|0.8|0.9|1.25|1.5|1.75|2|3|3.5|4|4.5|6|7|8|9|15|25|40|50)
                 replacement: $1
                 action: drop
-            {{ end }}
+            {{ stop enableAPIserverJob }}
             - job_name: serviceMonitor/default/kube-prometheus-stack-prometheus-node-exporter/0
               honor_timestamps: true
               scrape_interval: 30s
@@ -1607,6 +1607,15 @@ spec:
             - job_name: 'node-exporter'
               kubernetes_sd_configs:
                 - role: endpoints
+              ec2_sd_configs:
+              relabel_configs:
+                - source_labels: [ __address__ ]
+                  action: keep
+                  regex: '.*:9100$'
+                - action: replace
+                  source_labels: [__meta_kubernetes_endpoint_node_name]
+                  target_label: nodename
+            {{ start enableJavaMonJob }}
             - job_name: 'kubernetes-java-jmx'
               sample_limit: {{javaScrapeSampleLimit}}
               metrics_path: {{javaPrometheusMetricsEndpoint}}
@@ -1637,13 +1646,180 @@ spec:
                 - source_labels: [ __name__ ]
                   regex: 'jvm_gc_collection_seconds.*'
                   action: drop
+            {{ stop enableJavaMonJob }}
+
+            {{ start enableNginxMonJob }}
+            - job_name: 'kubernetes-nginx'
+              sample_limit: {{nginxScrapeSampleLimit}}
+              metrics_path: {{nginxPrometheusMetricsEndpoint}}
+              kubernetes_sd_configs:
+                - role: pod
+              relabel_configs:
+                - source_labels: [ __address__ ]
+                  action: keep
+                  regex: '.*:10254$'
+                - source_labels: [__meta_kubernetes_pod_container_name]
+                  target_label: container
+                  action: replace
+                - source_labels: [__meta_kubernetes_pod_node_name]
+                  target_label: host
+                  action: replace
+                - source_labels: [__meta_kubernetes_namespace]
+                  target_label: namespace
+                  action: replace
+              metric_relabel_configs:
+                - source_labels: [__name__]
+                  regex: 'go_memstats.*'
+                  action: drop
+                - source_labels: [__name__]
+                  regex: 'go_gc.*'
+                  action: drop
+                - source_labels: [__name__]
+                  regex: 'go_threads'
+                  action: drop
+                - regex: exported_host
+                  action: labeldrop
+            {{ stop enableNginxMonJob }}
+
+         {{ start enableIstioMonJob }}
+            - honor_labels: true
+              job_name: kubernetes-istio
+              kubernetes_sd_configs:
+              - role: pod
+              relabel_configs:
+              - action: keep
+                regex: true
+                source_labels:
+                - __meta_kubernetes_pod_annotation_prometheus_io_scrape
+              - action: drop
+                regex: true
+                source_labels:
+                - __meta_kubernetes_pod_annotation_prometheus_io_scrape_slow
+              - action: replace
+                regex: (https?)
+                source_labels:
+                - __meta_kubernetes_pod_annotation_prometheus_io_scheme
+                target_label: __scheme__
+              - action: replace
+                regex: (.+)
+                source_labels:
+                - __meta_kubernetes_pod_annotation_prometheus_io_path
+                target_label: __metrics_path__
+              - action: replace
+                regex: (\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4})
+                replacement: '[$$2]:$$1'
+                source_labels:
+                - __meta_kubernetes_pod_annotation_prometheus_io_port
+                - __meta_kubernetes_pod_ip
+                target_label: __address__
+              - action: replace
+                regex: (\d+);((([0-9]+?)(\.|$)){4})
+                replacement: $$2:$$1
+                source_labels:
+                - __meta_kubernetes_pod_annotation_prometheus_io_port
+                - __meta_kubernetes_pod_ip
+                target_label: __address__
+              - action: labelmap
+                regex: __meta_kubernetes_pod_annotation_prometheus_io_param_(.+)
+                replacement: __param_$1
+              - action: labelmap
+                regex: __meta_kubernetes_pod_label_(.+)
+              - action: replace
+                source_labels:
+                - __meta_kubernetes_namespace
+                target_label: namespace
+              - action: replace
+                source_labels:
+                - __meta_kubernetes_pod_name
+                target_label: pod
+              - action: keep
+                source_labels: [ __address__ ]
+                regex: '.*:15020$$'
+              - action: drop
+                regex: Pending|Succeeded|Failed|Completed
+                source_labels:
+                - __meta_kubernetes_pod_phase
+            {{ stop enableIstioMonJob }}
+      {{ start enableAdotContainerLogsReceiver }}
+      filelog:
+        include: 
+        - /var/log/pods/*/*/*.log 
+        include_file_name: false
+        include_file_path: true
+        operators:
+        - id: get-format
+          routes:
+          - expr: body matches "^\\{"
+            output: parser-docker
+          - expr: body matches "^[^ Z]+ "
+            output: parser-crio
+          - expr: body matches "^[^ Z]+Z"
+            output: parser-containerd
+          type: router
+        - id: parser-crio
+          output: extract_metadata_from_filepath
+          regex: ^(?P<time>[^ Z]+) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*) ?(?P<log>.*)$
+          timestamp:
+            layout: 2006-01-02T15:04:05.999999999Z07:00
+            layout_type: gotime
+            parse_from: attributes.time
+          type: regex_parser
+        - id: parser-containerd    
+          output: extract_metadata_from_filepath
+          regex: ^(?P<time>[^ ^Z]+Z) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*) ?(?P<log>.*)$
+          timestamp:
+            layout: '%Y-%m-%dT%H:%M:%S.%LZ'
+            parse_from: attributes.time
+          type: regex_parser
+        - id: parser-docker
+          output: extract_metadata_from_filepath
+          timestamp:
+            layout: '%Y-%m-%dT%H:%M:%S.%LZ'
+            parse_from: attributes.time
+          type: json_parser
+        - id: extract_metadata_from_filepath
+          parse_from: attributes["log.file.path"]
+          regex: ^.*\/(?P<namespace>[^_]+)_(?P<pod_name>[^_]+)_(?P<uid>[a-f0-9\-]+)\/(?P<container_name>[^\._]+)\/(?P<restart_count>\d+)\.log$
+          type: regex_parser
+        - from: attributes.stream
+          to: attributes["log.iostream"]
+          type: move
+        - from: attributes.container_name
+          to: resource["k8s.container.name"]
+          type: move
+        - from: attributes.namespace
+          to: resource["k8s.namespace.name"]
+          type: move
+        - from: attributes.pod_name
+          to: resource["k8s.pod.name"]
+          type: move
+        - from: attributes.restart_count
+          to: resource["k8s.container.restart_count"]
+          type: move
+        - from: attributes.uid
+          to: resource["k8s.pod.uid"]
+          type: move
+        - from: attributes.log
+          to: body
+          type: move
+        start_at: beginning
+      {{ stop enableAdotContainerLogsReceiver }}
     exporters:
       prometheusremotewrite:
         endpoint: "{{remoteWriteEndpoint}}"
         auth:
           authenticator: sigv4auth
       logging:
         loglevel: info
+      {{ start enableAdotContainerLogsExporter }}
+      awscloudwatchlogs:
+        log_group_name: "{{logGroupName}}"
+        log_stream_name: "{{logStreamName}}"
+        region: "{{awsRegion}}"
+        endpoint: "logs.{{awsRegion}}.amazonaws.com"
+        log_retention: {{logRetentionDays}}
+        raw_log: false
+      {{ stop enableAdotContainerLogsExporter }}
     extensions:
       sigv4auth:
         region: "{{awsRegion}}"
@@ -1665,3 +1841,4 @@ spec:
           address: 0.0.0.0:8888
           level: basic
       {{ stop enableAdotMetricsCollectionTelemetry }}
+      
diff --git a/lib/single-new-eks-awsnative-fargate-observability-construct/index.ts b/lib/single-new-eks-awsnative-fargate-observability-construct/index.ts
@@ -50,7 +50,7 @@ export default class SingleNewEksAWSNativeFargateobservabilityConstruct {
         };
 
         const coreDnsAddOnProps : blueprints.CoreDnsAddOnProps = {
-            version:"auto",
+            version:"v1.10.1-eksbuild.6",
             configurationValues:{
                 computeType: "Fargate" 
             }
@@ -64,7 +64,6 @@ export default class SingleNewEksAWSNativeFargateobservabilityConstruct {
             .withCertManagerProps(certManagerAddOnProps)
             .withCoreDnsProps(coreDnsAddOnProps)
             .enableFargatePatternAddOns()
-            .enableControlPlaneLogging()
             .clusterProvider(fargateClusterProvider)
             .addOns(...addOns)
             .build(scope, stackId);

diff --git a/lib/single-new-eks-awsnative-observability-pattern/index.ts b/lib/single-new-eks-awsnative-observability-pattern/index.ts
@@ -18,7 +18,6 @@ export default class SingleNewEksClusterAWSNativeobservabilityPattern {
             .region(region)
             .version('auto')
             .enableNativePatternAddOns()
-            .enableControlPlaneLogging()
             .addOns(...addOns)
             .build(scope, stackId);
     }