OSS pattern for EKS Fargate new cluster

bin/single-new-eks-fargate-opensource-observability.ts

@@ -0,0 +1,5 @@
+import SingleNewEksFargateOpenSourceObservabilityConstruct from '../lib/single-new-eks-fargate-opensource-observability-pattern';
+import { configureApp } from '../lib/common/construct-utils';
+
+const app = configureApp();
+new SingleNewEksFargateOpenSourceObservabilityConstruct(app, 'single-new-eks-fargate-opensource');

lib/common/resources/otel-collector-config-new.yml

@@ -9,7 +9,14 @@ metadata:
   namespace: "{{namespace}}"
 spec:
   mode: "{{deploymentMode}}"
-  image: public.ecr.aws/aws-observability/aws-otel-collector:v0.21.0
+  image: public.ecr.aws/aws-observability/aws-otel-collector:v0.33.1
+  resources:
+    limits:
+      cpu: "1"
+      memory: "2Gi"
+    requests:
+      cpu: "1"
+      memory: "2Gi"
   serviceAccount: adot-collector
   config: |
     receivers:
@@ -55,7 +62,31 @@ spec:
                 regex: (.+)
                 target_label: __metrics_path__
                 replacement: /api/v1/nodes/$${1}/proxy/metrics/cadvisor
-
+            - job_name: 'apiserver'
+              scheme: https
+              tls_config:
+                ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+                insecure_skip_verify: true
+              bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+              kubernetes_sd_configs:
+              - role: endpoints
+              relabel_configs:
+              - source_labels:
+                  [
+                    __meta_kubernetes_namespace,
+                    __meta_kubernetes_service_name,
+                    __meta_kubernetes_endpoint_port_name,
+                  ]
+                action: keep
+                regex: default;kubernetes;https
+              metric_relabel_configs:
+              - action: keep
+                source_labels: [__name__]
+              - source_labels: [__name__, le]
+                separator: ;
+                regex: apiserver_request_duration_seconds_bucket;(0.15|0.2|0.3|0.35|0.4|0.45|0.6|0.7|0.8|0.9|1.25|1.5|1.75|2|3|3.5|4|4.5|6|7|8|9|15|25|40|50)
+                replacement: $1
+                action: drop
             - job_name: serviceMonitor/default/kube-prometheus-stack-prometheus-node-exporter/0
               honor_timestamps: true
               scrape_interval: 30s

lib/common/resources/otel-collector-config.yml

@@ -9,7 +9,14 @@ metadata:
   namespace: "{{namespace}}"
 spec:
   mode: "{{deploymentMode}}"
-  image: public.ecr.aws/aws-observability/aws-otel-collector:v0.21.0
+  image: public.ecr.aws/aws-observability/aws-otel-collector:v0.33.1
+  resources:
+    limits:
+      cpu: "1"
+      memory: "2Gi"
+    requests:
+      cpu: "1"
+      memory: "2Gi"
   serviceAccount: adot-collector
   config: |
     receivers:

lib/single-new-eks-fargate-opensource-observability-pattern/index.ts

@@ -0,0 +1,144 @@
+import { Construct } from 'constructs';
+import { utils } from '@aws-quickstart/eks-blueprints';
+import * as blueprints from '@aws-quickstart/eks-blueprints';
+import { GrafanaOperatorSecretAddon } from '../single-new-eks-opensource-observability-pattern/grafanaoperatorsecretaddon';
+import * as amp from 'aws-cdk-lib/aws-aps';
+import { ObservabilityBuilder } from '@aws-quickstart/eks-blueprints';
+import * as eks from 'aws-cdk-lib/aws-eks';
+import * as fs from 'fs';
+
+export default class SingleNewEksFargateOpenSourceObservabilityConstruct {
+    constructor(scope: Construct, id: string) {
+        const stackId = `${id}-observability-accelerator`;
+
+        const account = process.env.COA_ACCOUNT_ID! || process.env.CDK_DEFAULT_ACCOUNT!;
+        const region = process.env.COA_AWS_REGION! || process.env.CDK_DEFAULT_REGION!;
+        const ampWorkspaceName = process.env.COA_AMP_WORKSPACE_NAME! || 'observability-amp-Workspace';
+        const ampWorkspace = blueprints.getNamedResource(ampWorkspaceName) as unknown as amp.CfnWorkspace;
+        const ampEndpoint = ampWorkspace.attrPrometheusEndpoint;
+        const ampWorkspaceArn = ampWorkspace.attrArn;
+
+        const amgEndpointUrl = process.env.COA_AMG_ENDPOINT_URL;
+        // All Grafana Dashboard URLs from `cdk.json`
+        const fluxRepository: blueprints.FluxGitRepo = utils.valueFromContext(scope, "fluxRepository", undefined);
+        fluxRepository.values!.AMG_AWS_REGION = region;
+        fluxRepository.values!.AMP_ENDPOINT_URL = ampEndpoint;
+        fluxRepository.values!.AMG_ENDPOINT_URL = amgEndpointUrl;
+
+        const ampAddOnProps: blueprints.AmpAddOnProps = {
+            ampPrometheusEndpoint: ampEndpoint,
+            ampRules: {
+                ampWorkspaceArn: ampWorkspaceArn,
+                ruleFilePaths: [
+                    __dirname + '/../common/resources/amp-config/alerting-rules.yml',
+                    __dirname + '/../common/resources/amp-config/recording-rules.yml'
+                ]
+            }
+        };
+
+        let doc = utils.readYamlDocument(__dirname + '/../common/resources/otel-collector-config.yml');
+        doc = utils.changeTextBetweenTokens(
+            doc,
+            "{{ if enableAPIserverJob }}",
+            "{{ end }}",
+            true
+        );
+        console.log(doc);
+        fs.writeFileSync(__dirname + '/../common/resources/otel-collector-config-new.yml', doc);
+
+
+        if (utils.valueFromContext(scope, "java.pattern.enabled", false)) {
+            ampAddOnProps.openTelemetryCollector = {
+                manifestPath: __dirname + '/../common/resources/otel-collector-config-new.yml',
+                manifestParameterMap: {
+                    javaScrapeSampleLimit: 1000,
+                    javaPrometheusMetricsEndpoint: "/metrics"
+                }
+            };
+            ampAddOnProps.ampRules?.ruleFilePaths.push(
+                __dirname + '/../common/resources/amp-config/java/alerting-rules.yml',
+                __dirname + '/../common/resources/amp-config/java/recording-rules.yml'
+            );
+        }
+
+        if (utils.valueFromContext(scope, "apiserver.pattern.enabled", false)) {
+            ampAddOnProps.enableAPIServerJob = true,
+            ampAddOnProps.ampRules?.ruleFilePaths.push(
+                __dirname + '/../common/resources/amp-config/apiserver/recording-rules.yml'
+            );
+        }
+
+        if (utils.valueFromContext(scope, "nginx.pattern.enabled", false)) {
+            ampAddOnProps.openTelemetryCollector = {
+                manifestPath: __dirname + '/../common/resources/otel-collector-config-new.yml',
+                manifestParameterMap: {
+                    javaScrapeSampleLimit: 1000,
+                    javaPrometheusMetricsEndpoint: "/metrics"
+                }
+            };
+            ampAddOnProps.ampRules?.ruleFilePaths.push(
+                __dirname + '/../common/resources/amp-config/nginx/alerting-rules.yml'
+            );
+        }
+
+        Reflect.defineMetadata("ordered", true, blueprints.addons.GrafanaOperatorAddon);
+        const addOns: Array<blueprints.ClusterAddOn> = [
+            new blueprints.addons.VpcCniAddOn(),
+            new blueprints.addons.CoreDnsAddOn({
+                version: "v1.10.1-eksbuild.1",
+                configurationValues: { computeType: "Fargate" }
+            }),
+            new blueprints.addons.KubeProxyAddOn(),
+            new blueprints.addons.AwsLoadBalancerControllerAddOn(),
+            new blueprints.addons.CertManagerAddOn({
+                installCRDs: true,
+                createNamespace: true,
+                namespace: "cert-manager",
+                values: { webhook: { securePort: 10260 } }
+            }),
+            new blueprints.addons.KubeStateMetricsAddOn(),
+            new blueprints.addons.MetricsServerAddOn(),
+            new blueprints.addons.CloudWatchLogsAddon({
+                logGroupPrefix: `/aws/eks/${stackId}`,
+                logRetentionDays: 30
+            }),
+            new blueprints.addons.ExternalsSecretsAddOn({
+                namespace: "external-secrets",
+                values: { webhook: { port: 9443 } }
+            }),
+            new blueprints.addons.GrafanaOperatorAddon(),
+            new blueprints.addons.FluxCDAddOn({"repositories": [fluxRepository]}),
+            new GrafanaOperatorSecretAddon(),
+            new blueprints.addons.AdotCollectorAddOn(),
+            new blueprints.addons.XrayAdotAddOn(),
+            new blueprints.addons.AmpAddOn(ampAddOnProps)
+        ];
+
+
+        const fargateProfiles: Map<string, eks.FargateProfileOptions> = new Map([
+            ["MyProfile", {
+                selectors: [
+                    { namespace: "cert-manager" },
+                    { namespace: "opentelemetry-operator-system" },
+                    { namespace: "external-secrets" },
+                    { namespace: "grafana-operator" },
+                    { namespace: "flux-system" }
+                ]
+            }]
+        ]);
+
+        // Define fargate cluster provider and pass the profile options
+        const fargateClusterProvider: blueprints.FargateClusterProvider = new blueprints.FargateClusterProvider({
+            fargateProfiles,
+            version: eks.KubernetesVersion.of("1.27")
+        });
+
+        ObservabilityBuilder.builder()
+            .account(account)
+            .region(region)
+            .clusterProvider(fargateClusterProvider)
+            .resourceProvider(ampWorkspaceName, new blueprints.CreateAmpProvider(ampWorkspaceName, ampWorkspaceName))
+            .addOns(...addOns)
+            .build(scope, stackId);
+    }
+}

mkdocs.yml

@@ -34,7 +34,9 @@ nav:
     - New Cluster:
       - AWS Native : patterns/single-new-eks-observability-accelerators/single-new-eks-awsnative-observability.md
       - Graviton OSS : patterns/single-new-eks-observability-accelerators/single-new-eks-graviton-opensource-observability.md
-      - Fargate AWS Native : patterns/single-new-eks-observability-accelerators/single-new-eks-awsnative-fargate-observability.md
+      - EKS Fargate :
+        - AWS Native : patterns/single-new-eks-observability-accelerators/single-new-eks-awsnative-fargate-observability.md
+        - OSS: patterns/single-new-eks-observability-accelerators/single-new-eks-fargate-opensource-observability.md 
       - Mixed : patterns/single-new-eks-observability-accelerators/single-new-eks-mixed-observability.md
       - Multi-Cluster Multi-Region Mon : patterns/multi-new-eks-observability-accelerators/multi-acc-new-eks-mixed-observability.md
       - OSS : patterns/single-new-eks-observability-accelerators/single-new-eks-opensource-observability.md