PR - Fargate OSS pattern logs enabled | FluentBit

docs/patterns/single-new-eks-observability-accelerators/single-new-eks-fargate-opensource-observability.md

@@ -194,8 +194,10 @@ You should now see a new dashboard named `Java/JMX`, under `Observability Accele
 
 ## Viewing Logs
 
-By default, we deploy a FluentBit daemon set in the cluster to collect worker logs for all namespaces. Logs are collected and exported to Amazon CloudWatch Logs, which enables you to centralize the logs from all of your systems, applications,
-and AWS services that you use, in a single, highly scalable service.
+Amazon EKS on Fargate offers a built-in log router based on Fluent Bit. This means that you don't explicitly run a Fluent Bit container as a sidecar, but Amazon runs it for you. All that you have to do is configure the log router. The configuration happens through a dedicated [`ConfigMap`](https://github.com/aws-observability/cdk-aws-observability-accelerator/tree/main/lib/single-new-eks-fargate-opensource-observability-pattern/fluentbitconfig.yml). Logs are collected and exported to Amazon CloudWatch Logs, which enables you to centralize the logs from all of your systems, applications,
+and AWS services that you use, in a single, highly scalable service. By default, the logs are exported to us-east-1 region but you can modify the `ConfigMap` for your region of choice. At least one supported `OUTPUT` plugin has to be provided in the `ConfigMap` to enable logging. You can also modify the destination from cloudwatch to Cloudwatch (default), Amazon OpenSearch Service or Kinesis Data Firehose. Read more about [EKS Fargate logging](https://docs.aws.amazon.com/eks/latest/userguide/fargate-logging.html).
+
+![fargate-fluentbit](../images/logs-fargate-fluentbit.png)
 
 ## Teardown
 

lib/single-new-eks-awsnative-fargate-observability-construct/index.ts

@@ -39,7 +39,7 @@ export default class SingleNewEksAWSNativeFargateobservabilityConstruct {
         // Define fargate cluster provider and pass the profile options
         const fargateClusterProvider : blueprints.FargateClusterProvider = new blueprints.FargateClusterProvider({
             fargateProfiles,
-            version: eks.KubernetesVersion.of("1.27")
+            version: eks.KubernetesVersion.of("1.28")
         });
 
         const certManagerAddOnProps : blueprints.CertManagerAddOnProps = {
@@ -50,7 +50,7 @@ export default class SingleNewEksAWSNativeFargateobservabilityConstruct {
         };
 
         const coreDnsAddOnProps : blueprints.CoreDnsAddOnProps = {
-            version:"v1.10.1-eksbuild.1",
+            version:"v1.10.1-eksbuild.6",
             configurationValues:{
                 computeType: "Fargate" 
             }
@@ -64,7 +64,6 @@ export default class SingleNewEksAWSNativeFargateobservabilityConstruct {
             .withCertManagerProps(certManagerAddOnProps)
             .withCoreDnsProps(coreDnsAddOnProps)
             .enableFargatePatternAddOns()
-            .enableControlPlaneLogging()
             .clusterProvider(fargateClusterProvider)
             .addOns(...addOns)
             .build(scope, stackId);

lib/single-new-eks-fargate-opensource-observability-pattern/fluentbitconfig.yml

@@ -0,0 +1,42 @@
+kind: Namespace
+apiVersion: v1
+metadata:
+  name: aws-observability
+  labels:
+    aws-observability: enabled
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: aws-logging
+  namespace: aws-observability
+data:
+  flb_log_cw: "true"  # Set to true to ship Fluent Bit process logs to CloudWatch.
+  filters.conf: |
+    [FILTER]
+        Name parser
+        Match *
+        Key_name log
+        Parser crio
+    [FILTER]
+        Name kubernetes
+        Match kube.*
+        Merge_Log On
+        Keep_Log Off
+        Buffer_Size 0
+        Kube_Meta_Cache_TTL 300s
+  output.conf: |
+    [OUTPUT]
+        Name cloudwatch_logs
+        Match   kube.*
+        region us-east-1
+        log_group_name fargate-observability
+        log_stream_prefix from-fluent-bit-
+        auto_create_group true
+  parsers.conf: |
+    [PARSER]
+        Name crio
+        Format Regex
+        Regex ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>P|F) (?<log>.*)$
+        Time_Key    time
+        Time_Format %Y-%m-%dT%H:%M:%S.%L%z

lib/single-new-eks-fargate-opensource-observability-pattern/fluentbitconfigmap.ts

@@ -0,0 +1,23 @@
+import 'source-map-support/register';
+import * as blueprints from '@aws-quickstart/eks-blueprints';
+import * as eks from "aws-cdk-lib/aws-eks";
+import { Construct } from 'constructs';
+import { loadYaml, readYamlDocument } from '@aws-quickstart/eks-blueprints/dist/utils';
+
+export class FluentBitConfigMap implements blueprints.ClusterAddOn {
+    id?: string | undefined;
+    deploy(clusterInfo: blueprints.ClusterInfo): void | Promise<Construct> {
+        const cluster = clusterInfo.cluster;
+
+        const doc = readYamlDocument(__dirname + '/fluentbitconfig.yml');
+        const manifest = doc.split("---").map(e => loadYaml(e));
+
+        const configMap = new eks.KubernetesManifest(cluster.stack, "aws-observability", {
+            cluster,
+            manifest,
+            overwrite: true
+        });
+
+        return Promise.resolve(configMap);
+    }
+}

lib/single-new-eks-fargate-opensource-observability-pattern/index.ts

@@ -6,6 +6,9 @@ import * as amp from 'aws-cdk-lib/aws-aps';
 import { ObservabilityBuilder } from '@aws-quickstart/eks-blueprints';
 import * as eks from 'aws-cdk-lib/aws-eks';
 import * as fs from 'fs';
+import { ManagedPolicy,Role,ServicePrincipal } from "aws-cdk-lib/aws-iam";
+import { FluentBitConfigMap } from './fluentbitconfigmap';
+
 
 export default class SingleNewEksFargateOpenSourceObservabilityConstruct {
     constructor(scope: Construct, id: string) {
@@ -115,7 +118,7 @@ export default class SingleNewEksFargateOpenSourceObservabilityConstruct {
         const addOns: Array<blueprints.ClusterAddOn> = [
             new blueprints.addons.VpcCniAddOn(),
             new blueprints.addons.CoreDnsAddOn({
-                version: "v1.10.1-eksbuild.1",
+                version: "v1.10.1-eksbuild.6",
                 configurationValues: { computeType: "Fargate" }
             }),
             new blueprints.addons.KubeProxyAddOn(),
@@ -128,10 +131,6 @@ export default class SingleNewEksFargateOpenSourceObservabilityConstruct {
             }),
             new blueprints.addons.KubeStateMetricsAddOn(),
             new blueprints.addons.MetricsServerAddOn(),
-            new blueprints.addons.CloudWatchLogsAddon({
-                logGroupPrefix: `/aws/eks/${stackId}`,
-                logRetentionDays: 30
-            }),
             new blueprints.addons.ExternalsSecretsAddOn({
                 namespace: "external-secrets",
                 values: { webhook: { port: 9443 } }
@@ -141,9 +140,17 @@ export default class SingleNewEksFargateOpenSourceObservabilityConstruct {
             new GrafanaOperatorSecretAddon(),
             new blueprints.addons.AdotCollectorAddOn(),
             new blueprints.addons.XrayAdotAddOn(),
-            new blueprints.addons.AmpAddOn(ampAddOnProps)
+            new blueprints.addons.AmpAddOn(ampAddOnProps),
+            new FluentBitConfigMap()
         ];
 
+        const nodeRole = new blueprints.CreateRoleProvider("blueprint-fargate-pod-role", new ServicePrincipal("eks-fargate-pods.amazonaws.com"),
+            [
+                ManagedPolicy.fromAwsManagedPolicyName("AmazonEKSFargatePodExecutionRolePolicy"),
+                ManagedPolicy.fromAwsManagedPolicyName("CloudWatchAgentServerPolicy"),
+            ]);
+
+        const podExecutionRole = blueprints.getNamedResource("blueprint-fargate-pod-role") as Role;
 
         const fargateProfiles: Map<string, eks.FargateProfileOptions> = new Map([
             ["MyProfile", {
@@ -153,22 +160,24 @@ export default class SingleNewEksFargateOpenSourceObservabilityConstruct {
                     { namespace: "external-secrets" },
                     { namespace: "grafana-operator" },
                     { namespace: "flux-system" }
-                ]
-            }]
+                ],  podExecutionRole : podExecutionRole
+            }],
         ]);
 
         // Define fargate cluster provider and pass the profile options
         const fargateClusterProvider: blueprints.FargateClusterProvider = new blueprints.FargateClusterProvider({
             fargateProfiles,
-            version: eks.KubernetesVersion.of("1.27")
+            version: eks.KubernetesVersion.of("1.28"),
         });
 
+
         ObservabilityBuilder.builder()
             .account(account)
             .region(region)
             .clusterProvider(fargateClusterProvider)
+            .resourceProvider("blueprint-fargate-pod-role", nodeRole)
             .resourceProvider(ampWorkspaceName, new blueprints.CreateAmpProvider(ampWorkspaceName, ampWorkspaceName))
             .addOns(...addOns)
             .build(scope, stackId);
     }
-}
+}