PR - Fargate OSS pattern logs enabled | FluentBit

lib/single-new-eks-awsnative-fargate-observability-construct/index.ts

@@ -39,7 +39,7 @@ export default class SingleNewEksAWSNativeFargateobservabilityConstruct {
         // Define fargate cluster provider and pass the profile options
         const fargateClusterProvider : blueprints.FargateClusterProvider = new blueprints.FargateClusterProvider({
             fargateProfiles,
-            version: eks.KubernetesVersion.of("1.27")
+            version: eks.KubernetesVersion.of("1.28")
         });
 
         const certManagerAddOnProps : blueprints.CertManagerAddOnProps = {
@@ -50,7 +50,7 @@ export default class SingleNewEksAWSNativeFargateobservabilityConstruct {
         };
 
         const coreDnsAddOnProps : blueprints.CoreDnsAddOnProps = {
-            version:"v1.10.1-eksbuild.1",
+            version:"v1.10.1-eksbuild.6",
             configurationValues:{
                 computeType: "Fargate" 
             }
@@ -64,7 +64,6 @@ export default class SingleNewEksAWSNativeFargateobservabilityConstruct {
             .withCertManagerProps(certManagerAddOnProps)
             .withCoreDnsProps(coreDnsAddOnProps)
             .enableFargatePatternAddOns()
-            .enableControlPlaneLogging()
             .clusterProvider(fargateClusterProvider)
             .addOns(...addOns)
             .build(scope, stackId);

lib/single-new-eks-fargate-opensource-observability-pattern/fluentbitconfig.yml

@@ -0,0 +1,42 @@
+kind: Namespace
+apiVersion: v1
+metadata:
+  name: aws-observability
+  labels:
+    aws-observability: enabled
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: aws-logging
+  namespace: aws-observability
+data:
+  flb_log_cw: "true"  # Set to true to ship Fluent Bit process logs to CloudWatch.
+  filters.conf: |
+    [FILTER]
+        Name parser
+        Match *
+        Key_name log
+        Parser crio
+    [FILTER]
+        Name kubernetes
+        Match kube.*
+        Merge_Log On
+        Keep_Log Off
+        Buffer_Size 0
+        Kube_Meta_Cache_TTL 300s
+  output.conf: |
+    [OUTPUT]
+        Name cloudwatch_logs
+        Match   kube.*
+        region us-east-1
+        log_group_name fargate-observability
+        log_stream_prefix from-fluent-bit-
+        auto_create_group true
+  parsers.conf: |
+    [PARSER]
+        Name crio
+        Format Regex
+        Regex ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>P|F) (?<log>.*)$
+        Time_Key    time
+        Time_Format %Y-%m-%dT%H:%M:%S.%L%z

lib/single-new-eks-fargate-opensource-observability-pattern/fluentbitconfigmap.ts

@@ -0,0 +1,23 @@
+import 'source-map-support/register';
+import * as blueprints from '@aws-quickstart/eks-blueprints';
+import * as eks from "aws-cdk-lib/aws-eks";
+import { Construct } from 'constructs';
+import { loadYaml, readYamlDocument } from '@aws-quickstart/eks-blueprints/dist/utils';
+
+export class FluentBitConfigMap implements blueprints.ClusterAddOn {
+    id?: string | undefined;
+    deploy(clusterInfo: blueprints.ClusterInfo): void | Promise<Construct> {
+        const cluster = clusterInfo.cluster;
+
+        const doc = readYamlDocument(__dirname + '/fluentbitconfig.yml');
+        const manifest = doc.split("---").map(e => loadYaml(e));
+
+        const configMap = new eks.KubernetesManifest(cluster.stack, "aws-observability", {
+            cluster,
+            manifest,
+            overwrite: true
+        });
+
+        return Promise.resolve(configMap);
+    }
+}

lib/single-new-eks-fargate-opensource-observability-pattern/index.ts

@@ -6,6 +6,9 @@ import * as amp from 'aws-cdk-lib/aws-aps';
 import { ObservabilityBuilder } from '@aws-quickstart/eks-blueprints';
 import * as eks from 'aws-cdk-lib/aws-eks';
 import * as fs from 'fs';
+import { ManagedPolicy,Role,ServicePrincipal } from "aws-cdk-lib/aws-iam";
+import { FluentBitConfigMap } from './fluentbitconfigmap';
+
 
 export default class SingleNewEksFargateOpenSourceObservabilityConstruct {
     constructor(scope: Construct, id: string) {
@@ -115,7 +118,7 @@ export default class SingleNewEksFargateOpenSourceObservabilityConstruct {
         const addOns: Array<blueprints.ClusterAddOn> = [
             new blueprints.addons.VpcCniAddOn(),
             new blueprints.addons.CoreDnsAddOn({
-                version: "v1.10.1-eksbuild.1",
+                version: "v1.10.1-eksbuild.6",
                 configurationValues: { computeType: "Fargate" }
             }),
             new blueprints.addons.KubeProxyAddOn(),
@@ -128,10 +131,6 @@ export default class SingleNewEksFargateOpenSourceObservabilityConstruct {
             }),
             new blueprints.addons.KubeStateMetricsAddOn(),
             new blueprints.addons.MetricsServerAddOn(),
-            new blueprints.addons.CloudWatchLogsAddon({
-                logGroupPrefix: `/aws/eks/${stackId}`,
-                logRetentionDays: 30
-            }),
             new blueprints.addons.ExternalsSecretsAddOn({
                 namespace: "external-secrets",
                 values: { webhook: { port: 9443 } }
@@ -141,9 +140,17 @@ export default class SingleNewEksFargateOpenSourceObservabilityConstruct {
             new GrafanaOperatorSecretAddon(),
             new blueprints.addons.AdotCollectorAddOn(),
             new blueprints.addons.XrayAdotAddOn(),
-            new blueprints.addons.AmpAddOn(ampAddOnProps)
+            new blueprints.addons.AmpAddOn(ampAddOnProps),
+            new FluentBitConfigMap()
         ];
 
+        const nodeRole = new blueprints.CreateRoleProvider("blueprint-fargate-pod-role", new ServicePrincipal("eks-fargate-pods.amazonaws.com"),
+            [
+                ManagedPolicy.fromAwsManagedPolicyName("AmazonEKSFargatePodExecutionRolePolicy"),
+                ManagedPolicy.fromAwsManagedPolicyName("CloudWatchAgentServerPolicy"),
+            ]);
+
+        const podExecutionRole = blueprints.getNamedResource("blueprint-fargate-pod-role") as Role;
 
         const fargateProfiles: Map<string, eks.FargateProfileOptions> = new Map([
             ["MyProfile", {
@@ -153,22 +160,24 @@ export default class SingleNewEksFargateOpenSourceObservabilityConstruct {
                     { namespace: "external-secrets" },
                     { namespace: "grafana-operator" },
                     { namespace: "flux-system" }
-                ]
-            }]
+                ],  podExecutionRole : podExecutionRole
+            }],
         ]);
 
         // Define fargate cluster provider and pass the profile options
         const fargateClusterProvider: blueprints.FargateClusterProvider = new blueprints.FargateClusterProvider({
             fargateProfiles,
-            version: eks.KubernetesVersion.of("1.27")
+            version: eks.KubernetesVersion.of("1.28"),
         });
 
+
         ObservabilityBuilder.builder()
             .account(account)
             .region(region)
             .clusterProvider(fargateClusterProvider)
+            .resourceProvider("blueprint-fargate-pod-role", nodeRole)
             .resourceProvider(ampWorkspaceName, new blueprints.CreateAmpProvider(ampWorkspaceName, ampWorkspaceName))
             .addOns(...addOns)
             .build(scope, stackId);
     }
-}
+}