parameterised fluent bit config for fargate oss pattern

lib/single-new-eks-fargate-opensource-observability-pattern/fluentbitconfig.yml → lib/common/resources/fluent-bit/fluent-bit-fargate-config.ytpl

@@ -11,7 +11,7 @@ metadata:
   name: aws-logging
   namespace: aws-observability
 data:
-  flb_log_cw: "true"  # Set to true to ship Fluent Bit process logs to CloudWatch.
+  flb_log_cw: "{{enableFlbProcessLogs}}"  # Set to true to ship Fluent Bit process logs to CloudWatch.
   filters.conf: |
     [FILTER]
         Name parser
@@ -29,9 +29,9 @@ data:
     [OUTPUT]
         Name cloudwatch_logs
         Match   kube.*
-        region us-east-1
-        log_group_name fargate-observability
-        log_stream_prefix from-fluent-bit-
+        region {{awsRegion}}
+        log_group_name {{logGroupName}}
+        log_stream_prefix {{log_stream_prefix}}
         auto_create_group true
   parsers.conf: |
     [PARSER]

lib/single-new-eks-fargate-opensource-observability-pattern/fluentbitconfigmap.ts

@@ -1,23 +1,79 @@
 import 'source-map-support/register';
 import * as blueprints from '@aws-quickstart/eks-blueprints';
-import * as eks from "aws-cdk-lib/aws-eks";
-import { Construct } from 'constructs';
+import { KubectlProvider, ManifestDeployment } from "@aws-quickstart/eks-blueprints/dist/addons/helm-addon/kubectl-provider";
 import { loadYaml, readYamlDocument } from '@aws-quickstart/eks-blueprints/dist/utils';
 
+/**
+ * Configuration options for the fluentbit configmap
+ */
+export interface FluentBitConfigMapProps {
+
+    /**
+     * Region to send cloudwatch logs.
+     */
+    awsRegion: string;
+
+    /**
+     * Log Group Name in cloudwatch
+     */
+    logGroupName: string
+
+    /**
+     * Prefix for logs stream
+     */
+    log_stream_prefix: string;
+
+    /**
+     * Enable logs from fluentBit process
+     */
+    enableFlbProcessLogs?: boolean
+}
+
+/**
+ * Default props for the add-on.
+ */
+const defaultProps: FluentBitConfigMapProps = {
+    awsRegion: "us-east-1",
+    logGroupName: "fargate-observability",
+    log_stream_prefix: "from-fluent-bit-",
+    enableFlbProcessLogs: false
+};
+
+/**
+ * Creates 'aws-observability' namespace and configurable ConfigMap
+ * to enable the Fargate built-in log router based on Fluent Bit 
+ * https://docs.aws.amazon.com/eks/latest/userguide/fargate-logging.html
+ */
 export class FluentBitConfigMap implements blueprints.ClusterAddOn {
     id?: string | undefined;
-    deploy(clusterInfo: blueprints.ClusterInfo): void | Promise<Construct> {
-        const cluster = clusterInfo.cluster;
+    readonly props: FluentBitConfigMapProps;
 
-        const doc = readYamlDocument(__dirname + '/fluentbitconfig.yml');
+    constructor(props?: FluentBitConfigMapProps) {
+        this.props = { ...defaultProps, ...props };
+    }
+
+    deploy(clusterInfo: blueprints.ClusterInfo): void {
+        const cluster = clusterInfo.cluster;
+
+        const doc = readYamlDocument(__dirname + '/../common/resources/fluent-bit/fluent-bit-fargate-config.ytpl');
         const manifest = doc.split("---").map(e => loadYaml(e));
 
-        const configMap = new eks.KubernetesManifest(cluster.stack, "aws-observability", {
-            cluster,
+        const values: blueprints.Values = {
+            awsRegion: this.props.awsRegion,
+            logGroupName: this.props.logGroupName,
+            log_stream_prefix: this.props.log_stream_prefix,
+            enableFlbProcessLogs: this.props.enableFlbProcessLogs,
+        };
+
+        const manifestDeployment: ManifestDeployment = {
+            name: 'aws-logging',
+            namespace: 'aws-observability',
             manifest,
-            overwrite: true
-        });
-
-        return Promise.resolve(configMap);
+            values
+        };
+
+        const kubectlProvider = new KubectlProvider(clusterInfo);
+        kubectlProvider.addManifest(manifestDeployment);
+
     }
 }

lib/single-new-eks-fargate-opensource-observability-pattern/index.ts

@@ -2,13 +2,12 @@ import { Construct } from 'constructs';
 import { utils } from '@aws-quickstart/eks-blueprints';
 import * as blueprints from '@aws-quickstart/eks-blueprints';
 import { GrafanaOperatorSecretAddon } from '../single-new-eks-opensource-observability-pattern/grafanaoperatorsecretaddon';
+import { FluentBitConfigMap, FluentBitConfigMapProps } from './fluentbitconfigmap';
 import * as amp from 'aws-cdk-lib/aws-aps';
 import { ObservabilityBuilder } from '@aws-quickstart/eks-blueprints';
 import * as eks from 'aws-cdk-lib/aws-eks';
 import * as fs from 'fs';
 import { ManagedPolicy,Role,ServicePrincipal } from "aws-cdk-lib/aws-iam";
-import { FluentBitConfigMap } from './fluentbitconfigmap';
-
 
 export default class SingleNewEksFargateOpenSourceObservabilityConstruct {
     constructor(scope: Construct, id: string) {
@@ -114,6 +113,12 @@ export default class SingleNewEksFargateOpenSourceObservabilityConstruct {
             );
         }
 
+        const fluentBitConfigMapProps = {
+            awsRegion: region,
+            logGroupName: "fargate-observability",
+            log_stream_prefix: "from-fluent-bit-",
+        } as FluentBitConfigMapProps;
+
         Reflect.defineMetadata("ordered", true, blueprints.addons.GrafanaOperatorAddon);
         const addOns: Array<blueprints.ClusterAddOn> = [
             new blueprints.addons.VpcCniAddOn(),
@@ -141,7 +146,7 @@ export default class SingleNewEksFargateOpenSourceObservabilityConstruct {
             new blueprints.addons.AdotCollectorAddOn(),
             new blueprints.addons.XrayAdotAddOn(),
             new blueprints.addons.AmpAddOn(ampAddOnProps),
-            new FluentBitConfigMap()
+            new FluentBitConfigMap(fluentBitConfigMapProps)
         ];
 
         const nodeRole = new blueprints.CreateRoleProvider("blueprint-fargate-pod-role", new ServicePrincipal("eks-fargate-pods.amazonaws.com"),