feat: kubeflow observability pattern

bin/single-new-eks-kubeflow-observability.ts

@@ -0,0 +1,6 @@
+import SingleNewEksKubeflowobservabilityPattern from '../lib/single-new-eks-kubeflow-observability-pattern';
+import { configureApp } from '../lib/common/construct-utils';
+
+const app = configureApp();
+
+new SingleNewEksKubeflowobservabilityPattern(app, 'single-new-eks-kubeflow');

cdk.json

@@ -40,6 +40,28 @@
         }
       ]
     },
+    "fluxIstioRepository": {
+      "name": "grafana-istio-dashboards",
+      "namespace": "grafana-operator",
+      "repository": {
+        "repoUrl": "https://github.com/arunvthangaraj/aws-observability-accelerator",
+        "name": "grafana-istio-dashboards",
+        "targetRevision": "main",
+        "path": "./artifacts/grafana-operator-manifests/eks/istio"
+      },
+      "values": {
+        "GRAFANA_ISTIO_CP_DASH_URL" : "https://raw.githubusercontent.com/arunvthangaraj/aws-observability-accelerator/main/artifacts/grafana-dashboards/eks/istio/istio-control-plane-dashboard.json",
+        "GRAFANA_ISTIO_MESH_DASH_URL" : "https://raw.githubusercontent.com/arunvthangaraj/aws-observability-accelerator/main/artifacts/grafana-dashboards/eks/istio/istio-mesh-dashboard.json",
+        "GRAFANA_ISTIO_PERF_DASH_URL" : "https://raw.githubusercontent.com/arunvthangaraj/aws-observability-accelerator/main/artifacts/grafana-dashboards/eks/istio/istio-performance-dashboard.json",
+        "GRAFANA_ISTIO_SERVICE_DASH_URL" : "https://raw.githubusercontent.com/arunvthangaraj/aws-observability-accelerator/main/artifacts/grafana-dashboards/eks/istio/istio-service-dashboard.json",
+        "GRAFANA_ISTIO_WORKLOAD_DASH_URL" : "https://raw.githubusercontent.com/arunvthangaraj/aws-observability-accelerator/main/artifacts/grafana-dashboards/eks/istio/istio-workload-dashboard.json"
+      },
+      "kustomizations": [
+        {
+          "kustomizationPath": "./artifacts/grafana-operator-manifests/eks/istio"
+        }
+      ]
+    },
     "gpuNodeGroup": {
       "instanceType": "g4dn.xlarge",
       "desiredSize": 2,

lib/single-new-eks-kubeflow-observability-pattern/argocd-cm.yml

@@ -0,0 +1,22 @@
+create: true
+admin.enabled: true
+application.instanceLabelKey: argocd.argoproj.io/instance
+exec.enabled: false
+server.rbac.log.enforce.enable: false
+timeout.hard.reconciliation: 0s
+timeout.reconciliation: 180s
+resource.customizations: |
+  argoproj.io/Application:
+    health.lua: |
+      hs = {}
+      hs.status = "Progressing"
+      hs.message = ""
+      if obj.status ~= nil then
+        if obj.status.health ~= nil then
+          hs.status = obj.status.health.status
+          if obj.status.health.message ~= nil then
+            hs.message = obj.status.health.message
+          end
+        end
+      end
+      return hs

lib/single-new-eks-kubeflow-observability-pattern/grafanaoperatorsecretaddon.ts

@@ -0,0 +1,75 @@
+import 'source-map-support/register';
+import * as blueprints from '@aws-quickstart/eks-blueprints';
+import * as eks from "aws-cdk-lib/aws-eks";
+import { Construct } from 'constructs';
+import { dependable } from '@aws-quickstart/eks-blueprints/dist/utils';
+
+export class GrafanaOperatorSecretAddon implements blueprints.ClusterAddOn {
+    id?: string | undefined;
+    @dependable(blueprints.addons.ExternalsSecretsAddOn.name, blueprints.addons.GrafanaOperatorAddon.name)
+    deploy(clusterInfo: blueprints.ClusterInfo): void | Promise<Construct> {
+        const cluster = clusterInfo.cluster;
+        const secretStore = new eks.KubernetesManifest(clusterInfo.cluster.stack, "ClusterSecretStore", {
+            cluster: cluster,
+            manifest: [
+                {
+                    apiVersion: "external-secrets.io/v1beta1",
+                    kind: "ClusterSecretStore",
+                    metadata: {
+                        name: "ssm-parameter-store",
+                        namespace: "default"
+                    },
+                    spec: {
+                        provider: {
+                            aws: {
+                                service: "ParameterStore",
+                                region: clusterInfo.cluster.stack.region,
+                                auth: {
+                                    jwt: {
+                                        serviceAccountRef: {
+                                            name: "external-secrets-sa",
+                                            namespace: "external-secrets",
+                                        },
+                                    },
+                                },
+                            },
+                        },
+                    },
+                },
+            ],
+        });
+
+        const externalSecret = new eks.KubernetesManifest(clusterInfo.cluster.stack, "ExternalSecret", {
+            cluster: cluster,
+            manifest: [
+                {
+                    apiVersion: "external-secrets.io/v1beta1",
+                    kind: "ExternalSecret",
+                    metadata: {
+                        name: "external-grafana-admin-credentials",
+                        namespace: "grafana-operator"
+                    },
+                    spec: {
+                        secretStoreRef: {
+                            name: "ssm-parameter-store",
+                            kind: "ClusterSecretStore",
+                        },
+                        target: {
+                            name: "grafana-admin-credentials"
+                        },
+                        data: [
+                            {
+                                secretKey: "GF_SECURITY_ADMIN_APIKEY",
+                                remoteRef: {
+                                    key: "/cdk-accelerator/grafana-api-key"
+                                },
+                            },
+                        ],
+                    },
+                },
+            ],
+        });
+        externalSecret.node.addDependency(secretStore);
+        return Promise.resolve(secretStore);
+    }
+}

lib/single-new-eks-kubeflow-observability-pattern/index.ts

@@ -0,0 +1,176 @@
+import { Construct } from 'constructs';
+import { utils } from '@aws-quickstart/eks-blueprints';
+import * as blueprints from '@aws-quickstart/eks-blueprints';
+import { GrafanaOperatorSecretAddon } from './grafanaoperatorsecretaddon';
+import * as eks from 'aws-cdk-lib/aws-eks';
+import * as ec2 from 'aws-cdk-lib/aws-ec2';
+import * as amp from 'aws-cdk-lib/aws-aps';
+import { ObservabilityBuilder } from '@aws-quickstart/eks-blueprints';
+import * as fs from 'fs';
+import { loadYaml, readYamlDocument } from '@aws-quickstart/eks-blueprints/dist/utils';
+
+export default class SingleNewEksKubeflowbservabilityPattern {
+    constructor(scope: Construct, id: string) {
+
+        const stackId = `${id}-observability-accelerator`;
+
+        const account = process.env.COA_ACCOUNT_ID! || process.env.CDK_DEFAULT_ACCOUNT!;
+        const region = process.env.COA_AWS_REGION! || process.env.CDK_DEFAULT_REGION!;
+        const ampWorkspaceName = process.env.COA_AMP_WORKSPACE_NAME! || 'observability-amp-Workspace';
+        const ampWorkspace = blueprints.getNamedResource(ampWorkspaceName) as unknown as amp.CfnWorkspace;
+        const ampEndpoint = ampWorkspace.attrPrometheusEndpoint;
+        const ampWorkspaceArn = ampWorkspace.attrArn;
+
+        const amgEndpointUrl = process.env.COA_AMG_ENDPOINT_URL;
+
+        // All Grafana Dashboard URLs from `cdk.json`
+        const fluxRepository: blueprints.FluxGitRepo = utils.valueFromContext(scope, "fluxRepository", undefined);
+        fluxRepository.values!.AMG_AWS_REGION = region;
+        fluxRepository.values!.AMP_ENDPOINT_URL = ampEndpoint;
+        fluxRepository.values!.AMG_ENDPOINT_URL = amgEndpointUrl;
+
+        const fluxIstioRepository: blueprints.FluxGitRepo = utils.valueFromContext(scope, "fluxIstioRepository", undefined);
+        fluxIstioRepository.values!.AMG_AWS_REGION = region;
+        fluxIstioRepository.values!.AMP_ENDPOINT_URL = ampEndpoint;
+        fluxIstioRepository.values!.AMG_ENDPOINT_URL = amgEndpointUrl;
+
+        const ampAddOnProps: blueprints.AmpAddOnProps = {
+            ampPrometheusEndpoint: ampEndpoint,
+            deploymentMode: blueprints.DeploymentMode.DAEMONSET,
+            ampRules: {
+                ampWorkspaceArn: ampWorkspaceArn,
+                ruleFilePaths: [
+                    __dirname + '/../common/resources/amp-config/alerting-rules.yml',
+                    __dirname + '/../common/resources/amp-config/recording-rules.yml'
+                ]
+            }
+        };
+
+        let doc = utils.readYamlDocument(__dirname + '/../common/resources/otel-collector-config.yml');
+
+        doc = utils.changeTextBetweenTokens(
+            doc,
+            "{{ start enableJavaMonJob }}",
+            "{{ stop enableJavaMonJob }}",
+            false
+        );
+        doc = utils.changeTextBetweenTokens(
+            doc,
+            "{{ start enableNginxMonJob }}",
+            "{{ stop enableNginxMonJob }}",
+            false
+        );
+        doc = utils.changeTextBetweenTokens(
+            doc,
+            "{{ start enableAdotMetricsCollectionJob }}",
+            "{{ stop enableAdotMetricsCollectionJob }}",
+            false
+        );
+        doc = utils.changeTextBetweenTokens(
+            doc,
+            "{{ start enableAdotMetricsCollectionTelemetry }}",
+            "{{ stop enableAdotMetricsCollectionTelemetry }}",
+            false
+        );
+
+        doc = utils.changeTextBetweenTokens(
+            doc,
+            "{{ start enableIstioMonJob }}",
+            "{{ stop enableIstioMonJob }}",
+            true
+        );
+        doc = utils.changeTextBetweenTokens(
+            doc,
+            "{{ start enableAPIserverJob }}",
+            "{{ stop enableAPIserverJob }}",
+            true
+        );
+        doc = utils.changeTextBetweenTokens(
+            doc,
+            "{{ start enableAdotContainerLogsReceiver }}",
+            "{{ stop enableAdotContainerLogsReceiver }}",
+            true
+        );
+        doc = utils.changeTextBetweenTokens(
+            doc,
+            "{{ start enableAdotContainerLogsExporter }}",
+            "{{ stop enableAdotContainerLogsExporter }}",
+            true
+        );
+        doc = utils.changeTextBetweenTokens(
+            doc,
+            "{{ start kubecostJob }}",
+            "{{ stop kubecostJob }}",
+            false
+        );
+        console.log(doc);
+        fs.writeFileSync(__dirname + '/../common/resources/otel-collector-config-new.yml', doc);
+
+        if (utils.valueFromContext(scope, "apiserver.pattern.enabled", false)) {
+            ampAddOnProps.enableAPIServerJob = true,
+            ampAddOnProps.ampRules?.ruleFilePaths.push(
+                __dirname + '/../common/resources/amp-config/apiserver/recording-rules.yml'
+            );
+        }
+
+        ampAddOnProps.openTelemetryCollector = {
+            manifestPath: __dirname + '/../common/resources/otel-collector-config-new.yml',
+            manifestParameterMap: {
+                logGroupName: `/aws/eks/${stackId}`,
+                logStreamName: `$NODE_NAME`,
+                logRetentionDays: 30,
+                awsRegion: region 
+            }
+        };
+
+        ampAddOnProps.ampRules?.ruleFilePaths.push(
+            __dirname + '/../common/resources/amp-config/istio/alerting-rules.yml',
+            __dirname + '/../common/resources/amp-config/istio/recording-rules.yml'
+        );
+
+        const argocdCmConfigData = loadYaml(readYamlDocument(__dirname + '/argocd-cm.yml'));
+
+        Reflect.defineMetadata("ordered", true, blueprints.addons.GrafanaOperatorAddon);
+        const addOns: Array<blueprints.ClusterAddOn> = [
+            new blueprints.VpcCniAddOn(),
+            new blueprints.EbsCsiDriverAddOn(),
+            new blueprints.addons.XrayAdotAddOn(),
+            new blueprints.addons.FluxCDAddOn({"repositories": [fluxRepository, fluxIstioRepository]}),
+            new GrafanaOperatorSecretAddon(),
+            new blueprints.ArgoCDAddOn(
+                {
+                    values: {
+                        "configs": {
+                            "cm": argocdCmConfigData
+                        }
+                    },
+                    bootstrapRepo: {
+                        repoUrl: 'https://github.com/arunvthangaraj/eks-blueprints-workloads.git',
+                        targetRevision: 'main',
+                        path: 'kubeflow-monitoring/envs/prod',
+                    }
+                }
+            )
+        ];
+
+        const mngProps: blueprints.MngClusterProviderProps = {
+            version: eks.KubernetesVersion.V1_29,
+            instanceTypes: [new ec2.InstanceType("m5.2xlarge")],
+            amiType: eks.NodegroupAmiType.AL2_X86_64,
+            desiredSize: 5,
+            maxSize: 10, 
+        };
+
+        ObservabilityBuilder.builder()
+            .account(account)
+            .region(region)
+            .clusterProvider(new blueprints.MngClusterProvider(mngProps))
+            .resourceProvider(ampWorkspaceName, new blueprints.CreateAmpProvider(ampWorkspaceName, ampWorkspaceName))
+            .version('auto')
+            .withAmpProps(ampAddOnProps)
+            .enableControlPlaneLogging()
+            .enableOpenSourcePatternAddOns()
+            .addOns(...addOns)
+            .build(scope, stackId);
+    }
+}