From 2dc5de22e3199865541e29fce5578090d162735e Mon Sep 17 00:00:00 2001 From: howlla Date: Wed, 14 Feb 2024 14:19:03 -0800 Subject: [PATCH 01/28] Cost monitoring pattern --- ...ingle-new-eks-awsnative-cost-monitoring.ts | 6 ++ .../grafanaoperatorsecretaddon.ts | 102 ++++++++++++++++++ .../index.ts | 101 +++++++++++++++++ package.json | 1 + 4 files changed, 210 insertions(+) create mode 100644 bin/single-new-eks-awsnative-cost-monitoring.ts create mode 100644 lib/single-new-eks-cost-monitoring-pattern/grafanaoperatorsecretaddon.ts create mode 100644 lib/single-new-eks-cost-monitoring-pattern/index.ts diff --git a/bin/single-new-eks-awsnative-cost-monitoring.ts b/bin/single-new-eks-awsnative-cost-monitoring.ts new file mode 100644 index 00000000..358861bd --- /dev/null +++ b/bin/single-new-eks-awsnative-cost-monitoring.ts @@ -0,0 +1,6 @@ +import SingleNewEksCostMonitoringPattern from '../lib/single-new-eks-cost-monitoring-pattern'; +import { configureApp } from '../lib/common/construct-utils'; + +const app = configureApp(); + +new SingleNewEksCostMonitoringPattern(app, "single-new-eks-awsnative-cost"); \ No newline at end of file diff --git a/lib/single-new-eks-cost-monitoring-pattern/grafanaoperatorsecretaddon.ts b/lib/single-new-eks-cost-monitoring-pattern/grafanaoperatorsecretaddon.ts new file mode 100644 index 00000000..6581d7bc --- /dev/null +++ b/lib/single-new-eks-cost-monitoring-pattern/grafanaoperatorsecretaddon.ts @@ -0,0 +1,102 @@ +import 'source-map-support/register'; +import * as blueprints from '@aws-quickstart/eks-blueprints'; +import * as eks from "aws-cdk-lib/aws-eks"; +import { ManagedPolicy } from "aws-cdk-lib/aws-iam"; +import { Construct } from 'constructs'; +import { createNamespace, dependable } from '@aws-quickstart/eks-blueprints/dist/utils'; + +export class GrafanaOperatorSecretAddon implements blueprints.ClusterAddOn { + id?: string | undefined; + @dependable(blueprints.addons.ExternalsSecretsAddOn.name, blueprints.addons.GrafanaOperatorAddon.name) + deploy(clusterInfo: blueprints.ClusterInfo): void | Promise { + const cluster = clusterInfo.cluster; + + const policyRead = ManagedPolicy.fromAwsManagedPolicyName("AmazonPrometheusQueryAccess"); + const policyWrite = ManagedPolicy.fromAwsManagedPolicyName("AmazonPrometheusRemoteWriteAccess"); + + const serviceAccount1 = cluster.addServiceAccount("kubecost-cost-analyzer-amp", { + name: "kubecost-cost-analyzer-amp", + namespace: "kubecost" + }); + + + serviceAccount1.role.addManagedPolicy(policyRead); + serviceAccount1.role.addManagedPolicy(policyWrite); + + const serviceAccount2 = cluster.addServiceAccount("kubecost-prometheus-server-amp", { + name: "kubecost-prometheus-server-amp", + namespace: "kubecost" + }); + + serviceAccount2.role.addManagedPolicy(policyRead); + serviceAccount2.role.addManagedPolicy(policyWrite); + + const namespace = createNamespace("kubecost",cluster); + + serviceAccount1.node.addDependency(namespace); + serviceAccount2.node.addDependency(namespace); + + const secretStore = new eks.KubernetesManifest(clusterInfo.cluster.stack, "ClusterSecretStore", { + cluster: cluster, + manifest: [ + { + apiVersion: "external-secrets.io/v1beta1", + kind: "ClusterSecretStore", + metadata: { + name: "ssm-parameter-store", + namespace: "default" + }, + spec: { + provider: { + aws: { + service: "ParameterStore", + region: clusterInfo.cluster.stack.region, + auth: { + jwt: { + serviceAccountRef: { + name: "external-secrets-sa", + namespace: "external-secrets", + }, + }, + }, + }, + }, + }, + }, + ], + }); + + const externalSecret = new eks.KubernetesManifest(clusterInfo.cluster.stack, "ExternalSecret", { + cluster: cluster, + manifest: [ + { + apiVersion: "external-secrets.io/v1beta1", + kind: "ExternalSecret", + metadata: { + name: "external-grafana-admin-credentials", + namespace: "grafana-operator" + }, + spec: { + secretStoreRef: { + name: "ssm-parameter-store", + kind: "ClusterSecretStore", + }, + target: { + name: "grafana-admin-credentials" + }, + data: [ + { + secretKey: "GF_SECURITY_ADMIN_APIKEY", + remoteRef: { + key: "/cdk-accelerator/grafana-api-key" + }, + }, + ], + }, + }, + ], + }); + externalSecret.node.addDependency(secretStore); + return Promise.resolve(secretStore); + } +} \ No newline at end of file diff --git a/lib/single-new-eks-cost-monitoring-pattern/index.ts b/lib/single-new-eks-cost-monitoring-pattern/index.ts new file mode 100644 index 00000000..11dd32ca --- /dev/null +++ b/lib/single-new-eks-cost-monitoring-pattern/index.ts @@ -0,0 +1,101 @@ +import { Construct } from 'constructs'; +import * as blueprints from '@aws-quickstart/eks-blueprints'; +import { ObservabilityBuilder } from '@aws-quickstart/eks-blueprints'; +import { GrafanaOperatorSecretAddon } from './grafanaoperatorsecretaddon'; +import { KubecostAddOn } from '@kubecost/kubecost-eks-blueprints-addon'; +import * as amp from 'aws-cdk-lib/aws-aps'; + +export default class SingleNewEksCostMonitoringPattern { + constructor(scope: Construct, id: string) { + + const stackId = `${id}-observability-accelerator`; + const account = process.env.COA_ACCOUNT_ID! || process.env.CDK_DEFAULT_ACCOUNT!; + const region = process.env.COA_AWS_REGION! || process.env.CDK_DEFAULT_REGION!; + + const ampWorkspaceName = process.env.COA_AMP_WORKSPACE_NAME! || 'observability-amp-workspace'; + const ampWorkspace = blueprints.getNamedResource(ampWorkspaceName) as unknown as amp.CfnWorkspace; + const ampEndpoint = ampWorkspace.attrPrometheusEndpoint; + const ampWorkspaceArn = ampWorkspace.attrArn; + + const queryUrl = `https://aps-workspaces.us-west-2.amazonaws.com/workspaces/ws-bd2b2b87-6484-4349-8753-fa46557e6031/api/v1/query`; + + const ampAddOnProps: blueprints.AmpAddOnProps = { + ampPrometheusEndpoint: ampEndpoint, + ampRules: { + ampWorkspaceArn: ampWorkspaceArn, + ruleFilePaths: [ + __dirname + '/../common/resources/amp-config/alerting-rules.yml', + __dirname + '/../common/resources/amp-config/recording-rules.yml' + ] + } + }; + + Reflect.defineMetadata("ordered", true, blueprints.addons.GrafanaOperatorAddon); + const addOns: Array = [ + new blueprints.addons.CloudWatchLogsAddon({ + logGroupPrefix: `/aws/eks/${stackId}`, + logRetentionDays: 30 + }), + new blueprints.addons.EbsCsiDriverAddOn(), + new blueprints.addons.ExternalsSecretsAddOn(), + new blueprints.SecretsStoreAddOn({ rotationPollInterval: "120s" }), + new blueprints.SSMAgentAddOn(), + new KubecostAddOn({ + namespace:"kubecost", + values: { + global: { + amp: { + enabled: true, + prometheusServerEndpoint: queryUrl, + remoteWriteService: ampEndpoint, + sigv4: { + region: region + } + } + }, + kubecostProductConfigs: { + clusterName: stackId, + projectID: account + }, + prometheus: { + server: { + global: { + external_labels: { + cluster_id: stackId + } + } + } + }, + serviceAccount:{ + name: "kubecost-cost-analyzer-amp", + create: false, + server: { + create: false, + name: "kubecost-prometheus-server-amp" + } + }, + federatedETL:{ + federator: { + useMultiClusterDB : true + } + } + } + + }), + new blueprints.addons.GrafanaOperatorAddon({ + createNamespace: true, + }), + new GrafanaOperatorSecretAddon() + ]; + + ObservabilityBuilder.builder() + .account(account) + .region(region) + .version('auto') + .withAmpProps(ampAddOnProps) + .resourceProvider(ampWorkspaceName, new blueprints.CreateAmpProvider(ampWorkspaceName, ampWorkspaceName)) + .enableNativePatternAddOns() + .addOns(...addOns) + .build(scope, stackId); + } +} \ No newline at end of file diff --git a/package.json b/package.json index 36567de9..3c174bc2 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,7 @@ }, "dependencies": { "@aws-quickstart/eks-blueprints": "^1.13.1", + "@kubecost/kubecost-eks-blueprints-addon": "^0.1.8", "aws-cdk": "2.114.1", "aws-sdk": "^2.1455.0", "constructs": "^10.3.0", From 8bd2c55c21d2fa3258a150040b11c528d43094e8 Mon Sep 17 00:00:00 2001 From: Arun Date: Mon, 4 Mar 2024 15:43:22 -0600 Subject: [PATCH 02/28] adds KubeCostExtensionAddOn --- bin/single-new-eks-cost-monitoring.ts | 6 +++++ .../index.ts | 26 +++++++++++++++---- 2 files changed, 27 insertions(+), 5 deletions(-) create mode 100644 bin/single-new-eks-cost-monitoring.ts diff --git a/bin/single-new-eks-cost-monitoring.ts b/bin/single-new-eks-cost-monitoring.ts new file mode 100644 index 00000000..8a0741c2 --- /dev/null +++ b/bin/single-new-eks-cost-monitoring.ts @@ -0,0 +1,6 @@ +import { configureApp } from '../lib/common/construct-utils'; +import SingleNewEksCostMonitoringPattern from '../lib/single-new-eks-cost-monitoring-pattern'; + +const app = configureApp(); + +new SingleNewEksCostMonitoringPattern(app, 'single-new-eks-cost-monitoring'); \ No newline at end of file diff --git a/lib/single-new-eks-cost-monitoring-pattern/index.ts b/lib/single-new-eks-cost-monitoring-pattern/index.ts index 11dd32ca..028f1f9b 100644 --- a/lib/single-new-eks-cost-monitoring-pattern/index.ts +++ b/lib/single-new-eks-cost-monitoring-pattern/index.ts @@ -2,8 +2,9 @@ import { Construct } from 'constructs'; import * as blueprints from '@aws-quickstart/eks-blueprints'; import { ObservabilityBuilder } from '@aws-quickstart/eks-blueprints'; import { GrafanaOperatorSecretAddon } from './grafanaoperatorsecretaddon'; -import { KubecostAddOn } from '@kubecost/kubecost-eks-blueprints-addon'; +import { KubecostAddOn, KubecostAddOnProps } from '@kubecost/kubecost-eks-blueprints-addon'; import * as amp from 'aws-cdk-lib/aws-aps'; +import { dependable, setPath } from '@aws-quickstart/eks-blueprints/dist/utils'; export default class SingleNewEksCostMonitoringPattern { constructor(scope: Construct, id: string) { @@ -17,7 +18,7 @@ export default class SingleNewEksCostMonitoringPattern { const ampEndpoint = ampWorkspace.attrPrometheusEndpoint; const ampWorkspaceArn = ampWorkspace.attrArn; - const queryUrl = `https://aps-workspaces.us-west-2.amazonaws.com/workspaces/ws-bd2b2b87-6484-4349-8753-fa46557e6031/api/v1/query`; + //const queryUrl = `${ampEndpoint}api/v1/query`; const ampAddOnProps: blueprints.AmpAddOnProps = { ampPrometheusEndpoint: ampEndpoint, @@ -40,14 +41,14 @@ export default class SingleNewEksCostMonitoringPattern { new blueprints.addons.ExternalsSecretsAddOn(), new blueprints.SecretsStoreAddOn({ rotationPollInterval: "120s" }), new blueprints.SSMAgentAddOn(), - new KubecostAddOn({ + new KubeCostExtensionAddon({ namespace:"kubecost", values: { global: { amp: { enabled: true, - prometheusServerEndpoint: queryUrl, - remoteWriteService: ampEndpoint, + prometheusServerEndpoint: ampEndpoint, + //remoteWriteService: `${ampEndpoint}api/v1/remote_write`, sigv4: { region: region } @@ -98,4 +99,19 @@ export default class SingleNewEksCostMonitoringPattern { .addOns(...addOns) .build(scope, stackId); } +} + +class KubeCostExtensionAddon extends KubecostAddOn { + constructor(props?: KubecostAddOnProps) { + super(props); + } + + deploy(clusterInfo: blueprints.ClusterInfo): Promise { + const ampWorkspaceName = process.env.COA_AMP_WORKSPACE_NAME! || 'observability-amp-workspace'; + const ampWorkspace = blueprints.getNamedResource(ampWorkspaceName) as unknown as amp.CfnWorkspace; + const ampEndpoint = ampWorkspace.attrPrometheusEndpoint; + const remoteWriteEndpoint = `${ampEndpoint}api/v1/remote_write`; + setPath(this.options!.values, "global.amp.remoteWriteService", remoteWriteEndpoint); + return super.deploy(clusterInfo); + } } \ No newline at end of file From cc9bb35deab8e8d5600c4bfcc1bc38b5369826d9 Mon Sep 17 00:00:00 2001 From: Arun Date: Tue, 5 Mar 2024 18:13:51 -0600 Subject: [PATCH 03/28] adds managed nodegroup --- .../index.ts | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/lib/single-new-eks-cost-monitoring-pattern/index.ts b/lib/single-new-eks-cost-monitoring-pattern/index.ts index 028f1f9b..5f8d5c33 100644 --- a/lib/single-new-eks-cost-monitoring-pattern/index.ts +++ b/lib/single-new-eks-cost-monitoring-pattern/index.ts @@ -4,7 +4,9 @@ import { ObservabilityBuilder } from '@aws-quickstart/eks-blueprints'; import { GrafanaOperatorSecretAddon } from './grafanaoperatorsecretaddon'; import { KubecostAddOn, KubecostAddOnProps } from '@kubecost/kubecost-eks-blueprints-addon'; import * as amp from 'aws-cdk-lib/aws-aps'; -import { dependable, setPath } from '@aws-quickstart/eks-blueprints/dist/utils'; +import * as eks from 'aws-cdk-lib/aws-eks'; +import * as ec2 from 'aws-cdk-lib/aws-ec2'; +import { setPath } from '@aws-quickstart/eks-blueprints/dist/utils'; export default class SingleNewEksCostMonitoringPattern { constructor(scope: Construct, id: string) { @@ -18,8 +20,6 @@ export default class SingleNewEksCostMonitoringPattern { const ampEndpoint = ampWorkspace.attrPrometheusEndpoint; const ampWorkspaceArn = ampWorkspace.attrArn; - //const queryUrl = `${ampEndpoint}api/v1/query`; - const ampAddOnProps: blueprints.AmpAddOnProps = { ampPrometheusEndpoint: ampEndpoint, ampRules: { @@ -46,9 +46,8 @@ export default class SingleNewEksCostMonitoringPattern { values: { global: { amp: { + prometheusServerEndpoint: ampWorkspace.attrWorkspaceId, enabled: true, - prometheusServerEndpoint: ampEndpoint, - //remoteWriteService: `${ampEndpoint}api/v1/remote_write`, sigv4: { region: region } @@ -89,9 +88,18 @@ export default class SingleNewEksCostMonitoringPattern { new GrafanaOperatorSecretAddon() ]; + const mngProps: blueprints.MngClusterProviderProps = { + version: eks.KubernetesVersion.of("1.28"), + instanceTypes: [new ec2.InstanceType("m5.2xlarge")], + amiType: eks.NodegroupAmiType.AL2_X86_64, + desiredSize: 2, + maxSize: 3, + }; + ObservabilityBuilder.builder() .account(account) .region(region) + .clusterProvider(new blueprints.MngClusterProvider(mngProps)) .version('auto') .withAmpProps(ampAddOnProps) .resourceProvider(ampWorkspaceName, new blueprints.CreateAmpProvider(ampWorkspaceName, ampWorkspaceName)) @@ -107,10 +115,11 @@ class KubeCostExtensionAddon extends KubecostAddOn { } deploy(clusterInfo: blueprints.ClusterInfo): Promise { - const ampWorkspaceName = process.env.COA_AMP_WORKSPACE_NAME! || 'observability-amp-workspace'; - const ampWorkspace = blueprints.getNamedResource(ampWorkspaceName) as unknown as amp.CfnWorkspace; - const ampEndpoint = ampWorkspace.attrPrometheusEndpoint; - const remoteWriteEndpoint = `${ampEndpoint}api/v1/remote_write`; + const region = process.env.COA_AWS_REGION! || process.env.CDK_DEFAULT_REGION!; + const ampWorkspaceId = this.options.values!.global.amp.prometheusServerEndpoint; + const prometheusServerEndpoint = 'http://localhost:8005/workspaces/' + ampWorkspaceId; + const remoteWriteEndpoint = `https://aps-workspaces.${region}.amazonaws.com/workspaces/${ampWorkspaceId}/api/v1/remote_write`; + setPath(this.options!.values, "global.amp.prometheusServerEndpoint", prometheusServerEndpoint); setPath(this.options!.values, "global.amp.remoteWriteService", remoteWriteEndpoint); return super.deploy(clusterInfo); } From d05c1318b83d7a4a42ba1ac30d7f910160733afc Mon Sep 17 00:00:00 2001 From: Arun Date: Fri, 8 Mar 2024 15:51:20 -0600 Subject: [PATCH 04/28] disable kubecost prometheus node exporter --- lib/single-new-eks-cost-monitoring-pattern/index.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/single-new-eks-cost-monitoring-pattern/index.ts b/lib/single-new-eks-cost-monitoring-pattern/index.ts index 5f8d5c33..8a485cee 100644 --- a/lib/single-new-eks-cost-monitoring-pattern/index.ts +++ b/lib/single-new-eks-cost-monitoring-pattern/index.ts @@ -58,6 +58,14 @@ export default class SingleNewEksCostMonitoringPattern { projectID: account }, prometheus: { + nodeExporter: { + enabled: false + }, + serviceAccounts:{ + nodeExporter:{ + create: false + } + }, server: { global: { external_labels: { From 67e08a4c07b7dce1f7919825213148b217c2bda0 Mon Sep 17 00:00:00 2001 From: Arun Date: Mon, 11 Mar 2024 23:28:37 -0500 Subject: [PATCH 05/28] adds cognito auth to kubecost dashboard --- bin/single-new-eks-cost-monitoring.ts | 9 +- .../cognito-idp-stack.ts | 129 ++++++++++++++++++ .../index.ts | 54 +++++++- .../lambda/lambda_function.py | 39 ++++++ 4 files changed, 224 insertions(+), 7 deletions(-) create mode 100644 lib/single-new-eks-cost-monitoring-pattern/cognito-idp-stack.ts create mode 100644 lib/single-new-eks-cost-monitoring-pattern/lambda/lambda_function.py diff --git a/bin/single-new-eks-cost-monitoring.ts b/bin/single-new-eks-cost-monitoring.ts index 8a0741c2..255ccceb 100644 --- a/bin/single-new-eks-cost-monitoring.ts +++ b/bin/single-new-eks-cost-monitoring.ts @@ -1,6 +1,11 @@ -import { configureApp } from '../lib/common/construct-utils'; +import { configureApp, errorHandler } from '../lib/common/construct-utils'; import SingleNewEksCostMonitoringPattern from '../lib/single-new-eks-cost-monitoring-pattern'; const app = configureApp(); -new SingleNewEksCostMonitoringPattern(app, 'single-new-eks-cost-monitoring'); \ No newline at end of file +new SingleNewEksCostMonitoringPattern() + .buildAsync(app, 'single-new-eks-cost-monitoring') + .catch((e) => { + errorHandler(app, "Secure Ingress Auth pattern is not setup due to missing secrets for ArgoCD admin pwd. \ + See Secure Ingress Auth in the readme for instructions", e); + }); \ No newline at end of file diff --git a/lib/single-new-eks-cost-monitoring-pattern/cognito-idp-stack.ts b/lib/single-new-eks-cost-monitoring-pattern/cognito-idp-stack.ts new file mode 100644 index 00000000..c071eba2 --- /dev/null +++ b/lib/single-new-eks-cost-monitoring-pattern/cognito-idp-stack.ts @@ -0,0 +1,129 @@ +import * as cdk from 'aws-cdk-lib'; +import * as blueprints from '@aws-quickstart/eks-blueprints'; +import { Construct } from 'constructs'; +import * as cognito from 'aws-cdk-lib/aws-cognito'; +import * as lambda from 'aws-cdk-lib/aws-lambda'; +import * as iam from 'aws-cdk-lib/aws-iam'; + +export default class CognitoIdpStack extends cdk.Stack { + + public readonly userPoolOut: cognito.UserPool; + public readonly userPoolClientOut: cognito.UserPoolClient; + public readonly userPoolDomainOut: cognito.UserPoolDomain; + + constructor(scope: Construct, id: string, subDomain: string, props?: cdk.StackProps) { + super(scope, id, props); + + const lambdaExecutionRole = new iam.Role(this, 'Lambda Execution Role', { + assumedBy: new iam.ServicePrincipal('lambda.amazonaws.com'), + }); + + lambdaExecutionRole.addManagedPolicy(iam.ManagedPolicy.fromAwsManagedPolicyName("service-role/AWSLambdaBasicExecutionRole")); + lambdaExecutionRole.addManagedPolicy(iam.ManagedPolicy.fromAwsManagedPolicyName("AmazonSSMReadOnlyAccess")); + + const authChallengeFn = new lambda.Function(this, 'authChallengeFn', { + runtime: lambda.Runtime.PYTHON_3_12, + code: lambda.Code.fromAsset('./lib/single-new-eks-cost-monitoring-pattern/lambda'), + handler: 'lambda_function.lambda_handler', + role: lambdaExecutionRole, + environment: { + "ALLOWED_DOMAINS_LIST": blueprints.utils.valueFromContext(scope, "allowed.domains.list", "example.com") + } + }); + + + // Cognito User Pool + const userPool = new cognito.UserPool(this, 'CognitoIDPUserPool', { + userPoolName: 'CognitoIDPUserPool', + selfSignUpEnabled: true, + signInAliases: { + email: true, + username: true + }, + standardAttributes: { + email: { + mutable: true, + required: true + }, + givenName: { + mutable: true, + required: true + }, + familyName: { + mutable: true, + required: true + } + }, + lambdaTriggers: { + preSignUp: authChallengeFn, + preAuthentication: authChallengeFn, + }, + }); + + + // Output the User Pool ID + + this.userPoolOut = userPool; + + new cdk.CfnOutput(this, 'CognitoIDPUserPoolOut', { + value: userPool.userPoolId, + exportName: 'CognitoIDPUserPoolId' + }); + + new cdk.CfnOutput(this, 'CognitoIDPUserPoolArnOut', { + value: userPool.userPoolArn, + exportName: 'CognitoIDPUserPoolArn' + }); + + + // We will ask the IDP to redirect back to our domain's index page + const redirectUri = `https://${subDomain}/oauth2/idpresponse`; + + // Configure the user pool client application + const userPoolClient = new cognito.UserPoolClient(this, 'CognitoAppClient', { + userPool, + authFlows: { + userPassword: true + }, + oAuth: { + flows: { + authorizationCodeGrant: true + }, + scopes: [ + cognito.OAuthScope.OPENID + ], + callbackUrls: [redirectUri] + // TODO - What about logoutUrls? + }, + generateSecret: true, + userPoolClientName: 'Web', + supportedIdentityProviders: [cognito.UserPoolClientIdentityProvider.COGNITO] + }); + + // Output the User Pool App Client ID + this.userPoolClientOut = userPoolClient; + + new cdk.CfnOutput(this, 'CognitoIDPUserPoolClientOut', { + value: userPoolClient.userPoolClientId, + exportName: 'CognitoIDPUserPoolClientId' + }); + + // Add the domain to the user pool + const randomText = (Math.random() + 1).toString(36).substring(7); + const userPoolDomain = userPool.addDomain('CognitoDomain', { + cognitoDomain: { + domainPrefix: `my-cdk-blueprint-${randomText}`, + }, + }); + + // Output the User Pool App Client ID + + this.userPoolDomainOut = userPoolDomain; + + new cdk.CfnOutput(this, 'CognitoIDPUserPoolDomainOut', { + value: userPoolDomain.domainName, + exportName: 'CognitoIDPUserPoolDomain' + }); + + } +} \ No newline at end of file diff --git a/lib/single-new-eks-cost-monitoring-pattern/index.ts b/lib/single-new-eks-cost-monitoring-pattern/index.ts index 8a485cee..cbde4f8e 100644 --- a/lib/single-new-eks-cost-monitoring-pattern/index.ts +++ b/lib/single-new-eks-cost-monitoring-pattern/index.ts @@ -1,15 +1,37 @@ import { Construct } from 'constructs'; +import * as cdk from "aws-cdk-lib"; import * as blueprints from '@aws-quickstart/eks-blueprints'; -import { ObservabilityBuilder } from '@aws-quickstart/eks-blueprints'; +import { GlobalResources, LookupHostedZoneProvider, ObservabilityBuilder } from '@aws-quickstart/eks-blueprints'; import { GrafanaOperatorSecretAddon } from './grafanaoperatorsecretaddon'; import { KubecostAddOn, KubecostAddOnProps } from '@kubecost/kubecost-eks-blueprints-addon'; import * as amp from 'aws-cdk-lib/aws-aps'; import * as eks from 'aws-cdk-lib/aws-eks'; import * as ec2 from 'aws-cdk-lib/aws-ec2'; +import { ICertificate } from 'aws-cdk-lib/aws-certificatemanager'; import { setPath } from '@aws-quickstart/eks-blueprints/dist/utils'; +import { prevalidateSecrets } from '../common/construct-utils'; +import CognitoIdpStack from './cognito-idp-stack'; + +const SECRET_ARGO_ADMIN_PWD = 'argo-admin-secret'; +const gitUrl = 'https://github.com/aws-samples/eks-blueprints-workloads.git'; + +export default class SingleNewEksCostMonitoringPattern extends cdk.Stack { + async buildAsync(scope: Construct, id: string) { + + await prevalidateSecrets(SingleNewEksCostMonitoringPattern.name, undefined, SECRET_ARGO_ADMIN_PWD); + + const subdomain: string = blueprints.utils.valueFromContext(scope, "dev.subzone.name", "dev.mycompany.a2z.com"); + const parentDomain = blueprints.utils.valueFromContext(scope, "parent.hostedzone.name", "mycompany.a2z.com"); + const certificate: ICertificate = blueprints.getNamedResource(GlobalResources.Certificate); + + const cognitoIdpStackOut = new CognitoIdpStack (scope,'cognito-idp-stack', subdomain, + { + env: { + account: process.env.CDK_DEFAULT_ACCOUNT, + region: process.env.CDK_DEFAULT_REGION, + }, + }); -export default class SingleNewEksCostMonitoringPattern { - constructor(scope: Construct, id: string) { const stackId = `${id}-observability-accelerator`; const account = process.env.COA_ACCOUNT_ID! || process.env.CDK_DEFAULT_ACCOUNT!; @@ -93,6 +115,26 @@ export default class SingleNewEksCostMonitoringPattern { new blueprints.addons.GrafanaOperatorAddon({ createNamespace: true, }), + new blueprints.ArgoCDAddOn({ + bootstrapRepo: { + repoUrl: gitUrl, + targetRevision: "main", + path: 'secure-ingress-cognito/envs/dev' + }, + bootstrapValues: { + spec: { + ingress: { + host: subdomain, + cognitoUserPoolArn: cognitoIdpStackOut.userPoolOut.userPoolArn, + cognitoUserPoolAppId: cognitoIdpStackOut.userPoolClientOut.userPoolClientId, + cognitoDomainName: cognitoIdpStackOut.userPoolDomainOut.domainName, + certificateArn: certificate.certificateArn, + region: process.env.CDK_DEFAULT_REGION, + } + }, + }, + adminPasswordSecretName: SECRET_ARGO_ADMIN_PWD, + }), new GrafanaOperatorSecretAddon() ]; @@ -104,16 +146,18 @@ export default class SingleNewEksCostMonitoringPattern { maxSize: 3, }; - ObservabilityBuilder.builder() + await ObservabilityBuilder.builder() .account(account) .region(region) .clusterProvider(new blueprints.MngClusterProvider(mngProps)) .version('auto') + .resourceProvider(GlobalResources.HostedZone, new LookupHostedZoneProvider(parentDomain)) + .resourceProvider(GlobalResources.Certificate, new blueprints.CreateCertificateProvider('secure-ingress-cert', `${subdomain}`, GlobalResources.HostedZone)) .withAmpProps(ampAddOnProps) .resourceProvider(ampWorkspaceName, new blueprints.CreateAmpProvider(ampWorkspaceName, ampWorkspaceName)) .enableNativePatternAddOns() .addOns(...addOns) - .build(scope, stackId); + .buildAsync(scope, stackId); } } diff --git a/lib/single-new-eks-cost-monitoring-pattern/lambda/lambda_function.py b/lib/single-new-eks-cost-monitoring-pattern/lambda/lambda_function.py new file mode 100644 index 00000000..46fdd8b5 --- /dev/null +++ b/lib/single-new-eks-cost-monitoring-pattern/lambda/lambda_function.py @@ -0,0 +1,39 @@ +import json +import os +import boto3 + +def lambda_handler(event, context): + print("Received event: " + json.dumps(event, indent=2)) + + ssmclient = boto3.client('ssm') + + try: + allowed_domains_list = os.environ.get("ALLOWED_DOMAINS_LIST", "example.com") + + except Exception as e: + print("Error in reading the SSM Parameter Store : {}".format(str(e))) + + triggerSource = event['triggerSource'] + + # Split the email address so we can compare domains + emailId = event['request']['userAttributes']['email'] + address = emailId.split('@') + #print("address={} allowed_domains_list={} auto_approved_domains_list={} email_allow_list={}".format(address, allowed_domains_list, auto_approved_domains_list, email_white_list)) + + emailDomain = address[1] + + print("Running the Validation for {} flow".format(triggerSource)) + + if triggerSource == 'PreSignUp_SignUp': + # It sets the user pool autoConfirmUser flag after validating the email domain + event['response']['autoConfirmUser'] = False + + # This example uses a custom attribute 'custom:domain' + if emailDomain not in allowed_domains_list: + raise Exception("Cannot register users with email domains other than allowed domains list={}".format(allowed_domains_list)) + else: + print("triggerSource={} is incorrect".format(triggerSource)) + + #print("Received event: " + json.dumps(event, indent=2)) + + return event \ No newline at end of file From 8271fb08949f504b72f297140e98171e06c0af30 Mon Sep 17 00:00:00 2001 From: howlla Date: Mon, 18 Mar 2024 13:56:39 -0700 Subject: [PATCH 06/28] Added helm chart values for sigv4 --- lib/single-new-eks-cost-monitoring-pattern/index.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/single-new-eks-cost-monitoring-pattern/index.ts b/lib/single-new-eks-cost-monitoring-pattern/index.ts index cbde4f8e..04a989ed 100644 --- a/lib/single-new-eks-cost-monitoring-pattern/index.ts +++ b/lib/single-new-eks-cost-monitoring-pattern/index.ts @@ -103,11 +103,6 @@ export default class SingleNewEksCostMonitoringPattern extends cdk.Stack { create: false, name: "kubecost-prometheus-server-amp" } - }, - federatedETL:{ - federator: { - useMultiClusterDB : true - } } } @@ -171,8 +166,13 @@ class KubeCostExtensionAddon extends KubecostAddOn { const ampWorkspaceId = this.options.values!.global.amp.prometheusServerEndpoint; const prometheusServerEndpoint = 'http://localhost:8005/workspaces/' + ampWorkspaceId; const remoteWriteEndpoint = `https://aps-workspaces.${region}.amazonaws.com/workspaces/${ampWorkspaceId}/api/v1/remote_write`; + const sigV4ProxyHost = `aps-workspaces.${region}.amazonaws.com` setPath(this.options!.values, "global.amp.prometheusServerEndpoint", prometheusServerEndpoint); setPath(this.options!.values, "global.amp.remoteWriteService", remoteWriteEndpoint); + setPath(this.options!.values, "global.amp.sigv4.region", region); + setPath(this.options!.values, "global.amp.enabled", true); + setPath(this.options!.values, "sigV4Proxy.region", region); + setPath(this.options!.values, "sigV4Proxy.host", sigV4ProxyHost); return super.deploy(clusterInfo); } } \ No newline at end of file From 4d38bcff3ddef9504698908636db359f18279c5d Mon Sep 17 00:00:00 2001 From: howlla Date: Wed, 20 Mar 2024 11:57:05 -0700 Subject: [PATCH 07/28] pushing version update --- lib/single-new-eks-cost-monitoring-pattern/index.ts | 6 ++++-- .../index.ts | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/single-new-eks-cost-monitoring-pattern/index.ts b/lib/single-new-eks-cost-monitoring-pattern/index.ts index 04a989ed..f388015a 100644 --- a/lib/single-new-eks-cost-monitoring-pattern/index.ts +++ b/lib/single-new-eks-cost-monitoring-pattern/index.ts @@ -20,8 +20,8 @@ export default class SingleNewEksCostMonitoringPattern extends cdk.Stack { await prevalidateSecrets(SingleNewEksCostMonitoringPattern.name, undefined, SECRET_ARGO_ADMIN_PWD); - const subdomain: string = blueprints.utils.valueFromContext(scope, "dev.subzone.name", "dev.mycompany.a2z.com"); - const parentDomain = blueprints.utils.valueFromContext(scope, "parent.hostedzone.name", "mycompany.a2z.com"); + const subdomain: string = blueprints.utils.valueFromContext(scope, "dev.subzone.name", "dashboard.kubecost.avt.eks.aws.dev"); + const parentDomain = blueprints.utils.valueFromContext(scope, "parent.hostedzone.name", "kubecost.avt.eks.aws.dev"); const certificate: ICertificate = blueprints.getNamedResource(GlobalResources.Certificate); const cognitoIdpStackOut = new CognitoIdpStack (scope,'cognito-idp-stack', subdomain, @@ -65,6 +65,8 @@ export default class SingleNewEksCostMonitoringPattern extends cdk.Stack { new blueprints.SSMAgentAddOn(), new KubeCostExtensionAddon({ namespace:"kubecost", + version:"1.108.1", + kubecostToken: "Z2dvZDk5OUBnbWFpbC5jb20=xm343yadf98", values: { global: { amp: { diff --git a/lib/single-new-eks-fargate-opensource-observability-pattern/index.ts b/lib/single-new-eks-fargate-opensource-observability-pattern/index.ts index 897588c9..c6497e13 100644 --- a/lib/single-new-eks-fargate-opensource-observability-pattern/index.ts +++ b/lib/single-new-eks-fargate-opensource-observability-pattern/index.ts @@ -81,7 +81,7 @@ export default class SingleNewEksFargateOpenSourceObservabilityConstruct { if (utils.valueFromContext(scope, "java.pattern.enabled", false)) { ampAddOnProps.openTelemetryCollector = { - manifestPath: __dirname + '/../common/resources/otel-collector-config-new.yml', + manifestPath: __ + '/../common/resources/otel-collector-config-new.yml', manifestParameterMap: { javaScrapeSampleLimit: 1000, javaPrometheusMetricsEndpoint: "/metrics" From bafdaba0a8857c15640a191fa584960391108947 Mon Sep 17 00:00:00 2001 From: howlla Date: Wed, 20 Mar 2024 12:04:48 -0700 Subject: [PATCH 08/28] typo fix --- .../index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/single-new-eks-fargate-opensource-observability-pattern/index.ts b/lib/single-new-eks-fargate-opensource-observability-pattern/index.ts index c6497e13..897588c9 100644 --- a/lib/single-new-eks-fargate-opensource-observability-pattern/index.ts +++ b/lib/single-new-eks-fargate-opensource-observability-pattern/index.ts @@ -81,7 +81,7 @@ export default class SingleNewEksFargateOpenSourceObservabilityConstruct { if (utils.valueFromContext(scope, "java.pattern.enabled", false)) { ampAddOnProps.openTelemetryCollector = { - manifestPath: __ + '/../common/resources/otel-collector-config-new.yml', + manifestPath: __dirname + '/../common/resources/otel-collector-config-new.yml', manifestParameterMap: { javaScrapeSampleLimit: 1000, javaPrometheusMetricsEndpoint: "/metrics" From dc0cb7d3d4630a751c42ec751bf65b0a3977997e Mon Sep 17 00:00:00 2001 From: howlla Date: Thu, 21 Mar 2024 10:29:19 -0700 Subject: [PATCH 09/28] change to oss pattern --- .../resources/otel-collector-config.yml | 11 +++ .../index.ts | 72 ++++++++++++++++++- 2 files changed, 81 insertions(+), 2 deletions(-) diff --git a/lib/common/resources/otel-collector-config.yml b/lib/common/resources/otel-collector-config.yml index 14b76d04..ec872ec9 100644 --- a/lib/common/resources/otel-collector-config.yml +++ b/lib/common/resources/otel-collector-config.yml @@ -44,6 +44,17 @@ spec: external_labels: cluster: "{{clusterName}}" scrape_configs: + - job_name: kubecost + honor_labels: true + scrape_interval: 1m + scrape_timeout: 10s + metrics_path: /metrics + scheme: http + dns_sd_configs: + - names: + - dashboard.kubecost.avt.eks.aws.dev + type: 'A' + port: 9003 {{ start enableAdotMetricsCollectionJob}} - job_name: otel-collector-metrics scrape_interval: 10s diff --git a/lib/single-new-eks-cost-monitoring-pattern/index.ts b/lib/single-new-eks-cost-monitoring-pattern/index.ts index f388015a..a0bef22e 100644 --- a/lib/single-new-eks-cost-monitoring-pattern/index.ts +++ b/lib/single-new-eks-cost-monitoring-pattern/index.ts @@ -11,6 +11,7 @@ import { ICertificate } from 'aws-cdk-lib/aws-certificatemanager'; import { setPath } from '@aws-quickstart/eks-blueprints/dist/utils'; import { prevalidateSecrets } from '../common/construct-utils'; import CognitoIdpStack from './cognito-idp-stack'; +import * as fs from 'fs'; const SECRET_ARGO_ADMIN_PWD = 'argo-admin-secret'; const gitUrl = 'https://github.com/aws-samples/eks-blueprints-workloads.git'; @@ -53,6 +54,69 @@ export default class SingleNewEksCostMonitoringPattern extends cdk.Stack { } }; + let doc = blueprints.utils.readYamlDocument(__dirname + '/../common/resources/otel-collector-config.yml'); + doc = blueprints.utils.changeTextBetweenTokens( + doc, + "{{ start enableJavaMonJob }}", + "{{ stop enableJavaMonJob }}", + false + ); + doc = blueprints.utils.changeTextBetweenTokens( + doc, + "{{ start enableNginxMonJob }}", + "{{ stop enableNginxMonJob }}", + false + ); + doc = blueprints.utils.changeTextBetweenTokens( + doc, + "{{ start enableIstioMonJob }}", + "{{ stop enableIstioMonJob }}", + false + ); + doc = blueprints.utils.changeTextBetweenTokens( + doc, + "{{ start enableAPIserverJob }}", + "{{ stop enableAPIserverJob }}", + false + ); + doc = blueprints.utils.changeTextBetweenTokens( + doc, + "{{ start enableAdotMetricsCollectionJob}}", + "{{ stop enableAdotMetricsCollectionJob }}", + false + ); + doc = blueprints.utils.changeTextBetweenTokens( + doc, + "{{ start enableAdotMetricsCollectionTelemetry }}", + "{{ stop enableAdotMetricsCollectionTelemetry }}", + true + ); + + doc = blueprints.utils.changeTextBetweenTokens( + doc, + "{{ start enableAdotContainerLogsReceiver }}", + "{{ stop enableAdotContainerLogsReceiver }}", + true + ); + doc = blueprints.utils.changeTextBetweenTokens( + doc, + "{{ start enableAdotContainerLogsE dxporter }}", + "{{ stop enableAdotContainerLogsExporter }}", + true + ); + + fs.writeFileSync(__dirname + '/../common/resources/otel-collector-config-new.yml', doc); + + ampAddOnProps.openTelemetryCollector = { + manifestPath: __dirname + '/../common/resources/otel-collector-config-new.yml', + manifestParameterMap: { + logGroupName: `/aws/eks/costmonitoring/${ampWorkspaceName}`, + logStreamName: `$NODE_NAME`, + logRetentionDays: 30, + awsRegion: region + } + }; + Reflect.defineMetadata("ordered", true, blueprints.addons.GrafanaOperatorAddon); const addOns: Array = [ new blueprints.addons.CloudWatchLogsAddon({ @@ -63,12 +127,16 @@ export default class SingleNewEksCostMonitoringPattern extends cdk.Stack { new blueprints.addons.ExternalsSecretsAddOn(), new blueprints.SecretsStoreAddOn({ rotationPollInterval: "120s" }), new blueprints.SSMAgentAddOn(), + new blueprints.AdotCollectorAddOn(), new KubeCostExtensionAddon({ namespace:"kubecost", version:"1.108.1", kubecostToken: "Z2dvZDk5OUBnbWFpbC5jb20=xm343yadf98", values: { global: { + prometheus: { + enabled: false, + }, amp: { prometheusServerEndpoint: ampWorkspace.attrWorkspaceId, enabled: true, @@ -152,7 +220,7 @@ export default class SingleNewEksCostMonitoringPattern extends cdk.Stack { .resourceProvider(GlobalResources.Certificate, new blueprints.CreateCertificateProvider('secure-ingress-cert', `${subdomain}`, GlobalResources.HostedZone)) .withAmpProps(ampAddOnProps) .resourceProvider(ampWorkspaceName, new blueprints.CreateAmpProvider(ampWorkspaceName, ampWorkspaceName)) - .enableNativePatternAddOns() + .enableOpenSourcePatternAddOns() .addOns(...addOns) .buildAsync(scope, stackId); } @@ -172,7 +240,7 @@ class KubeCostExtensionAddon extends KubecostAddOn { setPath(this.options!.values, "global.amp.prometheusServerEndpoint", prometheusServerEndpoint); setPath(this.options!.values, "global.amp.remoteWriteService", remoteWriteEndpoint); setPath(this.options!.values, "global.amp.sigv4.region", region); - setPath(this.options!.values, "global.amp.enabled", true); + setPath(this.options!.values, "global.prometheus.fqdn", remoteWriteEndpoint); setPath(this.options!.values, "sigV4Proxy.region", region); setPath(this.options!.values, "sigV4Proxy.host", sigV4ProxyHost); return super.deploy(clusterInfo); From 47da964b6144bd969c12b8aef3cc1a6a0e0c166e Mon Sep 17 00:00:00 2001 From: howlla Date: Thu, 21 Mar 2024 10:32:47 -0700 Subject: [PATCH 10/28] remove extra bin file --- bin/single-new-eks-awsnative-cost-monitoring.ts | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 bin/single-new-eks-awsnative-cost-monitoring.ts diff --git a/bin/single-new-eks-awsnative-cost-monitoring.ts b/bin/single-new-eks-awsnative-cost-monitoring.ts deleted file mode 100644 index 358861bd..00000000 --- a/bin/single-new-eks-awsnative-cost-monitoring.ts +++ /dev/null @@ -1,6 +0,0 @@ -import SingleNewEksCostMonitoringPattern from '../lib/single-new-eks-cost-monitoring-pattern'; -import { configureApp } from '../lib/common/construct-utils'; - -const app = configureApp(); - -new SingleNewEksCostMonitoringPattern(app, "single-new-eks-awsnative-cost"); \ No newline at end of file From fc9b667cc896efd428d79fa9152210a66dba1014 Mon Sep 17 00:00:00 2001 From: howlla Date: Thu, 21 Mar 2024 15:40:58 -0700 Subject: [PATCH 11/28] working otel collector --- a.out | 1716 +++++++++++++++++ a.yaml | 711 +++++++ .../resources/otel-collector-config.yml | 22 +- .../cognito-idp-stack.ts | 4 +- .../grafanaoperatorsecretaddon.ts | 64 +- .../index.ts | 37 +- 6 files changed, 2498 insertions(+), 56 deletions(-) create mode 100644 a.out create mode 100644 a.yaml diff --git a/a.out b/a.out new file mode 100644 index 00000000..19765757 --- /dev/null +++ b/a.out @@ -0,0 +1,1716 @@ +Name: otel-collector-amp-collector +Namespace: default +Labels: app.kubernetes.io/component=opentelemetry-collector + app.kubernetes.io/instance=default.otel-collector-amp + app.kubernetes.io/managed-by=opentelemetry-operator + app.kubernetes.io/name=otel-collector-amp-collector + app.kubernetes.io/part-of=opentelemetry + app.kubernetes.io/version=v0.37.0 + aws.cdk.eks/prune-c812b288c8c9222087922b1ab3c9aba798868b58a0= +Annotations: + +Data +==== +collector.yaml: +---- +receivers: + prometheus: + config: + global: + scrape_interval: 15s + scrape_timeout: 10s + external_labels: + cluster: "single-new-eks-cost-monitoring-observability-accelerator" + scrape_configs: + - job_name: kubecost + honor_labels: true + scrape_interval: 1m + scrape_timeout: 10s + metrics_path: /metrics + scheme: http + dns_sd_configs: + - names: + - dashboard.kubecost.avt.eks.aws.dev + type: 'A' + port: 9003 + + - job_name: 'kubernetes-kubelet' + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: true + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - target_label: __address__ + replacement: kubernetes.default.svc.cluster.local:443 + - source_labels: [__meta_kubernetes_node_name] + regex: (.+) + target_label: __metrics_path__ + replacement: /api/v1/nodes/$${1}/proxy/metrics + - job_name: 'kubelet' + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: true + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - target_label: __address__ + replacement: kubernetes.default.svc.cluster.local:443 + - source_labels: [__meta_kubernetes_node_name] + regex: (.+) + target_label: __metrics_path__ + replacement: /api/v1/nodes/$${1}/proxy/metrics/cadvisor + + - job_name: serviceMonitor/default/kube-prometheus-stack-prometheus-node-exporter/0 + honor_timestamps: true + scrape_interval: 30s + scrape_timeout: 10s + metrics_path: /metrics + scheme: http + follow_redirects: true + enable_http2: true + relabel_configs: + - source_labels: [job] + separator: ; + regex: (.*) + target_label: __tmp_prometheus_job_name + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_app, __meta_kubernetes_service_labelpresent_app] + separator: ; + regex: (prometheus-node-exporter);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_service_label_release, __meta_kubernetes_service_labelpresent_release] + separator: ; + regex: (kube-prometheus-stack);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_port_name] + separator: ; + regex: http-metrics + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Node;(.*) + target_label: node + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Pod;(.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_namespace] + separator: ; + regex: (.*) + target_label: namespace + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: service + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_name] + separator: ; + regex: (.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_container_name] + separator: ; + regex: (.*) + target_label: container + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: job + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_jobLabel] + separator: ; + regex: (.+) + target_label: job + replacement: $$1 + action: replace + - separator: ; + regex: (.*) + target_label: endpoint + replacement: http-metrics + action: replace + - source_labels: [__address__] + separator: ; + regex: (.*) + modulus: 1 + target_label: __tmp_hash + replacement: $$1 + action: hashmod + - source_labels: [__tmp_hash] + separator: ; + regex: "0" + replacement: $$1 + action: keep + kubernetes_sd_configs: + - role: endpoints + kubeconfig_file: "" + follow_redirects: true + enable_http2: true + namespaces: + own_namespace: false + names: + - default + - job_name: serviceMonitor/default/kube-prometheus-stack-prometheus/0 + honor_timestamps: true + scrape_interval: 30s + scrape_timeout: 10s + metrics_path: /metrics + scheme: http + follow_redirects: true + enable_http2: true + relabel_configs: + - source_labels: [job] + separator: ; + regex: (.*) + target_label: __tmp_prometheus_job_name + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_app, __meta_kubernetes_service_labelpresent_app] + separator: ; + regex: (kube-prometheus-stack-prometheus);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_service_label_release, __meta_kubernetes_service_labelpresent_release] + separator: ; + regex: (kube-prometheus-stack);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_service_label_self_monitor, __meta_kubernetes_service_labelpresent_self_monitor] + separator: ; + regex: (true);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_port_name] + separator: ; + regex: http-web + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Node;(.*) + target_label: node + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Pod;(.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_namespace] + separator: ; + regex: (.*) + target_label: namespace + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: service + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_name] + separator: ; + regex: (.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_container_name] + separator: ; + regex: (.*) + target_label: container + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: job + replacement: $$1 + action: replace + - separator: ; + regex: (.*) + target_label: endpoint + replacement: http-web + action: replace + - source_labels: [__address__] + separator: ; + regex: (.*) + modulus: 1 + target_label: __tmp_hash + replacement: $$1 + action: hashmod + - source_labels: [__tmp_hash] + separator: ; + regex: "0" + replacement: $$1 + action: keep + kubernetes_sd_configs: + - role: endpoints + kubeconfig_file: "" + follow_redirects: true + enable_http2: true + namespaces: + own_namespace: false + names: + - default + - job_name: serviceMonitor/default/kube-prometheus-stack-operator/0 + honor_labels: true + honor_timestamps: true + scrape_interval: 30s + scrape_timeout: 10s + metrics_path: /metrics + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: true + follow_redirects: true + enable_http2: true + relabel_configs: + - source_labels: [job] + separator: ; + regex: (.*) + target_label: __tmp_prometheus_job_name + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_app, __meta_kubernetes_service_labelpresent_app] + separator: ; + regex: (kube-prometheus-stack-operator);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_service_label_release, __meta_kubernetes_service_labelpresent_release] + separator: ; + regex: (kube-prometheus-stack);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_port_name] + separator: ; + regex: https + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Node;(.*) + target_label: node + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Pod;(.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_namespace] + separator: ; + regex: (.*) + target_label: namespace + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: service + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_name] + separator: ; + regex: (.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_container_name] + separator: ; + regex: (.*) + target_label: container + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: job + replacement: $$1 + action: replace + - separator: ; + regex: (.*) + target_label: endpoint + replacement: https + action: replace + - source_labels: [__address__] + separator: ; + regex: (.*) + modulus: 1 + target_label: __tmp_hash + replacement: $$1 + action: hashmod + - source_labels: [__tmp_hash] + separator: ; + regex: "0" + replacement: $$1 + action: keep + kubernetes_sd_configs: + - role: endpoints + kubeconfig_file: "" + follow_redirects: true + enable_http2: true + namespaces: + own_namespace: false + names: + - default + - job_name: serviceMonitor/default/kube-prometheus-stack-kubelet/2 + honor_labels: true + honor_timestamps: true + scrape_interval: 30s + scrape_timeout: 10s + metrics_path: /metrics/probes + scheme: https + authorization: + type: Bearer + credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: true + follow_redirects: true + enable_http2: true + relabel_configs: + - source_labels: [job] + separator: ; + regex: (.*) + target_label: __tmp_prometheus_job_name + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_app_kubernetes_io_name, __meta_kubernetes_service_labelpresent_app_kubernetes_io_name] + separator: ; + regex: (kubelet);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_service_label_k8s_app, __meta_kubernetes_service_labelpresent_k8s_app] + separator: ; + regex: (kubelet);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_port_name] + separator: ; + regex: https-metrics + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Node;(.*) + target_label: node + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Pod;(.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_namespace] + separator: ; + regex: (.*) + target_label: namespace + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: service + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_name] + separator: ; + regex: (.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_container_name] + separator: ; + regex: (.*) + target_label: container + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: job + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_k8s_app] + separator: ; + regex: (.+) + target_label: job + replacement: $$1 + action: replace + - separator: ; + regex: (.*) + target_label: endpoint + replacement: https-metrics + action: replace + - source_labels: [__metrics_path__] + separator: ; + regex: (.*) + target_label: metrics_path + replacement: $$1 + action: replace + - source_labels: [__address__] + separator: ; + regex: (.*) + modulus: 1 + target_label: __tmp_hash + replacement: $$1 + action: hashmod + - source_labels: [__tmp_hash] + separator: ; + regex: "0" + replacement: $$1 + action: keep + kubernetes_sd_configs: + - role: endpoints + kubeconfig_file: "" + follow_redirects: true + enable_http2: true + namespaces: + own_namespace: false + names: + - kube-system + - job_name: serviceMonitor/default/kube-prometheus-stack-kubelet/1 + honor_labels: true + honor_timestamps: true + scrape_interval: 30s + scrape_timeout: 10s + metrics_path: /metrics/cadvisor + scheme: https + authorization: + type: Bearer + credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: true + follow_redirects: true + enable_http2: true + relabel_configs: + - source_labels: [job] + separator: ; + regex: (.*) + target_label: __tmp_prometheus_job_name + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_app_kubernetes_io_name, __meta_kubernetes_service_labelpresent_app_kubernetes_io_name] + separator: ; + regex: (kubelet);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_service_label_k8s_app, __meta_kubernetes_service_labelpresent_k8s_app] + separator: ; + regex: (kubelet);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_port_name] + separator: ; + regex: https-metrics + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Node;(.*) + target_label: node + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Pod;(.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_namespace] + separator: ; + regex: (.*) + target_label: namespace + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: service + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_name] + separator: ; + regex: (.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_container_name] + separator: ; + regex: (.*) + target_label: container + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: job + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_k8s_app] + separator: ; + regex: (.+) + target_label: job + replacement: $$1 + action: replace + - separator: ; + regex: (.*) + target_label: endpoint + replacement: https-metrics + action: replace + - source_labels: [__metrics_path__] + separator: ; + regex: (.*) + target_label: metrics_path + replacement: $$1 + action: replace + - source_labels: [__address__] + separator: ; + regex: (.*) + modulus: 1 + target_label: __tmp_hash + replacement: $$1 + action: hashmod + - source_labels: [__tmp_hash] + separator: ; + regex: "0" + replacement: $$1 + action: keep + kubernetes_sd_configs: + - role: endpoints + kubeconfig_file: "" + follow_redirects: true + enable_http2: true + namespaces: + own_namespace: false + names: + - kube-system + - job_name: serviceMonitor/default/kube-prometheus-stack-kubelet/0 + honor_labels: true + honor_timestamps: true + scrape_interval: 30s + scrape_timeout: 10s + metrics_path: /metrics + scheme: https + authorization: + type: Bearer + credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: true + follow_redirects: true + enable_http2: true + relabel_configs: + - source_labels: [job] + separator: ; + regex: (.*) + target_label: __tmp_prometheus_job_name + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_app_kubernetes_io_name, __meta_kubernetes_service_labelpresent_app_kubernetes_io_name] + separator: ; + regex: (kubelet);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_service_label_k8s_app, __meta_kubernetes_service_labelpresent_k8s_app] + separator: ; + regex: (kubelet);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_port_name] + separator: ; + regex: https-metrics + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Node;(.*) + target_label: node + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Pod;(.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_namespace] + separator: ; + regex: (.*) + target_label: namespace + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: service + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_name] + separator: ; + regex: (.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_container_name] + separator: ; + regex: (.*) + target_label: container + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: job + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_k8s_app] + separator: ; + regex: (.+) + target_label: job + replacement: $$1 + action: replace + - separator: ; + regex: (.*) + target_label: endpoint + replacement: https-metrics + action: replace + - source_labels: [__metrics_path__] + separator: ; + regex: (.*) + target_label: metrics_path + replacement: $$1 + action: replace + - source_labels: [__address__] + separator: ; + regex: (.*) + modulus: 1 + target_label: __tmp_hash + replacement: $$1 + action: hashmod + - source_labels: [__tmp_hash] + separator: ; + regex: "0" + replacement: $$1 + action: keep + kubernetes_sd_configs: + - role: endpoints + kubeconfig_file: "" + follow_redirects: true + enable_http2: true + namespaces: + own_namespace: false + names: + - kube-system + - job_name: serviceMonitor/default/kube-prometheus-stack-kube-state-metrics/0 + honor_labels: true + honor_timestamps: true + scrape_interval: 30s + scrape_timeout: 10s + metrics_path: /metrics + scheme: http + follow_redirects: true + enable_http2: true + relabel_configs: + - source_labels: [job] + separator: ; + regex: (.*) + target_label: __tmp_prometheus_job_name + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_app_kubernetes_io_instance, __meta_kubernetes_service_labelpresent_app_kubernetes_io_instance] + separator: ; + regex: (kube-prometheus-stack);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_service_label_app_kubernetes_io_name, __meta_kubernetes_service_labelpresent_app_kubernetes_io_name] + separator: ; + regex: (kube-state-metrics);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_port_name] + separator: ; + regex: http + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Node;(.*) + target_label: node + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Pod;(.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_namespace] + separator: ; + regex: (.*) + target_label: namespace + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: service + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_name] + separator: ; + regex: (.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_container_name] + separator: ; + regex: (.*) + target_label: container + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: job + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_app_kubernetes_io_name] + separator: ; + regex: (.+) + target_label: job + replacement: $$1 + action: replace + - separator: ; + regex: (.*) + target_label: endpoint + replacement: http + action: replace + - source_labels: [__address__] + separator: ; + regex: (.*) + modulus: 1 + target_label: __tmp_hash + replacement: $$1 + action: hashmod + - source_labels: [__tmp_hash] + separator: ; + regex: "0" + replacement: $$1 + action: keep + kubernetes_sd_configs: + - role: endpoints + kubeconfig_file: "" + follow_redirects: true + enable_http2: true + namespaces: + own_namespace: false + names: + - default + - job_name: serviceMonitor/default/kube-prometheus-stack-kube-scheduler/0 + honor_timestamps: true + scrape_interval: 30s + scrape_timeout: 10s + metrics_path: /metrics + scheme: http + authorization: + type: Bearer + credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token + follow_redirects: true + enable_http2: true + relabel_configs: + - source_labels: [job] + separator: ; + regex: (.*) + target_label: __tmp_prometheus_job_name + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_app, __meta_kubernetes_service_labelpresent_app] + separator: ; + regex: (kube-prometheus-stack-kube-scheduler);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_service_label_release, __meta_kubernetes_service_labelpresent_release] + separator: ; + regex: (kube-prometheus-stack);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_port_name] + separator: ; + regex: http-metrics + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Node;(.*) + target_label: node + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Pod;(.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_namespace] + separator: ; + regex: (.*) + target_label: namespace + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: service + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_name] + separator: ; + regex: (.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_container_name] + separator: ; + regex: (.*) + target_label: container + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: job + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_jobLabel] + separator: ; + regex: (.+) + target_label: job + replacement: $$1 + action: replace + - separator: ; + regex: (.*) + target_label: endpoint + replacement: http-metrics + action: replace + - source_labels: [__address__] + separator: ; + regex: (.*) + modulus: 1 + target_label: __tmp_hash + replacement: $$1 + action: hashmod + - source_labels: [__tmp_hash] + separator: ; + regex: "0" + replacement: $$1 + action: keep + kubernetes_sd_configs: + - role: endpoints + kubeconfig_file: "" + follow_redirects: true + enable_http2: true + namespaces: + own_namespace: false + names: + - kube-system + - job_name: serviceMonitor/default/kube-prometheus-stack-kube-proxy/0 + honor_timestamps: true + scrape_interval: 30s + scrape_timeout: 10s + metrics_path: /metrics + scheme: http + authorization: + type: Bearer + credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token + follow_redirects: true + enable_http2: true + relabel_configs: + - source_labels: [job] + separator: ; + regex: (.*) + target_label: __tmp_prometheus_job_name + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_app, __meta_kubernetes_service_labelpresent_app] + separator: ; + regex: (kube-prometheus-stack-kube-proxy);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_service_label_release, __meta_kubernetes_service_labelpresent_release] + separator: ; + regex: (kube-prometheus-stack);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_port_name] + separator: ; + regex: http-metrics + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Node;(.*) + target_label: node + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Pod;(.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_namespace] + separator: ; + regex: (.*) + target_label: namespace + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: service + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_name] + separator: ; + regex: (.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_container_name] + separator: ; + regex: (.*) + target_label: container + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: job + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_jobLabel] + separator: ; + regex: (.+) + target_label: job + replacement: $$1 + action: replace + - separator: ; + regex: (.*) + target_label: endpoint + replacement: http-metrics + action: replace + - source_labels: [__address__] + separator: ; + regex: (.*) + modulus: 1 + target_label: __tmp_hash + replacement: $$1 + action: hashmod + - source_labels: [__tmp_hash] + separator: ; + regex: "0" + replacement: $$1 + action: keep + kubernetes_sd_configs: + - role: endpoints + kubeconfig_file: "" + follow_redirects: true + enable_http2: true + namespaces: + own_namespace: false + names: + - kube-system + - job_name: serviceMonitor/default/kube-prometheus-stack-kube-etcd/0 + honor_timestamps: true + scrape_interval: 30s + scrape_timeout: 10s + metrics_path: /metrics + scheme: http + authorization: + type: Bearer + credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token + follow_redirects: true + enable_http2: true + relabel_configs: + - source_labels: [job] + separator: ; + regex: (.*) + target_label: __tmp_prometheus_job_name + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_app, __meta_kubernetes_service_labelpresent_app] + separator: ; + regex: (kube-prometheus-stack-kube-etcd);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_service_label_release, __meta_kubernetes_service_labelpresent_release] + separator: ; + regex: (kube-prometheus-stack);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_port_name] + separator: ; + regex: http-metrics + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Node;(.*) + target_label: node + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Pod;(.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_namespace] + separator: ; + regex: (.*) + target_label: namespace + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: service + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_name] + separator: ; + regex: (.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_container_name] + separator: ; + regex: (.*) + target_label: container + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: job + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_jobLabel] + separator: ; + regex: (.+) + target_label: job + replacement: $$1 + action: replace + - separator: ; + regex: (.*) + target_label: endpoint + replacement: http-metrics + action: replace + - source_labels: [__address__] + separator: ; + regex: (.*) + modulus: 1 + target_label: __tmp_hash + replacement: $$1 + action: hashmod + - source_labels: [__tmp_hash] + separator: ; + regex: "0" + replacement: $$1 + action: keep + kubernetes_sd_configs: + - role: endpoints + kubeconfig_file: "" + follow_redirects: true + enable_http2: true + namespaces: + own_namespace: false + names: + - kube-system + - job_name: serviceMonitor/default/kube-prometheus-stack-kube-controller-manager/0 + honor_timestamps: true + scrape_interval: 30s + scrape_timeout: 10s + metrics_path: /metrics + scheme: http + authorization: + type: Bearer + credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token + follow_redirects: true + enable_http2: true + relabel_configs: + - source_labels: [job] + separator: ; + regex: (.*) + target_label: __tmp_prometheus_job_name + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_app, __meta_kubernetes_service_labelpresent_app] + separator: ; + regex: (kube-prometheus-stack-kube-controller-manager);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_service_label_release, __meta_kubernetes_service_labelpresent_release] + separator: ; + regex: (kube-prometheus-stack);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_port_name] + separator: ; + regex: http-metrics + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Node;(.*) + target_label: node + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Pod;(.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_namespace] + separator: ; + regex: (.*) + target_label: namespace + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: service + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_name] + separator: ; + regex: (.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_container_name] + separator: ; + regex: (.*) + target_label: container + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: job + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_jobLabel] + separator: ; + regex: (.+) + target_label: job + replacement: $$1 + action: replace + - separator: ; + regex: (.*) + target_label: endpoint + replacement: http-metrics + action: replace + - source_labels: [__address__] + separator: ; + regex: (.*) + modulus: 1 + target_label: __tmp_hash + replacement: $$1 + action: hashmod + - source_labels: [__tmp_hash] + separator: ; + regex: "0" + replacement: $$1 + action: keep + kubernetes_sd_configs: + - role: endpoints + kubeconfig_file: "" + follow_redirects: true + enable_http2: true + namespaces: + own_namespace: false + names: + - kube-system + - job_name: serviceMonitor/default/kube-prometheus-stack-coredns/0 + honor_timestamps: true + scrape_interval: 30s + scrape_timeout: 10s + metrics_path: /metrics + scheme: http + authorization: + type: Bearer + credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token + follow_redirects: true + enable_http2: true + relabel_configs: + - source_labels: [job] + separator: ; + regex: (.*) + target_label: __tmp_prometheus_job_name + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_app, __meta_kubernetes_service_labelpresent_app] + separator: ; + regex: (kube-prometheus-stack-coredns);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_service_label_release, __meta_kubernetes_service_labelpresent_release] + separator: ; + regex: (kube-prometheus-stack);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_port_name] + separator: ; + regex: http-metrics + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Node;(.*) + target_label: node + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Pod;(.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_namespace] + separator: ; + regex: (.*) + target_label: namespace + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: service + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_name] + separator: ; + regex: (.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_container_name] + separator: ; + regex: (.*) + target_label: container + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: job + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_jobLabel] + separator: ; + regex: (.+) + target_label: job + replacement: $$1 + action: replace + - separator: ; + regex: (.*) + target_label: endpoint + replacement: http-metrics + action: replace + - source_labels: [__address__] + separator: ; + regex: (.*) + modulus: 1 + target_label: __tmp_hash + replacement: $$1 + action: hashmod + - source_labels: [__tmp_hash] + separator: ; + regex: "0" + replacement: $$1 + action: keep + kubernetes_sd_configs: + - role: endpoints + kubeconfig_file: "" + namespaces: + own_namespace: false + names: + - kube-system + - job_name: serviceMonitor/default/kube-prometheus-stack-apiserver/0 + honor_timestamps: true + scrape_interval: 30s + scrape_timeout: 10s + metrics_path: /metrics + scheme: https + authorization: + type: Bearer + credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + server_name: kubernetes + follow_redirects: true + enable_http2: true + relabel_configs: + - source_labels: [job] + separator: ; + regex: (.*) + target_label: __tmp_prometheus_job_name + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_component, __meta_kubernetes_service_labelpresent_component] + separator: ; + regex: (kubernetes);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_port_name] + separator: ; + regex: https + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Node;(.*) + target_label: node + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Pod;(.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_namespace] + separator: ; + regex: (.*) + target_label: namespace + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: service + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_name] + separator: ; + regex: (.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_container_name] + separator: ; + regex: (.*) + target_label: container + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: job + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_component] + separator: ; + regex: (.+) + target_label: job + replacement: $$1 + action: replace + - separator: ; + regex: (.*) + target_label: endpoint + replacement: https + action: replace + - source_labels: [__address__] + separator: ; + regex: (.*) + modulus: 1 + target_label: __tmp_hash + replacement: $$1 + action: hashmod + - source_labels: [__tmp_hash] + separator: ; + regex: "0" + replacement: $$1 + action: keep + kubernetes_sd_configs: + - role: endpoints + kubeconfig_file: "" + follow_redirects: true + enable_http2: true + namespaces: + own_namespace: false + names: + - default + - job_name: serviceMonitor/default/kube-prometheus-stack-alertmanager/0 + honor_timestamps: true + scrape_interval: 30s + scrape_timeout: 10s + metrics_path: /metrics + scheme: http + follow_redirects: true + enable_http2: true + relabel_configs: + - source_labels: [job] + separator: ; + regex: (.*) + target_label: __tmp_prometheus_job_name + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_app, __meta_kubernetes_service_labelpresent_app] + separator: ; + regex: (kube-prometheus-stack-alertmanager);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_service_label_release, __meta_kubernetes_service_labelpresent_release] + separator: ; + regex: (kube-prometheus-stack);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_service_label_self_monitor, __meta_kubernetes_service_labelpresent_self_monitor] + separator: ; + regex: (true);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_port_name] + separator: ; + regex: http-web + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Node;(.*) + target_label: node + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Pod;(.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_namespace] + separator: ; + regex: (.*) + target_label: namespace + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: service + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_name] + separator: ; + regex: (.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_container_name] + separator: ; + regex: (.*) + target_label: container + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: job + replacement: $$1 + action: replace + - separator: ; + regex: (.*) + target_label: endpoint + replacement: http-web + action: replace + - source_labels: [__address__] + separator: ; + regex: (.*) + modulus: 1 + target_label: __tmp_hash + replacement: $$1 + action: hashmod + - source_labels: [__tmp_hash] + separator: ; + regex: "0" + replacement: $$1 + action: keep + kubernetes_sd_configs: + - role: endpoints + kubeconfig_file: "" + follow_redirects: true + enable_http2: true + namespaces: + own_namespace: false + names: + - default + - job_name: 'kube-state-metrics' + static_configs: + - targets: ['kube-state-metrics.kube-system.svc.cluster.local:8080'] + - job_name: 'node-exporter' + kubernetes_sd_configs: + - role: endpoints + ec2_sd_configs: + relabel_configs: + - source_labels: [ __address__ ] + action: keep + regex: '.*:9100$' + - action: replace + source_labels: [__meta_kubernetes_endpoint_node_name] + target_label: nodename + + + + + + filelog: + include: [ /var/log/pods/*/*/*.log ] + include_file_name: false + include_file_path: true + start_at: end + operators: + # Find out which format is used by kubernetes + - type: router + id: get-format + routes: + - output: parser-docker + expr: 'body matches "^\\{"' + - output: parser-crio + expr: 'body matches "^[^ Z]+ "' + - output: parser-containerd + expr: 'body matches "^[^ Z]+Z"' + # Parse CRI-O format + - type: regex_parser + id: parser-crio + regex: + '^(?P