From 7f61e881a1f4d8c3b2bb10375bb275cf162d0d15 Mon Sep 17 00:00:00 2001 From: hainenber Date: Sun, 1 Oct 2023 21:12:09 +0700 Subject: [PATCH 1/6] feat(lambda/promtail): support dropping labels --- tools/lambda-promtail/README.md | 4 +- tools/lambda-promtail/lambda-promtail/cw.go | 2 +- .../lambda-promtail/kinesis.go | 2 +- tools/lambda-promtail/lambda-promtail/main.go | 39 +++++++++++++------ .../lambda-promtail/main_test.go | 18 +++++++++ tools/lambda-promtail/lambda-promtail/s3.go | 17 ++++---- tools/lambda-promtail/main.tf | 1 + tools/lambda-promtail/variables.tf | 6 +++ 8 files changed, 66 insertions(+), 23 deletions(-) diff --git a/tools/lambda-promtail/README.md b/tools/lambda-promtail/README.md index 2f4ab27fd1215..d8fa8222806d0 100644 --- a/tools/lambda-promtail/README.md +++ b/tools/lambda-promtail/README.md @@ -54,12 +54,12 @@ Then use Terraform to deploy: ```bash ## use cloudwatch log group -terraform apply -var ":" -var "write_address=https://your-loki-url/loki/api/v1/push" -var "password=" -var "username=" -var 'bearer_token=' -var 'log_group_names=["log-group-01", "log-group-02"]' -var 'extra_labels="name1,value1,name2,value2"' -var "tenant_id=" -var 'skip_tls_verify="false"' +terraform apply -var ":" -var "write_address=https://your-loki-url/loki/api/v1/push" -var "password=" -var "username=" -var 'bearer_token=' -var 'log_group_names=["log-group-01", "log-group-02"]' -var 'extra_labels="name1,value1,name2,value2"' -var 'extra_labels="name1,name2"' -var "tenant_id=" -var 'skip_tls_verify="false"' ``` ```bash ## use kinesis data stream -terraform apply -var ":" -var "write_address=https://your-loki-url/loki/api/v1/push" -var "password=" -var "username=" -var 'kinesis_stream_name=["kinesis-stream-01", "kinesis-stream-02"]' -var 'extra_labels="name1,value1,name2,value2"' -var "tenant_id=" -var 'skip_tls_verify="false"' +terraform apply -var ":" -var "write_address=https://your-loki-url/loki/api/v1/push" -var "password=" -var "username=" -var 'kinesis_stream_name=["kinesis-stream-01", "kinesis-stream-02"]' -var 'extra_labels="name1,value1,name2,value2"' -var 'extra_labels="name1,name2"' -var "tenant_id=" -var 'skip_tls_verify="false"' ``` or CloudFormation: diff --git a/tools/lambda-promtail/lambda-promtail/cw.go b/tools/lambda-promtail/lambda-promtail/cw.go index 4238e09051ee6..895cd66c8f450 100644 --- a/tools/lambda-promtail/lambda-promtail/cw.go +++ b/tools/lambda-promtail/lambda-promtail/cw.go @@ -26,7 +26,7 @@ func parseCWEvent(ctx context.Context, b *batch, ev *events.CloudwatchLogsEvent) labels[model.LabelName("__aws_cloudwatch_log_stream")] = model.LabelValue(data.LogStream) } - labels = applyExtraLabels(labels) + labels = applyLabels(labels) for _, event := range data.LogEvents { timestamp := time.UnixMilli(event.Timestamp) diff --git a/tools/lambda-promtail/lambda-promtail/kinesis.go b/tools/lambda-promtail/lambda-promtail/kinesis.go index dd465021c9760..31b619284b06d 100644 --- a/tools/lambda-promtail/lambda-promtail/kinesis.go +++ b/tools/lambda-promtail/lambda-promtail/kinesis.go @@ -26,7 +26,7 @@ func parseKinesisEvent(ctx context.Context, b batchIf, ev *events.KinesisEvent) model.LabelName("__aws_kinesis_event_source_arn"): model.LabelValue(record.EventSourceArn), } - labels = applyExtraLabels(labels) + labels = applyLabels(labels) // Check if the data is gzipped by inspecting the 'data' field if isGzipped(record.Kinesis.Data) { diff --git a/tools/lambda-promtail/lambda-promtail/main.go b/tools/lambda-promtail/lambda-promtail/main.go index 3d230ca1334fc..7fdd4b0363793 100644 --- a/tools/lambda-promtail/lambda-promtail/main.go +++ b/tools/lambda-promtail/lambda-promtail/main.go @@ -25,18 +25,19 @@ const ( maxErrMsgLen = 1024 - invalidExtraLabelsError = "Invalid value for environment variable EXTRA_LABELS. Expected a comma separated list with an even number of entries. " + invalidExtraLabelsError = "invalid value for environment variable EXTRA_LABELS. Expected a comma separated list with an even number of entries. " ) var ( - writeAddress *url.URL - username, password, extraLabelsRaw, tenantID, bearerToken string - keepStream bool - batchSize int - s3Clients map[string]*s3.Client - extraLabels model.LabelSet - skipTlsVerify bool - printLogLine bool + writeAddress *url.URL + username, password, extraLabelsRaw, dropLabelsRaw, tenantID, bearerToken string + keepStream bool + batchSize int + s3Clients map[string]*s3.Client + extraLabels model.LabelSet + dropLabels []model.LabelName + skipTlsVerify bool + printLogLine bool ) func setupArguments() { @@ -60,6 +61,16 @@ func setupArguments() { panic(err) } + dropLabelsRaw = os.Getenv("DROP_LABELS") + dropLabelsRawSplit := strings.Split(dropLabelsRaw, ",") + for _, dropLabelRaw := range dropLabelsRawSplit { + dropLabel := model.LabelName(dropLabelRaw) + if !dropLabel.IsValid() { + panic(fmt.Errorf("invalid label name %s", dropLabelRaw)) + } + dropLabels = append(dropLabels, model.LabelName(dropLabel)) + } + username = os.Getenv("USERNAME") password = os.Getenv("PASSWORD") // If either username or password is set then both must be. @@ -128,8 +139,14 @@ func parseExtraLabels(extraLabelsRaw string, omitPrefix bool) (model.LabelSet, e return extractedLabels, nil } -func applyExtraLabels(labels model.LabelSet) model.LabelSet { - return labels.Merge(extraLabels) +func applyLabels(labels model.LabelSet) model.LabelSet { + finalLabels := labels.Merge(extraLabels) + + for _, dropLabel := range dropLabels { + delete(finalLabels, dropLabel) + } + + return finalLabels } func checkEventType(ev map[string]interface{}) (interface{}, error) { diff --git a/tools/lambda-promtail/lambda-promtail/main_test.go b/tools/lambda-promtail/lambda-promtail/main_test.go index 03cc81d48fb7b..a67cf83ff8de3 100644 --- a/tools/lambda-promtail/lambda-promtail/main_test.go +++ b/tools/lambda-promtail/lambda-promtail/main_test.go @@ -1,6 +1,7 @@ package main import ( + "os" "testing" "github.com/prometheus/common/model" @@ -34,3 +35,20 @@ func TestLambdaPromtail_TestParseLabelsNoneProvided(t *testing.T) { require.Len(t, extraLabels, 0) require.Nil(t, err) } + +func TestLambdaPromtail_TestSetupArgumentWithDropLabels(t *testing.T) { + os.Setenv("WRITE_ADDRESS", "https://localhost:3100/loki/api/v1/push") + os.Setenv("OMIT_EXTRA_LABELS_PREFIX", "true") + os.Setenv("EXTRA_LABELS", "A1,a,B2,b,C3,c,D4,d") + os.Setenv("DROP_LABELS", "A1") + require.NotPanics(t, func() { + setupArguments() + }) + + defaultLabelSet := model.LabelSet{ + model.LabelName("default"): model.LabelValue("default"), + } + modifiedLabels := applyLabels(defaultLabelSet) + require.Contains(t, modifiedLabels, model.LabelName("B2")) + require.NotContains(t, modifiedLabels, model.LabelName("A1")) +} diff --git a/tools/lambda-promtail/lambda-promtail/s3.go b/tools/lambda-promtail/lambda-promtail/s3.go index dc9707634e2d3..5dca5cf7d6090 100644 --- a/tools/lambda-promtail/lambda-promtail/s3.go +++ b/tools/lambda-promtail/lambda-promtail/s3.go @@ -162,7 +162,7 @@ func parseS3Log(ctx context.Context, b *batch, labels map[string]string, obj io. model.LabelName(fmt.Sprintf("__aws_%s_owner", parser.logTypeLabel)): model.LabelValue(labels[parser.ownerLabelKey]), } - ls = applyExtraLabels(ls) + ls = applyLabels(ls) // extract the timestamp of the nested event and sends the rest as raw json if labels["type"] == CLOUDTRAIL_LOG_TYPE { @@ -341,13 +341,14 @@ func stringToRawEvent(body string) (map[string]interface{}, error) { // It also makes use of the fact that the log10 of a number in base 10 is its number of digits - 1. // It returns early if the fractional seconds is 0 because getting the log10 of 0 results in -Inf. // For example, given a string 1234567890123: -// iLog10 = 12 // the parsed int is 13 digits long -// multiplier = 0.001 // to get the seconds part it must be divided by 1000 -// sec = 1234567890123 * 0.001 = 1234567890 // this is the seconds part of the Unix time -// fractionalSec = 123 // the rest of the parsed int -// fractionalSecLog10 = 2 // it is 3 digits long -// multiplier = 1000000 // nano is 10^-9, so the nanoseconds part is 9 digits long -// nsec = 123000000 // this is the nanoseconds part of the Unix time +// +// iLog10 = 12 // the parsed int is 13 digits long +// multiplier = 0.001 // to get the seconds part it must be divided by 1000 +// sec = 1234567890123 * 0.001 = 1234567890 // this is the seconds part of the Unix time +// fractionalSec = 123 // the rest of the parsed int +// fractionalSecLog10 = 2 // it is 3 digits long +// multiplier = 1000000 // nano is 10^-9, so the nanoseconds part is 9 digits long +// nsec = 123000000 // this is the nanoseconds part of the Unix time func getUnixSecNsec(s string) (sec int64, nsec int64, err error) { const ( UNIX_SEC_LOG10 = 9 diff --git a/tools/lambda-promtail/main.tf b/tools/lambda-promtail/main.tf index ebc9925cc9d40..1b91fdc797c11 100644 --- a/tools/lambda-promtail/main.tf +++ b/tools/lambda-promtail/main.tf @@ -174,6 +174,7 @@ resource "aws_lambda_function" "this" { KEEP_STREAM = var.keep_stream BATCH_SIZE = var.batch_size EXTRA_LABELS = var.extra_labels + DROP_LABELS = var.drop_labels OMIT_EXTRA_LABELS_PREFIX = var.omit_extra_labels_prefix ? "true" : "false" TENANT_ID = var.tenant_id SKIP_TLS_VERIFY = var.skip_tls_verify diff --git a/tools/lambda-promtail/variables.tf b/tools/lambda-promtail/variables.tf index 040f1961a8210..5dc4f01a2d29b 100644 --- a/tools/lambda-promtail/variables.tf +++ b/tools/lambda-promtail/variables.tf @@ -72,6 +72,12 @@ variable "extra_labels" { default = "" } +variable "drop_labels" { + type = string + description = "Comma separated list of labels to be drop, in the format 'name1,name2,...,nameN,valueN' to be omitted to entries forwarded by lambda-promtail." + default = "" +} + variable "omit_extra_labels_prefix" { type = bool description = "Whether or not to omit the prefix `__extra_` from extra labels defined in the variable `extra_labels`." From cf081711628775934078c0ae1f666967aca4d583 Mon Sep 17 00:00:00 2001 From: hainenber Date: Thu, 12 Oct 2023 22:53:51 +0700 Subject: [PATCH 2/6] chore(CHANGELOG): add new entry Signed-off-by: hainenber --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b00c5884feb6d..cf63f63f9f4a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,7 @@ * [10416](https://github.com/grafana/loki/pull/10416) **lpugoy**: Lambda-Promtail: Add support for WAF logs in S3 * [10301](https://github.com/grafana/loki/pull/10301) **wildum**: users can now define `additional_fields` in cloudflare configuration. +* [10755](https://github.com/grafana/loki/pull/10755) **hainenber**: Lambda-Promtail: Add support for dropping labels passed via env var ##### Changes From c28f2883ce8a5907ee31540c90cbcf1e03dc9520 Mon Sep 17 00:00:00 2001 From: hainenber Date: Fri, 13 Oct 2023 00:00:37 +0700 Subject: [PATCH 3/6] fix(lambda/promtail): modularize drop_labels getter Also prevent tainting global vars via test Signed-off-by: hainenber --- tools/lambda-promtail/lambda-promtail/main.go | 27 +++++++++++++------ .../lambda-promtail/main_test.go | 25 ++++++++++------- 2 files changed, 35 insertions(+), 17 deletions(-) diff --git a/tools/lambda-promtail/lambda-promtail/main.go b/tools/lambda-promtail/lambda-promtail/main.go index 7fdd4b0363793..14cef6732737a 100644 --- a/tools/lambda-promtail/lambda-promtail/main.go +++ b/tools/lambda-promtail/lambda-promtail/main.go @@ -61,14 +61,9 @@ func setupArguments() { panic(err) } - dropLabelsRaw = os.Getenv("DROP_LABELS") - dropLabelsRawSplit := strings.Split(dropLabelsRaw, ",") - for _, dropLabelRaw := range dropLabelsRawSplit { - dropLabel := model.LabelName(dropLabelRaw) - if !dropLabel.IsValid() { - panic(fmt.Errorf("invalid label name %s", dropLabelRaw)) - } - dropLabels = append(dropLabels, model.LabelName(dropLabel)) + dropLabels, err = getDropLabels() + if err != nil { + panic(err) } username = os.Getenv("USERNAME") @@ -139,6 +134,22 @@ func parseExtraLabels(extraLabelsRaw string, omitPrefix bool) (model.LabelSet, e return extractedLabels, nil } +func getDropLabels() ([]model.LabelName, error) { + var result []model.LabelName + + dropLabelsRaw = os.Getenv("DROP_LABELS") + dropLabelsRawSplit := strings.Split(dropLabelsRaw, ",") + for _, dropLabelRaw := range dropLabelsRawSplit { + dropLabel := model.LabelName(dropLabelRaw) + if !dropLabel.IsValid() { + return []model.LabelName{}, fmt.Errorf("invalid label name %s", dropLabelRaw) + } + result = append(result, dropLabel) + } + + return result, nil +} + func applyLabels(labels model.LabelSet) model.LabelSet { finalLabels := labels.Merge(extraLabels) diff --git a/tools/lambda-promtail/lambda-promtail/main_test.go b/tools/lambda-promtail/lambda-promtail/main_test.go index a67cf83ff8de3..21063b643c109 100644 --- a/tools/lambda-promtail/lambda-promtail/main_test.go +++ b/tools/lambda-promtail/lambda-promtail/main_test.go @@ -36,19 +36,26 @@ func TestLambdaPromtail_TestParseLabelsNoneProvided(t *testing.T) { require.Nil(t, err) } -func TestLambdaPromtail_TestSetupArgumentWithDropLabels(t *testing.T) { - os.Setenv("WRITE_ADDRESS", "https://localhost:3100/loki/api/v1/push") - os.Setenv("OMIT_EXTRA_LABELS_PREFIX", "true") - os.Setenv("EXTRA_LABELS", "A1,a,B2,b,C3,c,D4,d") - os.Setenv("DROP_LABELS", "A1") - require.NotPanics(t, func() { - setupArguments() - }) +func TestLambdaPromtail_TestDropLabels(t *testing.T) { + os.Setenv("DROP_LABELS", "A1,A2") + + // Reset the shared global variables + defer func() { + os.Unsetenv("DROP_LABELS") + dropLabels = []model.LabelName{} + }() + + var err error + dropLabels, err = getDropLabels() + require.Nil(t, err) + require.Contains(t, dropLabels, model.LabelName("A1")) defaultLabelSet := model.LabelSet{ model.LabelName("default"): model.LabelValue("default"), + model.LabelName("A1"): model.LabelValue("A1"), + model.LabelName("B2"): model.LabelValue("B2"), } modifiedLabels := applyLabels(defaultLabelSet) - require.Contains(t, modifiedLabels, model.LabelName("B2")) require.NotContains(t, modifiedLabels, model.LabelName("A1")) + require.Contains(t, modifiedLabels, model.LabelName("B2")) } From fa11c89f8a7726dc6c36b321613b7ca6a685172b Mon Sep 17 00:00:00 2001 From: hainenber Date: Fri, 13 Oct 2023 00:01:10 +0700 Subject: [PATCH 4/6] doc(README): correct TF var instruction for drop_label Signed-off-by: hainenber --- tools/lambda-promtail/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/lambda-promtail/README.md b/tools/lambda-promtail/README.md index d8fa8222806d0..32b52db38c6b3 100644 --- a/tools/lambda-promtail/README.md +++ b/tools/lambda-promtail/README.md @@ -54,12 +54,12 @@ Then use Terraform to deploy: ```bash ## use cloudwatch log group -terraform apply -var ":" -var "write_address=https://your-loki-url/loki/api/v1/push" -var "password=" -var "username=" -var 'bearer_token=' -var 'log_group_names=["log-group-01", "log-group-02"]' -var 'extra_labels="name1,value1,name2,value2"' -var 'extra_labels="name1,name2"' -var "tenant_id=" -var 'skip_tls_verify="false"' +terraform apply -var ":" -var "write_address=https://your-loki-url/loki/api/v1/push" -var "password=" -var "username=" -var 'bearer_token=' -var 'log_group_names=["log-group-01", "log-group-02"]' -var 'extra_labels="name1,value1,name2,value2"' -var 'drop_labels="name1,name2"' -var "tenant_id=" -var 'skip_tls_verify="false"' ``` ```bash ## use kinesis data stream -terraform apply -var ":" -var "write_address=https://your-loki-url/loki/api/v1/push" -var "password=" -var "username=" -var 'kinesis_stream_name=["kinesis-stream-01", "kinesis-stream-02"]' -var 'extra_labels="name1,value1,name2,value2"' -var 'extra_labels="name1,name2"' -var "tenant_id=" -var 'skip_tls_verify="false"' +terraform apply -var ":" -var "write_address=https://your-loki-url/loki/api/v1/push" -var "password=" -var "username=" -var 'kinesis_stream_name=["kinesis-stream-01", "kinesis-stream-02"]' -var 'extra_labels="name1,value1,name2,value2"' -var 'drop_labels="name1,name2"' -var "tenant_id=" -var 'skip_tls_verify="false"' ``` or CloudFormation: From 6bbbd84328c20e8d00fbe31aa69ad65f06dbcd3f Mon Sep 17 00:00:00 2001 From: hainenber Date: Fri, 13 Oct 2023 00:01:58 +0700 Subject: [PATCH 5/6] chore(lambda/promtail): nil check for Promtail client Signed-off-by: hainenber --- tools/lambda-promtail/lambda-promtail/promtail.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tools/lambda-promtail/lambda-promtail/promtail.go b/tools/lambda-promtail/lambda-promtail/promtail.go index 56020c30ea81f..c1d01c36b174b 100644 --- a/tools/lambda-promtail/lambda-promtail/promtail.go +++ b/tools/lambda-promtail/lambda-promtail/promtail.go @@ -126,9 +126,11 @@ func (b *batch) createPushRequest() (*logproto.PushRequest, int) { } func (b *batch) flushBatch(ctx context.Context) error { - err := b.client.sendToPromtail(ctx, b) - if err != nil { - return err + if b.client != nil { + err := b.client.sendToPromtail(ctx, b) + if err != nil { + return err + } } b.resetBatch() From 5d568adfa5ec65dc6763437c321f5ebee563929b Mon Sep 17 00:00:00 2001 From: hainenber Date: Fri, 13 Oct 2023 00:04:04 +0700 Subject: [PATCH 6/6] chore(lambda/promtail): correct desc for drop_labels var Signed-off-by: hainenber --- tools/lambda-promtail/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/lambda-promtail/variables.tf b/tools/lambda-promtail/variables.tf index 5dc4f01a2d29b..bda956bc855b3 100644 --- a/tools/lambda-promtail/variables.tf +++ b/tools/lambda-promtail/variables.tf @@ -74,7 +74,7 @@ variable "extra_labels" { variable "drop_labels" { type = string - description = "Comma separated list of labels to be drop, in the format 'name1,name2,...,nameN,valueN' to be omitted to entries forwarded by lambda-promtail." + description = "Comma separated list of labels to be drop, in the format 'name1,name2,...,nameN' to be omitted to entries forwarded by lambda-promtail." default = "" }