From 5d4c988dc8b74554abbc9edb4c3f7dc53b0872c8 Mon Sep 17 00:00:00 2001 From: Krzysztof Skitek Date: Tue, 10 Dec 2024 10:08:59 +0100 Subject: [PATCH] feat: Add AzureMonitor Logs aggegation validation --- manifest/v1alpha/slo/metrics_azure_monitor.go | 48 +++++++++++ .../v1alpha/slo/metrics_azure_monitor_test.go | 83 ++++++++++++++++++- 2 files changed, 130 insertions(+), 1 deletion(-) diff --git a/manifest/v1alpha/slo/metrics_azure_monitor.go b/manifest/v1alpha/slo/metrics_azure_monitor.go index 3bb17281..577124f6 100644 --- a/manifest/v1alpha/slo/metrics_azure_monitor.go +++ b/manifest/v1alpha/slo/metrics_azure_monitor.go @@ -1,10 +1,14 @@ package slo import ( + "fmt" "regexp" + "strings" + "time" "github.com/nobl9/govy/pkg/govy" "github.com/nobl9/govy/pkg/rules" + "github.com/pkg/errors" "github.com/nobl9/nobl9-go/manifest/v1alpha" ) @@ -122,6 +126,7 @@ var azureMonitorMetricLogsDataTypeValidation = govy.New[AzureMonitorMetric]( WithName("kqlQuery"). Required(). Rules( + azureMonitorkqlQueryRule, rules.StringMatchRegexp(regexp.MustCompile(`(?m)\bn9_time\b`)). WithDetails("n9_time is required"), rules.StringMatchRegexp(regexp.MustCompile(`(?m)\bn9_value\b`)). @@ -147,6 +152,49 @@ var azureMonitorMetricLogsDataTypeValidation = govy.New[AzureMonitorMetric]( govy.WhenDescription("dataType is '%s'", AzureMonitorDataTypeLogs), ) +var azureMonitorkqlQueryRule = govy.NewRule(func(kqlQuery string) error { + // supported formats: + // - summarize n9_value = () by bin(, ) + // - summarize n9_value = () + + parts := strings.Split(kqlQuery, "|") + summarizePart := "" + for _, part := range parts { + if strings.Contains(part, "summarize") { + summarizePart = part + // getting the last summarize as this will be our result resolution + } + } + + if summarizePart == "" { + return errors.New("summarize is required") + } + + binBy := regexp.MustCompile(`summarize.*by\s+bin\s*\(.* ([0-9]+\w+)\)`). + FindAllStringSubmatch(summarizePart, -1) + if len(binBy) == 1 { + const minResolution = time.Duration(15 * time.Second) + + binDuration, err := time.ParseDuration(binBy[0][1]) + if err != nil { + return fmt.Errorf("bin duration is required in short 'timespan' format. E.g. '15s'") + } + if binDuration < minResolution { + return fmt.Errorf( + "bin duration must be at least %s but was %s", + minResolution, + binDuration, + ) + } + } + + if strings.Contains(summarizePart, "by") && !strings.Contains(summarizePart, "bin") { + return errors.New("'summarize .* by' requires 'bin'(time, resolution) clause") + } + + return nil +}) + var azureMonitorMetricDimensionValidation = govy.New[AzureMonitorMetricDimension]( govy.ForPointer(func(a AzureMonitorMetricDimension) *string { return a.Name }). WithName("name"). diff --git a/manifest/v1alpha/slo/metrics_azure_monitor_test.go b/manifest/v1alpha/slo/metrics_azure_monitor_test.go index aab8a739..8f48e679 100644 --- a/manifest/v1alpha/slo/metrics_azure_monitor_test.go +++ b/manifest/v1alpha/slo/metrics_azure_monitor_test.go @@ -642,6 +642,87 @@ func TestAzureMonitor_ResourceID(t *testing.T) { } } +func TestAzureMonitor_kqlQuery(t *testing.T) { + testCases := []struct { + desc string + kqlQuery string + isValid bool + errorMessage string + }{ + { + "valid query without bin", + "Logs | summarize n9_value = max(value) | project TimeGenerated as n9_time, 1 as n9_value", + true, + "", + }, + { + "valid query with bin", + "Logs | summarize n9_value = max(value) by bin(time, 15s) | project TimeGenerated as n9_time, 1 as n9_value", + true, + "", + }, + { + "no summarize", + "Logs | project TimeGenerated as n9_time, 1 as n9_value", + false, + "summarize is required", + }, + { + "summarize without bin", + "Logs | summarize n9_value = avg(value) | project TimeGenerated as n9_time, 1 as n9_value", + true, + "", + }, + { + "summarize without bin with time aggregation", + "Logs | summarize n9_value = avg(value) by time | project TimeGenerated as n9_time, 1 as n9_value", + false, + "'summarize .* by' requires 'bin'(time, resolution) clause", + }, + { + "invalid aggregation resolution", + "Logs | summarize n9_value = avg(value) by bin(time, 15) | project TimeGenerated as n9_time, 1 as n9_value", + false, + "bin duration is required in short 'timespan' format. E.g. '15s'", + }, + { + "aggregation resolution to small", + "Logs | summarize n9_value = avg(value) by bin(time, 10ms) | project TimeGenerated as n9_time, 1 as n9_value", + false, + "bin duration must be at least 15s but was 10ms", + }, + { + "summarize used two times - valid", + "Logs | summarize n9_value = avg(value) by time | summarize n9_value = avg(value) | project TimeGenerated as n9_time, 1 as n9_value", + true, + "", + }, + { + "summarize used two times - invalid", + "Logs | summarize n9_value = avg(value) | summarize n9_value = avg(value) by time | project TimeGenerated as n9_time, 1 as n9_value", + false, + "'summarize .* by' requires 'bin'(time, resolution) clause", + }, + } + for _, tC := range testCases { + t.Run(tC.desc, func(t *testing.T) { + slo := validRawMetricSLO(v1alpha.AzureMonitor) + slo.Spec.Objectives[0].RawMetric.MetricQuery.AzureMonitor = getValidAzureMetric(AzureMonitorDataTypeLogs) + slo.Spec.Objectives[0].RawMetric.MetricQuery.AzureMonitor.KQLQuery = tC.kqlQuery + + err := validate(slo) + if tC.isValid { + testutils.AssertNoError(t, slo, err) + } else { + testutils.AssertContainsErrors(t, slo, err, 1, testutils.ExpectedError{ + Prop: "spec.objectives[0].rawMetric.query.azureMonitor.kqlQuery", + Message: tC.errorMessage, + }) + } + }) + } +} + func validAzureMonitorMetricsDataType() *AzureMonitorMetric { return &AzureMonitorMetric{DataType: AzureMonitorDataTypeMetrics, ResourceID: "/subscriptions/123/resourceGroups/azure-monitor-test-sources/providers/Microsoft.Web/sites/app", @@ -658,7 +739,7 @@ func validAzureMonitorLogsDataType() *AzureMonitorMetric { ResourceGroup: "rg", WorkspaceID: "11111111-1111-1111-1111-111111111111", }, - KQLQuery: "A | project TimeGenerated as n9_time, 1 as n9_value", + KQLQuery: "A | summarize n9_value = max(value) | project TimeGenerated as n9_time, 1 as n9_value", } }