From 0ce46f636a0e704599d34525f5c4f8c1bb9ca806 Mon Sep 17 00:00:00 2001 From: yincong Date: Wed, 3 Jul 2024 21:39:10 +0800 Subject: [PATCH 1/6] adhoc result in unexpected request --- .../lezer-metricsql/src/metricsql.grammar | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/packages/lezer-metricsql/src/metricsql.grammar b/packages/lezer-metricsql/src/metricsql.grammar index 64158831..0b840c68 100644 --- a/packages/lezer-metricsql/src/metricsql.grammar +++ b/packages/lezer-metricsql/src/metricsql.grammar @@ -43,7 +43,22 @@ AggregateOp { Stddev | Stdvar | Sum | - Topk + Topk | + Median | + Distinct | + Geomean | + Histogram | + Any | + Mad | + Mode | + Share | + Sum2 | + Zscore | + HistogramAvg | + HistogramQuantile | + HistogramStddev | + HistogramStdvar | + LimitK } AggregateModifier { @@ -133,7 +148,6 @@ FunctionIdentifier { Deriv | Exp | Floor | - HistogramQuantile | HoltWinters | Hour | Idelta | @@ -213,9 +227,6 @@ FunctionIdentifier { TmaxOverTime | TminOverTime | ZscoreOverTime | - HistogramAvg | - HistogramStddev | - HistogramStdvar | PrometheusBuckets | Interpolate | KeepLastValue | @@ -237,9 +248,6 @@ FunctionIdentifier { RunningMax | RunningMin | RunningSum | - Distinct | - Geomean | - Histogram | BottomkAvg | BottomkLast | BottomkMax | @@ -250,14 +258,6 @@ FunctionIdentifier { TopkMax | TopkMedian | TopkMin | - Any | - Limitk | - Mad | - Median | - Mode | - Share | - Sum2 | - Zscore | LabelDel | LabelKeep | LabelLowercase | From bfddc60d0dbb3417f608922c9d4b88ed6be0230e Mon Sep 17 00:00:00 2001 From: yincong Date: Thu, 4 Jul 2024 09:51:08 +0800 Subject: [PATCH 2/6] modify func name --- packages/lezer-metricsql/src/metricsql.grammar | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/lezer-metricsql/src/metricsql.grammar b/packages/lezer-metricsql/src/metricsql.grammar index 0b840c68..a5d6e0b8 100644 --- a/packages/lezer-metricsql/src/metricsql.grammar +++ b/packages/lezer-metricsql/src/metricsql.grammar @@ -58,7 +58,7 @@ AggregateOp { HistogramQuantile | HistogramStddev | HistogramStdvar | - LimitK + Limitk } AggregateModifier { From b62c92bca6069e0124010841800a2e9e7bc0a26d Mon Sep 17 00:00:00 2001 From: yincong Date: Thu, 4 Jul 2024 10:05:01 +0800 Subject: [PATCH 3/6] modify grammar file --- packages/lezer-metricsql/src/metricsql.grammar | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/lezer-metricsql/src/metricsql.grammar b/packages/lezer-metricsql/src/metricsql.grammar index a5d6e0b8..fd84aac7 100644 --- a/packages/lezer-metricsql/src/metricsql.grammar +++ b/packages/lezer-metricsql/src/metricsql.grammar @@ -54,10 +54,6 @@ AggregateOp { Share | Sum2 | Zscore | - HistogramAvg | - HistogramQuantile | - HistogramStddev | - HistogramStdvar | Limitk } @@ -148,6 +144,7 @@ FunctionIdentifier { Deriv | Exp | Floor | + HistogramQuantile | HoltWinters | Hour | Idelta | @@ -227,6 +224,9 @@ FunctionIdentifier { TmaxOverTime | TminOverTime | ZscoreOverTime | + HistogramAvg | + HistogramStddev | + HistogramStdvar | PrometheusBuckets | Interpolate | KeepLastValue | From 5d2ba820a045ba3d98d8f231cdb58f058a820c17 Mon Sep 17 00:00:00 2001 From: yincong Date: Thu, 4 Jul 2024 11:53:43 +0800 Subject: [PATCH 4/6] modify func type --- src/querybuilder/aggregations.ts | 3 +++ src/querybuilder/metricsql-functions/basic/index.ts | 3 --- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/querybuilder/aggregations.ts b/src/querybuilder/aggregations.ts index fef630fa..04b29ebe 100755 --- a/src/querybuilder/aggregations.ts +++ b/src/querybuilder/aggregations.ts @@ -15,6 +15,9 @@ export function getAggregationOperations(): QueryBuilderOperationDef[] { ...createAggregationOperation(PromOperationId.Min), ...createAggregationOperation(PromOperationId.Max), ...createAggregationOperation(PromOperationId.Count), + ...createAggregationOperation(PromOperationId.Distinct), + ...createAggregationOperation(PromOperationId.Geomean), + ...createAggregationOperation(PromOperationId.Histogram), ...createAggregationOperationWithParam(PromOperationId.TopK, { params: [{ name: 'K-value', type: 'number' }], defaultParams: [5], diff --git a/src/querybuilder/metricsql-functions/basic/index.ts b/src/querybuilder/metricsql-functions/basic/index.ts index 2065e810..67f26939 100644 --- a/src/querybuilder/metricsql-functions/basic/index.ts +++ b/src/querybuilder/metricsql-functions/basic/index.ts @@ -24,9 +24,6 @@ const basicFunctions = [ PromOperationId.RunningMax, PromOperationId.RunningMin, PromOperationId.RunningSum, - PromOperationId.Distinct, - PromOperationId.Geomean, - PromOperationId.Histogram ] export function getBasicFunctions(): QueryBuilderOperationDef[] { From 4de6a528513e5b0716a67f7e878ade2d236f2751 Mon Sep 17 00:00:00 2001 From: yincong Date: Wed, 28 Aug 2024 11:51:10 +0800 Subject: [PATCH 5/6] add use dot as label --- src/metric_find_query.test.ts | 381 ++++++++++++++++++++++++++++ src/metric_find_query.ts | 2 +- src/migrations/variableMigration.ts | 2 +- 3 files changed, 383 insertions(+), 2 deletions(-) create mode 100644 src/metric_find_query.test.ts diff --git a/src/metric_find_query.test.ts b/src/metric_find_query.test.ts new file mode 100644 index 00000000..0fc72bc7 --- /dev/null +++ b/src/metric_find_query.test.ts @@ -0,0 +1,381 @@ +// Core Grafana history https://github.com/grafana/grafana/blob/v11.0.0-preview/public/app/plugins/datasource/prometheus/metric_find_query.test.ts +import { Observable, of } from 'rxjs'; + +import { DataSourceInstanceSettings, TimeRange, toUtc } from '@grafana/data'; +import { BackendDataSourceResponse, BackendSrvRequest, FetchResponse, TemplateSrv } from '@grafana/runtime'; + +import { PrometheusDatasource } from './datasource'; +import {PromOptions} from "./types"; +import PrometheusMetricFindQuery from "./metric_find_query"; + +const fetchMock = jest.fn((options: BackendSrvRequest): Observable> => { + return of({} as unknown as FetchResponse); +}); + +jest.mock('@grafana/runtime', () => ({ + ...jest.requireActual('@grafana/runtime'), + getBackendSrv: () => { + return { + fetch: fetchMock, + }; + }, +})); + +const instanceSettings = { + url: 'proxied', + id: 1, + uid: 'ABCDEF', + user: 'test', + password: 'mupp', + jsonData: { + httpMethod: 'GET', + prometheusVersion: '2.20.0', + }, +} as Partial> as DataSourceInstanceSettings; +const raw: TimeRange = { + from: toUtc('2018-04-25 10:00'), + to: toUtc('2018-04-25 11:00'), + raw: { + from: '2018-04-25 10:00', + to: '2018-04-25 11:00', + }, +}; + +const templateSrvStub = { + getAdhocFilters: jest.fn().mockImplementation(() => []), + replace: jest.fn().mockImplementation((a: string) => a), +} as unknown as TemplateSrv; + +beforeEach(() => { + jest.clearAllMocks(); +}); + +describe('PrometheusMetricFindQuery', () => { + let legacyPrometheusDatasource: PrometheusDatasource; + let prometheusDatasource: PrometheusDatasource; + beforeEach(() => { + legacyPrometheusDatasource = new PrometheusDatasource(instanceSettings, templateSrvStub); + prometheusDatasource = new PrometheusDatasource( + { + ...instanceSettings, + jsonData: { ...instanceSettings.jsonData }, + }, + templateSrvStub + ); + }); + + const setupMetricFindQuery = ( + data: { + query: string; + response: { + data: unknown; + }; + }, + datasource?: PrometheusDatasource + ) => { + fetchMock.mockImplementation(() => of({ status: 'success', data: data.response } as unknown as FetchResponse)); + return new PrometheusMetricFindQuery(datasource ?? legacyPrometheusDatasource, data.query); + }; + + describe('When performing metricFindQuery', () => { + it('label_names() should generate label name search query', async () => { + const query = setupMetricFindQuery({ + query: 'label_names()', + response: { + data: ['name1', 'name2', 'name3'], + }, + }); + const results = await query.process(raw); + + expect(results).toHaveLength(3); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(fetchMock).toHaveBeenCalledWith({ + method: 'GET', + url: `/api/datasources/proxy/1/api/v1/labels?limit=0`, + hideFromInspector: true, + showErrorAlert: false, + headers: {}, + }); + }); + + it('label_values(resource) should generate label search query', async () => { + const query = setupMetricFindQuery({ + query: 'label_values(resource)', + response: { + data: ['value1', 'value2', 'value3'], + }, + }); + const results = await query.process(raw); + + expect(results).toHaveLength(3); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(fetchMock).toHaveBeenCalledWith({ + method: 'GET', + url: `/api/datasources/proxy/1/api/v1/label/resource/values?start=${raw.from.unix()}&end=${raw.to.unix()}`, + hideFromInspector: true, + headers: {}, + }); + }); + + const emptyFilters = ['{}', '{ }', ' { } ', ' {} ']; + + emptyFilters.forEach((emptyFilter) => { + const queryString = `label_values(${emptyFilter}, resource)`; + it(`Empty filter, query, ${queryString} should just generate label search query`, async () => { + const query = setupMetricFindQuery({ + query: queryString, + response: { + data: ['value1', 'value2', 'value3'], + }, + }); + const results = await query.process(raw); + + expect(results).toHaveLength(3); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(fetchMock).toHaveBeenCalledWith({ + method: 'GET', + url: `/api/datasources/proxy/1/api/v1/label/resource/values?start=${raw.from.unix()}&end=${raw.to.unix()}`, + hideFromInspector: true, + headers: {}, + }); + }); + }); + + // + it('label_values(metric, resource) should generate series query with correct time', async () => { + const query = setupMetricFindQuery({ + query: 'label_values(metric, resource)', + response: { + data: [ + { __name__: 'metric', resource: 'value1' }, + { __name__: 'metric', resource: 'value2' }, + { __name__: 'metric', resource: 'value3' }, + ], + }, + }); + const results = await query.process(raw); + + expect(results).toHaveLength(3); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(fetchMock).toHaveBeenCalledWith({ + method: 'GET', + url: `/api/datasources/proxy/1/api/v1/label/resource/values?match${encodeURIComponent( + '[]' + )}=metric&start=${raw.from.unix()}&end=${raw.to.unix()}`, + hideFromInspector: true, + headers: {}, + }); + }); + + it('label_values(metric{label1="foo", label2="bar", label3="baz"}, resource) should generate series query with correct time', async () => { + const query = setupMetricFindQuery({ + query: 'label_values(metric{label1="foo", label2="bar", label3="baz"}, resource)', + response: { + data: [ + { __name__: 'metric', resource: 'value1' }, + { __name__: 'metric', resource: 'value2' }, + { __name__: 'metric', resource: 'value3' }, + ], + }, + }); + const results = await query.process(raw); + + expect(results).toHaveLength(3); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(fetchMock).toHaveBeenCalledWith({ + method: 'GET', + url: '/api/datasources/proxy/1/api/v1/label/resource/values?match%5B%5D=metric%7Blabel1%3D%22foo%22%2C%20label2%3D%22bar%22%2C%20label3%3D%22baz%22%7D&start=1524650400&end=1524654000', + hideFromInspector: true, + headers: {}, + }); + }); + + it('metrics(metric.*) should generate metric name query', async () => { + const query = setupMetricFindQuery({ + query: 'metrics(metric.*)', + response: { + data: ['metric1', 'metric2', 'metric3', 'nomatch'], + }, + }); + const results = await query.process(raw); + + expect(results).toHaveLength(3); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(fetchMock).toHaveBeenCalledWith({ + method: 'GET', + url: `/api/datasources/proxy/1/api/v1/label/__name__/values?start=${raw.from.unix()}&end=${raw.to.unix()}`, + hideFromInspector: true, + headers: {}, + }); + }); + + it('query_result(metric) should generate metric name query', async () => { + const query = setupMetricFindQuery({ + query: 'query_result(metric)', + response: { + data: { + resultType: 'vector', + result: [ + { + metric: { __name__: 'metric', job: 'testjob' }, + value: [1443454528.0, '3846'], + }, + ], + }, + }, + }); + const results = await query.process(raw); + + expect(results).toHaveLength(1); + expect(results[0].text).toBe('metric{job="testjob"} 3846 1443454528000'); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(fetchMock).toHaveBeenCalledWith({ + method: 'GET', + url: `/api/datasources/proxy/1/api/v1/query?query=metric&time=${raw.to.unix()}`, + headers: {}, + hideFromInspector: true, + showErrorAlert: false, + }); + }); + + it('query_result(metric) should pass time parameter to datasource.metric_find_query', async () => { + const query = setupMetricFindQuery({ + query: 'query_result(metric)', + response: { + data: { + resultType: 'vector', + result: [ + { + metric: { __name__: 'metric', job: 'testjob' }, + value: [1443454528.0, '3846'], + }, + ], + }, + }, + }); + const results = await query.process(raw); + + const expectedTime = prometheusDatasource.getPrometheusTime(raw.to, true); + + expect(results).toHaveLength(1); + expect(results[0].text).toBe('metric{job="testjob"} 3846 1443454528000'); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(fetchMock).toHaveBeenCalledWith({ + method: 'GET', + url: `/api/datasources/proxy/1/api/v1/query?query=metric&time=${expectedTime}`, + headers: {}, + hideFromInspector: true, + showErrorAlert: false, + }); + }); + + it('query_result(metric) should handle scalar resultTypes separately', async () => { + const query = setupMetricFindQuery({ + query: 'query_result(1+1)', + response: { + data: { + resultType: 'scalar', + result: [1443454528.0, '2'], + }, + }, + }); + const results = await query.process(raw); + expect(results).toHaveLength(1); + expect(results[0].text).toBe('2'); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(fetchMock).toHaveBeenCalledWith({ + method: 'GET', + url: `/api/datasources/proxy/1/api/v1/query?query=1%2B1&time=${raw.to.unix()}`, + headers: {}, + hideFromInspector: true, + showErrorAlert: false, + }); + }); + + it('up{job="job1"} should fallback using generate series query', async () => { + const query = setupMetricFindQuery({ + query: 'up{job="job1"}', + response: { + data: [ + { __name__: 'up', instance: '127.0.0.1:1234', job: 'job1' }, + { __name__: 'up', instance: '127.0.0.1:5678', job: 'job1' }, + { __name__: 'up', instance: '127.0.0.1:9102', job: 'job1' }, + ], + }, + }); + const results = await query.process(raw); + + expect(results).toHaveLength(3); + expect(results[0].text).toBe('up{instance="127.0.0.1:1234",job="job1"}'); + expect(results[1].text).toBe('up{instance="127.0.0.1:5678",job="job1"}'); + expect(results[2].text).toBe('up{instance="127.0.0.1:9102",job="job1"}'); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(fetchMock).toHaveBeenCalledWith({ + method: 'GET', + url: `/api/datasources/proxy/1/api/v1/series?match${encodeURIComponent('[]')}=${encodeURIComponent( + 'up{job="job1"}' + )}&start=${raw.from.unix()}&end=${raw.to.unix()}`, + hideFromInspector: true, + showErrorAlert: false, + headers: {}, + }); + }); + + it('label_values(metric, resource) should generate label values query with correct time', async () => { + const metricName = 'metricName'; + const resourceName = 'resourceName.test.data'; + const query = setupMetricFindQuery( + { + query: `label_values(${metricName}, ${resourceName})`, + response: { + data: [ + { __name__: `${metricName}`, resourceName: 'value1' }, + { __name__: `${metricName}`, resourceName: 'value2' }, + { __name__: `${metricName}`, resourceName: 'value3' }, + ], + }, + }, + prometheusDatasource + ); + const results = await query.process(raw); + + expect(results).toHaveLength(3); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(fetchMock).toHaveBeenCalledWith({ + method: 'GET', + url: `/api/datasources/proxy/1/api/v1/label/${resourceName}/values?match${encodeURIComponent( + '[]' + )}=${metricName}&start=${raw.from.unix()}&end=${raw.to.unix()}`, + hideFromInspector: true, + headers: {}, + }); + }); + + it('label_values(metric{label1="foo", label2="bar", label3="baz"}, resource) should generate label values query with correct time', async () => { + const metricName = 'metricName'; + const resourceName = 'resourceName'; + const label1Name = 'label1'; + const label1Value = 'label1Value'; + const query = setupMetricFindQuery( + { + query: `label_values(${metricName}{${label1Name}="${label1Value}"}, ${resourceName})`, + response: { + data: [{ __name__: metricName, resourceName: label1Value }], + }, + }, + prometheusDatasource + ); + const results = await query.process(raw); + + expect(results).toHaveLength(1); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(fetchMock).toHaveBeenCalledWith({ + method: 'GET', + url: `/api/datasources/proxy/1/api/v1/label/${resourceName}/values?match%5B%5D=${metricName}%7B${label1Name}%3D%22${label1Value}%22%7D&start=1524650400&end=1524654000`, + hideFromInspector: true, + headers: {}, + }); + }); + + }); +}); diff --git a/src/metric_find_query.ts b/src/metric_find_query.ts index aed5bd39..4e3fcd34 100755 --- a/src/metric_find_query.ts +++ b/src/metric_find_query.ts @@ -45,7 +45,7 @@ export default class PrometheusMetricFindQuery { this.range = timeRange; const labelNamesRegex = PrometheusLabelNamesRegex; const labelNamesRegexWithMatch = PrometheusLabelNamesRegexWithMatch; - const labelValuesRegex = /^label_values\((?:(.+),\s*)?([a-zA-Z_][a-zA-Z0-9_]*)\)\s*$/; + const labelValuesRegex = /^label_values\((?:(.+),\s*)?([a-zA-Z_][a-zA-Z0-9_.]*)\)\s*$/; const metricNamesRegex = PrometheusMetricNamesRegex; const queryResultRegex = PrometheusQueryResultRegex; const labelNamesQuery = this.query.match(labelNamesRegex); diff --git a/src/migrations/variableMigration.ts b/src/migrations/variableMigration.ts index 734eef72..c9d2037c 100644 --- a/src/migrations/variableMigration.ts +++ b/src/migrations/variableMigration.ts @@ -21,7 +21,7 @@ import { PromVariableQuery, PromVariableQueryType as QueryType } from '../types' export const PrometheusLabelNamesRegex = /^label_names\(\)\s*$/; // Note that this regex is different from the one in metric_find_query.ts because this is used pre-interpolation -export const PrometheusLabelValuesRegex = /^label_values\((?:(.+),\s*)?([a-zA-Z_$][a-zA-Z0-9_]*)\)\s*$/; +export const PrometheusLabelValuesRegex = /^label_values\((?:(.+),\s*)?([a-zA-Z_$][a-zA-Z0-9_.]*)\)\s*$/; export const PrometheusMetricNamesRegex = /^metrics\((.+)\)\s*$/; export const PrometheusQueryResultRegex = /^query_result\((.+)\)\s*$/; export const PrometheusLabelNamesRegexWithMatch = /^label_names\((.+)\)\s*$/; From 5a3addcc4e7d5a60417e14c61bbd4ce46ee1167a Mon Sep 17 00:00:00 2001 From: yincong Date: Wed, 28 Aug 2024 19:18:10 +0800 Subject: [PATCH 6/6] modify annotation --- src/metric_find_query.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/metric_find_query.test.ts b/src/metric_find_query.test.ts index 0fc72bc7..7047eb0c 100644 --- a/src/metric_find_query.test.ts +++ b/src/metric_find_query.test.ts @@ -1,4 +1,3 @@ -// Core Grafana history https://github.com/grafana/grafana/blob/v11.0.0-preview/public/app/plugins/datasource/prometheus/metric_find_query.test.ts import { Observable, of } from 'rxjs'; import { DataSourceInstanceSettings, TimeRange, toUtc } from '@grafana/data';