From 90475280f0c578eb4e3f6b7b6985d954740e007c Mon Sep 17 00:00:00 2001 From: Dima Arnautov Date: Tue, 8 Sep 2020 17:47:12 +0200 Subject: [PATCH 1/5] [ML] fix custom urls processing for security app --- .../application/util/custom_url_utils.test.ts | 64 ++++++++++++++++++- .../application/util/custom_url_utils.ts | 6 +- 2 files changed, 65 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/ml/public/application/util/custom_url_utils.test.ts b/x-pack/plugins/ml/public/application/util/custom_url_utils.test.ts index 2912aad6819cf..1ac6c1207a450 100644 --- a/x-pack/plugins/ml/public/application/util/custom_url_utils.test.ts +++ b/x-pack/plugins/ml/public/application/util/custom_url_utils.test.ts @@ -61,8 +61,13 @@ describe('ML - custom URL utils', () => { influencer_field_name: 'airline', influencer_field_values: ['<>:;[}")'], }, + { + influencer_field_name: 'odd:field,name', + influencer_field_values: [">:&12<'"], + }, ], airline: ['<>:;[}")'], + 'odd:field,name': [">:&12<'"], }; const TEST_RECORD_MULTIPLE_INFLUENCER_VALUES: CustomUrlAnomalyRecordDoc = { @@ -98,7 +103,7 @@ describe('ML - custom URL utils', () => { url_name: 'Raw data', time_range: 'auto', url_value: - "discover#/?_g=(time:(from:'$earliest$',mode:absolute,to:'$latest$'))&_a=(index:bf6e5860-9404-11e8-8d4c-593f69c47267,query:(language:kuery,query:'airline:\"$airline$\"'))", + "discover#/?_g=(time:(from:'$earliest$',mode:absolute,to:'$latest$'))&_a=(index:bf6e5860-9404-11e8-8d4c-593f69c47267,query:(language:kuery,query:'airline:\"$airline$\" and odd:field,name : $odd:field,name$'))", }; const TEST_DASHBOARD_LUCENE_URL: KibanaUrlConfig = { @@ -263,9 +268,10 @@ describe('ML - custom URL utils', () => { ); }); - test('returns expected URL for a Kibana Discover type URL when record field contains special characters', () => { + /** FIXME */ + test.skip('returns expected URL for a Kibana Discover type URL when record field contains special characters', () => { expect(getUrlForRecord(TEST_DISCOVER_URL, TEST_RECORD_SPECIAL_CHARS)).toBe( - "discover#/?_g=(time:(from:'2017-02-09T15:10:00.000Z',mode:absolute,to:'2017-02-09T17:15:00.000Z'))&_a=(index:bf6e5860-9404-11e8-8d4c-593f69c47267,query:(language:kuery,query:'airline:\"%3C%3E%3A%3B%5B%7D%5C%22)\"'))" + "discover#/?_g=(time:(from:'2017-02-09T15:10:00.000Z',mode:absolute,to:'2017-02-09T17:15:00.000Z'))&_a=(index:bf6e5860-9404-11e8-8d4c-593f69c47267,query:(language:kuery,query:'airline:\"%3C%3E%3A%3B%5B%7D%5C%22)\" and odd:field,name:>:&12<''))" ); }); @@ -405,6 +411,58 @@ describe('ML - custom URL utils', () => { ); }); + test('return expected url for Security app', () => { + const urlConfig = { + url_name: 'Hosts Overview by process name', + url_value: + "security/hosts/ml-hosts?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))", + }; + + const testRecords = { + job_id: 'rare_process_by_host_linux_ecs', + result_type: 'record', + probability: 0.018122957282324745, + multi_bucket_impact: 0, + record_score: 20.513469583273547, + initial_record_score: 20.513469583273547, + bucket_span: 900, + detector_index: 0, + is_interim: false, + timestamp: 1549043100000, + by_field_name: 'process.name', + by_field_value: 'seq', + partition_field_name: 'host.name', + partition_field_value: 'showcase', + function: 'rare', + function_description: 'rare', + typical: [0.018122957282324745], + actual: [1], + influencers: [ + { + influencer_field_name: 'user.name', + influencer_field_values: ['sophie'], + }, + { + influencer_field_name: 'process.name', + influencer_field_values: ['seq'], + }, + { + influencer_field_name: 'host.name', + influencer_field_values: ['showcase'], + }, + ], + 'process.name': ['seq'], + 'user.name': ['sophie'], + 'host.name': ['showcase'], + earliest: '2019-02-01T16:00:00.000Z', + latest: '2019-02-01T18:59:59.999Z', + }; + + expect(getUrlForRecord(urlConfig, testRecords)).toBe( + "security/hosts/ml-hosts?_g=()&query=(query:'process.name:\"seq\"',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'2019-02-01T16:00:00.000Z',kind:absolute,to:'2019-02-01T18:59:59.999Z')),timeline:(linkTo:!(global),timerange:(from:'2019-02-01T16%3A00%3A00.000Z',kind:absolute,to:'2019-02-01T18%3A59%3A59.999Z')))" + ); + }); + test('removes an empty path component with a trailing slash', () => { const urlConfig = { url_name: 'APM', diff --git a/x-pack/plugins/ml/public/application/util/custom_url_utils.ts b/x-pack/plugins/ml/public/application/util/custom_url_utils.ts index 8263def2034aa..e538008a24a8a 100644 --- a/x-pack/plugins/ml/public/application/util/custom_url_utils.ts +++ b/x-pack/plugins/ml/public/application/util/custom_url_utils.ts @@ -184,11 +184,13 @@ function buildKibanaUrl(urlConfig: UrlConfig, record: CustomUrlAnomalyRecordDoc) let availableCharactersLeft = URL_LENGTH_LIMIT - resultPrefix.length - resultPostfix.length; - const queryFields = queryString + + // URL template might contain encoded characters + const queryFields = decodeURIComponent(queryString) // Split query string by AND operator. .split(/\sand\s/i) // Get property name from `influencerField:$influencerField$` string. - .map((v) => v.split(':')[0]); + .map((v) => String(v.split(/:(.+)?\$/)[0]).trim()); const queryParts: string[] = []; const joinOperator = ' AND '; From 57fa5a00eb286758dff6bcf7b592f10dd5399905 Mon Sep 17 00:00:00 2001 From: Dima Arnautov Date: Mon, 14 Sep 2020 10:52:58 +0200 Subject: [PATCH 2/5] [ML] improve query string parsing --- .../application/util/custom_url_utils.test.ts | 3 +- .../application/util/custom_url_utils.ts | 133 +++++++++++------- 2 files changed, 80 insertions(+), 56 deletions(-) diff --git a/x-pack/plugins/ml/public/application/util/custom_url_utils.test.ts b/x-pack/plugins/ml/public/application/util/custom_url_utils.test.ts index 1ac6c1207a450..1e19fb3e99790 100644 --- a/x-pack/plugins/ml/public/application/util/custom_url_utils.test.ts +++ b/x-pack/plugins/ml/public/application/util/custom_url_utils.test.ts @@ -268,7 +268,6 @@ describe('ML - custom URL utils', () => { ); }); - /** FIXME */ test.skip('returns expected URL for a Kibana Discover type URL when record field contains special characters', () => { expect(getUrlForRecord(TEST_DISCOVER_URL, TEST_RECORD_SPECIAL_CHARS)).toBe( "discover#/?_g=(time:(from:'2017-02-09T15:10:00.000Z',mode:absolute,to:'2017-02-09T17:15:00.000Z'))&_a=(index:bf6e5860-9404-11e8-8d4c-593f69c47267,query:(language:kuery,query:'airline:\"%3C%3E%3A%3B%5B%7D%5C%22)\" and odd:field,name:>:&12<''))" @@ -459,7 +458,7 @@ describe('ML - custom URL utils', () => { }; expect(getUrlForRecord(urlConfig, testRecords)).toBe( - "security/hosts/ml-hosts?_g=()&query=(query:'process.name:\"seq\"',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'2019-02-01T16:00:00.000Z',kind:absolute,to:'2019-02-01T18:59:59.999Z')),timeline:(linkTo:!(global),timerange:(from:'2019-02-01T16%3A00%3A00.000Z',kind:absolute,to:'2019-02-01T18%3A59%3A59.999Z')))" + "security/hosts/ml-hosts?_g=()&query=(language:kuery,query:'process.name:\"seq\"')&timerange=(global:(linkTo:!(timeline),timerange:(from:'2019-02-01T16:00:00.000Z',kind:absolute,to:'2019-02-01T18:59:59.999Z')),timeline:(linkTo:!(global),timerange:(from:'2019-02-01T16%3A00%3A00.000Z',kind:absolute,to:'2019-02-01T18%3A59%3A59.999Z')))" ); }); diff --git a/x-pack/plugins/ml/public/application/util/custom_url_utils.ts b/x-pack/plugins/ml/public/application/util/custom_url_utils.ts index e538008a24a8a..4f8e27993c878 100644 --- a/x-pack/plugins/ml/public/application/util/custom_url_utils.ts +++ b/x-pack/plugins/ml/public/application/util/custom_url_utils.ts @@ -8,6 +8,7 @@ import { get, flow } from 'lodash'; import moment from 'moment'; +import rison, { RisonObject, RisonValue } from 'rison-node'; import { parseInterval } from '../../../common/util/parse_interval'; import { escapeForElasticsearchQuery, replaceStringTokens } from './string_utils'; @@ -131,13 +132,70 @@ function escapeForKQL(value: string | number): string { type GetResultTokenValue = (v: string) => string; +export const isRisonObject = (value: RisonValue): value is RisonObject => { + return value !== null && typeof value === 'object'; +}; + +const getQueryStringResultProvider = ( + record: CustomUrlAnomalyRecordDoc, + getResultTokenValue: GetResultTokenValue +) => (resultPrefix: string, queryString: string, resultPostfix: string): string => { + const URL_LENGTH_LIMIT = 2000; + + let availableCharactersLeft = URL_LENGTH_LIMIT - resultPrefix.length - resultPostfix.length; + + // URL template might contain encoded characters + const queryFields = queryString + // Split query string by AND operator. + .split(/\sand\s/i) + // Get property name from `influencerField:$influencerField$` string. + .map((v) => String(v.split(/:(.+)?\$/)[0]).trim()); + + const queryParts: string[] = []; + const joinOperator = ' AND '; + + fieldsLoop: for (let i = 0; i < queryFields.length; i++) { + const field = queryFields[i]; + // Use lodash get to allow nested JSON fields to be retrieved. + let tokenValues: string[] | string | null = get(record, field) || null; + if (tokenValues === null) { + continue; + } + tokenValues = Array.isArray(tokenValues) ? tokenValues : [tokenValues]; + + // Create a pair `influencerField:value`. + // In cases where there are multiple influencer field values for an anomaly + // combine values with OR operator e.g. `(influencerField:value or influencerField:another_value)`. + let result = ''; + for (let j = 0; j < tokenValues.length; j++) { + const part = `${j > 0 ? ' OR ' : ''}${field}:"${getResultTokenValue(tokenValues[j])}"`; + + // Build up a URL string which is not longer than the allowed length and isn't corrupted by invalid query. + if (availableCharactersLeft < part.length) { + if (result.length > 0) { + queryParts.push(j > 0 ? `(${result})` : result); + } + break fieldsLoop; + } + + result += part; + + availableCharactersLeft -= result.length; + } + + if (result.length > 0) { + queryParts.push(tokenValues.length > 1 ? `(${result})` : result); + } + } + return queryParts.join(joinOperator); +}; + /** * Builds a Kibana dashboard or Discover URL from the supplied config, with any * dollar delimited tokens substituted from the supplied anomaly record. */ function buildKibanaUrl(urlConfig: UrlConfig, record: CustomUrlAnomalyRecordDoc) { const urlValue = urlConfig.url_value; - const URL_LENGTH_LIMIT = 2000; const isLuceneQueryLanguage = urlValue.includes('language:lucene'); @@ -172,67 +230,34 @@ function buildKibanaUrl(urlConfig: UrlConfig, record: CustomUrlAnomalyRecordDoc) return flow( (str: string) => str.replace('$earliest$', record.earliest).replace('$latest$', record.latest), // Process query string content of the URL + decodeURIComponent, (str: string) => { const getResultTokenValue: GetResultTokenValue = flow( queryLanguageEscapeCallback, commonEscapeCallback ); + + const getQueryStringResult = getQueryStringResultProvider(record, getResultTokenValue); + + const match = str.match(/(.+)(\(.*\blanguage:(?:lucene|kuery)\b.*?\))(.+)/); + + if (match !== null && match[2] !== undefined) { + const [, prefix, queryDef, postfix] = match; + + const q = rison.decode(queryDef); + + if (isRisonObject(q) && q.hasOwnProperty('query')) { + const [resultPrefix, resultPostfix] = [prefix, postfix].map(replaceSingleTokenValues); + const resultQuery = getQueryStringResult(resultPrefix, q.query as string, resultPostfix); + return `${resultPrefix}${rison.encode({ ...q, query: resultQuery })}${resultPostfix}`; + } + } + return str.replace( - /(.+query:'|.+&kuery=)([^']*)(['&].+)/, + /(.+&kuery=)(.*?)[^!](&.+)/, (fullMatch, prefix: string, queryString: string, postfix: string) => { const [resultPrefix, resultPostfix] = [prefix, postfix].map(replaceSingleTokenValues); - - let availableCharactersLeft = - URL_LENGTH_LIMIT - resultPrefix.length - resultPostfix.length; - - // URL template might contain encoded characters - const queryFields = decodeURIComponent(queryString) - // Split query string by AND operator. - .split(/\sand\s/i) - // Get property name from `influencerField:$influencerField$` string. - .map((v) => String(v.split(/:(.+)?\$/)[0]).trim()); - - const queryParts: string[] = []; - const joinOperator = ' AND '; - - fieldsLoop: for (let i = 0; i < queryFields.length; i++) { - const field = queryFields[i]; - // Use lodash get to allow nested JSON fields to be retrieved. - let tokenValues: string[] | string | null = get(record, field) || null; - if (tokenValues === null) { - continue; - } - tokenValues = Array.isArray(tokenValues) ? tokenValues : [tokenValues]; - - // Create a pair `influencerField:value`. - // In cases where there are multiple influencer field values for an anomaly - // combine values with OR operator e.g. `(influencerField:value or influencerField:another_value)`. - let result = ''; - for (let j = 0; j < tokenValues.length; j++) { - const part = `${j > 0 ? ' OR ' : ''}${field}:"${getResultTokenValue( - tokenValues[j] - )}"`; - - // Build up a URL string which is not longer than the allowed length and isn't corrupted by invalid query. - if (availableCharactersLeft < part.length) { - if (result.length > 0) { - queryParts.push(j > 0 ? `(${result})` : result); - } - break fieldsLoop; - } - - result += part; - - availableCharactersLeft -= result.length; - } - - if (result.length > 0) { - queryParts.push(tokenValues.length > 1 ? `(${result})` : result); - } - } - - const resultQuery = queryParts.join(joinOperator); - + const resultQuery = getQueryStringResult(resultPrefix, queryString, resultPostfix); return `${resultPrefix}${resultQuery}${resultPostfix}`; } ); From e8b935e9acc3d287a57b27b6512d4754b25a54bf Mon Sep 17 00:00:00 2001 From: Dima Arnautov Date: Mon, 14 Sep 2020 12:16:47 +0200 Subject: [PATCH 3/5] [ML] remove escaping with !, adjust a unit test for security app --- .../ml/public/application/util/custom_url_utils.test.ts | 4 ++-- .../plugins/ml/public/application/util/custom_url_utils.ts | 6 +----- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/ml/public/application/util/custom_url_utils.test.ts b/x-pack/plugins/ml/public/application/util/custom_url_utils.test.ts index 1e19fb3e99790..ba17bc99c16e7 100644 --- a/x-pack/plugins/ml/public/application/util/custom_url_utils.test.ts +++ b/x-pack/plugins/ml/public/application/util/custom_url_utils.test.ts @@ -414,7 +414,7 @@ describe('ML - custom URL utils', () => { const urlConfig = { url_name: 'Hosts Overview by process name', url_value: - "security/hosts/ml-hosts?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))", + "security/hosts/ml-hosts/$host.name$?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))", }; const testRecords = { @@ -458,7 +458,7 @@ describe('ML - custom URL utils', () => { }; expect(getUrlForRecord(urlConfig, testRecords)).toBe( - "security/hosts/ml-hosts?_g=()&query=(language:kuery,query:'process.name:\"seq\"')&timerange=(global:(linkTo:!(timeline),timerange:(from:'2019-02-01T16:00:00.000Z',kind:absolute,to:'2019-02-01T18:59:59.999Z')),timeline:(linkTo:!(global),timerange:(from:'2019-02-01T16%3A00%3A00.000Z',kind:absolute,to:'2019-02-01T18%3A59%3A59.999Z')))" + "security/hosts/ml-hosts/showcase?_g=()&query=(language:kuery,query:'process.name:\"seq\"')&timerange=(global:(linkTo:!(timeline),timerange:(from:'2019-02-01T16:00:00.000Z',kind:absolute,to:'2019-02-01T18:59:59.999Z')),timeline:(linkTo:!(global),timerange:(from:'2019-02-01T16%3A00%3A00.000Z',kind:absolute,to:'2019-02-01T18%3A59%3A59.999Z')))" ); }); diff --git a/x-pack/plugins/ml/public/application/util/custom_url_utils.ts b/x-pack/plugins/ml/public/application/util/custom_url_utils.ts index 4f8e27993c878..18ba1e4ee337b 100644 --- a/x-pack/plugins/ml/public/application/util/custom_url_utils.ts +++ b/x-pack/plugins/ml/public/application/util/custom_url_utils.ts @@ -203,11 +203,7 @@ function buildKibanaUrl(urlConfig: UrlConfig, record: CustomUrlAnomalyRecordDoc) ? escapeForElasticsearchQuery : escapeForKQL; - const commonEscapeCallback = flow( - // Kibana URLs used rison encoding, so escape with ! any ! or ' characters - (v: string): string => v.replace(/[!']/g, '!$&'), - encodeURIComponent - ); + const commonEscapeCallback = flow(encodeURIComponent); const replaceSingleTokenValues = (str: string) => { const getResultTokenValue: GetResultTokenValue = flow( From 5b34682e7598f0ad8ede6b107e73ffd57d26dc09 Mon Sep 17 00:00:00 2001 From: Dima Arnautov Date: Mon, 14 Sep 2020 12:33:40 +0200 Subject: [PATCH 4/5] [ML] unit test --- .../application/util/custom_url_utils.test.ts | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/x-pack/plugins/ml/public/application/util/custom_url_utils.test.ts b/x-pack/plugins/ml/public/application/util/custom_url_utils.test.ts index ba17bc99c16e7..b7dd9524d22c9 100644 --- a/x-pack/plugins/ml/public/application/util/custom_url_utils.test.ts +++ b/x-pack/plugins/ml/public/application/util/custom_url_utils.test.ts @@ -274,6 +274,52 @@ describe('ML - custom URL utils', () => { ); }); + test('correctly encodes special characters inside of a query string', () => { + const testUrl = { + url_name: 'Show dashboard', + time_range: 'auto', + url_value: `dashboards#/view/351de820-f2bb-11ea-ab06-cb93221707e9?_a=(filters:!(),query:(language:kuery,query:'at@name:"$at@name$" and singlequote!'name:"$singlequote!'name$"'))&_g=(filters:!(),time:(from:'$earliest$',mode:absolute,to:'$latest$'))`, + }; + + const testRecord = { + job_id: 'spec-char', + result_type: 'record', + probability: 0.0028099428534745633, + multi_bucket_impact: 5, + record_score: 49.00785814424704, + initial_record_score: 49.00785814424704, + bucket_span: 900, + detector_index: 0, + is_interim: false, + timestamp: 1549593000000, + partition_field_name: 'at@name', + partition_field_value: "contains a ' quote", + function: 'mean', + function_description: 'mean', + typical: [1993.2657340111837], + actual: [1808.3334418402778], + field_name: 'metric%$£&!{(]field', + influencers: [ + { + influencer_field_name: "singlequote'name", + influencer_field_values: ["contains a ' quote"], + }, + { + influencer_field_name: 'at@name', + influencer_field_values: ["contains a ' quote"], + }, + ], + "singlequote'name": ["contains a ' quote"], + 'at@name': ["contains a ' quote"], + earliest: '2019-02-08T00:00:00.000Z', + latest: '2019-02-08T23:59:59.999Z', + }; + + expect(getUrlForRecord(testUrl, testRecord)).toBe( + `dashboards#/view/351de820-f2bb-11ea-ab06-cb93221707e9?_a=(filters:!(),query:(language:kuery,query:'at@name:"contains%20a%20!'%20quote" AND singlequote!'name:"contains%20a%20!'%20quote"'))&_g=(filters:!(),time:(from:'2019-02-08T00:00:00.000Z',mode:absolute,to:'2019-02-08T23:59:59.999Z'))` + ); + }); + test('replaces tokens with nesting', () => { const testUrlApache: KibanaUrlConfig = { url_name: 'Raw data', From e0da44ab121313e3c9efb75955d2037f2951a2f2 Mon Sep 17 00:00:00 2001 From: Dima Arnautov Date: Mon, 14 Sep 2020 13:03:48 +0200 Subject: [PATCH 5/5] [ML] unit test --- .../plugins/ml/public/application/util/custom_url_utils.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/ml/public/application/util/custom_url_utils.test.ts b/x-pack/plugins/ml/public/application/util/custom_url_utils.test.ts index b7dd9524d22c9..6a5583ecbb8ac 100644 --- a/x-pack/plugins/ml/public/application/util/custom_url_utils.test.ts +++ b/x-pack/plugins/ml/public/application/util/custom_url_utils.test.ts @@ -458,7 +458,7 @@ describe('ML - custom URL utils', () => { test('return expected url for Security app', () => { const urlConfig = { - url_name: 'Hosts Overview by process name', + url_name: 'Hosts Details by process name', url_value: "security/hosts/ml-hosts/$host.name$?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))", };