From 9eda1edb12be6c68af0f5fa70a220cf3a3ed3772 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Wed, 12 Feb 2020 11:28:12 -0700 Subject: [PATCH 1/4] [Maps] Fix document source top hits split by scripted field --- .../es_search_source/es_search_source.js | 44 +++++++-- .../apps/maps/documents_source/top_hits.js | 95 +++++++++++-------- .../es_archives/maps/kibana/data.json | 56 +++++++++++ 3 files changed, 144 insertions(+), 51 deletions(-) diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js index b8644adddcf7e..3cb50e960411f 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js @@ -241,8 +241,21 @@ export class ESSearchSource extends AbstractESSource { }); } + _getField(indexPattern, fieldName) { + const field = indexPattern.fields.getByName(fieldName); + if (!field) { + throw new Error( + i18n.translate('xpack.maps.source.esSearch.fieldNotFoundMsg', { + defaultMessage: `Unable to find '{fieldName}'' in index-pattern 'indexPatternTitle'.`, + values: { fieldName, indexPatternTitle: indexPattern.title }, + }) + ); + } + return field; + } + async _getTopHits(layerName, searchFilters, registerCancelCallback) { - const { topHitsSplitField, topHitsSize } = this._descriptor; + const { topHitsSplitField: topHitsSplitFieldName, topHitsSize } = this._descriptor; const indexPattern = await this.getIndexPattern(); const geoField = await this._getGeoField(); @@ -279,20 +292,31 @@ export class ESSearchSource extends AbstractESSource { }; } + const cardinalityAgg = { precision_threshold: 1 }; + const termsAgg = { + size: DEFAULT_MAX_BUCKETS_LIMIT, + shard_size: DEFAULT_MAX_BUCKETS_LIMIT, + }; + const topHitsSplitField = this._getField(indexPattern, topHitsSplitFieldName); + if (topHitsSplitField.scripted) { + const script = { + source: topHitsSplitField.script, + lang: topHitsSplitField.lang, + }; + cardinalityAgg.script = script; + termsAgg.script = script; + } else { + cardinalityAgg.field = topHitsSplitFieldName; + termsAgg.field = topHitsSplitFieldName; + } + const searchSource = await this._makeSearchSource(searchFilters, 0); searchSource.setField('aggs', { totalEntities: { - cardinality: { - field: topHitsSplitField, - precision_threshold: 1, - }, + cardinality: cardinalityAgg, }, entitySplit: { - terms: { - field: topHitsSplitField, - size: DEFAULT_MAX_BUCKETS_LIMIT, - shard_size: DEFAULT_MAX_BUCKETS_LIMIT, - }, + terms: termsAgg, aggs: { entityHits: { top_hits: topHits, diff --git a/x-pack/test/functional/apps/maps/documents_source/top_hits.js b/x-pack/test/functional/apps/maps/documents_source/top_hits.js index 65b008e3f07c9..59a4bea27ebab 100644 --- a/x-pack/test/functional/apps/maps/documents_source/top_hits.js +++ b/x-pack/test/functional/apps/maps/documents_source/top_hits.js @@ -13,60 +13,73 @@ export default function({ getPageObjects, getService }) { const inspector = getService('inspector'); const find = getService('find'); - describe('top hits', () => { - before(async () => { - await PageObjects.maps.loadSavedMap('document example top hits'); - }); - - it('should not fetch any search hits', async () => { - await inspector.open(); - await inspector.openInspectorRequestsView(); - const requestStats = await inspector.getTableData(); - const hits = PageObjects.maps.getInspectorStatRowHit(requestStats, 'Hits'); - expect(hits).to.equal('0'); // aggregation requests do not return any documents - }); - - it('should display top hits per entity', async () => { - const mapboxStyle = await PageObjects.maps.getMapboxStyle(); - expect(mapboxStyle.sources[VECTOR_SOURCE_ID].data.features.length).to.equal(10); - }); - - describe('configuration', () => { + describe('geo top hits', () => { + describe('split on string field', () => { before(async () => { - await PageObjects.maps.openLayerPanel('logstash'); - // Can not use testSubjects because data-test-subj is placed range input and number input - const sizeInput = await find.byCssSelector( - `input[data-test-subj="layerPanelTopHitsSize"][type='number']` - ); - await sizeInput.click(); - await sizeInput.clearValue(); - await sizeInput.type('3'); - await PageObjects.maps.waitForLayersToLoad(); + await PageObjects.maps.loadSavedMap('document example top hits'); }); - after(async () => { - await PageObjects.maps.closeLayerPanel(); + it('should not fetch any search hits', async () => { + await inspector.open(); + await inspector.openInspectorRequestsView(); + const requestStats = await inspector.getTableData(); + const hits = PageObjects.maps.getInspectorStatRowHit(requestStats, 'Hits'); + expect(hits).to.equal('0'); // aggregation requests do not return any documents }); - it('should update top hits when configation changes', async () => { + it('should display top hits per entity', async () => { const mapboxStyle = await PageObjects.maps.getMapboxStyle(); - expect(mapboxStyle.sources[VECTOR_SOURCE_ID].data.features.length).to.equal(15); + expect(mapboxStyle.sources[VECTOR_SOURCE_ID].data.features.length).to.equal(10); }); - }); - describe('query', () => { - before(async () => { - await PageObjects.maps.setAndSubmitQuery('machine.os.raw : "win 8"'); + describe('configuration', () => { + before(async () => { + await PageObjects.maps.openLayerPanel('logstash'); + // Can not use testSubjects because data-test-subj is placed range input and number input + const sizeInput = await find.byCssSelector( + `input[data-test-subj="layerPanelTopHitsSize"][type='number']` + ); + await sizeInput.click(); + await sizeInput.clearValue(); + await sizeInput.type('3'); + await PageObjects.maps.waitForLayersToLoad(); + }); + + after(async () => { + await PageObjects.maps.closeLayerPanel(); + }); + + it('should update top hits when configation changes', async () => { + const mapboxStyle = await PageObjects.maps.getMapboxStyle(); + expect(mapboxStyle.sources[VECTOR_SOURCE_ID].data.features.length).to.equal(15); + }); + }); + + describe('query', () => { + before(async () => { + await PageObjects.maps.setAndSubmitQuery('machine.os.raw : "win 8"'); + }); + + after(async () => { + await PageObjects.maps.setAndSubmitQuery(''); + }); + + it('should apply query to top hits request', async () => { + await PageObjects.maps.setAndSubmitQuery('machine.os.raw : "win 8"'); + const mapboxStyle = await PageObjects.maps.getMapboxStyle(); + expect(mapboxStyle.sources[VECTOR_SOURCE_ID].data.features.length).to.equal(2); + }); }); + }); - after(async () => { - await PageObjects.maps.setAndSubmitQuery(''); + describe('split on scripted field', () => { + before(async () => { + await PageObjects.maps.loadSavedMap('document example top hits split with scripted field'); }); - it('should apply query to top hits request', async () => { - await PageObjects.maps.setAndSubmitQuery('machine.os.raw : "win 8"'); + it('should display top hits per entity', async () => { const mapboxStyle = await PageObjects.maps.getMapboxStyle(); - expect(mapboxStyle.sources[VECTOR_SOURCE_ID].data.features.length).to.equal(2); + expect(mapboxStyle.sources[VECTOR_SOURCE_ID].data.features.length).to.equal(24); }); }); }); diff --git a/x-pack/test/functional/es_archives/maps/kibana/data.json b/x-pack/test/functional/es_archives/maps/kibana/data.json index e24734841bf55..e50ec593cc990 100644 --- a/x-pack/test/functional/es_archives/maps/kibana/data.json +++ b/x-pack/test/functional/es_archives/maps/kibana/data.json @@ -278,6 +278,62 @@ } } +{ + "type": "doc", + "value": { + "id": "map:4ea1e4f0-4dba-11ea-b554-4ba0def79f86", + "index": ".kibana", + "source": { + "map": { + "title" : "document example top hits split with scripted field", + "description" : "", + "mapStateJSON" : "{\"zoom\":4.1,\"center\":{\"lon\":-100.61091,\"lat\":33.23887},\"timeFilters\":{\"from\":\"2015-09-20T00:00:00.000Z\",\"to\":\"2015-09-24T01:00:00.000Z\"},\"refreshConfig\":{\"isPaused\":true,\"interval\":1000},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[]}", + "layerListJSON" : "[{\"id\":\"0hmz5\",\"sourceDescriptor\":{\"type\":\"EMS_TMS\",\"id\":\"road_map\"},\"visible\":true,\"temporary\":false,\"style\":{\"type\":\"TILE\",\"properties\":{}},\"type\":\"VECTOR_TILE\",\"minZoom\":0,\"maxZoom\":24},{\"id\":\"z52lq\",\"label\":\"logstash\",\"minZoom\":0,\"maxZoom\":24,\"sourceDescriptor\":{\"id\":\"e1a5e1a6-676c-4a89-8ea9-0d91d64b73c6\",\"type\":\"ES_SEARCH\",\"geoField\":\"geo.coordinates\",\"limit\":2048,\"filterByMapBounds\":true,\"showTooltip\":true,\"tooltipProperties\":[],\"useTopHits\":true,\"topHitsSplitField\":\"hour_of_day\",\"topHitsSize\":1,\"sortField\":\"@timestamp\",\"sortOrder\":\"desc\",\"applyGlobalQuery\":true,\"indexPatternRefName\":\"layer_1_source_index_pattern\"},\"visible\":true,\"temporary\":false,\"style\":{\"type\":\"VECTOR\",\"properties\":{\"fillColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#e6194b\"}},\"lineColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#FFFFFF\"}},\"lineWidth\":{\"type\":\"STATIC\",\"options\":{\"size\":1}},\"iconSize\":{\"type\":\"STATIC\",\"options\":{\"size\":10}},\"symbolizeAs\":{\"options\":{\"value\":\"circle\"}},\"icon\":{\"type\":\"STATIC\",\"options\":{\"value\":\"airfield\"}}},\"previousStyle\":null},\"type\":\"VECTOR\"}]", + "uiStateJSON" : "{\"isLayerTOCOpen\":true,\"openTOCDetails\":[]}", + "bounds" : { + "type" : "Polygon", + "coordinates" : [ + [ + [ + -141.61334, + 47.30762 + ], + [ + -141.61334, + 16.49119 + ], + [ + -59.60848, + 16.49119 + ], + [ + -59.60848, + 47.30762 + ], + [ + -141.61334, + 47.30762 + ] + ] + ] + } + }, + "type" : "map", + "references" : [ + { + "name" : "layer_1_source_index_pattern", + "type" : "index-pattern", + "id" : "c698b940-e149-11e8-a35a-370a8516603a" + } + ], + "migrationVersion" : { + "map" : "7.7.0" + }, + "updated_at" : "2020-02-12T17:08:36.671Z" + } + } +} + { "type": "doc", "value": { From e05ac8ed09164aad99d4b83f9bee5c01b32d6fe5 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Wed, 12 Feb 2020 12:58:03 -0700 Subject: [PATCH 2/4] fix i18n message --- .../public/layers/sources/es_search_source/es_search_source.js | 2 +- .../public/layers/styles/vector/components/_stop_input.scss | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 x-pack/legacy/plugins/maps/public/layers/styles/vector/components/_stop_input.scss diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js index 3cb50e960411f..332b7e7a6c423 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js @@ -246,7 +246,7 @@ export class ESSearchSource extends AbstractESSource { if (!field) { throw new Error( i18n.translate('xpack.maps.source.esSearch.fieldNotFoundMsg', { - defaultMessage: `Unable to find '{fieldName}'' in index-pattern 'indexPatternTitle'.`, + defaultMessage: `Unable to find '{fieldName}' in index-pattern '{indexPatternTitle}'.`, values: { fieldName, indexPatternTitle: indexPattern.title }, }) ); diff --git a/x-pack/legacy/plugins/maps/public/layers/styles/vector/components/_stop_input.scss b/x-pack/legacy/plugins/maps/public/layers/styles/vector/components/_stop_input.scss new file mode 100644 index 0000000000000..037578fcf39e8 --- /dev/null +++ b/x-pack/legacy/plugins/maps/public/layers/styles/vector/components/_stop_input.scss @@ -0,0 +1,3 @@ +.mapStopInputComboBox { + max-width: $euiSizeXL * 4; +} From 33991ca2b9d5d44016e8122cb173fbf6b669eed4 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Wed, 12 Feb 2020 14:45:06 -0700 Subject: [PATCH 3/4] review feedback --- .../es_search_source/es_search_source.js | 55 ++++++++++--------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js index 332b7e7a6c423..93ef40162a584 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js @@ -29,6 +29,31 @@ import { loadIndexSettings } from './load_index_settings'; import { DEFAULT_FILTER_BY_MAP_BOUNDS } from './constants'; import { ESDocField } from '../../fields/es_doc_field'; +function getField(indexPattern, fieldName) { + const field = indexPattern.fields.getByName(fieldName); + if (!field) { + throw new Error( + i18n.translate('xpack.maps.source.esSearch.fieldNotFoundMsg', { + defaultMessage: `Unable to find '{fieldName}' in index-pattern '{indexPatternTitle}'.`, + values: { fieldName, indexPatternTitle: indexPattern.title }, + }) + ); + } + return field; +} + +function addFieldToDSL(dsl, field) { + return !field.scripted + ? { ...dsl, field: field.name } + : { + ...dsl, + script: { + source: field.script, + lang: field.lang, + }, + }; +} + export class ESSearchSource extends AbstractESSource { static type = ES_SEARCH; static title = i18n.translate('xpack.maps.source.esSearchTitle', { @@ -241,19 +266,6 @@ export class ESSearchSource extends AbstractESSource { }); } - _getField(indexPattern, fieldName) { - const field = indexPattern.fields.getByName(fieldName); - if (!field) { - throw new Error( - i18n.translate('xpack.maps.source.esSearch.fieldNotFoundMsg', { - defaultMessage: `Unable to find '{fieldName}' in index-pattern '{indexPatternTitle}'.`, - values: { fieldName, indexPatternTitle: indexPattern.title }, - }) - ); - } - return field; - } - async _getTopHits(layerName, searchFilters, registerCancelCallback) { const { topHitsSplitField: topHitsSplitFieldName, topHitsSize } = this._descriptor; @@ -292,31 +304,20 @@ export class ESSearchSource extends AbstractESSource { }; } + const topHitsSplitField = getField(indexPattern, topHitsSplitFieldName); const cardinalityAgg = { precision_threshold: 1 }; const termsAgg = { size: DEFAULT_MAX_BUCKETS_LIMIT, shard_size: DEFAULT_MAX_BUCKETS_LIMIT, }; - const topHitsSplitField = this._getField(indexPattern, topHitsSplitFieldName); - if (topHitsSplitField.scripted) { - const script = { - source: topHitsSplitField.script, - lang: topHitsSplitField.lang, - }; - cardinalityAgg.script = script; - termsAgg.script = script; - } else { - cardinalityAgg.field = topHitsSplitFieldName; - termsAgg.field = topHitsSplitFieldName; - } const searchSource = await this._makeSearchSource(searchFilters, 0); searchSource.setField('aggs', { totalEntities: { - cardinality: cardinalityAgg, + cardinality: addFieldToDSL(cardinalityAgg, topHitsSplitField), }, entitySplit: { - terms: termsAgg, + terms: addFieldToDSL(termsAgg, topHitsSplitField), aggs: { entityHits: { top_hits: topHits, From 240395b3f58942fda16fee7f273a83e54fff79bf Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Wed, 12 Feb 2020 14:47:00 -0700 Subject: [PATCH 4/4] remove unneeded scss file --- .../public/layers/styles/vector/components/_stop_input.scss | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 x-pack/legacy/plugins/maps/public/layers/styles/vector/components/_stop_input.scss diff --git a/x-pack/legacy/plugins/maps/public/layers/styles/vector/components/_stop_input.scss b/x-pack/legacy/plugins/maps/public/layers/styles/vector/components/_stop_input.scss deleted file mode 100644 index 037578fcf39e8..0000000000000 --- a/x-pack/legacy/plugins/maps/public/layers/styles/vector/components/_stop_input.scss +++ /dev/null @@ -1,3 +0,0 @@ -.mapStopInputComboBox { - max-width: $euiSizeXL * 4; -}