From dbda7f4a52f4c35b9dd1df517803de5808bfe785 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Fri, 20 Sep 2019 16:46:01 +0200 Subject: [PATCH] Add numeric_type option for correct sort order on mixed date and date_nanos fields (#44212) (#46212) * Implement getSortForSearchSource for add-on of 'numeric_type' to the ES request. Then sorting on a field that can be of date or date_nanos type works correctly * Add functional test --- .../stubbed_logstash_index_pattern.js | 1 + .../public/discover/controllers/discover.js | 8 +- .../lib/get_sort_for_search_source.ts | 45 +++++++++ .../discover/embeddable/search_embeddable.ts | 14 +-- .../apps/discover/_date_nanos_mixed.js | 54 +++++++++++ test/functional/apps/discover/index.js | 1 + .../es_archiver/date_nanos_mixed/data.json | 95 +++++++++++++++++++ .../date_nanos_mixed/mappings.json | 43 +++++++++ 8 files changed, 252 insertions(+), 9 deletions(-) create mode 100644 src/legacy/core_plugins/kibana/public/discover/doc_table/lib/get_sort_for_search_source.ts create mode 100644 test/functional/apps/discover/_date_nanos_mixed.js create mode 100644 test/functional/fixtures/es_archiver/date_nanos_mixed/data.json create mode 100644 test/functional/fixtures/es_archiver/date_nanos_mixed/mappings.json diff --git a/src/fixtures/stubbed_logstash_index_pattern.js b/src/fixtures/stubbed_logstash_index_pattern.js index dd0381c440bd6..effb48a43a081 100644 --- a/src/fixtures/stubbed_logstash_index_pattern.js +++ b/src/fixtures/stubbed_logstash_index_pattern.js @@ -42,6 +42,7 @@ export default function stubbedLogstashIndexPatternService(Private) { const indexPattern = new StubIndexPattern('logstash-*', cfg => cfg, 'time', fields); indexPattern.id = 'logstash-*'; + indexPattern.isTimeNanosBased = () => false; return indexPattern; diff --git a/src/legacy/core_plugins/kibana/public/discover/controllers/discover.js b/src/legacy/core_plugins/kibana/public/discover/controllers/discover.js index a184f6c9d4ff8..fbd86b00fa5c1 100644 --- a/src/legacy/core_plugins/kibana/public/discover/controllers/discover.js +++ b/src/legacy/core_plugins/kibana/public/discover/controllers/discover.js @@ -29,6 +29,7 @@ import dateMath from '@elastic/datemath'; // doc table import '../doc_table'; import { getSort } from '../doc_table/lib/get_sort'; +import { getSortForSearchSource } from '../doc_table/lib/get_sort_for_search_source'; import * as columnActions from '../doc_table/actions/columns'; import * as filterActions from '../doc_table/actions/filter'; @@ -479,7 +480,7 @@ function discoverController( const { searchFields, selectFields } = await getSharingDataFields(); searchSource.setField('fields', searchFields); - searchSource.setField('sort', getSort($state.sort, $scope.indexPattern)); + searchSource.setField('sort', getSortForSearchSource($state.sort, $scope.indexPattern)); searchSource.setField('highlight', null); searchSource.setField('highlightAll', null); searchSource.setField('aggs', null); @@ -879,9 +880,10 @@ function discoverController( }; $scope.updateDataSource = Promise.method(function updateDataSource() { - $scope.searchSource + const { indexPattern, searchSource } = $scope; + searchSource .setField('size', $scope.opts.sampleSize) - .setField('sort', getSort($state.sort, $scope.indexPattern)) + .setField('sort', getSortForSearchSource($state.sort, indexPattern)) .setField('query', !$state.query ? null : $state.query) .setField('filter', queryFilter.getFilters()); }); diff --git a/src/legacy/core_plugins/kibana/public/discover/doc_table/lib/get_sort_for_search_source.ts b/src/legacy/core_plugins/kibana/public/discover/doc_table/lib/get_sort_for_search_source.ts new file mode 100644 index 0000000000000..95adc20d89f81 --- /dev/null +++ b/src/legacy/core_plugins/kibana/public/discover/doc_table/lib/get_sort_for_search_source.ts @@ -0,0 +1,45 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { IndexPattern } from 'ui/index_patterns'; +import { SortOrder } from '../components/table_header/helpers'; +import { getSort } from './get_sort'; + +/** + * prepares sort for search source, that's sending the request to ES + * handles the special case when there's sorting by date_nanos typed fields + * the addon of the numeric_type guarantees the right sort order + * when there are indices with date and indices with date_nanos field + */ +export function getSortForSearchSource(sort?: SortOrder[], indexPattern?: IndexPattern) { + if (!sort || !indexPattern) { + return []; + } + const { timeFieldName } = indexPattern; + return getSort(sort, indexPattern).map((sortPair: Record) => { + if (indexPattern.isTimeNanosBased() && timeFieldName && sortPair[timeFieldName]) { + return { + [timeFieldName]: { + order: sortPair[timeFieldName], + numeric_type: 'date_nanos', + }, + }; + } + return sortPair; + }); +} diff --git a/src/legacy/core_plugins/kibana/public/discover/embeddable/search_embeddable.ts b/src/legacy/core_plugins/kibana/public/discover/embeddable/search_embeddable.ts index e307d5da7ebe8..9d862a1c36b70 100644 --- a/src/legacy/core_plugins/kibana/public/discover/embeddable/search_embeddable.ts +++ b/src/legacy/core_plugins/kibana/public/discover/embeddable/search_embeddable.ts @@ -26,7 +26,7 @@ import { getRequestInspectorStats, getResponseInspectorStats, } from 'ui/courier/utils/courier_inspector_utils'; -import { StaticIndexPattern } from 'ui/index_patterns'; +import { IndexPattern } from 'ui/index_patterns'; import { RequestAdapter } from 'ui/inspector/adapters'; import { Adapters } from 'ui/inspector/types'; import { Subscription } from 'rxjs'; @@ -48,8 +48,8 @@ import * as columnActions from '../doc_table/actions/columns'; import { SavedSearch } from '../types'; import searchTemplate from './search_template.html'; import { ISearchEmbeddable, SearchInput, SearchOutput } from './types'; -import { getSort } from '../doc_table/lib/get_sort'; import { SortOrder } from '../doc_table/components/table_header/helpers'; +import { getSortForSearchSource } from '../doc_table/lib/get_sort_for_search_source'; const config = chrome.getUiSettingsClient(); @@ -65,7 +65,7 @@ interface SearchScope extends ng.IScope { moveColumn?: (column: string, index: number) => void; filter?: (field: { name: string; scripted: boolean }, value: string[], operator: string) => void; hits?: any[]; - indexPattern?: StaticIndexPattern; + indexPattern?: IndexPattern; totalHitCount?: number; isLoading?: boolean; } @@ -87,7 +87,7 @@ interface SearchEmbeddableConfig { $compile: ng.ICompileService; savedSearch: SavedSearch; editUrl: string; - indexPatterns?: StaticIndexPattern[]; + indexPatterns?: IndexPattern[]; editable: boolean; queryFilter: unknown; } @@ -163,7 +163,6 @@ export class SearchEmbeddable extends Embeddable /** * * @param {Element} domNode - * @param {ContainerState} containerState */ public render(domNode: HTMLElement) { if (!this.searchScope) { @@ -275,7 +274,10 @@ export class SearchEmbeddable extends Embeddable searchSource.cancelQueued(); searchSource.setField('size', config.get('discover:sampleSize')); - searchSource.setField('sort', getSort(this.searchScope.sort, this.searchScope.indexPattern)); + searchSource.setField( + 'sort', + getSortForSearchSource(this.searchScope.sort, this.searchScope.indexPattern) + ); // Log request to inspector this.inspectorAdaptors.requests.reset(); diff --git a/test/functional/apps/discover/_date_nanos_mixed.js b/test/functional/apps/discover/_date_nanos_mixed.js new file mode 100644 index 0000000000000..8c9a7eb4d5f13 --- /dev/null +++ b/test/functional/apps/discover/_date_nanos_mixed.js @@ -0,0 +1,54 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import expect from '@kbn/expect'; + +export default function ({ getService, getPageObjects }) { + const esArchiver = getService('esArchiver'); + const PageObjects = getPageObjects(['common', 'timePicker', 'discover']); + const kibanaServer = getService('kibanaServer'); + const fromTime = '2019-01-01 00:00:00.000'; + const toTime = '2019-01-01 23:59:59.999'; + + describe('date_nanos_mixed', function () { + + before(async function () { + await esArchiver.loadIfNeeded('date_nanos_mixed'); + await kibanaServer.uiSettings.replace({ 'defaultIndex': 'timestamp-*' }); + await PageObjects.common.navigateToApp('discover'); + await PageObjects.timePicker.setAbsoluteRange(fromTime, toTime); + }); + + after(function unloadMakelogs() { + return esArchiver.unload('date_nanos_mixed'); + }); + + it('shows a list of records of indices with date & date_nanos fields in the right order', async function () { + const rowData1 = await PageObjects.discover.getDocTableIndex(1); + expect(rowData1.startsWith('Jan 1, 2019 @ 12:10:30.124000000')).to.be.ok(); + const rowData2 = await PageObjects.discover.getDocTableIndex(3); + expect(rowData2.startsWith('Jan 1, 2019 @ 12:10:30.123498765')).to.be.ok(); + const rowData3 = await PageObjects.discover.getDocTableIndex(5); + expect(rowData3.startsWith('Jan 1, 2019 @ 12:10:30.123456789')).to.be.ok(); + const rowData4 = await PageObjects.discover.getDocTableIndex(7); + expect(rowData4.startsWith('Jan 1, 2019 @ 12:10:30.123000000')).to.be.ok(); + }); + }); + +} diff --git a/test/functional/apps/discover/index.js b/test/functional/apps/discover/index.js index 9e4430ca71c1f..902490bebd1ac 100644 --- a/test/functional/apps/discover/index.js +++ b/test/functional/apps/discover/index.js @@ -43,5 +43,6 @@ export default function ({ getService, loadTestFile }) { loadTestFile(require.resolve('./_inspector')); loadTestFile(require.resolve('./_doc_navigation')); loadTestFile(require.resolve('./_date_nanos')); + loadTestFile(require.resolve('./_date_nanos_mixed')); }); } diff --git a/test/functional/fixtures/es_archiver/date_nanos_mixed/data.json b/test/functional/fixtures/es_archiver/date_nanos_mixed/data.json new file mode 100644 index 0000000000000..abde15e2b08c4 --- /dev/null +++ b/test/functional/fixtures/es_archiver/date_nanos_mixed/data.json @@ -0,0 +1,95 @@ +{ + "type": "doc", + "value": { + "id": "index-pattern:timestamp-*", + "index": ".kibana", + "source": { + "index-pattern": { + "fields": "[{\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"esTypes\":[\"_type\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"timestamp\",\"type\":\"date\",\"esTypes\":[\"date\",\"date_nanos\"],\"count\":2,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]", + "timeFieldName": "timestamp", + "title": "timestamp-*", + "fieldFormatMap": "{\"timestamp\":{\"id\":\"date_nanos\"}}" + }, + "type": "index-pattern" + }, + "type": "_doc" + } +} + +{ + "type": "doc", + "value": { + "id": "search:82116b30-d407-11e9-8004-932185690e7b", + "index": ".kibana", + "source": { + "search": { + "columns": [ + "_source" + ], + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"highlightAll\":true,\"version\":true,\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[{\"$state\":{\"store\":\"appState\"},\"meta\":{\"alias\":null,\"disabled\":false,\"key\":\"number\",\"negate\":false,\"params\":{\"query\":123},\"type\":\"phrase\",\"value\":\"123\",\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match\":{\"number\":{\"query\":123,\"type\":\"phrase\"}}}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + }, + "sort": [ + [ + "@timestamp", + "desc" + ] + ], + "title": "New Saved Search", + "version": 1 + }, + "type": "search" + }, + "type": "_doc" + } +} + +{ + "type": "doc", + "value": { + "id": "2", + "index": "timestamp-millis", + "source": { + "timestamp": "2019-01-01T12:10:30.124Z" + }, + "type": "_doc" + } +} + +{ + "type": "doc", + "value": { + "id": "1", + "index": "timestamp-millis", + "source": { + "timestamp": "2019-01-01T12:10:30.123Z" + }, + "type": "_doc" + } +} + +{ + "type": "doc", + "value": { + "id": "3", + "index": "timestamp-nanos", + "source": { + "timestamp": "2019-01-01T12:10:30.123456789Z" + }, + "type": "_doc" + } +} + +{ + "type": "doc", + "value": { + "id": "4", + "index": "timestamp-nanos", + "source": { + "timestamp": "2019-01-01T12:10:30.123498765Z" + }, + "type": "_doc" + } +} diff --git a/test/functional/fixtures/es_archiver/date_nanos_mixed/mappings.json b/test/functional/fixtures/es_archiver/date_nanos_mixed/mappings.json new file mode 100644 index 0000000000000..c62918abced58 --- /dev/null +++ b/test/functional/fixtures/es_archiver/date_nanos_mixed/mappings.json @@ -0,0 +1,43 @@ +{ + "type": "index", + "value": { + "aliases": { + }, + "index": "timestamp-millis", + "mappings": { + "properties": { + "timestamp": { + "type": "date" + } + } + }, + "settings": { + "index": { + "number_of_replicas": "1", + "number_of_shards": "1" + } + } + } +} + +{ + "type": "index", + "value": { + "aliases": { + }, + "index": "timestamp-nanos", + "mappings": { + "properties": { + "timestamp": { + "type": "date_nanos" + } + } + }, + "settings": { + "index": { + "number_of_replicas": "1", + "number_of_shards": "1" + } + } + } +}