From ac150da49def86136bfa3bb523ac28c37c90ae18 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Wed, 11 Nov 2020 06:36:04 -0500 Subject: [PATCH 001/104] [Lens] Functional tests for drag and drop (#82796) --- test/functional/services/common/browser.ts | 74 ++++++++++++++-- .../lens/public/drag_drop/drag_drop.test.tsx | 24 ++++-- .../lens/public/drag_drop/drag_drop.tsx | 12 ++- .../functional/apps/lens/drag_and_drop.ts | 83 ++++++++++++++++++ x-pack/test/functional/apps/lens/index.ts | 1 + .../test/functional/apps/lens/smokescreen.ts | 7 +- .../test/functional/page_objects/lens_page.ts | 85 +++++++++++++++++-- 7 files changed, 258 insertions(+), 28 deletions(-) create mode 100644 x-pack/test/functional/apps/lens/drag_and_drop.ts diff --git a/test/functional/services/common/browser.ts b/test/functional/services/common/browser.ts index f5fb54c72177f..b3b7fd32eae19 100644 --- a/test/functional/services/common/browser.ts +++ b/test/functional/services/common/browser.ts @@ -216,13 +216,17 @@ export async function BrowserProvider({ getService }: FtrProviderContext) { * Does a drag-and-drop action from one point to another * https://seleniumhq.github.io/selenium/docs/api/javascript/module/selenium-webdriver/lib/input_exports_Actions.html#dragAndDrop * - * @param {{element: WebElementWrapper | {x: number, y: number}, offset: {x: number, y: number}}} from - * @param {{element: WebElementWrapper | {x: number, y: number}, offset: {x: number, y: number}}} to * @return {Promise} */ public async dragAndDrop( - from: { offset?: { x: any; y: any }; location: any }, - to: { offset?: { x: any; y: any }; location: any } + from: { + location: WebElementWrapper | { x?: number; y?: number }; + offset?: { x?: number; y?: number }; + }, + to: { + location: WebElementWrapper | { x?: number; y?: number }; + offset?: { x?: number; y?: number }; + } ) { // The offset should be specified in pixels relative to the center of the element's bounding box const getW3CPoint = (data: any) => { @@ -230,7 +234,11 @@ export async function BrowserProvider({ getService }: FtrProviderContext) { data.offset = {}; } return data.location instanceof WebElementWrapper - ? { x: data.offset.x || 0, y: data.offset.y || 0, origin: data.location._webElement } + ? { + x: data.offset.x || 0, + y: data.offset.y || 0, + origin: data.location._webElement, + } : { x: data.location.x, y: data.location.y, origin: Origin.POINTER }; }; @@ -240,6 +248,62 @@ export async function BrowserProvider({ getService }: FtrProviderContext) { return await this.getActions().move(startPoint).press().move(endPoint).release().perform(); } + /** + * Performs drag and drop for html5 native drag and drop implementation + * There's a bug in Chromedriver for html5 dnd that doesn't allow to use the method `dragAndDrop` defined above + * https://github.com/SeleniumHQ/selenium/issues/6235 + * This implementation simulates user's action by calling the drag and drop specific events directly. + * + * @param {string} from html selector + * @param {string} to html selector + * @return {Promise} + */ + public async html5DragAndDrop(from: string, to: string) { + await this.execute( + ` + function createEvent(typeOfEvent) { + const event = document.createEvent("CustomEvent"); + event.initCustomEvent(typeOfEvent, true, true, null); + event.dataTransfer = { + data: {}, + setData: function (key, value) { + this.data[key] = value; + }, + getData: function (key) { + return this.data[key]; + } + }; + return event; + } + function dispatchEvent(element, event, transferData) { + if (transferData !== undefined) { + event.dataTransfer = transferData; + } + if (element.dispatchEvent) { + element.dispatchEvent(event); + } else if (element.fireEvent) { + element.fireEvent("on" + event.type, event); + } + } + + const origin = document.querySelector(arguments[0]); + const target = document.querySelector(arguments[1]); + + const dragStartEvent = createEvent('dragstart'); + dispatchEvent(origin, dragStartEvent); + + setTimeout(() => { + const dropEvent = createEvent('drop'); + dispatchEvent(target, dropEvent, dragStartEvent.dataTransfer); + const dragEndEvent = createEvent('dragend'); + dispatchEvent(origin, dragEndEvent, dropEvent.dataTransfer); + }, 50); + `, + from, + to + ); + } + /** * Reloads the current browser window/frame. * https://seleniumhq.github.io/selenium/docs/api/javascript/module/selenium-webdriver/lib/webdriver_exports_Navigation.html#refresh diff --git a/x-pack/plugins/lens/public/drag_drop/drag_drop.test.tsx b/x-pack/plugins/lens/public/drag_drop/drag_drop.test.tsx index 8d381dff351c9..07b489d29ad06 100644 --- a/x-pack/plugins/lens/public/drag_drop/drag_drop.test.tsx +++ b/x-pack/plugins/lens/public/drag_drop/drag_drop.test.tsx @@ -300,17 +300,27 @@ describe('DragDrop', () => { jest.runAllTimers(); component.find('[data-test-subj="lnsDragDrop-reorderableDrop"]').at(2).simulate('dragover'); - expect(component.find('[data-test-subj="lnsDragDrop"]').at(0).prop('style')).toEqual({}); - expect(component.find('[data-test-subj="lnsDragDrop"]').at(1).prop('style')).toEqual({ - transform: 'translateY(-8px)', + expect( + component.find('[data-test-subj="lnsDragDrop-reorderableDrag"]').at(0).prop('style') + ).toEqual({}); + expect( + component.find('[data-test-subj="lnsDragDrop-reorderableDrag"]').at(1).prop('style') + ).toEqual({ + transform: 'translateY(-40px)', }); - expect(component.find('[data-test-subj="lnsDragDrop"]').at(2).prop('style')).toEqual({ - transform: 'translateY(-8px)', + expect( + component.find('[data-test-subj="lnsDragDrop-reorderableDrag"]').at(2).prop('style') + ).toEqual({ + transform: 'translateY(-40px)', }); component.find('[data-test-subj="lnsDragDrop-reorderableDrop"]').at(2).simulate('dragleave'); - expect(component.find('[data-test-subj="lnsDragDrop"]').at(1).prop('style')).toEqual({}); - expect(component.find('[data-test-subj="lnsDragDrop"]').at(2).prop('style')).toEqual({}); + expect( + component.find('[data-test-subj="lnsDragDrop-reorderableDrag"]').at(1).prop('style') + ).toEqual({}); + expect( + component.find('[data-test-subj="lnsDragDrop-reorderableDrag"]').at(2).prop('style') + ).toEqual({}); }); test(`Dropping an item runs onDrop function`, () => { const preventDefault = jest.fn(); diff --git a/x-pack/plugins/lens/public/drag_drop/drag_drop.tsx b/x-pack/plugins/lens/public/drag_drop/drag_drop.tsx index c0e01ad93fe83..32facbf8e84a8 100644 --- a/x-pack/plugins/lens/public/drag_drop/drag_drop.tsx +++ b/x-pack/plugins/lens/public/drag_drop/drag_drop.tsx @@ -271,12 +271,12 @@ const DragDropInner = React.memo(function DragDropInner( dropTo={dropTo} label={label} className={className} + dataTestSubj={props['data-test-subj'] || 'lnsDragDrop'} draggingProps={{ className: classNames(children.props.className, classes), draggable, onDragEnd: dragEnd, onDragStart: dragStart, - dataTestSubj: props['data-test-subj'] || 'lnsDragDrop', isReorderDragging, }} dropProps={{ @@ -338,13 +338,13 @@ export const ReorderableDragDrop = ({ label, dropTo, className, + dataTestSubj, }: { draggingProps: { className: string; draggable: Props['draggable']; onDragEnd: (e: DroppableEvent) => void; onDragStart: (e: DroppableEvent) => void; - dataTestSubj: string; isReorderDragging: boolean; }; dropProps: { @@ -361,6 +361,7 @@ export const ReorderableDragDrop = ({ label: string; dropTo: DropToHandler; className?: string; + dataTestSubj: string; }) => { const { itemsInGroup, dragging, id, droppable } = dropProps; const { reorderState, setReorderState } = useContext(ReorderContext); @@ -378,7 +379,10 @@ export const ReorderableDragDrop = ({ ); return ( -
+
diff --git a/src/plugins/embeddable/public/public.api.md b/src/plugins/embeddable/public/public.api.md index 4406dded98547..6a2565edf2f67 100644 --- a/src/plugins/embeddable/public/public.api.md +++ b/src/plugins/embeddable/public/public.api.md @@ -80,6 +80,7 @@ import { TransportRequestParams } from '@elastic/elasticsearch/lib/Transport'; import { TransportRequestPromise } from '@elastic/elasticsearch/lib/Transport'; import { TypeOf } from '@kbn/config-schema'; import { UiComponent } from 'src/plugins/kibana_utils/public'; +import { UiStatsMetricType } from '@kbn/analytics'; import { UnregisterCallback } from 'history'; import { UserProvidedValues } from 'src/core/server/types'; From a50960e2fdcaf989b96b4bfb1d79b4aa99fba1c6 Mon Sep 17 00:00:00 2001 From: Marta Bondyra Date: Wed, 11 Nov 2020 13:26:53 +0100 Subject: [PATCH 003/104] [Lens] Implement counter rate expression (#82948) --- .../common/expression_functions/index.ts | 1 + .../{specs => }/series_calculation_helpers.ts | 2 +- .../specs/cumulative_sum.ts | 2 +- .../expression_functions/specs/derivative.ts | 2 +- .../specs/moving_average.ts | 2 +- .../counter_rate/counter_rate.test.ts | 409 ++++++++++++++++++ .../counter_rate/index.ts | 142 ++++++ .../indexpattern_datasource/time_scale.ts | 35 +- 8 files changed, 564 insertions(+), 31 deletions(-) rename src/plugins/expressions/common/expression_functions/{specs => }/series_calculation_helpers.ts (97%) create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/counter_rate/counter_rate.test.ts create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/counter_rate/index.ts diff --git a/src/plugins/expressions/common/expression_functions/index.ts b/src/plugins/expressions/common/expression_functions/index.ts index b29e6b78b8f4d..094fbe83efd22 100644 --- a/src/plugins/expressions/common/expression_functions/index.ts +++ b/src/plugins/expressions/common/expression_functions/index.ts @@ -22,3 +22,4 @@ export * from './arguments'; export * from './expression_function_parameter'; export * from './expression_function'; export * from './specs'; +export * from './series_calculation_helpers'; diff --git a/src/plugins/expressions/common/expression_functions/specs/series_calculation_helpers.ts b/src/plugins/expressions/common/expression_functions/series_calculation_helpers.ts similarity index 97% rename from src/plugins/expressions/common/expression_functions/specs/series_calculation_helpers.ts rename to src/plugins/expressions/common/expression_functions/series_calculation_helpers.ts index 8ba9d527d4c59..99ad2098ab10a 100644 --- a/src/plugins/expressions/common/expression_functions/specs/series_calculation_helpers.ts +++ b/src/plugins/expressions/common/expression_functions/series_calculation_helpers.ts @@ -18,7 +18,7 @@ */ import { i18n } from '@kbn/i18n'; -import { Datatable, DatatableRow } from '../../expression_types'; +import { Datatable, DatatableRow } from '../expression_types'; /** * Returns a string identifying the group of a row by a list of columns to group by diff --git a/src/plugins/expressions/common/expression_functions/specs/cumulative_sum.ts b/src/plugins/expressions/common/expression_functions/specs/cumulative_sum.ts index 672abadd3c016..0d9547f70dd3b 100644 --- a/src/plugins/expressions/common/expression_functions/specs/cumulative_sum.ts +++ b/src/plugins/expressions/common/expression_functions/specs/cumulative_sum.ts @@ -20,7 +20,7 @@ import { i18n } from '@kbn/i18n'; import { ExpressionFunctionDefinition } from '../types'; import { Datatable } from '../../expression_types'; -import { buildResultColumns, getBucketIdentifier } from './series_calculation_helpers'; +import { buildResultColumns, getBucketIdentifier } from '../series_calculation_helpers'; export interface CumulativeSumArgs { by?: string[]; diff --git a/src/plugins/expressions/common/expression_functions/specs/derivative.ts b/src/plugins/expressions/common/expression_functions/specs/derivative.ts index 44ac198e2d17c..320a254bf94a9 100644 --- a/src/plugins/expressions/common/expression_functions/specs/derivative.ts +++ b/src/plugins/expressions/common/expression_functions/specs/derivative.ts @@ -20,7 +20,7 @@ import { i18n } from '@kbn/i18n'; import { ExpressionFunctionDefinition } from '../types'; import { Datatable } from '../../expression_types'; -import { buildResultColumns, getBucketIdentifier } from './series_calculation_helpers'; +import { buildResultColumns, getBucketIdentifier } from '../series_calculation_helpers'; export interface DerivativeArgs { by?: string[]; diff --git a/src/plugins/expressions/common/expression_functions/specs/moving_average.ts b/src/plugins/expressions/common/expression_functions/specs/moving_average.ts index 00a4d8c45839e..c4887e0240ec0 100644 --- a/src/plugins/expressions/common/expression_functions/specs/moving_average.ts +++ b/src/plugins/expressions/common/expression_functions/specs/moving_average.ts @@ -20,7 +20,7 @@ import { i18n } from '@kbn/i18n'; import { ExpressionFunctionDefinition } from '../types'; import { Datatable } from '../../expression_types'; -import { buildResultColumns, getBucketIdentifier } from './series_calculation_helpers'; +import { buildResultColumns, getBucketIdentifier } from '../series_calculation_helpers'; export interface MovingAverageArgs { by?: string[]; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/counter_rate/counter_rate.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/counter_rate/counter_rate.test.ts new file mode 100644 index 0000000000000..ccabb7147d2e4 --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/counter_rate/counter_rate.test.ts @@ -0,0 +1,409 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { counterRate, CounterRateArgs } from '../counter_rate'; + +import { Datatable } from 'src/plugins/expressions/public'; +import { functionWrapper } from 'src/plugins/expressions/common/expression_functions/specs/tests/utils'; + +describe('lens_counter_rate', () => { + const fn = functionWrapper(counterRate); + const runFn = (input: Datatable, args: CounterRateArgs) => fn(input, args) as Datatable; + + it('calculates counter rate', () => { + const result = runFn( + { + type: 'datatable', + columns: [{ id: 'val', name: 'val', meta: { type: 'number' } }], + rows: [{ val: 5 }, { val: 5 }, { val: 7 }, { val: 3 }, { val: 2 }], + }, + { inputColumnId: 'val', outputColumnId: 'output' } + ); + expect(result.columns).toContainEqual({ + id: 'output', + name: 'output', + meta: { type: 'number' }, + }); + expect(result.rows.map((row) => row.output)).toEqual([undefined, 0, 2, 3, 2]); + }); + + it('calculates counter rate with decreasing values in input', () => { + const result = runFn( + { + type: 'datatable', + columns: [{ id: 'val', name: 'val', meta: { type: 'number' } }], + rows: [{ val: 7 }, { val: 6 }, { val: 5 }, { val: 4 }], + }, + { inputColumnId: 'val', outputColumnId: 'output' } + ); + expect(result.columns).toContainEqual({ + id: 'output', + name: 'output', + meta: { type: 'number' }, + }); + expect(result.rows.map((row) => row.output)).toEqual([undefined, 6, 5, 4]); + }); + + it('skips null or undefined values until there is real data', () => { + const result = runFn( + { + type: 'datatable', + columns: [{ id: 'val', name: 'val', meta: { type: 'number' } }], + rows: [ + {}, + { val: null }, + { val: undefined }, + { val: 1 }, + { val: 2 }, + { val: undefined }, + { val: undefined }, + { val: 4 }, + { val: 8 }, + ], + }, + { inputColumnId: 'val', outputColumnId: 'output' } + ); + expect(result.columns).toContainEqual({ + id: 'output', + name: 'output', + meta: { type: 'number' }, + }); + expect(result.rows.map((row) => row.output)).toEqual([ + undefined, + undefined, + undefined, + undefined, + 2 - 1, + undefined, + undefined, + undefined, + 8 - 4, + ]); + }); + + it('treats 0 as real data', () => { + const result = runFn( + { + type: 'datatable', + columns: [{ id: 'val', name: 'val', meta: { type: 'number' } }], + rows: [ + {}, + { val: null }, + { val: undefined }, + { val: 1 }, + { val: 2 }, + { val: 0 }, + { val: undefined }, + { val: 0 }, + { val: undefined }, + { val: 0 }, + { val: 8 }, + { val: 0 }, + ], + }, + { inputColumnId: 'val', outputColumnId: 'output' } + ); + expect(result.rows.map((row) => row.output)).toEqual([ + undefined, + undefined, + undefined, + undefined, + 2 - 1, + 0, + undefined, + undefined, + undefined, + undefined, + 8 - 0, + 0, + ]); + }); + + it('calculates counter rate for multiple series', () => { + const result = runFn( + { + type: 'datatable', + columns: [ + { id: 'val', name: 'val', meta: { type: 'number' } }, + { id: 'split', name: 'split', meta: { type: 'string' } }, + ], + rows: [ + { val: 1, split: 'A' }, + { val: 2, split: 'B' }, + { val: 3, split: 'B' }, + { val: 4, split: 'A' }, + { val: 5, split: 'A' }, + { val: 6, split: 'A' }, + { val: 7, split: 'B' }, + { val: 8, split: 'B' }, + ], + }, + { inputColumnId: 'val', outputColumnId: 'output', by: ['split'] } + ); + + expect(result.rows.map((row) => row.output)).toEqual([ + undefined, + undefined, + 3 - 2, + 4 - 1, + 5 - 4, + 6 - 5, + 7 - 3, + 8 - 7, + ]); + }); + + it('treats missing split column as separate series', () => { + const result = runFn( + { + type: 'datatable', + columns: [ + { id: 'val', name: 'val', meta: { type: 'number' } }, + { id: 'split', name: 'split', meta: { type: 'string' } }, + ], + rows: [ + { val: 1, split: 'A' }, + { val: 2, split: 'B' }, + { val: 3 }, + { val: 4, split: 'A' }, + { val: 5 }, + { val: 6, split: 'A' }, + { val: 7, split: 'B' }, + { val: 8, split: 'B' }, + ], + }, + { inputColumnId: 'val', outputColumnId: 'output', by: ['split'] } + ); + expect(result.rows.map((row) => row.output)).toEqual([ + undefined, + undefined, + undefined, + 4 - 1, + 5 - 3, + 6 - 4, + 7 - 2, + 8 - 7, + ]); + }); + + it('treats null like undefined and empty string for split columns', () => { + const result = runFn( + { + type: 'datatable', + columns: [ + { id: 'val', name: 'val', meta: { type: 'number' } }, + { id: 'split', name: 'split', meta: { type: 'string' } }, + ], + rows: [ + { val: 1, split: 'A' }, + { val: 2, split: 'B' }, + { val: 3 }, + { val: 4, split: 'A' }, + { val: 5 }, + { val: 6, split: 'A' }, + { val: 7, split: null }, + { val: 8, split: 'B' }, + { val: 9, split: '' }, + ], + }, + { inputColumnId: 'val', outputColumnId: 'output', by: ['split'] } + ); + expect(result.rows.map((row) => row.output)).toEqual([ + undefined, + undefined, + undefined, + 4 - 1, + 5 - 3, + 6 - 4, + 7 - 5, + 8 - 2, + 9 - 7, + ]); + }); + + it('calculates counter rate for multiple series by multiple split columns', () => { + const result = runFn( + { + type: 'datatable', + columns: [ + { id: 'val', name: 'val', meta: { type: 'number' } }, + { id: 'split', name: 'split', meta: { type: 'string' } }, + { id: 'split2', name: 'split2', meta: { type: 'string' } }, + ], + rows: [ + { val: 1, split: 'A', split2: 'C' }, + { val: 2, split: 'B', split2: 'C' }, + { val: 3, split2: 'C' }, + { val: 4, split: 'A', split2: 'C' }, + { val: 5 }, + { val: 6, split: 'A', split2: 'D' }, + { val: 7, split: 'B', split2: 'D' }, + { val: 8, split: 'B', split2: 'D' }, + ], + }, + { inputColumnId: 'val', outputColumnId: 'output', by: ['split', 'split2'] } + ); + expect(result.rows.map((row) => row.output)).toEqual([ + undefined, + undefined, + undefined, + 4 - 1, + undefined, + undefined, + undefined, + 8 - 7, + ]); + }); + + it('splits separate series by the string representation of the cell values', () => { + const result = runFn( + { + type: 'datatable', + columns: [ + { id: 'val', name: 'val', meta: { type: 'number' } }, + { id: 'split', name: 'split', meta: { type: 'string' } }, + ], + rows: [ + { val: 1, split: { anObj: 3 } }, + { val: 2, split: { anotherObj: 5 } }, + { val: 10, split: 5 }, + { val: 11, split: '5' }, + ], + }, + { inputColumnId: 'val', outputColumnId: 'output', by: ['split'] } + ); + + expect(result.rows.map((row) => row.output)).toEqual([undefined, 2 - 1, undefined, 11 - 10]); + }); + + it('casts values to number before calculating counter rate', () => { + const result = runFn( + { + type: 'datatable', + columns: [{ id: 'val', name: 'val', meta: { type: 'number' } }], + rows: [{ val: 5 }, { val: '7' }, { val: '3' }, { val: 2 }], + }, + { inputColumnId: 'val', outputColumnId: 'output' } + ); + expect(result.rows.map((row) => row.output)).toEqual([undefined, 7 - 5, 3, 2]); + }); + + it('casts values to number before calculating counter rate for NaN like values', () => { + const result = runFn( + { + type: 'datatable', + columns: [{ id: 'val', name: 'val', meta: { type: 'number' } }], + rows: [{ val: 5 }, { val: '7' }, { val: {} }, { val: 2 }, { val: 5 }], + }, + { inputColumnId: 'val', outputColumnId: 'output' } + ); + expect(result.rows.map((row) => row.output)).toEqual([undefined, 7 - 5, NaN, 2, 5 - 2]); + }); + + it('copies over meta information from the source column', () => { + const result = runFn( + { + type: 'datatable', + columns: [ + { + id: 'val', + name: 'val', + meta: { + type: 'number', + + field: 'afield', + index: 'anindex', + params: { id: 'number', params: { pattern: '000' } }, + source: 'synthetic', + sourceParams: { + some: 'params', + }, + }, + }, + ], + rows: [{ val: 5 }], + }, + { inputColumnId: 'val', outputColumnId: 'output' } + ); + expect(result.columns).toContainEqual({ + id: 'output', + name: 'output', + meta: { + type: 'number', + + field: 'afield', + index: 'anindex', + params: { id: 'number', params: { pattern: '000' } }, + source: 'synthetic', + sourceParams: { + some: 'params', + }, + }, + }); + }); + + it('sets output name on output column if specified', () => { + const result = runFn( + { + type: 'datatable', + columns: [ + { + id: 'val', + name: 'val', + meta: { + type: 'number', + }, + }, + ], + rows: [{ val: 5 }], + }, + { inputColumnId: 'val', outputColumnId: 'output', outputColumnName: 'Output name' } + ); + expect(result.columns).toContainEqual({ + id: 'output', + name: 'Output name', + meta: { type: 'number' }, + }); + }); + + it('returns source table if input column does not exist', () => { + const input: Datatable = { + type: 'datatable', + columns: [ + { + id: 'val', + name: 'val', + meta: { + type: 'number', + }, + }, + ], + rows: [{ val: 5 }], + }; + expect(runFn(input, { inputColumnId: 'nonexisting', outputColumnId: 'output' })).toBe(input); + }); + + it('throws an error if output column exists already', () => { + expect(() => + runFn( + { + type: 'datatable', + columns: [ + { + id: 'val', + name: 'val', + meta: { + type: 'number', + }, + }, + ], + rows: [{ val: 5 }], + }, + { inputColumnId: 'val', outputColumnId: 'val' } + ) + ).toThrow(); + }); +}); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/counter_rate/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/counter_rate/index.ts new file mode 100644 index 0000000000000..45a8265ffdc4c --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/counter_rate/index.ts @@ -0,0 +1,142 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { i18n } from '@kbn/i18n'; +import { ExpressionFunctionDefinition, Datatable } from 'src/plugins/expressions/public'; +import { + getBucketIdentifier, + buildResultColumns, +} from '../../../../../../src/plugins/expressions/common'; + +export interface CounterRateArgs { + by?: string[]; + inputColumnId: string; + outputColumnId: string; + outputColumnName?: string; +} + +export type ExpressionFunctionCounterRate = ExpressionFunctionDefinition< + 'lens_counter_rate', + Datatable, + CounterRateArgs, + Datatable +>; + +/** + * Calculates the counter rate of a specified column in the data table. + * + * Also supports multiple series in a single data table - use the `by` argument + * to specify the columns to split the calculation by. + * For each unique combination of all `by` columns a separate counter rate will be calculated. + * The order of rows won't be changed - this function is not modifying any existing columns, it's only + * adding the specified `outputColumnId` column to every row of the table without adding or removing rows. + * + * Behavior: + * * Will write the counter rate of `inputColumnId` into `outputColumnId` + * * If provided will use `outputColumnName` as name for the newly created column. Otherwise falls back to `outputColumnId` + * * Counter rate always start with an undefined value for the first row of a series. + * * If the value of the current cell is not smaller than the previous one, an output cell will contain + * * its own value minus the value of the previous cell of the same series. If the value is smaller, + * * an output cell will contain its own value + * + * Edge cases: + * * Will return the input table if `inputColumnId` does not exist + * * Will throw an error if `outputColumnId` exists already in provided data table + * * If there is no previous row of the current series with a non `null` or `undefined` value, the output cell of the current row + * will be set to `undefined`. + * * If the row value contains `null` or `undefined`, it will be ignored and the output cell will be set to `undefined` + * * If the value of the previous row of the same series contains `null` or `undefined`, the output cell of the current row will be set to `undefined` as well + * * For all values besides `null` and `undefined`, the value will be cast to a number before it's used in the + * calculation of the current series even if this results in `NaN` (like in case of objects). + * * To determine separate series defined by the `by` columns, the values of these columns will be cast to strings + * before comparison. If the values are objects, the return value of their `toString` method will be used for comparison. + * Missing values (`null` and `undefined`) will be treated as empty strings. + */ +export const counterRate: ExpressionFunctionCounterRate = { + name: 'lens_counter_rate', + type: 'datatable', + + inputTypes: ['datatable'], + + help: i18n.translate('xpack.lens.functions.counterRate.help', { + defaultMessage: 'Calculates the counter rate of a column in a data table', + }), + + args: { + by: { + help: i18n.translate('xpack.lens.functions.counterRate.args.byHelpText', { + defaultMessage: 'Column to split the counter rate calculation by', + }), + multi: true, + types: ['string'], + required: false, + }, + inputColumnId: { + help: i18n.translate('xpack.lens.functions.counterRate.args.inputColumnIdHelpText', { + defaultMessage: 'Column to calculate the counter rate of', + }), + types: ['string'], + required: true, + }, + outputColumnId: { + help: i18n.translate('xpack.lens.functions.counterRate.args.outputColumnIdHelpText', { + defaultMessage: 'Column to store the resulting counter rate in', + }), + types: ['string'], + required: true, + }, + outputColumnName: { + help: i18n.translate('xpack.lens.functions.counterRate.args.outputColumnNameHelpText', { + defaultMessage: 'Name of the column to store the resulting counter rate in', + }), + types: ['string'], + required: false, + }, + }, + + fn(input, { by, inputColumnId, outputColumnId, outputColumnName }) { + const resultColumns = buildResultColumns( + input, + outputColumnId, + inputColumnId, + outputColumnName + ); + + if (!resultColumns) { + return input; + } + const previousValues: Partial> = {}; + return { + ...input, + columns: resultColumns, + rows: input.rows.map((row) => { + const newRow = { ...row }; + + const bucketIdentifier = getBucketIdentifier(row, by); + const previousValue = previousValues[bucketIdentifier]; + const currentValue = newRow[inputColumnId]; + if (currentValue != null && previousValue != null) { + const currentValueAsNumber = Number(currentValue); + if (currentValueAsNumber >= previousValue) { + newRow[outputColumnId] = currentValueAsNumber - previousValue; + } else { + newRow[outputColumnId] = currentValueAsNumber; + } + } else { + newRow[outputColumnId] = undefined; + } + + if (currentValue != null) { + previousValues[bucketIdentifier] = Number(currentValue); + } else { + previousValues[bucketIdentifier] = undefined; + } + + return newRow; + }), + }; + }, +}; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/time_scale.ts b/x-pack/plugins/lens/public/indexpattern_datasource/time_scale.ts index 0937f40eeb6d3..7a4e8f6bc0638 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/time_scale.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/time_scale.ts @@ -9,6 +9,7 @@ import { i18n } from '@kbn/i18n'; import { ExpressionFunctionDefinition, Datatable } from 'src/plugins/expressions/public'; import { DataPublicPluginStart } from 'src/plugins/data/public'; import { search } from '../../../../../src/plugins/data/public'; +import { buildResultColumns } from '../../../../../src/plugins/expressions/common'; type TimeScaleUnit = 's' | 'm' | 'h' | 'd'; @@ -69,17 +70,6 @@ export function getTimeScaleFunction(data: DataPublicPluginStart) { input, { dateColumnId, inputColumnId, outputColumnId, outputColumnName, targetUnit }: TimeScaleArgs ) { - if (input.columns.some((column) => column.id === outputColumnId)) { - throw new Error( - i18n.translate('xpack.lens.functions.timeScale.columnConflictMessage', { - defaultMessage: 'Specified outputColumnId {columnId} already exists.', - values: { - columnId: outputColumnId, - }, - }) - ); - } - const dateColumnDefinition = input.columns.find((column) => column.id === dateColumnId); if (!dateColumnDefinition) { @@ -93,26 +83,17 @@ export function getTimeScaleFunction(data: DataPublicPluginStart) { ); } - const inputColumnDefinition = input.columns.find((column) => column.id === inputColumnId); + const resultColumns = buildResultColumns( + input, + outputColumnId, + inputColumnId, + outputColumnName + ); - if (!inputColumnDefinition) { + if (!resultColumns) { return input; } - const outputColumnDefinition = { - ...inputColumnDefinition, - id: outputColumnId, - name: outputColumnName || outputColumnId, - }; - - const resultColumns = [...input.columns]; - // add output column after input column in the table - resultColumns.splice( - resultColumns.indexOf(inputColumnDefinition) + 1, - 0, - outputColumnDefinition - ); - const targetUnitInMs = unitInMs[targetUnit]; const timeInfo = await data.search.aggs.getDateMetaByDatatableColumn(dateColumnDefinition); const intervalDuration = timeInfo && search.aggs.parseInterval(timeInfo.interval); From 7fdd0a1b813a87b9e344b0f0861284fc6a37aaf1 Mon Sep 17 00:00:00 2001 From: Anton Dosov Date: Wed, 11 Nov 2020 14:03:10 +0100 Subject: [PATCH 004/104] [Search][Discover] Restore searchSessionId from URL (#81633) --- .../public/application/angular/discover.js | 30 +- .../discover/public/url_generator.test.ts | 9 + src/plugins/discover/public/url_generator.ts | 12 + .../functional/apps/discover/async_search.ts | 58 ++++ x-pack/test/functional/apps/discover/index.ts | 1 + .../es_archives/discover/default/data.json.gz | Bin 0 -> 1120 bytes .../discover/default/mappings.json | 273 ++++++++++++++++++ 7 files changed, 378 insertions(+), 5 deletions(-) create mode 100644 x-pack/test/functional/apps/discover/async_search.ts create mode 100644 x-pack/test/functional/es_archives/discover/default/data.json.gz create mode 100644 x-pack/test/functional/es_archives/discover/default/mappings.json diff --git a/src/plugins/discover/public/application/angular/discover.js b/src/plugins/discover/public/application/angular/discover.js index af763240bccfd..9319c58db3e33 100644 --- a/src/plugins/discover/public/application/angular/discover.js +++ b/src/plugins/discover/public/application/angular/discover.js @@ -83,6 +83,8 @@ import { MODIFY_COLUMNS_ON_SWITCH, } from '../../../common'; import { METRIC_TYPE } from '@kbn/analytics'; +import { SEARCH_SESSION_ID_QUERY_PARAM } from '../../url_generator'; +import { removeQueryParam, getQueryParams } from '../../../../kibana_utils/public'; const fetchStatuses = { UNINITIALIZED: 'uninitialized', @@ -91,6 +93,9 @@ const fetchStatuses = { ERROR: 'error', }; +const getSearchSessionIdFromURL = (history) => + getQueryParams(history.location)[SEARCH_SESSION_ID_QUERY_PARAM]; + const app = getAngularModule(); app.config(($routeProvider) => { @@ -208,6 +213,8 @@ function discoverController($element, $route, $scope, $timeout, $window, Promise }; const history = getHistory(); + // used for restoring background session + let isInitialSearch = true; const { appStateContainer, @@ -798,17 +805,30 @@ function discoverController($element, $route, $scope, $timeout, $window, Promise if (abortController) abortController.abort(); abortController = new AbortController(); - const sessionId = data.search.session.start(); + const searchSessionId = (() => { + const searchSessionIdFromURL = getSearchSessionIdFromURL(history); + if (searchSessionIdFromURL) { + if (isInitialSearch) { + data.search.session.restore(searchSessionIdFromURL); + isInitialSearch = false; + return searchSessionIdFromURL; + } else { + // navigating away from background search + removeQueryParam(history, SEARCH_SESSION_ID_QUERY_PARAM); + } + } + return data.search.session.start(); + })(); $scope .updateDataSource() .then(setupVisualization) .then(function () { $scope.fetchStatus = fetchStatuses.LOADING; - logInspectorRequest(); + logInspectorRequest({ searchSessionId }); return $scope.searchSource.fetch({ abortSignal: abortController.signal, - sessionId, + sessionId: searchSessionId, }); }) .then(onResults) @@ -900,7 +920,7 @@ function discoverController($element, $route, $scope, $timeout, $window, Promise $scope.fetchStatus = fetchStatuses.COMPLETE; } - function logInspectorRequest() { + function logInspectorRequest({ searchSessionId = null } = { searchSessionId: null }) { inspectorAdapters.requests.reset(); const title = i18n.translate('discover.inspectorRequestDataTitle', { defaultMessage: 'data', @@ -908,7 +928,7 @@ function discoverController($element, $route, $scope, $timeout, $window, Promise const description = i18n.translate('discover.inspectorRequestDescription', { defaultMessage: 'This request queries Elasticsearch to fetch the data for the search.', }); - inspectorRequest = inspectorAdapters.requests.start(title, { description }); + inspectorRequest = inspectorAdapters.requests.start(title, { description, searchSessionId }); inspectorRequest.stats(getRequestInspectorStats($scope.searchSource)); $scope.searchSource.getSearchRequestBody().then((body) => { inspectorRequest.json(body); diff --git a/src/plugins/discover/public/url_generator.test.ts b/src/plugins/discover/public/url_generator.test.ts index a18ee486ab007..98b7625e63c72 100644 --- a/src/plugins/discover/public/url_generator.test.ts +++ b/src/plugins/discover/public/url_generator.test.ts @@ -212,6 +212,15 @@ describe('Discover url generator', () => { }); }); + test('can specify a search session id', async () => { + const { generator } = await setup(); + const url = await generator.createUrl({ + searchSessionId: '__test__', + }); + expect(url).toMatchInlineSnapshot(`"xyz/app/discover#/?_g=()&_a=()&searchSessionId=__test__"`); + expect(url).toContain('__test__'); + }); + describe('useHash property', () => { describe('when default useHash is set to false', () => { test('when using default, sets index pattern ID in the generated URL', async () => { diff --git a/src/plugins/discover/public/url_generator.ts b/src/plugins/discover/public/url_generator.ts index c7f2e2147e819..df9b16a4627ec 100644 --- a/src/plugins/discover/public/url_generator.ts +++ b/src/plugins/discover/public/url_generator.ts @@ -67,6 +67,11 @@ export interface DiscoverUrlGeneratorState { * whether to hash the data in the url to avoid url length issues. */ useHash?: boolean; + + /** + * Background search session id + */ + searchSessionId?: string; } interface Params { @@ -74,6 +79,8 @@ interface Params { useHash: boolean; } +export const SEARCH_SESSION_ID_QUERY_PARAM = 'searchSessionId'; + export class DiscoverUrlGenerator implements UrlGeneratorsDefinition { constructor(private readonly params: Params) {} @@ -88,6 +95,7 @@ export class DiscoverUrlGenerator savedSearchId, timeRange, useHash = this.params.useHash, + searchSessionId, }: DiscoverUrlGeneratorState): Promise => { const savedSearchPath = savedSearchId ? encodeURIComponent(savedSearchId) : ''; const appState: { @@ -111,6 +119,10 @@ export class DiscoverUrlGenerator url = setStateToKbnUrl('_g', queryState, { useHash }, url); url = setStateToKbnUrl('_a', appState, { useHash }, url); + if (searchSessionId) { + url = `${url}&${SEARCH_SESSION_ID_QUERY_PARAM}=${searchSessionId}`; + } + return url; }; } diff --git a/x-pack/test/functional/apps/discover/async_search.ts b/x-pack/test/functional/apps/discover/async_search.ts new file mode 100644 index 0000000000000..77d60f4036a3b --- /dev/null +++ b/x-pack/test/functional/apps/discover/async_search.ts @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../ftr_provider_context'; + +export default function ({ getPageObjects, getService }: FtrProviderContext) { + const esArchiver = getService('esArchiver'); + const queryBar = getService('queryBar'); + const testSubjects = getService('testSubjects'); + const browser = getService('browser'); + const inspector = getService('inspector'); + const PageObjects = getPageObjects(['discover', 'common', 'timePicker', 'header']); + + describe('discover async search', () => { + before(async () => { + await esArchiver.loadIfNeeded('logstash_functional'); + await esArchiver.load('discover/default'); + await PageObjects.common.navigateToApp('discover'); + await PageObjects.timePicker.setDefaultAbsoluteRange(); + await PageObjects.header.waitUntilLoadingHasFinished(); + }); + + it('search session id should change between searches', async () => { + const searchSessionId1 = await getSearchSessionId(); + expect(searchSessionId1).not.to.be.empty(); + await queryBar.clickQuerySubmitButton(); + const searchSessionId2 = await getSearchSessionId(); + expect(searchSessionId2).not.to.be(searchSessionId1); + }); + + // NOTE: this test will be revised when + // `searchSessionId` functionality actually works + it('search session id should be picked up from the URL', async () => { + const url = await browser.getCurrentUrl(); + const fakeSearchSessionId = '__test__'; + const savedSessionURL = url + `&searchSessionId=${fakeSearchSessionId}`; + await browser.navigateTo(savedSessionURL); + await PageObjects.header.waitUntilLoadingHasFinished(); + const searchSessionId1 = await getSearchSessionId(); + expect(searchSessionId1).to.be(fakeSearchSessionId); + await queryBar.clickQuerySubmitButton(); + const searchSessionId2 = await getSearchSessionId(); + expect(searchSessionId2).not.to.be(searchSessionId1); + }); + }); + + async function getSearchSessionId(): Promise { + await inspector.open(); + const searchSessionId = await ( + await testSubjects.find('inspectorRequestSearchSessionId') + ).getAttribute('data-search-session-id'); + await inspector.close(); + return searchSessionId; + } +} diff --git a/x-pack/test/functional/apps/discover/index.ts b/x-pack/test/functional/apps/discover/index.ts index 93179ac68a038..fc91a72c3950f 100644 --- a/x-pack/test/functional/apps/discover/index.ts +++ b/x-pack/test/functional/apps/discover/index.ts @@ -16,5 +16,6 @@ export default function ({ loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./error_handling')); loadTestFile(require.resolve('./visualize_field')); loadTestFile(require.resolve('./value_suggestions')); + loadTestFile(require.resolve('./async_search')); }); } diff --git a/x-pack/test/functional/es_archives/discover/default/data.json.gz b/x-pack/test/functional/es_archives/discover/default/data.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..047d890f6d410178d6c0c385c8bdde46f804bc33 GIT binary patch literal 1120 zcmV-m1fTmKiwFo&46QOZ*Bn1TVHSEHVl97r!f3DG(po2w_Tok*szBI zyL;%~E`_1UOst)3`6Ri27Wli5l5EFLQ>^7-C2^P_OXU^nWVYm z>A54^G1)lE4lnRhAB#*OlIw@NEk!YW;^QVko>pjYYn%A@r}W3{NZp2k&EStMTIf5Ww?E#!E) zVbTDSyg*N}x^V=ACm;Ww+=DNl`S&*6ZSDCO0o#I;@b>aM%0#?>ZYp1cWH{(As*p}VFC$030iAhreN%A!#36J=iN8jKgB0-U!c zVIGj8`eq2VAacT&zs7V&y;`Y@x`a~c$9NV4}OZ;D_{ zd@+Qc@^5N*c{z-HM*FxzxsG&05NiP*&?bhu$V9xB%-(1z@OE=#9L3^`t#Q;QpN19M zC6ucIa-}rJe8hDS^^uv?N_58(Ctt6iW{3lguPSw@;&GbZqph%#J|R6horwaKsa0jD zVO*DB4E>^^FMz3tR=fvA^>~1Vh)is;2SaHpU#cXZ!AIi?ai%ny6Cat;`{uCtjwY&6 zM|oG=CO%$?Z?+ak)2|6wi^oE@u`hELHDce_%stk zko?WdUpHvu5eEJ%8h16{-zBO$l^s(Nwcv?QA3w_as<$f<)H)|(6V8Kmuc zG~s{ETs!v^;wo+1_O|P%_iggdgFWjrzLD#e-{+6%ZJ6^Nd(^Gpaohg3q5FT@mG8Oj znftY8XSHkS8~O_~h3Scl1yptdnvvA_ctu z;V{!-*3Krb`p21DjC{EmiTvXA?J3ES>Oghx=p|*DcwE6a~x+Q zq;-=8-}p9AzBk}byHN1{GV8r~LJiUeA)UKs_c#Yak7y8t%r^Ymy$0LSn;WI=p~{as mVW;imb{HIVYJsuV??9Virp2?a+quQWXa50HZ?gK@G5`QD?ly%0 literal 0 HcmV?d00001 diff --git a/x-pack/test/functional/es_archives/discover/default/mappings.json b/x-pack/test/functional/es_archives/discover/default/mappings.json new file mode 100644 index 0000000000000..82002c095bcc5 --- /dev/null +++ b/x-pack/test/functional/es_archives/discover/default/mappings.json @@ -0,0 +1,273 @@ +{ + "type": "index", + "value": { + "index": ".kibana", + "mappings": { + "properties": { + "config": { + "dynamic": "true", + "properties": { + "buildNum": { + "type": "keyword" + } + } + }, + "dashboard": { + "dynamic": "strict", + "properties": { + "description": { + "type": "text" + }, + "hits": { + "type": "integer" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" + } + } + }, + "optionsJSON": { + "type": "text" + }, + "panelsJSON": { + "type": "text" + }, + "refreshInterval": { + "properties": { + "display": { + "type": "keyword" + }, + "pause": { + "type": "boolean" + }, + "section": { + "type": "integer" + }, + "value": { + "type": "integer" + } + } + }, + "timeFrom": { + "type": "keyword" + }, + "timeRestore": { + "type": "boolean" + }, + "timeTo": { + "type": "keyword" + }, + "title": { + "type": "text" + }, + "uiStateJSON": { + "type": "text" + }, + "version": { + "type": "integer" + } + } + }, + "index-pattern": { + "dynamic": "strict", + "properties": { + "fieldFormatMap": { + "type": "text" + }, + "fields": { + "type": "text" + }, + "intervalName": { + "type": "keyword" + }, + "notExpandable": { + "type": "boolean" + }, + "sourceFilters": { + "type": "text" + }, + "timeFieldName": { + "type": "keyword" + }, + "title": { + "type": "text" + } + } + }, + "search": { + "dynamic": "strict", + "properties": { + "columns": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "hits": { + "type": "integer" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" + } + } + }, + "sort": { + "type": "keyword" + }, + "title": { + "type": "text" + }, + "version": { + "type": "integer" + } + } + }, + "server": { + "dynamic": "strict", + "properties": { + "uuid": { + "type": "keyword" + } + } + }, + "timelion-sheet": { + "dynamic": "strict", + "properties": { + "description": { + "type": "text" + }, + "hits": { + "type": "integer" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" + } + } + }, + "timelion_chart_height": { + "type": "integer" + }, + "timelion_columns": { + "type": "integer" + }, + "timelion_interval": { + "type": "keyword" + }, + "timelion_other_interval": { + "type": "keyword" + }, + "timelion_rows": { + "type": "integer" + }, + "timelion_sheet": { + "type": "text" + }, + "title": { + "type": "text" + }, + "version": { + "type": "integer" + } + } + }, + "type": { + "type": "keyword" + }, + "url": { + "dynamic": "strict", + "properties": { + "accessCount": { + "type": "long" + }, + "accessDate": { + "type": "date" + }, + "createDate": { + "type": "date" + }, + "url": { + "fields": { + "keyword": { + "ignore_above": 2048, + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "visualization": { + "dynamic": "strict", + "properties": { + "description": { + "type": "text" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" + } + } + }, + "savedSearchId": { + "type": "keyword" + }, + "title": { + "type": "text" + }, + "uiStateJSON": { + "type": "text" + }, + "version": { + "type": "integer" + }, + "visState": { + "type": "text" + } + } + }, + "query": { + "properties": { + "title": { + "type": "text" + }, + "description": { + "type": "text" + }, + "query": { + "properties": { + "language": { + "type": "keyword" + }, + "query": { + "type": "keyword", + "index": false + } + } + }, + "filters": { + "type": "object", + "enabled": false + }, + "timefilter": { + "type": "object", + "enabled": false + } + } + } + } + }, + "settings": { + "index": { + "number_of_replicas": "1", + "number_of_shards": "1" + } + } + } +} From 7e24ae6e70a250f4cd573688acec551643ad514a Mon Sep 17 00:00:00 2001 From: Dan Panzarella Date: Wed, 11 Nov 2020 09:12:25 -0500 Subject: [PATCH 005/104] [Security] Report accurate endpoint count (#83092) --- .../server/endpoint/routes/metadata/query_builders.test.ts | 4 ++++ .../endpoint/routes/metadata/support/query_strategies.ts | 3 +++ 2 files changed, 7 insertions(+) diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/query_builders.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/query_builders.test.ts index ac1de377124f0..88b141120a904 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/query_builders.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/query_builders.test.ts @@ -37,6 +37,7 @@ describe('query builder', () => { }, }, ], + track_total_hits: true, }, from: 0, size: 10, @@ -84,6 +85,7 @@ describe('query builder', () => { }, }, ], + track_total_hits: true, }, from: 0, size: 10, @@ -142,6 +144,7 @@ describe('query builder', () => { }, }, ], + track_total_hits: true, }, from: 0, size: 10, @@ -213,6 +216,7 @@ describe('query builder', () => { }, }, ], + track_total_hits: true, }, from: 0, size: 10, diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/support/query_strategies.ts b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/support/query_strategies.ts index f1614cc19e8c8..5c25ff0b99faa 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/support/query_strategies.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/support/query_strategies.ts @@ -86,6 +86,9 @@ export function metadataQueryStrategyV2(): MetadataQueryStrategy { }, }, ], + extraBodyProperties: { + track_total_hits: true, + }, queryResponseToHostListResult: ( searchResponse: SearchResponse ): HostListQueryResult => { From c32215d7ff38cc198ca818829c6aa6c2c01d9361 Mon Sep 17 00:00:00 2001 From: Dima Arnautov Date: Wed, 11 Nov 2020 15:42:53 +0100 Subject: [PATCH 006/104] [ML] Fix apiDocs extractor script (#82582) * [ML] fix serializer script * [ML] fix extractor * [ML] bump apiDoc version * [ML] update yarn.lock --- package.json | 4 +- x-pack/plugins/ml/server/routes/apidoc.json | 2 +- .../apidoc_scripts/schema_extractor.test.ts | 209 ++++++++++++++++++ .../routes/apidoc_scripts/schema_extractor.ts | 78 ++++--- .../routes/apidoc_scripts/schema_worker.ts | 4 +- yarn.lock | 139 +++++++----- 6 files changed, 347 insertions(+), 89 deletions(-) create mode 100644 x-pack/plugins/ml/server/routes/apidoc_scripts/schema_extractor.test.ts diff --git a/package.json b/package.json index 38cf150094677..7edbcb03be542 100644 --- a/package.json +++ b/package.json @@ -577,8 +577,8 @@ "angular-recursion": "^1.0.5", "angular-route": "^1.8.0", "angular-sortable-view": "^0.0.17", - "apidoc": "^0.20.1", - "apidoc-markdown": "^5.0.0", + "apidoc": "^0.25.0", + "apidoc-markdown": "^5.1.8", "apollo-link": "^1.2.3", "apollo-link-error": "^1.1.7", "apollo-link-state": "^0.4.1", diff --git a/x-pack/plugins/ml/server/routes/apidoc.json b/x-pack/plugins/ml/server/routes/apidoc.json index 8d09bdd568993..780835e2a300b 100644 --- a/x-pack/plugins/ml/server/routes/apidoc.json +++ b/x-pack/plugins/ml/server/routes/apidoc.json @@ -1,6 +1,6 @@ { "name": "ml_kibana_api", - "version": "7.8.0", + "version": "7.11.0", "description": "This is the documentation of the REST API provided by the Machine Learning Kibana plugin. Each API is experimental and can include breaking changes in any version.", "title": "ML Kibana API", "order": [ diff --git a/x-pack/plugins/ml/server/routes/apidoc_scripts/schema_extractor.test.ts b/x-pack/plugins/ml/server/routes/apidoc_scripts/schema_extractor.test.ts new file mode 100644 index 0000000000000..be89cd57c43ac --- /dev/null +++ b/x-pack/plugins/ml/server/routes/apidoc_scripts/schema_extractor.test.ts @@ -0,0 +1,209 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { extractDocumentation } from './schema_extractor'; +import * as path from 'path'; + +describe('schema_extractor', () => { + it('should serialize schema definition', () => { + const result = extractDocumentation([ + path.resolve(__dirname, '..', 'schemas', 'datafeeds_schema.ts'), + ]); + + expect(result.get('startDatafeedSchema')).toEqual([ + { + name: 'start', + documentation: '', + type: 'string | number', + }, + { + name: 'end', + documentation: '', + type: 'string | number', + }, + { + name: 'timeout', + documentation: '', + type: 'any', + }, + ]); + + expect(result.get('datafeedConfigSchema')).toEqual([ + { + name: 'datafeed_id', + documentation: '', + type: 'string', + }, + { + name: 'feed_id', + documentation: '', + type: 'string', + }, + { + name: 'aggregations', + documentation: '', + type: 'any', + }, + { + name: 'aggs', + documentation: '', + type: 'any', + }, + { + name: 'chunking_config', + documentation: '', + type: 'chunking_config', + nested: [ + { + name: 'mode', + documentation: '', + type: 'string', + }, + { + name: 'time_span', + documentation: '', + type: 'string', + }, + ], + }, + { + name: 'frequency', + documentation: '', + type: 'string', + }, + { + name: 'indices', + documentation: '', + type: 'string[]', + }, + { + name: 'indexes', + documentation: '', + type: 'string[]', + }, + { + name: 'job_id', + documentation: '', + type: 'string', + }, + { + name: 'query', + documentation: '', + type: 'any', + }, + { + name: 'max_empty_searches', + documentation: '', + type: 'number', + }, + { + name: 'query_delay', + documentation: '', + type: 'string', + }, + { + name: 'script_fields', + documentation: '', + type: 'any', + }, + { + name: 'scroll_size', + documentation: '', + type: 'number', + }, + { + name: 'delayed_data_check_config', + documentation: '', + type: 'any', + }, + { + name: 'indices_options', + documentation: '', + type: 'indices_options', + nested: [ + { + name: 'expand_wildcards', + documentation: '', + type: 'string[]', + }, + { + name: 'ignore_unavailable', + documentation: '', + type: 'boolean', + }, + { + name: 'allow_no_indices', + documentation: '', + type: 'boolean', + }, + { + name: 'ignore_throttled', + documentation: '', + type: 'boolean', + }, + ], + }, + ]); + + expect(result.get('deleteDatafeedQuerySchema')).toEqual([ + { + name: 'force', + documentation: '', + type: 'any', // string + }, + ]); + }); + + it('serializes schema with nested objects and nullable', () => { + const result = extractDocumentation([ + path.resolve(__dirname, '..', 'schemas', 'results_service_schema.ts'), + ]); + expect(result.get('getCategorizerStatsSchema')).toEqual([ + { + name: 'partitionByValue', + documentation: + 'Optional value to fetch the categorizer stats where results are filtered by partition_by_value = value', + type: 'any', // FIXME string + }, + ]); + + // @ts-ignore + expect(result.get('partitionFieldValuesSchema')![5].nested[0]).toEqual({ + name: 'partition_field', + documentation: '', + type: 'partition_field', + nested: [ + { + name: 'applyTimeRange', + documentation: '', + type: 'boolean', + }, + { + name: 'anomalousOnly', + documentation: '', + type: 'boolean', + }, + { + name: 'sort', + documentation: '', + type: 'sort', + nested: [ + { + name: 'by', + documentation: '', + type: 'string', + }, + { + name: 'order', + documentation: '', + type: 'string', + }, + ], + }, + ], + }); + }); +}); diff --git a/x-pack/plugins/ml/server/routes/apidoc_scripts/schema_extractor.ts b/x-pack/plugins/ml/server/routes/apidoc_scripts/schema_extractor.ts index 7116d1c23b515..e309372d0b2f4 100644 --- a/x-pack/plugins/ml/server/routes/apidoc_scripts/schema_extractor.ts +++ b/x-pack/plugins/ml/server/routes/apidoc_scripts/schema_extractor.ts @@ -90,65 +90,77 @@ export function extractDocumentation( return members; } - function resolveTypeArgument(type: ts.Type): ts.SymbolTable | string { - // required to extract members - type.getProperty('type'); - - // @ts-ignores - let members = type.members; + /** + * Extracts properties of the type. + * @param type + */ + function resolveTypeProperties(type: ts.Type): ts.Symbol[] { + let props = type.getProperties(); const typeArguments = checker.getTypeArguments((type as unknown) as ts.TypeReference); if (type.aliasTypeArguments) { // @ts-ignores - members = type.aliasTypeArguments[0].members; + props = type.aliasTypeArguments[0].getProperties(); } if (typeArguments.length > 0) { - members = resolveTypeArgument(typeArguments[0]); - } - - if (members === undefined) { - members = checker.typeToString(type); + props = resolveTypeProperties(typeArguments[0]); } - return members; + return props; } function serializeProperty(symbol: ts.Symbol): DocEntry { // @ts-ignore const typeOfSymbol = symbol.type; - const typeArguments = checker.getTypeArguments((typeOfSymbol as unknown) as ts.TypeReference); + if (typeOfSymbol === undefined) { + return { + name: symbol.getName(), + documentation: getCommentString(symbol), + type: 'any', + }; + } - let resultType: ts.Type = typeOfSymbol; + let targetType: ts.TypeReference | ts.Type = + typeOfSymbol.getProperty('type')?.type ?? typeOfSymbol; - let members; - if (typeArguments.length > 0) { - members = resolveTypeArgument(typeArguments[0]); - resultType = typeArguments[0]; + const isArrayOf = targetType.symbol?.name === 'Array'; + if (isArrayOf) { + targetType = checker.getTypeArguments(targetType as ts.TypeReference)[0]; } - let typeAsString = checker.typeToString(resultType); - + let typeAsString = checker.typeToString(targetType); const nestedEntries: DocEntry[] = []; - if (members && typeof members !== 'string' && members.size > 0) { - // we hit an object or collection - typeAsString = - resultType.symbol.name === 'Array' || typeOfSymbol.symbol.name === 'Array' - ? `${symbol.getName()}[]` - : symbol.getName(); - - members.forEach((member) => { - nestedEntries.push(serializeProperty(member)); - }); + + if ( + targetType.aliasTypeArguments || + checker.getTypeArguments(targetType as ts.TypeReference).length > 0 + ) { + // Resolve complex types, objects and arrays, that contain nested properties + const typeProperties = resolveTypeProperties(targetType); + + if (Array.isArray(typeProperties) && typeProperties.length > 0) { + // we hit an object or collection + typeAsString = + targetType.symbol?.name === 'Array' || typeOfSymbol.symbol?.name === 'Array' + ? `${symbol.getName()}[]` + : symbol.getName(); + + typeProperties.forEach((member) => { + nestedEntries.push(serializeProperty(member)); + }); + } } - return { + const res = { name: symbol.getName(), documentation: getCommentString(symbol), - type: typeAsString, + type: isArrayOf ? `${typeAsString}[]` : typeAsString, ...(nestedEntries.length > 0 ? { nested: nestedEntries } : {}), }; + + return res; } function getCommentString(symbol: ts.Symbol): string { diff --git a/x-pack/plugins/ml/server/routes/apidoc_scripts/schema_worker.ts b/x-pack/plugins/ml/server/routes/apidoc_scripts/schema_worker.ts index 8b9b35f7a0d3c..1853c5644d9df 100644 --- a/x-pack/plugins/ml/server/routes/apidoc_scripts/schema_worker.ts +++ b/x-pack/plugins/ml/server/routes/apidoc_scripts/schema_worker.ts @@ -10,10 +10,10 @@ import { DocEntry, extractDocumentation } from './schema_extractor'; import { ApiParameter, Block } from './types'; export function postProcess(parsedFiles: any[]): void { - const schemasDirPath = `${__dirname}${path.sep}..${path.sep}..${path.sep}schemas${path.sep}`; + const schemasDirPath = path.resolve(__dirname, '..', '..', 'schemas'); const schemaFiles = fs .readdirSync(schemasDirPath) - .map((filename) => path.resolve(schemasDirPath + filename)); + .map((filename) => path.resolve(schemasDirPath, filename)); const schemaDocs = extractDocumentation(schemaFiles); diff --git a/yarn.lock b/yarn.lock index b59c134968c18..32ab10c12fa12 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6765,39 +6765,41 @@ anymatch@^3.0.3, anymatch@~3.1.1: normalize-path "^3.0.0" picomatch "^2.0.4" -apidoc-core@^0.11.1: - version "0.11.1" - resolved "https://registry.yarnpkg.com/apidoc-core/-/apidoc-core-0.11.1.tgz#b04a7e0292e4ac0d714b40789f1b92f414486c81" - integrity sha512-pt/ICBdFQCZTgL38Aw1XB3G9AajDU1JA5E3yoDEgg0mqbPTCkOL8AyWdysjvNtQS/kkXgSPazCZaZzZYqrPHog== +apidoc-core@^0.12.0: + version "0.12.0" + resolved "https://registry.yarnpkg.com/apidoc-core/-/apidoc-core-0.12.0.tgz#96e5c04c17b92c288664ef3c8aa4c78cbfacccb6" + integrity sha512-VMhkJWz5IAyvWM0RnEbKNi1qe8se+id3/Ki3H/ePM8ih0KYTfaaSDxqo2w4uIVB1UVVKFvrTWyYUyQs7CEcoKQ== dependencies: - fs-extra "^8.1.0" - glob "^7.1.4" - iconv-lite "^0.5.0" + fs-extra "^9.0.1" + glob "^7.1.6" + iconv-lite "^0.6.2" klaw-sync "^6.0.0" - lodash "~4.17.15" - semver "~6.3.0" + lodash "^4.17.20" + semver "~7.3.2" -apidoc-markdown@^5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/apidoc-markdown/-/apidoc-markdown-5.0.0.tgz#e2d59d7cbbaa10402b09cec3e8ec17a03a27be59" - integrity sha512-gp4I4MvtgJvZPikEd7lwn149jjnC454CanPhm5demROdHCuakY+3YtIKEgVrJOqnS2iwbeeF+u4riB9CoO11+A== +apidoc-markdown@^5.1.8: + version "5.1.8" + resolved "https://registry.yarnpkg.com/apidoc-markdown/-/apidoc-markdown-5.1.8.tgz#521c1c5b387b8491f1ad58f26405ba8b5c6effe7" + integrity sha512-S92k1aJVSmWIW/HyKtU0NBcSWCmU5j/uNBF7OfWYgLFPryNnyjkYBZxVv4s+7MxRJaiinHXx1WFcxqnDq8Q2Gg== dependencies: - ejs "^3.0.1" - semver "^7.1.3" - yargs "^15.1.0" + ejs "^3.1.5" + semver "^7.3.2" + update-notifier "^4.1.1" + yargs "^16.0.3" -apidoc@^0.20.1: - version "0.20.1" - resolved "https://registry.yarnpkg.com/apidoc/-/apidoc-0.20.1.tgz#b29a2e2ae47e2df6a29e1f1527b94edf91853072" - integrity sha512-V54vkZ2lDFBiGn0qusZmHbMi4svuFBq0rjZAIe3nwYvBY7iztW78vKOyHyTr9ASaTB7EGe8hhLbpEnYAIO31TQ== +apidoc@^0.25.0: + version "0.25.0" + resolved "https://registry.yarnpkg.com/apidoc/-/apidoc-0.25.0.tgz#d59ad8724e091f4a1e349ff50cf4fb4dd3de6142" + integrity sha512-5g9fp8OffXZOdBTzm4BBvV5Vw54s+NmKnGZIUKuH+gRTqqJuRJpcGN6sz6WnjJ+NcvXhB7rIRp6FhtJahazx2Q== dependencies: - apidoc-core "^0.11.1" + apidoc-core "^0.12.0" commander "^2.20.0" - fs-extra "^8.1.0" - lodash "^4.17.15" - markdown-it "^10.0.0" - nodemon "^2.0.2" - winston "^3.2.1" + fs-extra "^9.0.1" + handlebars "^4.7.6" + lodash "^4.17.20" + markdown-it "^11.0.0" + nodemon "^2.0.4" + winston "^3.3.3" apollo-cache-control@^0.1.0: version "0.1.1" @@ -12054,7 +12056,7 @@ ee-first@1.1.1: resolved "https://registry.yarnpkg.com/ee-first/-/ee-first-1.1.1.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d" integrity sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0= -ejs@^3.0.1, ejs@^3.1.2, ejs@^3.1.5: +ejs@^3.1.2, ejs@^3.1.5: version "3.1.5" resolved "https://registry.yarnpkg.com/ejs/-/ejs-3.1.5.tgz#aed723844dc20acb4b170cd9ab1017e476a0d93b" integrity sha512-dldq3ZfFtgVTJMLjOe+/3sROTzALlL9E34V4/sDtUd/KlBSS0s6U1/+WPE1B4sj9CXHJpL1M6rhNJnc9Wbal9w== @@ -14135,15 +14137,6 @@ fs-extra@^7.0.0, fs-extra@^7.0.1, fs-extra@~7.0.1: jsonfile "^4.0.0" universalify "^0.1.0" -fs-extra@^8.1.0: - version "8.1.0" - resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-8.1.0.tgz#49d43c45a88cd9677668cb7be1b46efdb8d2e1c0" - integrity sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g== - dependencies: - graceful-fs "^4.2.0" - jsonfile "^4.0.0" - universalify "^0.1.0" - fs-extra@^9.0.0, fs-extra@^9.0.1: version "9.0.1" resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-9.0.1.tgz#910da0062437ba4c39fedd863f1675ccfefcb9fc" @@ -15353,7 +15346,7 @@ handle-thing@^2.0.0: resolved "https://registry.yarnpkg.com/handle-thing/-/handle-thing-2.0.0.tgz#0e039695ff50c93fc288557d696f3c1dc6776754" integrity sha512-d4sze1JNC454Wdo2fkuyzCr6aHcbL6PGGuFAz0Li/NcOm1tCHGnWDRmJP85dh9IhQErTc2svWFEX5xHIOo//kQ== -handlebars@4.7.6: +handlebars@4.7.6, handlebars@^4.7.6: version "4.7.6" resolved "https://registry.yarnpkg.com/handlebars/-/handlebars-4.7.6.tgz#d4c05c1baf90e9945f77aa68a7a219aa4a7df74e" integrity sha512-1f2BACcBfiwAfStCKZNrUCgqNZkGsAT7UM3kkYtXuLo0KnaVfjKOyf7PRzB6++aK9STyT1Pd2ZCPe3EGOXleXA== @@ -16040,13 +16033,20 @@ iconv-lite@0.4, iconv-lite@0.4.24, iconv-lite@^0.4.24, iconv-lite@~0.4.13: dependencies: safer-buffer ">= 2.1.2 < 3" -iconv-lite@^0.5.0, iconv-lite@^0.5.1: +iconv-lite@^0.5.1: version "0.5.1" resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.5.1.tgz#b2425d3c7b18f7219f2ca663d103bddb91718d64" integrity sha512-ONHr16SQvKZNSqjQT9gy5z24Jw+uqfO02/ngBSBoqChZ+W8qXX7GPRa1RoUnzGADw8K63R1BXUMzarCVQBpY8Q== dependencies: safer-buffer ">= 2.1.2 < 3" +iconv-lite@^0.6.2: + version "0.6.2" + resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.2.tgz#ce13d1875b0c3a674bd6a04b7f76b01b1b6ded01" + integrity sha512-2y91h5OpQlolefMPmUlivelittSWy0rP+oYVpn6A7GwVHNE8AWzoYOBNmlwks3LobaJxgHCYZAnyNo2GgpNRNQ== + dependencies: + safer-buffer ">= 2.1.2 < 3.0.0" + icss-utils@^4.0.0, icss-utils@^4.1.1: version "4.1.1" resolved "https://registry.yarnpkg.com/icss-utils/-/icss-utils-4.1.1.tgz#21170b53789ee27447c2f47dd683081403f9a467" @@ -18737,6 +18737,13 @@ linkify-it@^2.0.0: dependencies: uc.micro "^1.0.1" +linkify-it@^3.0.1: + version "3.0.2" + resolved "https://registry.yarnpkg.com/linkify-it/-/linkify-it-3.0.2.tgz#f55eeb8bc1d3ae754049e124ab3bb56d97797fb8" + integrity sha512-gDBO4aHNZS6coiZCKVhSNh43F9ioIL4JwRjLZPkoLIY4yZFwg264Y5lu2x6rb1Js42Gh6Yqm2f6L2AJcnkzinQ== + dependencies: + uc.micro "^1.0.1" + listr-silent-renderer@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/listr-silent-renderer/-/listr-silent-renderer-1.1.1.tgz#924b5a3757153770bf1a8e3fbf74b8bbf3f9242e" @@ -19527,6 +19534,17 @@ markdown-it@^10.0.0: mdurl "^1.0.1" uc.micro "^1.0.5" +markdown-it@^11.0.0: + version "11.0.1" + resolved "https://registry.yarnpkg.com/markdown-it/-/markdown-it-11.0.1.tgz#b54f15ec2a2193efa66dda1eb4173baea08993d6" + integrity sha512-aU1TzmBKcWNNYvH9pjq6u92BML+Hz3h5S/QpfTFwiQF852pLT+9qHsrhM9JYipkOXZxGn+sGH8oyJE9FD9WezQ== + dependencies: + argparse "^1.0.7" + entities "~2.0.0" + linkify-it "^3.0.1" + mdurl "^1.0.1" + uc.micro "^1.0.5" + markdown-to-jsx@^6.11.4: version "6.11.4" resolved "https://registry.yarnpkg.com/markdown-to-jsx/-/markdown-to-jsx-6.11.4.tgz#b4528b1ab668aef7fe61c1535c27e837819392c5" @@ -20766,10 +20784,10 @@ nodemailer@^4.7.0: resolved "https://registry.yarnpkg.com/nodemailer/-/nodemailer-4.7.0.tgz#4420e06abfffd77d0618f184ea49047db84f4ad8" integrity sha512-IludxDypFpYw4xpzKdMAozBSkzKHmNBvGanUREjJItgJ2NYcK/s8+PggVhj7c2yGFQykKsnnmv1+Aqo0ZfjHmw== -nodemon@^2.0.2: - version "2.0.3" - resolved "https://registry.yarnpkg.com/nodemon/-/nodemon-2.0.3.tgz#e9c64df8740ceaef1cb00e1f3da57c0a93ef3714" - integrity sha512-lLQLPS90Lqwc99IHe0U94rDgvjo+G9I4uEIxRG3evSLROcqQ9hwc0AxlSHKS4T1JW/IMj/7N5mthiN58NL/5kw== +nodemon@^2.0.4: + version "2.0.6" + resolved "https://registry.yarnpkg.com/nodemon/-/nodemon-2.0.6.tgz#1abe1937b463aaf62f0d52e2b7eaadf28cc2240d" + integrity sha512-4I3YDSKXg6ltYpcnZeHompqac4E6JeAMpGm8tJnB9Y3T0ehasLa4139dJOcCrB93HHrUMsCrKtoAlXTqT5n4AQ== dependencies: chokidar "^3.2.2" debug "^3.2.6" @@ -20779,8 +20797,8 @@ nodemon@^2.0.2: semver "^5.7.1" supports-color "^5.5.0" touch "^3.1.0" - undefsafe "^2.0.2" - update-notifier "^4.0.0" + undefsafe "^2.0.3" + update-notifier "^4.1.0" "nomnom@>= 1.5.x": version "1.8.1" @@ -24890,7 +24908,7 @@ safefs@^4.1.0: editions "^1.1.1" graceful-fs "^4.1.4" -"safer-buffer@>= 2.1.2 < 3", safer-buffer@^2.0.2, safer-buffer@^2.1.0, safer-buffer@~2.1.0: +"safer-buffer@>= 2.1.2 < 3", "safer-buffer@>= 2.1.2 < 3.0.0", safer-buffer@^2.0.2, safer-buffer@^2.1.0, safer-buffer@~2.1.0: version "2.1.2" resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== @@ -25129,12 +25147,12 @@ semver@7.0.0: resolved "https://registry.yarnpkg.com/semver/-/semver-7.0.0.tgz#5f3ca35761e47e05b206c6daff2cf814f0316b8e" integrity sha512-+GB6zVA9LWh6zovYQLALHwv5rb2PHGlJi3lfiqIHxR0uuwCgefcOJc59v9fv1w8GbStwxuuqqAjI9NMAOOgq1A== -semver@7.3.2, semver@^7.1.3, semver@^7.3.2: +semver@7.3.2, semver@^7.3.2, semver@~7.3.2: version "7.3.2" resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.2.tgz#604962b052b81ed0786aae84389ffba70ffd3938" integrity sha512-OrOb32TeeambH6UrhtShmF7CRDqhL6/5XpPNp2DuRH6+9QLw/orhp72j87v8Qa1ScDkvrrBNpZcDejAirJmfXQ== -semver@^6.0.0, semver@^6.1.0, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0, semver@~6.3.0: +semver@^6.0.0, semver@^6.1.0, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0: version "6.3.0" resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.0.tgz#ee0a64c8af5e8ceea67687b133761e1becbd1d3d" integrity sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw== @@ -27690,7 +27708,7 @@ undeclared-identifiers@^1.1.2: simple-concat "^1.0.0" xtend "^4.0.1" -undefsafe@^2.0.2: +undefsafe@^2.0.3: version "2.0.3" resolved "https://registry.yarnpkg.com/undefsafe/-/undefsafe-2.0.3.tgz#6b166e7094ad46313b2202da7ecc2cd7cc6e7aae" integrity sha512-nrXZwwXrD/T/JXeygJqdCO6NZZ1L66HrxM/Z7mIq2oPanoN0F1nLx3lwJMu6AwJY69hdixaFQOuoYsMjE5/C2A== @@ -28095,7 +28113,7 @@ update-notifier@^0.5.0: semver-diff "^2.0.0" string-length "^1.0.0" -update-notifier@^4.0.0, update-notifier@^4.1.0: +update-notifier@^4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/update-notifier/-/update-notifier-4.1.0.tgz#4866b98c3bc5b5473c020b1250583628f9a328f3" integrity sha512-w3doE1qtI0/ZmgeoDoARmI5fjDoT93IfKgEGqm26dGUOh8oNpaSTsGNdYRN/SjOuo10jcJGwkEL3mroKzktkew== @@ -28114,6 +28132,25 @@ update-notifier@^4.0.0, update-notifier@^4.1.0: semver-diff "^3.1.1" xdg-basedir "^4.0.0" +update-notifier@^4.1.1: + version "4.1.3" + resolved "https://registry.yarnpkg.com/update-notifier/-/update-notifier-4.1.3.tgz#be86ee13e8ce48fb50043ff72057b5bd598e1ea3" + integrity sha512-Yld6Z0RyCYGB6ckIjffGOSOmHXj1gMeE7aROz4MG+XMkmixBX4jUngrGXNYz7wPKBmtoD4MnBa2Anu7RSKht/A== + dependencies: + boxen "^4.2.0" + chalk "^3.0.0" + configstore "^5.0.1" + has-yarn "^2.1.0" + import-lazy "^2.1.0" + is-ci "^2.0.0" + is-installed-globally "^0.3.1" + is-npm "^4.0.0" + is-yarn-global "^0.3.0" + latest-version "^5.0.0" + pupa "^2.0.1" + semver-diff "^3.1.1" + xdg-basedir "^4.0.0" + upper-case-first@^1.1.0, upper-case-first@^1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/upper-case-first/-/upper-case-first-1.1.2.tgz#5d79bedcff14419518fd2edb0a0507c9b6859115" @@ -29450,7 +29487,7 @@ winston@3.2.1: triple-beam "^1.3.0" winston-transport "^4.3.0" -winston@^3.0.0, winston@^3.2.1, winston@^3.3.3: +winston@^3.0.0, winston@^3.3.3: version "3.3.3" resolved "https://registry.yarnpkg.com/winston/-/winston-3.3.3.tgz#ae6172042cafb29786afa3d09c8ff833ab7c9170" integrity sha512-oEXTISQnC8VlSAKf1KYSSd7J6IWuRPQqDdo8eoRNaYKLvwSb5+79Z3Yi1lrl6KDpU6/VWaxpakDAtb1oQ4n9aw== @@ -29816,7 +29853,7 @@ yargs@13.3.2, yargs@^13.2.2, yargs@^13.3.0, yargs@^13.3.2: y18n "^4.0.0" yargs-parser "^13.1.2" -yargs@^15.0.2, yargs@^15.1.0, yargs@^15.3.1, yargs@^15.4.1: +yargs@^15.0.2, yargs@^15.3.1, yargs@^15.4.1: version "15.4.1" resolved "https://registry.yarnpkg.com/yargs/-/yargs-15.4.1.tgz#0d87a16de01aee9d8bec2bfbf74f67851730f4f8" integrity sha512-aePbxDmcYW++PaqBsJ+HYUFwCdv4LVvdnhBy78E57PIor8/OVvhMrADFFEDh8DHDFRv/O9i3lPhsENjO7QX0+A== From 9038e5c397e45e3d102f42af39c46823637d3ca2 Mon Sep 17 00:00:00 2001 From: Jean-Louis Leysens Date: Wed, 11 Nov 2020 15:43:17 +0100 Subject: [PATCH 007/104] [Grokdebugger] Fix simulate error handling (#83036) * Fix detection of 4xx errors in grokdebugger simulate endpoint - removed code to call simulate endpoint from "KibanaFramework" - fixed throwing of string value - using new elasticsearch client instead of legacy - handle error with shared error handling logic * added deprecation notice to register route on KibanaFramework * remove deprecation notice --- .../grokdebugger/grokdebugger_service.js | 2 +- .../server/lib/kibana_framework.ts | 26 +------------------ ...ute.js => register_grok_simulate_route.ts} | 21 +++++++++------ .../grokdebugger/server/shared_imports.ts | 7 +++++ 4 files changed, 22 insertions(+), 34 deletions(-) rename x-pack/plugins/grokdebugger/server/routes/api/grokdebugger/{register_grok_simulate_route.js => register_grok_simulate_route.ts} (73%) create mode 100644 x-pack/plugins/grokdebugger/server/shared_imports.ts diff --git a/x-pack/plugins/grokdebugger/public/services/grokdebugger/grokdebugger_service.js b/x-pack/plugins/grokdebugger/public/services/grokdebugger/grokdebugger_service.js index 207093e72ca2c..7f4ac35bd4eef 100644 --- a/x-pack/plugins/grokdebugger/public/services/grokdebugger/grokdebugger_service.js +++ b/x-pack/plugins/grokdebugger/public/services/grokdebugger/grokdebugger_service.js @@ -21,7 +21,7 @@ export class GrokdebuggerService { return GrokdebuggerResponse.fromUpstreamJSON(response); }) .catch((e) => { - throw e.body.message; + throw new Error(e.body.message); }); } } diff --git a/x-pack/plugins/grokdebugger/server/lib/kibana_framework.ts b/x-pack/plugins/grokdebugger/server/lib/kibana_framework.ts index ee7fa74022fd5..94b7f8a9af796 100644 --- a/x-pack/plugins/grokdebugger/server/lib/kibana_framework.ts +++ b/x-pack/plugins/grokdebugger/server/lib/kibana_framework.ts @@ -6,14 +6,7 @@ import { i18n } from '@kbn/i18n'; -import { - CoreSetup, - IRouter, - RequestHandlerContext, - RouteMethod, - RouteConfig, - RequestHandler, -} from 'src/core/server'; +import { CoreSetup, IRouter, RouteMethod, RouteConfig, RequestHandler } from 'src/core/server'; import { ILicense } from '../../../licensing/server'; @@ -83,21 +76,4 @@ export class KibanaFramework { break; } } - - callWithRequest( - requestContext: RequestHandlerContext, - endpoint: 'ingest.simulate', - options?: { - body: any; - } - ): Promise; - - public async callWithRequest( - requestContext: RequestHandlerContext, - endpoint: string, - options?: any - ) { - const { elasticsearch } = requestContext.core; - return elasticsearch.legacy.client.callAsCurrentUser(endpoint, options); - } } diff --git a/x-pack/plugins/grokdebugger/server/routes/api/grokdebugger/register_grok_simulate_route.js b/x-pack/plugins/grokdebugger/server/routes/api/grokdebugger/register_grok_simulate_route.ts similarity index 73% rename from x-pack/plugins/grokdebugger/server/routes/api/grokdebugger/register_grok_simulate_route.js rename to x-pack/plugins/grokdebugger/server/routes/api/grokdebugger/register_grok_simulate_route.ts index f953bc64c3b4f..3525de8da87d1 100644 --- a/x-pack/plugins/grokdebugger/server/routes/api/grokdebugger/register_grok_simulate_route.js +++ b/x-pack/plugins/grokdebugger/server/routes/api/grokdebugger/register_grok_simulate_route.ts @@ -5,9 +5,16 @@ */ import { schema } from '@kbn/config-schema'; + +// @ts-ignore import { GrokdebuggerRequest } from '../../../models/grokdebugger_request'; +// @ts-ignore import { GrokdebuggerResponse } from '../../../models/grokdebugger_response'; +import { handleEsError } from '../../../shared_imports'; + +import { KibanaFramework } from '../../../lib/kibana_framework'; + const requestBodySchema = schema.object({ pattern: schema.string(), rawEvent: schema.string(), @@ -15,7 +22,7 @@ const requestBodySchema = schema.object({ customPatterns: schema.object({}, { unknowns: 'allow' }), }); -export function registerGrokSimulateRoute(framework) { +export function registerGrokSimulateRoute(framework: KibanaFramework) { framework.registerRoute( { method: 'post', @@ -27,19 +34,17 @@ export function registerGrokSimulateRoute(framework) { async (requestContext, request, response) => { try { const grokdebuggerRequest = GrokdebuggerRequest.fromDownstreamJSON(request.body); - const simulateResponseFromES = await framework.callWithRequest( - requestContext, - 'ingest.simulate', + const simulateResponseFromES = await requestContext.core.elasticsearch.client.asCurrentUser.ingest.simulate( { body: grokdebuggerRequest.upstreamJSON } ); - const grokdebuggerResponse = GrokdebuggerResponse.fromUpstreamJSON(simulateResponseFromES); + const grokdebuggerResponse = GrokdebuggerResponse.fromUpstreamJSON( + simulateResponseFromES.body + ); return response.ok({ body: grokdebuggerResponse, }); } catch (error) { - return response.internalError({ - body: error.message, - }); + return handleEsError({ error, response }); } } ); diff --git a/x-pack/plugins/grokdebugger/server/shared_imports.ts b/x-pack/plugins/grokdebugger/server/shared_imports.ts new file mode 100644 index 0000000000000..068cddcee4c86 --- /dev/null +++ b/x-pack/plugins/grokdebugger/server/shared_imports.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export { handleEsError } from '../../../../src/plugins/es_ui_shared/server'; From 244ce85fb4474461c190650f6335fa717d55f536 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Loix?= Date: Wed, 11 Nov 2020 15:51:58 +0100 Subject: [PATCH 008/104] [Index management] Fix test in index template wizard (#83150) --- .../index_template_wizard/template_create.test.tsx | 4 +--- .../index_template_wizard/template_form.helpers.ts | 2 ++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/index_management/__jest__/client_integration/index_template_wizard/template_create.test.tsx b/x-pack/plugins/index_management/__jest__/client_integration/index_template_wizard/template_create.test.tsx index b67e503f8d3e2..da74dcabaf1e8 100644 --- a/x-pack/plugins/index_management/__jest__/client_integration/index_template_wizard/template_create.test.tsx +++ b/x-pack/plugins/index_management/__jest__/client_integration/index_template_wizard/template_create.test.tsx @@ -266,9 +266,7 @@ describe('', () => { it('should not allow invalid json', async () => { const { form, actions } = testBed; - await act(async () => { - actions.completeStepThree('{ invalidJsonString '); - }); + await actions.completeStepThree('{ invalidJsonString '); expect(form.getErrorsMessages()).toContain('Invalid JSON format.'); }); diff --git a/x-pack/plugins/index_management/__jest__/client_integration/index_template_wizard/template_form.helpers.ts b/x-pack/plugins/index_management/__jest__/client_integration/index_template_wizard/template_form.helpers.ts index 025410129a002..2ce5aa2fccfc4 100644 --- a/x-pack/plugins/index_management/__jest__/client_integration/index_template_wizard/template_form.helpers.ts +++ b/x-pack/plugins/index_management/__jest__/client_integration/index_template_wizard/template_form.helpers.ts @@ -208,7 +208,9 @@ export const formSetup = async (initTestBed: SetupFunc) => { jsonString: settings, }); // Using mocked EuiCodeEditor } + }); + await act(async () => { clickNextButton(); }); From 352a65507f5d1d4e0df1a0458c9cc770aedefa43 Mon Sep 17 00:00:00 2001 From: Ryan Keairns Date: Wed, 11 Nov 2020 09:55:15 -0600 Subject: [PATCH 009/104] Change left nav category name to Analytics (#83132) * Change left nav category name to Analytics * update snapshot * update x-pack snapshot * remove existing default translations --- .../__snapshots__/collapsible_nav.test.tsx.snap | 16 ++++++++-------- src/core/utils/default_app_categories.ts | 2 +- .../__snapshots__/enabled_features.test.tsx.snap | 4 ++-- .../plugins/translations/translations/ja-JP.json | 1 - .../plugins/translations/translations/zh-CN.json | 1 - 5 files changed, 11 insertions(+), 13 deletions(-) diff --git a/src/core/public/chrome/ui/header/__snapshots__/collapsible_nav.test.tsx.snap b/src/core/public/chrome/ui/header/__snapshots__/collapsible_nav.test.tsx.snap index cf734f33cc3e4..b01dd205440a9 100644 --- a/src/core/public/chrome/ui/header/__snapshots__/collapsible_nav.test.tsx.snap +++ b/src/core/public/chrome/ui/header/__snapshots__/collapsible_nav.test.tsx.snap @@ -131,7 +131,7 @@ exports[`CollapsibleNav renders links grouped by category 1`] = ` "category": Object { "euiIconType": "logoKibana", "id": "kibana", - "label": "Kibana", + "label": "Analytics", "order": 1000, }, "data-test-subj": "discover", @@ -187,7 +187,7 @@ exports[`CollapsibleNav renders links grouped by category 1`] = ` "category": Object { "euiIconType": "logoKibana", "id": "kibana", - "label": "Kibana", + "label": "Analytics", "order": 1000, }, "data-test-subj": "visualize", @@ -201,7 +201,7 @@ exports[`CollapsibleNav renders links grouped by category 1`] = ` "category": Object { "euiIconType": "logoKibana", "id": "kibana", - "label": "Kibana", + "label": "Analytics", "order": 1000, }, "data-test-subj": "dashboard", @@ -859,7 +859,7 @@ exports[`CollapsibleNav renders links grouped by category 1`] = ` isCollapsible={true} key="kibana" onToggle={[Function]} - title="Kibana" + title="Analytics" > - Kibana + Analytics @@ -971,7 +971,7 @@ exports[`CollapsibleNav renders links grouped by category 1`] = ` className="euiTitle euiTitle--xxsmall euiCollapsibleNavGroup__title" id="mockId__title" > - Kibana + Analytics @@ -996,7 +996,7 @@ exports[`CollapsibleNav renders links grouped by category 1`] = ` className="euiCollapsibleNavGroup__children" >