From c74f8559e990ba3f3a27888691cda5e4e92678bc Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Thu, 29 Aug 2019 15:56:19 -0400 Subject: [PATCH 1/9] [lens] Calculate existence of fields in datasource --- src/core/server/index.ts | 2 +- src/fixtures/logstash_fields.js | 3 +- x-pack/legacy/plugins/lens/index.ts | 18 +- .../editor_frame/data_panel_wrapper.tsx | 7 +- .../editor_frame/editor_frame.tsx | 2 + .../indexpattern_plugin/datapanel.test.tsx | 320 ++++++++++++++---- .../public/indexpattern_plugin/datapanel.tsx | 264 ++++++++++++--- .../dimension_panel/_popover.scss | 4 + .../dimension_panel/dimension_panel.test.tsx | 7 + .../dimension_panel/field_select.tsx | 24 +- .../dimension_panel/popover_editor.tsx | 3 +- .../public/indexpattern_plugin/field_icon.tsx | 20 +- .../public/indexpattern_plugin/field_item.tsx | 28 +- .../indexpattern_plugin/indexpattern.scss | 7 +- .../indexpattern_plugin/indexpattern.test.ts | 6 + .../indexpattern_plugin/indexpattern.tsx | 24 +- .../indexpattern_suggestions.test.tsx | 3 + .../indexpattern_plugin/layerpanel.test.tsx | 1 + .../lens/public/indexpattern_plugin/loader.ts | 3 + .../definitions/date_histogram.test.tsx | 1 + .../definitions/filter_ratio.test.tsx | 1 + .../operations/definitions/terms.test.tsx | 1 + .../operations/operations.test.ts | 1 + .../indexpattern_plugin/state_helpers.test.ts | 6 + x-pack/legacy/plugins/lens/public/types.ts | 4 +- x-pack/legacy/plugins/lens/readme.md | 3 + x-pack/legacy/plugins/lens/server/index.ts | 11 + x-pack/legacy/plugins/lens/server/plugin.tsx | 24 ++ .../plugins/lens/server/routes/index.ts | 12 + .../lens/server/routes/index_stats.test.ts | 56 +++ .../plugins/lens/server/routes/index_stats.ts | 177 ++++++++++ x-pack/test/api_integration/apis/index.js | 1 + .../test/api_integration/apis/lens/index.ts | 13 + .../api_integration/apis/lens/index_stats.ts | 110 ++++++ x-pack/test/functional/apps/lens/index.ts | 5 + 35 files changed, 1013 insertions(+), 159 deletions(-) create mode 100644 x-pack/legacy/plugins/lens/server/index.ts create mode 100644 x-pack/legacy/plugins/lens/server/plugin.tsx create mode 100644 x-pack/legacy/plugins/lens/server/routes/index.ts create mode 100644 x-pack/legacy/plugins/lens/server/routes/index_stats.test.ts create mode 100644 x-pack/legacy/plugins/lens/server/routes/index_stats.ts create mode 100644 x-pack/test/api_integration/apis/lens/index.ts create mode 100644 x-pack/test/api_integration/apis/lens/index_stats.ts diff --git a/src/core/server/index.ts b/src/core/server/index.ts index ef31804be62b2..61266016089b9 100644 --- a/src/core/server/index.ts +++ b/src/core/server/index.ts @@ -211,7 +211,7 @@ export interface CoreSetup { name: T, provider: RequestHandlerContextProvider ) => RequestHandlerContextContainer; - createRouter: () => IRouter; + createRouter: (basePath: string) => IRouter; }; } diff --git a/src/fixtures/logstash_fields.js b/src/fixtures/logstash_fields.js index 1bd2c050b4563..5771a01047c2e 100644 --- a/src/fixtures/logstash_fields.js +++ b/src/fixtures/logstash_fields.js @@ -39,7 +39,8 @@ function stubbedLogstashFields() { ['area', 'geo_shape', true, true ], ['hashed', 'murmur3', false, true ], ['geo.coordinates', 'geo_point', true, true ], - ['extension', 'keyword', true, true ], + ['extension', 'text', true, true], + ['extension.keyword', 'keyword', true, true, {}, 'extension', 'multi' ], ['machine.os', 'text', true, true ], ['machine.os.raw', 'keyword', true, true, {}, 'machine.os', 'multi' ], ['geo.src', 'keyword', true, true ], diff --git a/x-pack/legacy/plugins/lens/index.ts b/x-pack/legacy/plugins/lens/index.ts index 36cc18c86b549..39b5a8f6820c6 100644 --- a/x-pack/legacy/plugins/lens/index.ts +++ b/x-pack/legacy/plugins/lens/index.ts @@ -5,11 +5,13 @@ */ import * as Joi from 'joi'; -import { Server } from 'hapi'; import { resolve } from 'path'; import { LegacyPluginInitializer } from 'src/legacy/types'; +import KbnServer, { Server } from 'src/legacy/server/kbn_server'; +import { CoreSetup } from 'src/core/server'; import mappings from './mappings.json'; import { PLUGIN_ID, getEditPath } from './common'; +import { lensServerPlugin } from './server'; const NOT_INTERNATIONALIZED_PRODUCT_NAME = 'Lens Visualizations'; @@ -49,7 +51,9 @@ export const lens: LegacyPluginInitializer = kibana => { }).default(); }, - init(server: Server) { + async init(server: Server) { + const kbnServer = (server as unknown) as KbnServer; + server.plugins.xpack_main.registerFeature({ id: PLUGIN_ID, name: NOT_INTERNATIONALIZED_PRODUCT_NAME, @@ -77,6 +81,16 @@ export const lens: LegacyPluginInitializer = kibana => { }, }, }); + + // Set up with the new platform plugin lifecycle API. + const plugin = lensServerPlugin(); + await plugin.setup(({ + http: kbnServer.newPlatform.setup.core.http, + } as unknown) as CoreSetup); + + server.events.on('stop', async () => { + await plugin.stop(); + }); }, }); }; diff --git a/x-pack/legacy/plugins/lens/public/editor_frame_plugin/editor_frame/data_panel_wrapper.tsx b/x-pack/legacy/plugins/lens/public/editor_frame_plugin/editor_frame/data_panel_wrapper.tsx index a013d9b1bceae..d3beddc0689c1 100644 --- a/x-pack/legacy/plugins/lens/public/editor_frame_plugin/editor_frame/data_panel_wrapper.tsx +++ b/x-pack/legacy/plugins/lens/public/editor_frame_plugin/editor_frame/data_panel_wrapper.tsx @@ -7,11 +7,12 @@ import React, { useMemo, memo, useContext, useState } from 'react'; import { i18n } from '@kbn/i18n'; import { EuiPopover, EuiButtonIcon, EuiContextMenuPanel, EuiContextMenuItem } from '@elastic/eui'; +import { Query } from 'src/plugins/data/common'; import { DatasourceDataPanelProps, Datasource } from '../../../public'; import { NativeRenderer } from '../../native_renderer'; import { Action } from './state_management'; import { DragContext } from '../../drag_drop'; -import { StateSetter } from '../../types'; +import { StateSetter, FramePublicAPI } from '../../types'; interface DataPanelWrapperProps { datasourceState: unknown; @@ -19,6 +20,8 @@ interface DataPanelWrapperProps { activeDatasource: string | null; datasourceIsLoading: boolean; dispatch: (action: Action) => void; + query: Query; + dateRange: FramePublicAPI['dateRange']; } export const DataPanelWrapper = memo((props: DataPanelWrapperProps) => { @@ -37,6 +40,8 @@ export const DataPanelWrapper = memo((props: DataPanelWrapperProps) => { dragDropContext: useContext(DragContext), state: props.datasourceState, setState: setDatasourceState, + query: props.query, + dateRange: props.dateRange, }; const [showDatasourceSwitcher, setDatasourceSwitcher] = useState(false); diff --git a/x-pack/legacy/plugins/lens/public/editor_frame_plugin/editor_frame/editor_frame.tsx b/x-pack/legacy/plugins/lens/public/editor_frame_plugin/editor_frame/editor_frame.tsx index 927a1f1abf3dc..f7837bc02b9be 100644 --- a/x-pack/legacy/plugins/lens/public/editor_frame_plugin/editor_frame/editor_frame.tsx +++ b/x-pack/legacy/plugins/lens/public/editor_frame_plugin/editor_frame/editor_frame.tsx @@ -219,6 +219,8 @@ export function EditorFrame(props: EditorFrameProps) { : true } dispatch={dispatch} + query={props.query} + dateRange={props.dateRange} /> } configPanel={ diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/datapanel.test.tsx b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/datapanel.test.tsx index 7f74fd8c1f0b8..c380646c33a30 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/datapanel.test.tsx +++ b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/datapanel.test.tsx @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { shallow } from 'enzyme'; +import { shallow, mount } from 'enzyme'; import React, { ChangeEvent, ReactElement } from 'react'; import { EuiComboBox, EuiFieldSearch, EuiContextMenuPanel } from '@elastic/eui'; import { IndexPatternPrivateState, IndexPatternColumn } from './indexpattern'; @@ -12,12 +12,16 @@ import { createMockedDragDropContext } from './mocks'; import { InnerIndexPatternDataPanel, IndexPatternDataPanel, MemoizedDataPanel } from './datapanel'; import { FieldItem } from './field_item'; import { act } from 'react-dom/test-utils'; +import { npStart as npStartMock } from 'ui/new_platform'; jest.mock('ui/new_platform'); jest.mock('./loader'); +const waitForPromises = () => new Promise(resolve => setTimeout(resolve)); + const initialState: IndexPatternPrivateState = { currentIndexPatternId: '1', + showEmptyFields: false, layers: { first: { indexPatternId: '1', @@ -207,6 +211,13 @@ describe('IndexPattern Data Panel', () => { showIndexPatternSwitcher: false, setShowIndexPatternSwitcher: jest.fn(), onChangeIndexPattern: jest.fn(), + dateRange: { + fromDate: 'now-7d', + toDate: 'now', + }, + query: { query: '', language: 'lucene' }, + showEmptyFields: false, + onToggleEmptyFields: jest.fn(), }; }); @@ -214,6 +225,7 @@ describe('IndexPattern Data Panel', () => { const setStateSpy = jest.fn(); const wrapper = shallow( { }; const wrapper = shallow( {} }} @@ -274,6 +287,7 @@ describe('IndexPattern Data Panel', () => { }; const wrapper = shallow( {} }} @@ -316,98 +330,260 @@ describe('IndexPattern Data Panel', () => { expect(defaultProps.onChangeIndexPattern).toHaveBeenCalledWith('2'); }); - it('should list all supported fields in the pattern sorted alphabetically', async () => { - const wrapper = shallow(); + describe('loading existence data', () => { + beforeEach(() => { + (npStartMock.core.http.post as jest.Mock).mockClear(); + }); - expect(wrapper.find(FieldItem).map(fieldItem => fieldItem.prop('field').name)).toEqual([ - 'bytes', - 'memory', - 'source', - 'timestamp', - ]); - }); + it('loads existence data and updates the index pattern', async () => { + (npStartMock.core.http.post as jest.Mock).mockResolvedValue({ + timestamp: { + exists: true, + cardinality: 500, + count: 500, + }, + }); + const updateFields = jest.fn(); + mount(); - it('should filter down by name', async () => { - const wrapper = shallow(); + await waitForPromises(); - act(() => { - wrapper.find(EuiFieldSearch).prop('onChange')!({ target: { value: 'mem' } } as ChangeEvent< - HTMLInputElement - >); + expect(npStartMock.core.http.post as jest.Mock).toHaveBeenCalledWith( + `/api/lens/index_stats/my-fake-index-pattern`, + { + body: JSON.stringify({ + earliest: 'now-7d', + latest: 'now', + size: 500, + timeFieldName: 'timestamp', + fields: [ + { + name: 'timestamp', + type: 'date', + }, + { + name: 'bytes', + type: 'number', + }, + { + name: 'memory', + type: 'number', + }, + { + name: 'unsupported', + type: 'geo', + }, + { + name: 'source', + type: 'string', + }, + ], + }), + } + ); + + expect(updateFields).toHaveBeenCalledWith('1', [ + { + name: 'timestamp', + type: 'date', + exists: true, + cardinality: 500, + count: 500, + aggregatable: true, + searchable: true, + }, + ...defaultProps.indexPatterns['1'].fields + .slice(1) + .map(field => ({ ...field, exists: false })), + ]); }); - expect(wrapper.find(FieldItem).map(fieldItem => fieldItem.prop('field').name)).toEqual([ - 'memory', - ]); - }); + it('does not attempt to load existence data if the index pattern has it', async () => { + const updateFields = jest.fn(); + const newIndexPatterns = { + ...defaultProps.indexPatterns, + '1': { + ...defaultProps.indexPatterns['1'], + hasExistence: true, + }, + }; - it('should filter down by type', async () => { - const wrapper = shallow(); + const props = { ...defaultProps, indexPatterns: newIndexPatterns }; - act(() => { - (wrapper - .find(EuiContextMenuPanel) - .prop('items')! - .find( - item => (item as ReactElement).props['data-test-subj'] === 'typeFilter-number' - )! as ReactElement).props.onClick(); - }); + mount(); - expect(wrapper.find(FieldItem).map(fieldItem => fieldItem.prop('field').name)).toEqual([ - 'bytes', - 'memory', - ]); + await waitForPromises(); + + expect(npStartMock.core.http.post as jest.Mock).not.toHaveBeenCalled(); + }); }); - it('should toggle type if clicked again', async () => { - const wrapper = shallow(); + describe('while showing empty fields', () => { + it('should list all supported fields in the pattern sorted alphabetically', async () => { + const wrapper = shallow( + + ); + + expect(wrapper.find(FieldItem).map(fieldItem => fieldItem.prop('field').name)).toEqual([ + 'bytes', + 'memory', + 'source', + 'timestamp', + ]); + }); - act(() => { - (wrapper - .find(EuiContextMenuPanel) - .prop('items')! - .find( - item => (item as ReactElement).props['data-test-subj'] === 'typeFilter-number' - )! as ReactElement).props.onClick(); + it('should filter down by name', async () => { + const wrapper = shallow( + + ); + + act(() => { + wrapper.find(EuiFieldSearch).prop('onChange')!({ target: { value: 'mem' } } as ChangeEvent< + HTMLInputElement + >); + }); + + expect(wrapper.find(FieldItem).map(fieldItem => fieldItem.prop('field').name)).toEqual([ + 'memory', + ]); }); - act(() => { - (wrapper - .find(EuiContextMenuPanel) - .prop('items')! - .find( - item => (item as ReactElement).props['data-test-subj'] === 'typeFilter-number' - )! as ReactElement).props.onClick(); + it('should filter down by type', async () => { + const wrapper = shallow( + + ); + + act(() => { + (wrapper + .find(EuiContextMenuPanel) + .prop('items')! + .find( + item => (item as ReactElement).props['data-test-subj'] === 'typeFilter-number' + )! as ReactElement).props.onClick(); + }); + + expect(wrapper.find(FieldItem).map(fieldItem => fieldItem.prop('field').name)).toEqual([ + 'bytes', + 'memory', + ]); }); - expect(wrapper.find(FieldItem).map(fieldItem => fieldItem.prop('field').name)).toEqual([ - 'bytes', - 'memory', - 'source', - 'timestamp', - ]); + it('should toggle type if clicked again', async () => { + const wrapper = shallow( + + ); + + act(() => { + (wrapper + .find(EuiContextMenuPanel) + .prop('items')! + .find( + item => (item as ReactElement).props['data-test-subj'] === 'typeFilter-number' + )! as ReactElement).props.onClick(); + }); + + act(() => { + (wrapper + .find(EuiContextMenuPanel) + .prop('items')! + .find( + item => (item as ReactElement).props['data-test-subj'] === 'typeFilter-number' + )! as ReactElement).props.onClick(); + }); + + expect(wrapper.find(FieldItem).map(fieldItem => fieldItem.prop('field').name)).toEqual([ + 'bytes', + 'memory', + 'source', + 'timestamp', + ]); + }); + + it('should filter down by type and by name', async () => { + const wrapper = shallow( + + ); + + act(() => { + wrapper.find(EuiFieldSearch).prop('onChange')!({ target: { value: 'mem' } } as ChangeEvent< + HTMLInputElement + >); + }); + + act(() => { + (wrapper + .find(EuiContextMenuPanel) + .prop('items')! + .find( + item => (item as ReactElement).props['data-test-subj'] === 'typeFilter-number' + )! as ReactElement).props.onClick(); + }); + + expect(wrapper.find(FieldItem).map(fieldItem => fieldItem.prop('field').name)).toEqual([ + 'memory', + ]); + }); }); - it('should filter down by type and by name', async () => { - const wrapper = shallow(); + describe('filtering out empty fields', () => { + let emptyFieldsTestProps: typeof defaultProps; + + beforeEach(() => { + emptyFieldsTestProps = { + ...defaultProps, + indexPatterns: { + ...defaultProps.indexPatterns, + '1': { + ...defaultProps.indexPatterns['1'], + hasExistence: true, + fields: defaultProps.indexPatterns['1'].fields.map(field => ({ + ...field, + exists: field.type === 'number', + })), + }, + }, + onToggleEmptyFields: jest.fn(), + }; + }); - act(() => { - wrapper.find(EuiFieldSearch).prop('onChange')!({ target: { value: 'mem' } } as ChangeEvent< - HTMLInputElement - >); + it('should list all supported fields in the pattern sorted alphabetically', async () => { + const wrapper = shallow(); + + expect(wrapper.find(FieldItem).map(fieldItem => fieldItem.prop('field').name)).toEqual([ + 'bytes', + 'memory', + ]); }); - act(() => { - (wrapper - .find(EuiContextMenuPanel) - .prop('items')! - .find( - item => (item as ReactElement).props['data-test-subj'] === 'typeFilter-number' - )! as ReactElement).props.onClick(); + it('should filter down by name', async () => { + const wrapper = shallow( + + ); + + act(() => { + wrapper.find(EuiFieldSearch).prop('onChange')!({ target: { value: 'mem' } } as ChangeEvent< + HTMLInputElement + >); + }); + + expect(wrapper.find(FieldItem).map(fieldItem => fieldItem.prop('field').name)).toEqual([ + 'memory', + ]); }); - expect(wrapper.find(FieldItem).map(fieldItem => fieldItem.prop('field').name)).toEqual([ - 'memory', - ]); + it('should allow removing the filter for data', async () => { + const wrapper = shallow(); + + act(() => { + (wrapper + .find(EuiContextMenuPanel) + .prop('items')! + .find( + item => (item as ReactElement).props['data-test-subj'] === 'emptyFilter' + )! as ReactElement).props.onClick(); + }); + + expect(emptyFieldsTestProps.onToggleEmptyFields).toHaveBeenCalled(); + }); }); }); diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/datapanel.tsx b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/datapanel.tsx index 74c19ad2cf0e3..78fbcc6e4cef9 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/datapanel.tsx +++ b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/datapanel.tsx @@ -9,6 +9,7 @@ import React, { useState, useEffect, memo, useCallback } from 'react'; import { EuiComboBox, EuiFieldSearch, + EuiLoadingSpinner, // @ts-ignore EuiHighlight, EuiFlexGroup, @@ -25,6 +26,8 @@ import { } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; +import { npStart } from 'ui/new_platform'; +import { Query } from 'src/plugins/data/common'; import { DatasourceDataPanelProps, DataType } from '../types'; import { IndexPatternPrivateState, IndexPatternField, IndexPattern } from './indexpattern'; import { ChildDragDropProvider, DragContextState } from '../drag_drop'; @@ -60,6 +63,8 @@ export function IndexPatternDataPanel({ setState, state, dragDropContext, + query, + dateRange, }: DatasourceDataPanelProps) { const { indexPatterns, currentIndexPatternId } = state; const [showIndexPatternSwitcher, setShowIndexPatternSwitcher] = useState(false); @@ -79,37 +84,93 @@ export function IndexPatternDataPanel({ [state, setState] ); + const updateFieldsWithCounts = useCallback( + (indexPatternId: string, allFields: IndexPattern['fields']) => { + setState(prevState => { + return { + ...prevState, + indexPatterns: { + ...prevState.indexPatterns, + [indexPatternId]: { + ...prevState.indexPatterns[indexPatternId], + hasExistence: true, + fields: allFields, + }, + }, + }; + }); + }, + [currentIndexPatternId, indexPatterns[currentIndexPatternId]] + ); + + const onToggleEmptyFields = useCallback(() => { + setState(prevState => ({ ...prevState, showEmptyFields: !prevState.showEmptyFields })); + }, [state, setState]); + return ( ); } +type OverallFields = Record< + string, + { + count: number; + cardinality: number; + } +>; + +interface DataPanelState { + isLoading: boolean; + nameFilter: string; + typeFilter: DataType[]; + isTypeFilterOpen: boolean; +} + export const InnerIndexPatternDataPanel = function InnerIndexPatternDataPanel({ currentIndexPatternId, indexPatterns, + query, + dateRange, dragDropContext, showIndexPatternSwitcher, setShowIndexPatternSwitcher, onChangeIndexPattern, -}: { + updateFieldsWithCounts, + showEmptyFields, + onToggleEmptyFields, +}: Partial & { currentIndexPatternId: string; indexPatterns: Record; + dateRange: DatasourceDataPanelProps['dateRange']; + query: Query; dragDropContext: DragContextState; showIndexPatternSwitcher: boolean; setShowIndexPatternSwitcher: (show: boolean) => void; + showEmptyFields: boolean; + onToggleEmptyFields: () => void; onChangeIndexPattern?: (newId: string) => void; + updateFieldsWithCounts?: (indexPatternId: string, fields: IndexPattern['fields']) => void; }) { - const [state, setState] = useState({ + const [localState, setLocalState] = useState({ + isLoading: false, nameFilter: '', - typeFilter: [] as DataType[], + typeFilter: [], isTypeFilterOpen: false, }); const [pageSize, setPageSize] = useState(PAGINATION_SIZE); @@ -131,7 +192,7 @@ export const InnerIndexPatternDataPanel = function InnerIndexPatternDataPanel({ setPageSize(PAGINATION_SIZE); lazyScroll(); } - }, [state.nameFilter, state.typeFilter, currentIndexPatternId]); + }, [localState.nameFilter, localState.typeFilter, currentIndexPatternId, showEmptyFields]); if (Object.keys(indexPatterns).length === 0) { return ( @@ -163,18 +224,96 @@ export const InnerIndexPatternDataPanel = function InnerIndexPatternDataPanel({ } const allFields = indexPatterns[currentIndexPatternId].fields; - const filteredFields = allFields + + const availableFieldTypes = uniq(allFields.map(({ type }) => type)).filter( + type => type in fieldTypeNames + ); + + const displayedFields = allFields + .filter(field => { + if (!showEmptyFields) { + const indexField = + indexPatterns[currentIndexPatternId] && + indexPatterns[currentIndexPatternId].hasExistence && + indexPatterns[currentIndexPatternId].fields.find(f => f.name === field.name); + if (localState.typeFilter.length > 0) { + return ( + indexField && + indexField.exists && + localState.typeFilter.includes(field.type as DataType) + ); + } + return indexField && indexField.exists; + } + if (localState.typeFilter.length > 0) { + return localState.typeFilter.includes(field.type as DataType); + } + return true; + }) .filter( (field: IndexPatternField) => - field.name.toLowerCase().includes(state.nameFilter.toLowerCase()) && + field.name.toLowerCase().includes(localState.nameFilter.toLowerCase()) && supportedFieldTypes.includes(field.type) - ) - .slice(0, pageSize); + ); - const availableFieldTypes = uniq(filteredFields.map(({ type }) => type)); - const availableFilteredTypes = state.typeFilter.filter(type => - availableFieldTypes.includes(type) - ); + const paginatedFields = displayedFields.sort(sortFields).slice(0, pageSize); + + useEffect(() => { + if ( + localState.isLoading || + indexPatterns[currentIndexPatternId].hasExistence || + !updateFieldsWithCounts + ) { + return; + } + + setLocalState(s => ({ ...s, isLoading: true })); + + npStart.core.http + .post(`/api/lens/index_stats/${indexPatterns[currentIndexPatternId].title}`, { + body: JSON.stringify({ + earliest: dateRange.fromDate, + latest: dateRange.toDate, + size: 500, + timeFieldName: indexPatterns[currentIndexPatternId].timeFieldName, + fields: allFields + .filter(field => field.aggregatable) + .map(field => ({ + name: field.name, + type: field.type, + })), + }), + }) + .then((results: OverallFields) => { + setLocalState(s => ({ + ...s, + isLoading: false, + })); + + if (!updateFieldsWithCounts) { + return; + } + + updateFieldsWithCounts( + currentIndexPatternId, + indexPatterns[currentIndexPatternId].fields.map(field => { + const matching = results[field.name]; + if (!matching) { + return { ...field, exists: false }; + } + return { + ...field, + exists: true, + cardinality: matching.cardinality, + count: matching.count, + }; + }) + ); + }) + .catch(() => { + setLocalState(s => ({ ...s, isLoading: false })); + }); + }, [currentIndexPatternId]); return ( @@ -240,11 +379,11 @@ export const InnerIndexPatternDataPanel = function InnerIndexPatternDataPanel({ onChange={choices => { onChangeIndexPattern!(choices[0].value as string); - setState({ - ...state, + setLocalState(s => ({ + ...s, nameFilter: '', typeFilter: [], - }); + })); setShowIndexPatternSwitcher(false); }} @@ -265,9 +404,9 @@ export const InnerIndexPatternDataPanel = function InnerIndexPatternDataPanel({ description: 'Search the list of fields in the index pattern for the provided text', })} - value={state.nameFilter} + value={localState.nameFilter} onChange={e => { - setState({ ...state, nameFilter: e.target.value }); + setLocalState({ ...localState, nameFilter: e.target.value }); }} aria-label={i18n.translate('xpack.lens.indexPatterns.filterByNameAriaLabel', { defaultMessage: 'Search fields', @@ -280,19 +419,24 @@ export const InnerIndexPatternDataPanel = function InnerIndexPatternDataPanel({ id="dataPanelTypeFilter" panelClassName="euiFilterGroup__popoverPanel" panelPaddingSize="none" - isOpen={state.isTypeFilterOpen} - closePopover={() => setState({ ...state, isTypeFilterOpen: false })} + isOpen={localState.isTypeFilterOpen} + closePopover={() => + setLocalState(s => ({ ...localState, isTypeFilterOpen: false })) + } button={ - setState({ ...state, isTypeFilterOpen: !state.isTypeFilterOpen }) + setLocalState(s => ({ + ...s, + isTypeFilterOpen: !localState.isTypeFilterOpen, + })) } iconType="arrowDown" data-test-subj="indexPatternTypeFilterButton" - isSelected={state.isTypeFilterOpen} - numFilters={availableFieldTypes.length} - hasActiveFilters={availableFilteredTypes.length > 0} - numActiveFilters={availableFilteredTypes.length} + isSelected={localState.isTypeFilterOpen} + numFilters={availableFieldTypes.length + 1} + hasActiveFilters={localState.typeFilter.length > 0 || showEmptyFields} + numActiveFilters={localState.typeFilter.length + (showEmptyFields ? 1 : 0)} > {i18n.translate('xpack.lens.indexPatterns.typeFilterLabel', { defaultMessage: 'Types', @@ -302,23 +446,38 @@ export const InnerIndexPatternDataPanel = function InnerIndexPatternDataPanel({ > ( - - setState({ - ...state, - typeFilter: state.typeFilter.includes(type) - ? state.typeFilter.filter(t => t !== type) - : [...state.typeFilter, type], - }) - } - > - {fieldTypeNames[type]} - - ))} + items={(availableFieldTypes as DataType[]) + .map(type => ( + + setLocalState(s => ({ + ...s, + typeFilter: localState.typeFilter.includes(type) + ? localState.typeFilter.filter(t => t !== type) + : [...localState.typeFilter, type], + })) + } + > + {fieldTypeNames[type]} + + )) + .concat([ + { + onToggleEmptyFields(); + }} + > + {i18n.translate('xpack.lens.datatypes.hiddenFields', { + defaultMessage: 'Only show fields that contain data', + })} + , + ])} /> @@ -335,21 +494,22 @@ export const InnerIndexPatternDataPanel = function InnerIndexPatternDataPanel({ onScroll={lazyScroll} >
- {filteredFields - .filter( - field => - state.typeFilter.length === 0 || - state.typeFilter.includes(field.type as DataType) - ) - .sort(sortFields) - .map(field => ( + {localState.isLoading && } + + {paginatedFields.map(field => { + const overallField = indexPatterns[currentIndexPatternId].fields.find( + f => f.name === field.name + ); + return ( - ))} + ); + })}
diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/dimension_panel/_popover.scss b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/dimension_panel/_popover.scss index b5701daf31d7e..42f9366bd1f1e 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/dimension_panel/_popover.scss +++ b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/dimension_panel/_popover.scss @@ -16,6 +16,10 @@ color: $euiColorLightShade; } +.lnsConfigPanel__fieldOption--nonExistant { + background-color: $euiColorLightestShade; +} + .lnsConfigPanel__operation { padding: $euiSizeXS; font-size: 0.875rem; diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/dimension_panel/dimension_panel.test.tsx b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/dimension_panel/dimension_panel.test.tsx index f45b674e0c19b..1bbf0265eacd5 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/dimension_panel/dimension_panel.test.tsx +++ b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/dimension_panel/dimension_panel.test.tsx @@ -33,30 +33,35 @@ const expectedIndexPatterns = { id: '1', title: 'my-fake-index-pattern', timeFieldName: 'timestamp', + hasExistence: true, fields: [ { name: 'timestamp', type: 'date', aggregatable: true, searchable: true, + exists: true, }, { name: 'bytes', type: 'number', aggregatable: true, searchable: true, + exists: true, }, { name: 'memory', type: 'number', aggregatable: true, searchable: true, + exists: true, }, { name: 'source', type: 'string', aggregatable: true, searchable: true, + exists: true, }, ], }, @@ -80,6 +85,7 @@ describe('IndexPatternDimensionPanel', () => { state = { indexPatterns: expectedIndexPatterns, currentIndexPatternId: '1', + showEmptyFields: false, layers: { first: { indexPatternId: '1', @@ -956,6 +962,7 @@ describe('IndexPatternDimensionPanel', () => { }, }, currentIndexPatternId: '1', + showEmptyFields: false, layers: { myLayer: { indexPatternId: 'foo', diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/dimension_panel/field_select.tsx b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/dimension_panel/field_select.tsx index 2874449bcb4ff..db15407adcb29 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/dimension_panel/field_select.tsx +++ b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/dimension_panel/field_select.tsx @@ -24,6 +24,7 @@ export type FieldChoice = export interface FieldSelectProps { currentIndexPattern: IndexPattern; + showEmptyFields: boolean; fieldMap: Record; incompatibleSelectedOperationType: OperationType | null; selectedColumnOperationType?: OperationType; @@ -34,16 +35,18 @@ export interface FieldSelectProps { } export function FieldSelect({ + currentIndexPattern, + showEmptyFields, + fieldMap, incompatibleSelectedOperationType, selectedColumnOperationType, selectedColumnSourceField, operationFieldSupportMatrix, - currentIndexPattern, - fieldMap, onChoose, onDeleteColumn, }: FieldSelectProps) { const { operationByDocument, operationByField } = operationFieldSupportMatrix; + const memoizedFieldOptions = useMemo(() => { const fields = Object.keys(operationByField).sort(); @@ -97,25 +100,31 @@ export function FieldSelect({ ? selectedColumnOperationType : undefined, }, + exists: fieldMap[field].exists || false, compatible: isCompatibleWithCurrentOperation(field), })) - .sort(({ compatible: a }, { compatible: b }) => { - if (a && !b) { + .filter(field => (showEmptyFields ? true : field.exists)) + .sort((a, b) => { + if (a.compatible && !b.compatible) { return -1; } - if (!a && b) { + if (!a.compatible && b.compatible) { return 1; } return 0; }) - .map(({ label, value, compatible }) => ({ + .map(({ label, value, compatible, exists }) => ({ label, value, - className: classNames({ 'lnsConfigPanel__fieldOption--incompatible': !compatible }), + className: classNames({ + 'lnsConfigPanel__fieldOption--incompatible': !compatible, + 'lnsConfigPanel__fieldOption--nonExistant': !exists, + }), 'data-test-subj': `lns-fieldOption${compatible ? '' : 'Incompatible'}-${label}`, })), }); } + return fieldOptions; }, [ incompatibleSelectedOperationType, @@ -124,6 +133,7 @@ export function FieldSelect({ operationFieldSupportMatrix, currentIndexPattern, fieldMap, + showEmptyFields, ]); return ( diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/dimension_panel/popover_editor.tsx b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/dimension_panel/popover_editor.tsx index 960e81b98699b..7eb03152b341f 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/dimension_panel/popover_editor.tsx +++ b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/dimension_panel/popover_editor.tsx @@ -258,8 +258,9 @@ export function PopoverEditor(props: PopoverEditorProps) { acc + ch.charCodeAt(0), 1); } -export type UnwrapArray = T extends Array ? P : T; - -export function FieldIcon({ type }: { type: DataType }) { - const icons: Partial>> = { +function getIconForDataType(dataType: string) { + const icons: Partial>> = { boolean: 'invert', date: 'calendar', }; + return icons[dataType] || ICON_TYPES.find(t => t === dataType) || 'empty'; +} - const iconType = icons[type] || ICON_TYPES.find(t => t === type) || 'empty'; +export function getColorForDataType(type: string) { + const iconType = getIconForDataType(type); const { colors } = palettes.euiPaletteColorBlind; const colorIndex = stringToNum(iconType) % colors.length; + return colors[colorIndex]; +} + +export type UnwrapArray = T extends Array ? P : T; + +export function FieldIcon({ type }: { type: DataType }) { + const iconType = getIconForDataType(type); const classes = classNames( 'lnsFieldListPanel__fieldIcon', `lnsFieldListPanel__fieldIcon--${type}` ); - return ; + return ; } diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/field_item.tsx b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/field_item.tsx index 51d519da21693..9756bd9d4a6ee 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/field_item.tsx +++ b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/field_item.tsx @@ -5,15 +5,18 @@ */ import React from 'react'; -import { IndexPatternField, DraggedField } from './indexpattern'; +// @ts-ignore +import { fieldFormats } from '../../../../../../src/legacy/ui/public/registry/field_formats'; +import { IndexPattern, IndexPatternField, DraggedField } from './indexpattern'; import { DragDrop } from '../drag_drop'; import { FieldIcon } from './field_icon'; import { DataType } from '..'; export interface FieldItemProps { field: IndexPatternField; - indexPatternId: string; + indexPattern: IndexPattern; highlight?: string; + exists: boolean; } function wrapOnDot(str?: string) { @@ -23,8 +26,8 @@ function wrapOnDot(str?: string) { return str ? str.replace(/\./g, '.\u200B') : ''; } -export function FieldItem({ field, indexPatternId, highlight }: FieldItemProps) { - const wrappableName = wrapOnDot(field.name); +export function FieldItem({ field, indexPattern, highlight, exists }: FieldItemProps) { + const wrappableName = wrapOnDot(field.name)!; const wrappableHighlight = wrapOnDot(highlight); const highlightIndex = wrappableHighlight ? wrappableName.toLowerCase().indexOf(wrappableHighlight.toLowerCase()) @@ -42,15 +45,20 @@ export function FieldItem({ field, indexPatternId, highlight }: FieldItemProps) return ( - - - {wrappableHighlightableFieldName} - +
+ + + + {wrappableHighlightableFieldName} + +
); } diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/indexpattern.scss b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/indexpattern.scss index adbbcacaa8f30..3dfd3c2c90756 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/indexpattern.scss +++ b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/indexpattern.scss @@ -53,10 +53,7 @@ background: $euiColorEmptyShade; border-radius: $euiBorderRadius; padding: $euiSizeS; - display: flex; - align-items: center; margin-bottom: $euiSizeXS; - font-weight: $euiFontWeightMedium; transition: box-shadow $euiAnimSpeedFast $euiAnimSlightResistance; &:hover { @@ -66,6 +63,10 @@ } } +.lnsFieldListPanel__field-missing { + background: $euiColorLightestShade; +} + .lnsFieldListPanel__fieldName { margin-left: $euiSizeXS; } diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/indexpattern.test.ts b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/indexpattern.test.ts index 8307b8e0ab828..4d2175aa91872 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/indexpattern.test.ts +++ b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/indexpattern.test.ts @@ -177,6 +177,7 @@ describe('IndexPattern Data Source', () => { currentIndexPatternId: '1', indexPatterns: expectedIndexPatterns, layers: {}, + showEmptyFields: false, }); }); @@ -185,6 +186,7 @@ describe('IndexPattern Data Source', () => { expect(state).toEqual({ ...persistedState, indexPatterns: expectedIndexPatterns, + showEmptyFields: false, }); }); }); @@ -266,6 +268,7 @@ describe('IndexPattern Data Source', () => { }, }, currentIndexPatternId: '1', + showEmptyFields: false, }; expect(indexPatternDatasource.insertLayer(state, 'newLayer')).toEqual({ ...state, @@ -284,6 +287,7 @@ describe('IndexPattern Data Source', () => { describe('#removeLayer', () => { it('should remove a layer', () => { const state = { + showEmptyFields: false, indexPatterns: expectedIndexPatterns, layers: { first: { @@ -316,6 +320,7 @@ describe('IndexPattern Data Source', () => { it('should list the current layers', () => { expect( indexPatternDatasource.getLayers({ + showEmptyFields: false, indexPatterns: expectedIndexPatterns, layers: { first: { @@ -339,6 +344,7 @@ describe('IndexPattern Data Source', () => { it('should return the title of the index patterns', () => { expect( indexPatternDatasource.getMetaData({ + showEmptyFields: false, indexPatterns: expectedIndexPatterns, layers: { first: { diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/indexpattern.tsx b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/indexpattern.tsx index a7cbdd59c6e0f..c2ac476ebc99c 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/indexpattern.tsx +++ b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/indexpattern.tsx @@ -28,7 +28,7 @@ import { import { isDraggedField } from './utils'; import { LayerPanel } from './layerpanel'; import { IndexPatternColumn } from './operations'; -import { Datasource } from '..'; +import { Datasource, StateSetter } from '..'; export { OperationType, IndexPatternColumn } from './operations'; @@ -37,6 +37,14 @@ export interface IndexPattern { fields: IndexPatternField[]; title: string; timeFieldName?: string | null; + hasExistence?: boolean; + fieldFormatMap?: Record< + string, + { + id: string; + params: unknown; + } + >; } export interface IndexPatternField { @@ -58,6 +66,11 @@ export interface IndexPatternField { } > >; + + // Loaded separately + exists?: boolean; + cardinality?: number; + count?: number; } export interface DraggedField { @@ -78,6 +91,7 @@ export interface IndexPatternPersistedState { export type IndexPatternPrivateState = IndexPatternPersistedState & { indexPatterns: Record; + showEmptyFields: boolean; }; export function columnToOperation(column: IndexPatternColumn): Operation { @@ -155,12 +169,14 @@ export function getIndexPatternDatasource({ return { ...state, indexPatterns, + showEmptyFields: false, }; } return { currentIndexPatternId: indexPatternObjects ? indexPatternObjects[0].id : '', indexPatterns, layers: {}, + showEmptyFields: false, }; }, @@ -223,7 +239,11 @@ export function getIndexPatternDatasource({ ); }, - getPublicAPI(state, setState, layerId) { + getPublicAPI( + state: IndexPatternPrivateState, + setState: StateSetter, + layerId: string + ) { return { getTableSpec: () => { return state.layers[layerId].columnOrder.map(colId => ({ columnId: colId })); diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/indexpattern_suggestions.test.tsx b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/indexpattern_suggestions.test.tsx index 37fee8f279d7a..47d04abdd3566 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/indexpattern_suggestions.test.tsx +++ b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/indexpattern_suggestions.test.tsx @@ -306,6 +306,7 @@ describe('IndexPattern Data Source suggestions', () => { it('should not make any suggestions for a number without a time field', async () => { const state: IndexPatternPrivateState = { currentIndexPatternId: '1', + showEmptyFields: false, indexPatterns: { 1: { id: '1', @@ -478,6 +479,7 @@ describe('IndexPattern Data Source suggestions', () => { it('should not make any suggestions for a number without a time field', async () => { const state: IndexPatternPrivateState = { currentIndexPatternId: '1', + showEmptyFields: false, indexPatterns: { 1: { id: '1', @@ -864,6 +866,7 @@ describe('IndexPattern Data Source suggestions', () => { it('returns no suggestions if there are no columns', () => { expect( indexPatternDatasource.getDatasourceSuggestionsFromCurrentState({ + showEmptyFields: false, indexPatterns: expectedIndexPatterns, layers: { first: { diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/layerpanel.test.tsx b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/layerpanel.test.tsx index 0faa6b4725896..e3806a0b63d63 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/layerpanel.test.tsx +++ b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/layerpanel.test.tsx @@ -18,6 +18,7 @@ jest.mock('./state_helpers'); const initialState: IndexPatternPrivateState = { currentIndexPatternId: '1', + showEmptyFields: false, layers: { first: { indexPatternId: '1', diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/loader.ts b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/loader.ts index 8ed19d098134e..79fa208e16cf9 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/loader.ts +++ b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/loader.ts @@ -57,6 +57,9 @@ export const getIndexPatterns = (chrome: Chrome, toastNotifications: ToastNotifi typeMeta: attributes.typeMeta ? (JSON.parse(attributes.typeMeta) as SavedRestrictionsInfo) : undefined, + fieldFormatMap: attributes.fieldFormatMap + ? JSON.parse(attributes.fieldFormatMap) + : undefined, }; }); }) diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/operations/definitions/date_histogram.test.tsx b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/operations/definitions/date_histogram.test.tsx index 12015a281eaa9..e21c0e58299c3 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/operations/definitions/date_histogram.test.tsx +++ b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/operations/definitions/date_histogram.test.tsx @@ -23,6 +23,7 @@ describe('date_histogram', () => { beforeEach(() => { state = { currentIndexPatternId: '1', + showEmptyFields: false, indexPatterns: { 1: { id: '1', diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/operations/definitions/filter_ratio.test.tsx b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/operations/definitions/filter_ratio.test.tsx index 94fe425543936..c6ba2e52fcd9d 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/operations/definitions/filter_ratio.test.tsx +++ b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/operations/definitions/filter_ratio.test.tsx @@ -32,6 +32,7 @@ describe('filter_ratio', () => { }, }, currentIndexPatternId: '1', + showEmptyFields: false, layers: { first: { indexPatternId: '1', diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/operations/definitions/terms.test.tsx b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/operations/definitions/terms.test.tsx index b6883a0cc3709..6b18d3d54a106 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/operations/definitions/terms.test.tsx +++ b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/operations/definitions/terms.test.tsx @@ -24,6 +24,7 @@ describe('terms', () => { state = { indexPatterns: {}, currentIndexPatternId: '1', + showEmptyFields: false, layers: { first: { indexPatternId: '1', diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/operations/operations.test.ts b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/operations/operations.test.ts index 15a3b8ab19296..45ba303293669 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/operations/operations.test.ts +++ b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/operations/operations.test.ts @@ -146,6 +146,7 @@ describe('getOperationTypesForField', () => { describe('buildColumn', () => { const state: IndexPatternPrivateState = { currentIndexPatternId: '1', + showEmptyFields: false, indexPatterns: expectedIndexPatterns, layers: { first: { diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/state_helpers.test.ts b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/state_helpers.test.ts index 1d366b931b89b..8d8b24a273d1d 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/state_helpers.test.ts +++ b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/state_helpers.test.ts @@ -42,6 +42,7 @@ describe('state_helpers', () => { const state: IndexPatternPrivateState = { indexPatterns: {}, currentIndexPatternId: '1', + showEmptyFields: false, layers: { first: { indexPatternId: '1', @@ -89,6 +90,7 @@ describe('state_helpers', () => { const state: IndexPatternPrivateState = { indexPatterns: {}, currentIndexPatternId: '1', + showEmptyFields: false, layers: { first: { indexPatternId: '1', @@ -140,6 +142,7 @@ describe('state_helpers', () => { const state: IndexPatternPrivateState = { indexPatterns: {}, currentIndexPatternId: '1', + showEmptyFields: false, layers: { first: { indexPatternId: '1', @@ -171,6 +174,7 @@ describe('state_helpers', () => { const state: IndexPatternPrivateState = { indexPatterns: {}, currentIndexPatternId: '1', + showEmptyFields: false, layers: { first: { indexPatternId: '1', @@ -233,6 +237,7 @@ describe('state_helpers', () => { const state: IndexPatternPrivateState = { indexPatterns: {}, currentIndexPatternId: '1', + showEmptyFields: false, layers: { first: { indexPatternId: '1', @@ -312,6 +317,7 @@ describe('state_helpers', () => { const state: IndexPatternPrivateState = { indexPatterns: {}, currentIndexPatternId: '1', + showEmptyFields: false, layers: { first: { indexPatternId: '1', diff --git a/x-pack/legacy/plugins/lens/public/types.ts b/x-pack/legacy/plugins/lens/public/types.ts index 8a71f36ea2f92..8ce2164eddc2a 100644 --- a/x-pack/legacy/plugins/lens/public/types.ts +++ b/x-pack/legacy/plugins/lens/public/types.ts @@ -121,6 +121,8 @@ export interface DatasourceDataPanelProps { state: T; dragDropContext: DragContextState; setState: StateSetter; + query: Query; + dateRange: FramePublicAPI['dateRange']; } // The only way a visualization has to restrict the query building @@ -174,7 +176,7 @@ export interface VisualizationProps { dragDropContext: DragContextState; frame: FramePublicAPI; state: T; - setState: (newState: T) => void; + setState: StateSetter; } export interface SuggestionRequest { diff --git a/x-pack/legacy/plugins/lens/readme.md b/x-pack/legacy/plugins/lens/readme.md index 2bc027121fed5..36cc4574b8360 100644 --- a/x-pack/legacy/plugins/lens/readme.md +++ b/x-pack/legacy/plugins/lens/readme.md @@ -7,3 +7,6 @@ - Run `node scripts/functional_tests_server` - Run `node ../scripts/functional_test_runner.js --config ./test/functional/config.js` - You may want to comment out all imports except for Lens in the config file. +- API Functional tests: + - Run `node scripts/functional_tests_server` + - Run `node ../scripts/functional_test_runner.js --config ./test/api_integration/config.js --grep=Lens` diff --git a/x-pack/legacy/plugins/lens/server/index.ts b/x-pack/legacy/plugins/lens/server/index.ts new file mode 100644 index 0000000000000..ae14d1c5a0052 --- /dev/null +++ b/x-pack/legacy/plugins/lens/server/index.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { LensServer } from './plugin'; + +export * from './plugin'; + +export const lensServerPlugin = () => new LensServer(); diff --git a/x-pack/legacy/plugins/lens/server/plugin.tsx b/x-pack/legacy/plugins/lens/server/plugin.tsx new file mode 100644 index 0000000000000..6ccfa2399b253 --- /dev/null +++ b/x-pack/legacy/plugins/lens/server/plugin.tsx @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Plugin, CoreSetup } from 'src/core/server'; +import { setupRoutes } from './routes'; + +export class LensServer implements Plugin<{}, {}, {}, {}> { + constructor() {} + + async setup(core: CoreSetup) { + setupRoutes(core); + + return {}; + } + + start() { + return {}; + } + + async stop() {} +} diff --git a/x-pack/legacy/plugins/lens/server/routes/index.ts b/x-pack/legacy/plugins/lens/server/routes/index.ts new file mode 100644 index 0000000000000..9a957765cc87d --- /dev/null +++ b/x-pack/legacy/plugins/lens/server/routes/index.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { CoreSetup } from 'src/core/server'; +import { initStatsRoute } from './index_stats'; + +export function setupRoutes(setup: CoreSetup) { + initStatsRoute(setup); +} diff --git a/x-pack/legacy/plugins/lens/server/routes/index_stats.test.ts b/x-pack/legacy/plugins/lens/server/routes/index_stats.test.ts new file mode 100644 index 0000000000000..1afa7ea363c84 --- /dev/null +++ b/x-pack/legacy/plugins/lens/server/routes/index_stats.test.ts @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +// @ts-ignore +import realHits from '../../../../../../src/fixtures/real_hits'; +// @ts-ignore +import stubbedLogstashFields from '../../../../../../src/fixtures/logstash_fields'; +import { recursiveFlatten } from './index_stats'; + +describe('Index Stats route', () => { + it('should ignore falsy fields', () => { + const results = recursiveFlatten( + [{ _source: {} }, { _source: { bytes: false } }], + stubbedLogstashFields(), + [ + { + name: 'extension.keyword', + type: 'string', + }, + { + name: 'bytes', + type: 'number', + }, + ] + ); + + expect(results).toEqual({}); + }); + + it('should find existing fields based on mapping', () => { + const results = recursiveFlatten(realHits, stubbedLogstashFields(), [ + { + name: 'extension.keyword', + type: 'string', + }, + { + name: 'bytes', + type: 'number', + }, + ]); + + expect(results).toEqual({ + bytes: { + count: 20, + cardinality: 16, + }, + 'extension.keyword': { + count: 20, + cardinality: 4, + }, + }); + }); +}); diff --git a/x-pack/legacy/plugins/lens/server/routes/index_stats.ts b/x-pack/legacy/plugins/lens/server/routes/index_stats.ts new file mode 100644 index 0000000000000..4cc9f27a44c71 --- /dev/null +++ b/x-pack/legacy/plugins/lens/server/routes/index_stats.ts @@ -0,0 +1,177 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import Boom from 'boom'; +import { get, uniq } from 'lodash'; +import { schema } from '@kbn/config-schema'; +import { SearchResponse } from 'elasticsearch'; +import { CoreSetup } from 'src/core/server'; +import { + IndexPatternsService, + FieldDescriptor, +} from '../../../../../../src/legacy/server/index_patterns/service'; + +type Document = Record; + +type Fields = Array<{ name: string; type: string; esTypes?: string[] }>; + +export async function initStatsRoute(setup: CoreSetup) { + const router = setup.http.createRouter('/api/lens/index_stats/{indexPatternTitle}'); + router.post( + { + path: '', + validate: { + params: schema.object({ + indexPatternTitle: schema.string(), + }), + body: schema.object({ + earliest: schema.string(), + latest: schema.string(), + timeZone: schema.maybe(schema.string()), + timeFieldName: schema.string(), + size: schema.number(), + fields: schema.arrayOf( + schema.object({ + name: schema.string(), + type: schema.string(), + }) + ), + }), + }, + }, + async (context, req, res) => { + const requestClient = context.core.elasticsearch.dataClient; + + const indexPatternsService = new IndexPatternsService(requestClient.callAsCurrentUser); + + const { earliest, latest, timeZone, timeFieldName, fields, size } = req.body; + + try { + const indexPattern = await indexPatternsService.getFieldsForWildcard({ + pattern: req.params.indexPatternTitle, + // TODO: Pull this from kibana advanced settings + metaFields: ['_source', '_id', '_type', '_index', '_score'], + }); + + const results = (await requestClient.callAsCurrentUser('search', { + index: req.params.indexPatternTitle, + body: { + query: { + bool: { + filter: [ + { + range: { + [timeFieldName]: { + gte: earliest, + lte: latest, + time_zone: timeZone, + }, + }, + }, + ], + }, + }, + size, + }, + })) as SearchResponse; + + if (results.hits.hits.length) { + return res.ok({ + body: recursiveFlatten(results.hits.hits, indexPattern, fields), + }); + } + return res.ok({ body: {} }); + } catch (e) { + if (e.isBoom) { + return res.internalError(e); + } else { + return res.internalError({ + body: Boom.internal(e.message || e.name), + }); + } + } + } + ); +} + +export function recursiveFlatten( + docs: Array<{ + _source: Document; + }>, + indexPattern: FieldDescriptor[], + fields: Fields +): Record< + string, + { + count: number; + cardinality: number; + } +> { + const overallKeys: Record< + string, + { + count: number; + samples: unknown[]; + } + > = {}; + + const expectedKeys: Record = {}; + fields.forEach(field => { + expectedKeys[field.name] = true; + }); + + // TODO: Alias types + indexPattern.forEach(field => { + if (!expectedKeys[field.name]) { + return; + } + + let matches; + if (field.parent) { + matches = docs.map(doc => { + if (!doc) { + return; + } + return get(doc._source, field.parent!); + }); + } else { + matches = docs.map(doc => { + if (!doc) { + return; + } + return get(doc._source, field.name); + }); + } + + matches.forEach(match => { + const record = overallKeys[field.name]; + if (record) { + record.count += 1; + record.samples.push(match); + } else if (match) { + overallKeys[field.name] = { + count: 1, + samples: [match], + }; + } + }); + }); + + const returnTypes: Record< + string, + { + count: number; + cardinality: number; + } + > = {}; + Object.entries(overallKeys).forEach(([key, value]) => { + returnTypes[key] = { + count: value.count, + cardinality: uniq(value.samples).length, + }; + }); + return returnTypes; +} diff --git a/x-pack/test/api_integration/apis/index.js b/x-pack/test/api_integration/apis/index.js index 99694c250ef3f..87d64126571b3 100644 --- a/x-pack/test/api_integration/apis/index.js +++ b/x-pack/test/api_integration/apis/index.js @@ -26,5 +26,6 @@ export default function ({ loadTestFile }) { loadTestFile(require.resolve('./siem')); loadTestFile(require.resolve('./code')); loadTestFile(require.resolve('./short_urls')); + loadTestFile(require.resolve('./lens')); }); } diff --git a/x-pack/test/api_integration/apis/lens/index.ts b/x-pack/test/api_integration/apis/lens/index.ts new file mode 100644 index 0000000000000..8b890c99a77b9 --- /dev/null +++ b/x-pack/test/api_integration/apis/lens/index.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { FtrProviderContext } from '../../ftr_provider_context'; + +export default function consoleApiIntegrationTests({ loadTestFile }: FtrProviderContext) { + describe('Lens', () => { + loadTestFile(require.resolve('./index_stats')); + }); +} diff --git a/x-pack/test/api_integration/apis/lens/index_stats.ts b/x-pack/test/api_integration/apis/lens/index_stats.ts new file mode 100644 index 0000000000000..286fe2f971621 --- /dev/null +++ b/x-pack/test/api_integration/apis/lens/index_stats.ts @@ -0,0 +1,110 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from '@kbn/expect'; + +import { FtrProviderContext } from '../../ftr_provider_context'; + +const TEST_START_TIME = '2015-09-19T06:31:44.000'; +const TEST_END_TIME = '2015-09-23T18:31:44.000'; +const COMMON_HEADERS = { + 'kbn-xsrf': 'some-xsrf-token', +}; + +const fieldsNotInPattern = [ + { name: 'geo', type: 'object' }, + { name: 'id', type: 'string' }, + { name: 'machine', type: 'object' }, +]; + +const fieldsNotInDocuments = [ + { name: 'meta', type: 'object' }, + { name: 'meta.char', type: 'string' }, + { name: 'meta.related', type: 'string' }, + { name: 'meta.user', type: 'object' }, + { name: 'meta.user.firstname', type: 'string' }, + { name: 'meta.user.lastname', type: 'string' }, +]; + +const fieldsWithData = [ + { name: '@message', type: 'string' }, + { name: '@message.raw', type: 'string' }, + { name: '@tags', type: 'string' }, + { name: '@tags.raw', type: 'string' }, + { name: '@timestamp', type: 'date' }, + { name: 'agent', type: 'string' }, + { name: 'agent.raw', type: 'string' }, + { name: 'bytes', type: 'number' }, + { name: 'clientip', type: 'ip' }, + { name: 'extension', type: 'string' }, + { name: 'extension.raw', type: 'string' }, + { name: 'geo.coordinates', type: 'geo_point' }, + { name: 'geo.dest', type: 'string' }, + { name: 'geo.src', type: 'string' }, + { name: 'geo.srcdest', type: 'string' }, + { name: 'headings', type: 'string' }, + { name: 'headings.raw', type: 'string' }, + { name: 'host', type: 'string' }, + { name: 'host.raw', type: 'string' }, + { name: 'index', type: 'string' }, + { name: 'index.raw', type: 'string' }, + { name: 'ip', type: 'ip' }, + { name: 'links', type: 'string' }, + { name: 'links.raw', type: 'string' }, + { name: 'machine.os', type: 'string' }, + { name: 'machine.os.raw', type: 'string' }, + { name: 'machine.ram', type: 'string' }, + { name: 'memory', type: 'string' }, + { name: 'phpmemory', type: 'string' }, + { name: 'referer', type: 'string' }, + { name: 'request', type: 'string' }, + { name: 'request.raw', type: 'string' }, + { name: 'response', type: 'string' }, + { name: 'response.raw', type: 'string' }, + { name: 'spaces', type: 'string' }, + { name: 'spaces.raw', type: 'string' }, + { name: 'type', type: 'string' }, + { name: 'url', type: 'string' }, + { name: 'url.raw', type: 'string' }, + { name: 'utc_time', type: 'string' }, + { name: 'xss', type: 'string' }, + { name: 'xss.raw', type: 'string' }, +]; + +// eslint-disable-next-line import/no-default-export +export default ({ getService }: FtrProviderContext) => { + const esArchiver = getService('esArchiver'); + const supertest = getService('supertest'); + + describe('index stats apis', () => { + before(async () => { + await esArchiver.loadIfNeeded('logstash_functional'); + await esArchiver.loadIfNeeded('visualize/default'); + }); + after(async () => { + await esArchiver.unload('logstash_functional'); + await esArchiver.unload('visualize/default'); + }); + + describe('existence', () => { + it('should find which fields exist in the sample documents', async () => { + const { body } = await supertest + .post('/api/lens/index_stats/logstash-2015.09.22') + .set(COMMON_HEADERS) + .send({ + earliest: TEST_START_TIME, + latest: TEST_END_TIME, + timeFieldName: '@timestamp', + size: 500, + fields: fieldsWithData.concat(fieldsNotInDocuments, fieldsNotInPattern), + }) + .expect(200); + + expect(Object.keys(body)).to.eql(fieldsWithData.map(field => field.name)); + }); + }); + }); +}; diff --git a/x-pack/test/functional/apps/lens/index.ts b/x-pack/test/functional/apps/lens/index.ts index 085950b9f5f6b..352e7b2d5de80 100644 --- a/x-pack/test/functional/apps/lens/index.ts +++ b/x-pack/test/functional/apps/lens/index.ts @@ -20,6 +20,11 @@ export default function({ getService, loadTestFile }: FtrProviderContext) { await esArchiver.loadIfNeeded('visualize/default'); }); + after(async () => { + await esArchiver.unload('logstash_functional'); + await esArchiver.unload('visualize/default'); + }); + describe('', function() { this.tags(['ciGroup4', 'skipFirefox']); From 73f0550e78097de29cd45dbb6f4d77eed7b11589 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Thu, 29 Aug 2019 16:50:00 -0400 Subject: [PATCH 2/9] Fix route registration --- src/core/server/index.ts | 2 +- x-pack/legacy/plugins/lens/index.ts | 5 ++++- x-pack/legacy/plugins/lens/server/routes/index_stats.ts | 4 ++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/core/server/index.ts b/src/core/server/index.ts index 61266016089b9..ef31804be62b2 100644 --- a/src/core/server/index.ts +++ b/src/core/server/index.ts @@ -211,7 +211,7 @@ export interface CoreSetup { name: T, provider: RequestHandlerContextProvider ) => RequestHandlerContextContainer; - createRouter: (basePath: string) => IRouter; + createRouter: () => IRouter; }; } diff --git a/x-pack/legacy/plugins/lens/index.ts b/x-pack/legacy/plugins/lens/index.ts index 39b5a8f6820c6..cda72d2e71de2 100644 --- a/x-pack/legacy/plugins/lens/index.ts +++ b/x-pack/legacy/plugins/lens/index.ts @@ -85,7 +85,10 @@ export const lens: LegacyPluginInitializer = kibana => { // Set up with the new platform plugin lifecycle API. const plugin = lensServerPlugin(); await plugin.setup(({ - http: kbnServer.newPlatform.setup.core.http, + http: { + ...kbnServer.newPlatform.setup.core.http, + createRouter: () => kbnServer.newPlatform.setup.core.http.createRouter('/api/lens'), + }, } as unknown) as CoreSetup); server.events.on('stop', async () => { diff --git a/x-pack/legacy/plugins/lens/server/routes/index_stats.ts b/x-pack/legacy/plugins/lens/server/routes/index_stats.ts index 4cc9f27a44c71..0ce566f30c597 100644 --- a/x-pack/legacy/plugins/lens/server/routes/index_stats.ts +++ b/x-pack/legacy/plugins/lens/server/routes/index_stats.ts @@ -19,10 +19,10 @@ type Document = Record; type Fields = Array<{ name: string; type: string; esTypes?: string[] }>; export async function initStatsRoute(setup: CoreSetup) { - const router = setup.http.createRouter('/api/lens/index_stats/{indexPatternTitle}'); + const router = setup.http.createRouter(); router.post( { - path: '', + path: '/index_stats/{indexPatternTitle}', validate: { params: schema.object({ indexPatternTitle: schema.string(), From ba15002fa1ac47a00324ec2b0d1c8606ac62987e Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Thu, 29 Aug 2019 17:49:28 -0400 Subject: [PATCH 3/9] Add page object and use existence in functional test --- .../public/indexpattern_plugin/datapanel.tsx | 19 ++++++++++++-- x-pack/legacy/plugins/lens/readme.md | 2 ++ .../apps/lens/indexpattern_datapanel.ts | 10 +++++--- x-pack/test/functional/page_objects/index.ts | 2 ++ x-pack/test/functional/page_objects/lens.ts | 25 +++++++++++++++++++ 5 files changed, 52 insertions(+), 6 deletions(-) create mode 100644 x-pack/test/functional/page_objects/lens.ts diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/datapanel.tsx b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/datapanel.tsx index 78fbcc6e4cef9..f0cc25f6b5755 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/datapanel.tsx +++ b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/datapanel.tsx @@ -23,6 +23,7 @@ import { EuiContextMenuPanelProps, EuiPopover, EuiCallOut, + EuiText, } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; @@ -438,8 +439,8 @@ export const InnerIndexPatternDataPanel = function InnerIndexPatternDataPanel({ hasActiveFilters={localState.typeFilter.length > 0 || showEmptyFields} numActiveFilters={localState.typeFilter.length + (showEmptyFields ? 1 : 0)} > - {i18n.translate('xpack.lens.indexPatterns.typeFilterLabel', { - defaultMessage: 'Types', + {i18n.translate('xpack.lens.indexPatterns.filtersLabel', { + defaultMessage: 'Filters', })} } @@ -510,6 +511,20 @@ export const InnerIndexPatternDataPanel = function InnerIndexPatternDataPanel({ /> ); })} + + {!localState.isLoading && paginatedFields.length === 0 && ( + + {showEmptyFields + ? i18n.translate('xpack.lens.indexPatterns.hiddenFieldsLabel', { + defaultMessage: + 'No fields have data with the current filters. You can show fields without data using the filters above.', + }) + : i18n.translate('xpack.lens.indexPatterns.noFieldsLabel', { + defaultMessage: 'No fields can be visualized from {title}', + values: { title: indexPatterns[currentIndexPatternId].title }, + })} + + )} diff --git a/x-pack/legacy/plugins/lens/readme.md b/x-pack/legacy/plugins/lens/readme.md index 36cc4574b8360..0ea0778dd17ef 100644 --- a/x-pack/legacy/plugins/lens/readme.md +++ b/x-pack/legacy/plugins/lens/readme.md @@ -2,6 +2,8 @@ ## Testing +Run all tests from the `x-pack` root directory + - Unit tests: `node scripts/jest --watch lens` - Functional tests: - Run `node scripts/functional_tests_server` diff --git a/x-pack/test/functional/apps/lens/indexpattern_datapanel.ts b/x-pack/test/functional/apps/lens/indexpattern_datapanel.ts index f0fc37a2c18d9..4c89ef4d51cc9 100644 --- a/x-pack/test/functional/apps/lens/indexpattern_datapanel.ts +++ b/x-pack/test/functional/apps/lens/indexpattern_datapanel.ts @@ -8,9 +8,8 @@ import expect from '@kbn/expect'; import { FtrProviderContext } from '../../ftr_provider_context'; // eslint-disable-next-line import/no-default-export -export default function({ getService, getPageObjects }: FtrProviderContext) { - const PageObjects = getPageObjects(['header', 'common']); - const find = getService('find'); +export default function({ getPageObjects }: FtrProviderContext) { + const PageObjects = getPageObjects(['header', 'common', 'lens']); describe('indexpattern_datapanel', () => { beforeEach(async () => { @@ -18,7 +17,10 @@ export default function({ getService, getPageObjects }: FtrProviderContext) { }); it('should list the index pattern fields', async () => { - const fields = await find.allByCssSelector('[data-test-subj="lnsFieldListPanelField"]'); + await PageObjects.lens.openIndexPatternFiltersPopover(); + await PageObjects.lens.toggleExistenceFilter(); + + const fields = await PageObjects.lens.findAllFields(); const fieldText = await Promise.all(fields.map(field => field.getVisibleText())); expect(fieldText).to.eql([ '_score', diff --git a/x-pack/test/functional/page_objects/index.ts b/x-pack/test/functional/page_objects/index.ts index 690a77ff00aa1..e22e36fcd73aa 100644 --- a/x-pack/test/functional/page_objects/index.ts +++ b/x-pack/test/functional/page_objects/index.ts @@ -44,6 +44,7 @@ import { SnapshotRestorePageProvider } from './snapshot_restore_page'; import { CrossClusterReplicationPageProvider } from './cross_cluster_replication_page'; import { RemoteClustersPageProvider } from './remote_clusters_page'; import { CopySavedObjectsToSpacePageProvider } from './copy_saved_objects_to_space_page'; +import { LensPageProvider } from './lens'; // just like services, PageObjects are defined as a map of // names to Providers. Merge in Kibana's or pick specific ones @@ -74,4 +75,5 @@ export const pageObjects = { crossClusterReplication: CrossClusterReplicationPageProvider, remoteClusters: RemoteClustersPageProvider, copySavedObjectsToSpace: CopySavedObjectsToSpacePageProvider, + lens: LensPageProvider, }; diff --git a/x-pack/test/functional/page_objects/lens.ts b/x-pack/test/functional/page_objects/lens.ts new file mode 100644 index 0000000000000..b45987b94ca83 --- /dev/null +++ b/x-pack/test/functional/page_objects/lens.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { FtrProviderContext } from '../ftr_provider_context'; + +export function LensPageProvider({ getService }: FtrProviderContext) { + const testSubjects = getService('testSubjects'); + + return { + async openIndexPatternFiltersPopover() { + await testSubjects.click('indexPatternTypeFilterButton'); + }, + + async toggleExistenceFilter() { + await testSubjects.click('emptyFilter'); + }, + + async findAllFields() { + return await testSubjects.findAll('lnsFieldListPanelField'); + }, + }; +} From 75f13f9517066e9dc26c9a2ac3738ee1f1f34677 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Fri, 30 Aug 2019 12:54:36 -0400 Subject: [PATCH 4/9] Simplify layout of filters for index pattern --- .../public/indexpattern_plugin/datapanel.tsx | 169 ++++++++++-------- .../public/indexpattern_plugin/field_item.tsx | 2 - x-pack/test/functional/page_objects/lens.ts | 4 +- 3 files changed, 101 insertions(+), 74 deletions(-) diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/datapanel.tsx b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/datapanel.tsx index f0cc25f6b5755..d312a6d78601f 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/datapanel.tsx +++ b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/datapanel.tsx @@ -8,7 +8,6 @@ import { mapValues, uniq } from 'lodash'; import React, { useState, useEffect, memo, useCallback } from 'react'; import { EuiComboBox, - EuiFieldSearch, EuiLoadingSpinner, // @ts-ignore EuiHighlight, @@ -17,13 +16,17 @@ import { EuiTitle, EuiButtonEmpty, EuiFilterGroup, - EuiFilterButton, EuiContextMenuPanel, EuiContextMenuItem, EuiContextMenuPanelProps, EuiPopover, + EuiPopoverTitle, + EuiPopoverFooter, EuiCallOut, EuiText, + EuiFormControlLayout, + EuiSwitch, + EuiIcon, } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; @@ -198,7 +201,7 @@ export const InnerIndexPatternDataPanel = function InnerIndexPatternDataPanel({ if (Object.keys(indexPatterns).length === 0) { return ( - { - setLocalState({ ...localState, nameFilter: e.target.value }); - }} - aria-label={i18n.translate('xpack.lens.indexPatterns.filterByNameAriaLabel', { - defaultMessage: 'Search fields', - })} - /> - - - - - setLocalState(s => ({ ...localState, isTypeFilterOpen: false })) - } - button={ - - setLocalState(s => ({ - ...s, - isTypeFilterOpen: !localState.isTypeFilterOpen, - })) - } - iconType="arrowDown" - data-test-subj="indexPatternTypeFilterButton" - isSelected={localState.isTypeFilterOpen} - numFilters={availableFieldTypes.length + 1} - hasActiveFilters={localState.typeFilter.length > 0 || showEmptyFields} - numActiveFilters={localState.typeFilter.length + (showEmptyFields ? 1 : 0)} - > - {i18n.translate('xpack.lens.indexPatterns.filtersLabel', { - defaultMessage: 'Filters', + + setLocalState(s => ({ ...localState, isTypeFilterOpen: false })) + } + button={ + { + setLocalState(s => ({ + ...s, + isTypeFilterOpen: !localState.isTypeFilterOpen, + })); + }} + data-test-subj="lnsIndexPatternTypeFilterButton" + title={i18n.translate('xpack.lens.indexPatterns.toggleFiltersPopover', { + defaultMessage: 'Toggle filters for index pattern', + })} + aria-label={i18n.translate( + 'xpack.lens.indexPatterns.toggleFiltersPopover', + { + defaultMessage: 'Toggle filters for index pattern', + } + )} + > + + + } + > + + {i18n.translate('xpack.lens.indexPatterns.filterByTypeLabel', { + defaultMessage: 'Filter by type', })} - - } - > - ( + + ( {fieldTypeNames[type]} - )) - .concat([ - { - onToggleEmptyFields(); - }} - > - {i18n.translate('xpack.lens.datatypes.hiddenFields', { - defaultMessage: 'Only show fields that contain data', - })} - , - ])} - /> - - + ))} + /> + + { + onToggleEmptyFields(); + }} + label={i18n.translate('xpack.lens.datatypes.hiddenFields', { + defaultMessage: 'Only show fields that contain data', + })} + data-test-subj="lnsEmptyFilter" + /> + + + } + clear={{ + title: i18n.translate('xpack.lens.indexPatterns.clearFiltersLabel', { + defaultMessage: 'Clear name and type filters', + }), + 'aria-label': i18n.translate('xpack.lens.indexPatterns.clearFiltersLabel', { + defaultMessage: 'Clear name and type filters', + }), + onClick: () => { + setLocalState(s => ({ + ...s, + nameFilter: '', + typeFilter: [], + })); + }, + }} + > + { + setLocalState({ ...localState, nameFilter: e.target.value }); + }} + aria-label={i18n.translate('xpack.lens.indexPatterns.filterByNameAriaLabel', { + defaultMessage: 'Search fields', + })} + /> + + + +
Date: Fri, 30 Aug 2019 16:36:17 -0400 Subject: [PATCH 5/9] Respond to review feedback --- .../legacy/plugins/lens/common/constants.ts | 5 +- x-pack/legacy/plugins/lens/index.ts | 12 +- .../indexpattern_plugin/datapanel.test.tsx | 119 +++++++------- .../public/indexpattern_plugin/datapanel.tsx | 155 +++++++++--------- .../dimension_panel/field_select.tsx | 2 +- .../indexpattern_plugin/indexpattern.tsx | 8 +- .../lens/public/register_vis_type_alias.ts | 3 +- x-pack/legacy/plugins/lens/public/types.ts | 2 +- x-pack/legacy/plugins/lens/server/plugin.tsx | 4 +- .../lens/server/routes/index_stats.test.ts | 53 +++++- .../plugins/lens/server/routes/index_stats.ts | 45 ++--- .../test/api_integration/apis/lens/index.ts | 2 +- x-pack/test/functional/page_objects/lens.ts | 2 +- 13 files changed, 227 insertions(+), 185 deletions(-) diff --git a/x-pack/legacy/plugins/lens/common/constants.ts b/x-pack/legacy/plugins/lens/common/constants.ts index edcc34c58881a..a7b59b5142d83 100644 --- a/x-pack/legacy/plugins/lens/common/constants.ts +++ b/x-pack/legacy/plugins/lens/common/constants.ts @@ -6,8 +6,9 @@ export const PLUGIN_ID = 'lens'; -export const BASE_URL = 'app/lens'; +export const BASE_APP_URL = 'app/lens'; +export const BASE_API_URL = '/api/lens'; export function getEditPath(id: string) { - return `/${BASE_URL}#/edit/${encodeURIComponent(id)}`; + return `/${BASE_APP_URL}#/edit/${encodeURIComponent(id)}`; } diff --git a/x-pack/legacy/plugins/lens/index.ts b/x-pack/legacy/plugins/lens/index.ts index cda72d2e71de2..fb44bd21f5588 100644 --- a/x-pack/legacy/plugins/lens/index.ts +++ b/x-pack/legacy/plugins/lens/index.ts @@ -10,7 +10,7 @@ import { LegacyPluginInitializer } from 'src/legacy/types'; import KbnServer, { Server } from 'src/legacy/server/kbn_server'; import { CoreSetup } from 'src/core/server'; import mappings from './mappings.json'; -import { PLUGIN_ID, getEditPath } from './common'; +import { PLUGIN_ID, getEditPath, BASE_API_URL } from './common'; import { lensServerPlugin } from './server'; const NOT_INTERNATIONALIZED_PRODUCT_NAME = 'Lens Visualizations'; @@ -51,7 +51,7 @@ export const lens: LegacyPluginInitializer = kibana => { }).default(); }, - async init(server: Server) { + init(server: Server) { const kbnServer = (server as unknown) as KbnServer; server.plugins.xpack_main.registerFeature({ @@ -84,15 +84,15 @@ export const lens: LegacyPluginInitializer = kibana => { // Set up with the new platform plugin lifecycle API. const plugin = lensServerPlugin(); - await plugin.setup(({ + plugin.setup(({ http: { ...kbnServer.newPlatform.setup.core.http, - createRouter: () => kbnServer.newPlatform.setup.core.http.createRouter('/api/lens'), + createRouter: () => kbnServer.newPlatform.setup.core.http.createRouter(BASE_API_URL), }, } as unknown) as CoreSetup); - server.events.on('stop', async () => { - await plugin.stop(); + server.events.on('stop', () => { + plugin.stop(); }); }, }); diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/datapanel.test.tsx b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/datapanel.test.tsx index c380646c33a30..d308d74d05a1d 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/datapanel.test.tsx +++ b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/datapanel.test.tsx @@ -6,7 +6,7 @@ import { shallow, mount } from 'enzyme'; import React, { ChangeEvent, ReactElement } from 'react'; -import { EuiComboBox, EuiFieldSearch, EuiContextMenuPanel } from '@elastic/eui'; +import { EuiComboBox, EuiContextMenuPanel } from '@elastic/eui'; import { IndexPatternPrivateState, IndexPatternColumn } from './indexpattern'; import { createMockedDragDropContext } from './mocks'; import { InnerIndexPatternDataPanel, IndexPatternDataPanel, MemoizedDataPanel } from './datapanel'; @@ -432,15 +432,15 @@ describe('IndexPattern Data Panel', () => { ]); }); - it('should filter down by name', async () => { + it('should filter down by name', () => { const wrapper = shallow( ); act(() => { - wrapper.find(EuiFieldSearch).prop('onChange')!({ target: { value: 'mem' } } as ChangeEvent< - HTMLInputElement - >); + wrapper.find('[data-test-subj="lnsIndexPatternFieldSearch"]').prop('onChange')!({ + target: { value: 'mem' }, + } as ChangeEvent); }); expect(wrapper.find(FieldItem).map(fieldItem => fieldItem.prop('field').name)).toEqual([ @@ -448,19 +448,20 @@ describe('IndexPattern Data Panel', () => { ]); }); - it('should filter down by type', async () => { - const wrapper = shallow( + it('should filter down by type', () => { + const wrapper = mount( ); - act(() => { - (wrapper - .find(EuiContextMenuPanel) - .prop('items')! - .find( - item => (item as ReactElement).props['data-test-subj'] === 'typeFilter-number' - )! as ReactElement).props.onClick(); - }); + wrapper + .find('[data-test-subj="lnsIndexPatternFiltersToggle"]') + .first() + .simulate('click'); + + wrapper + .find('[data-test-subj="typeFilter-number"]') + .first() + .simulate('click'); expect(wrapper.find(FieldItem).map(fieldItem => fieldItem.prop('field').name)).toEqual([ 'bytes', @@ -468,28 +469,24 @@ describe('IndexPattern Data Panel', () => { ]); }); - it('should toggle type if clicked again', async () => { - const wrapper = shallow( + it('should toggle type if clicked again', () => { + const wrapper = mount( ); - act(() => { - (wrapper - .find(EuiContextMenuPanel) - .prop('items')! - .find( - item => (item as ReactElement).props['data-test-subj'] === 'typeFilter-number' - )! as ReactElement).props.onClick(); - }); + wrapper + .find('[data-test-subj="lnsIndexPatternFiltersToggle"]') + .first() + .simulate('click'); - act(() => { - (wrapper - .find(EuiContextMenuPanel) - .prop('items')! - .find( - item => (item as ReactElement).props['data-test-subj'] === 'typeFilter-number' - )! as ReactElement).props.onClick(); - }); + wrapper + .find('[data-test-subj="typeFilter-number"]') + .first() + .simulate('click'); + wrapper + .find('[data-test-subj="typeFilter-number"]') + .first() + .simulate('click'); expect(wrapper.find(FieldItem).map(fieldItem => fieldItem.prop('field').name)).toEqual([ 'bytes', @@ -499,25 +496,26 @@ describe('IndexPattern Data Panel', () => { ]); }); - it('should filter down by type and by name', async () => { - const wrapper = shallow( + it('should filter down by type and by name', () => { + const wrapper = mount( ); act(() => { - wrapper.find(EuiFieldSearch).prop('onChange')!({ target: { value: 'mem' } } as ChangeEvent< - HTMLInputElement - >); + wrapper.find('[data-test-subj="lnsIndexPatternFieldSearch"]').prop('onChange')!({ + target: { value: 'mem' }, + } as ChangeEvent); }); - act(() => { - (wrapper - .find(EuiContextMenuPanel) - .prop('items')! - .find( - item => (item as ReactElement).props['data-test-subj'] === 'typeFilter-number' - )! as ReactElement).props.onClick(); - }); + wrapper + .find('[data-test-subj="lnsIndexPatternFiltersToggle"]') + .first() + .simulate('click'); + + wrapper + .find('[data-test-subj="typeFilter-number"]') + .first() + .simulate('click'); expect(wrapper.find(FieldItem).map(fieldItem => fieldItem.prop('field').name)).toEqual([ 'memory', @@ -555,15 +553,15 @@ describe('IndexPattern Data Panel', () => { ]); }); - it('should filter down by name', async () => { + it('should filter down by name', () => { const wrapper = shallow( ); act(() => { - wrapper.find(EuiFieldSearch).prop('onChange')!({ target: { value: 'mem' } } as ChangeEvent< - HTMLInputElement - >); + wrapper.find('[data-test-subj="lnsIndexPatternFieldSearch"]').prop('onChange')!({ + target: { value: 'mem' }, + } as ChangeEvent); }); expect(wrapper.find(FieldItem).map(fieldItem => fieldItem.prop('field').name)).toEqual([ @@ -571,17 +569,18 @@ describe('IndexPattern Data Panel', () => { ]); }); - it('should allow removing the filter for data', async () => { - const wrapper = shallow(); + it('should allow removing the filter for data', () => { + const wrapper = mount(); - act(() => { - (wrapper - .find(EuiContextMenuPanel) - .prop('items')! - .find( - item => (item as ReactElement).props['data-test-subj'] === 'emptyFilter' - )! as ReactElement).props.onClick(); - }); + wrapper + .find('[data-test-subj="lnsIndexPatternFiltersToggle"]') + .first() + .simulate('click'); + + wrapper + .find('[data-test-subj="lnsEmptyFilter"]') + .first() + .prop('onChange')!({} as ChangeEvent); expect(emptyFieldsTestProps.onToggleEmptyFields).toHaveBeenCalled(); }); diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/datapanel.tsx b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/datapanel.tsx index d312a6d78601f..832dfaf47a950 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/datapanel.tsx +++ b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/datapanel.tsx @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { mapValues, uniq } from 'lodash'; +import { mapValues, uniq, indexBy } from 'lodash'; import React, { useState, useEffect, memo, useCallback } from 'react'; import { EuiComboBox, @@ -15,7 +15,6 @@ import { EuiFlexItem, EuiTitle, EuiButtonEmpty, - EuiFilterGroup, EuiContextMenuPanel, EuiContextMenuItem, EuiContextMenuPanelProps, @@ -171,33 +170,6 @@ export const InnerIndexPatternDataPanel = function InnerIndexPatternDataPanel({ onChangeIndexPattern?: (newId: string) => void; updateFieldsWithCounts?: (indexPatternId: string, fields: IndexPattern['fields']) => void; }) { - const [localState, setLocalState] = useState({ - isLoading: false, - nameFilter: '', - typeFilter: [], - isTypeFilterOpen: false, - }); - const [pageSize, setPageSize] = useState(PAGINATION_SIZE); - const [scrollContainer, setScrollContainer] = useState(undefined); - const lazyScroll = () => { - if (scrollContainer) { - const nearBottom = - scrollContainer.scrollTop + scrollContainer.clientHeight > - scrollContainer.scrollHeight * 0.9; - if (nearBottom) { - setPageSize(Math.min(pageSize * 1.5, allFields.length)); - } - } - }; - - useEffect(() => { - if (scrollContainer) { - scrollContainer.scrollTop = 0; - setPageSize(PAGINATION_SIZE); - lazyScroll(); - } - }, [localState.nameFilter, localState.typeFilter, currentIndexPatternId, showEmptyFields]); - if (Object.keys(indexPatterns).length === 0) { return ( ({ + isLoading: false, + nameFilter: '', + typeFilter: [], + isTypeFilterOpen: false, + }); + const [pageSize, setPageSize] = useState(PAGINATION_SIZE); + const [scrollContainer, setScrollContainer] = useState(undefined); + + const currentIndexPattern = indexPatterns[currentIndexPatternId]; + const allFields = currentIndexPattern.fields; + const fieldByName = indexBy(allFields, 'name'); + + const lazyScroll = () => { + if (scrollContainer) { + const nearBottom = + scrollContainer.scrollTop + scrollContainer.clientHeight > + scrollContainer.scrollHeight * 0.9; + if (nearBottom) { + setPageSize(Math.min(pageSize * 1.5, allFields.length)); + } + } + }; + + useEffect(() => { + if (scrollContainer) { + scrollContainer.scrollTop = 0; + setPageSize(PAGINATION_SIZE); + lazyScroll(); + } + }, [localState.nameFilter, localState.typeFilter, currentIndexPatternId, showEmptyFields]); const availableFieldTypes = uniq(allFields.map(({ type }) => type)).filter( type => type in fieldTypeNames ); - const displayedFields = allFields - .filter(field => { - if (!showEmptyFields) { - const indexField = - indexPatterns[currentIndexPatternId] && - indexPatterns[currentIndexPatternId].hasExistence && - indexPatterns[currentIndexPatternId].fields.find(f => f.name === field.name); - if (localState.typeFilter.length > 0) { - return ( - indexField && - indexField.exists && - localState.typeFilter.includes(field.type as DataType) - ); - } - return indexField && indexField.exists; - } + const displayedFields = allFields.filter(field => { + if (!supportedFieldTypes.includes(field.type)) { + return false; + } + + if ( + localState.nameFilter.length && + !field.name.toLowerCase().includes(localState.nameFilter.toLowerCase()) + ) { + return false; + } + + if (!showEmptyFields) { + const indexField = + currentIndexPattern && currentIndexPattern.hasExistence && fieldByName[field.name]; if (localState.typeFilter.length > 0) { - return localState.typeFilter.includes(field.type as DataType); + return ( + indexField && indexField.exists && localState.typeFilter.includes(field.type as DataType) + ); } - return true; - }) - .filter( - (field: IndexPatternField) => - field.name.toLowerCase().includes(localState.nameFilter.toLowerCase()) && - supportedFieldTypes.includes(field.type) - ); + return indexField && indexField.exists; + } + + if (localState.typeFilter.length > 0) { + return localState.typeFilter.includes(field.type as DataType); + } + + return true; + }); const paginatedFields = displayedFields.sort(sortFields).slice(0, pageSize); + // Side effect: Fetch field existence data when the index pattern is switched useEffect(() => { - if ( - localState.isLoading || - indexPatterns[currentIndexPatternId].hasExistence || - !updateFieldsWithCounts - ) { + if (localState.isLoading || currentIndexPattern.hasExistence || !updateFieldsWithCounts) { return; } setLocalState(s => ({ ...s, isLoading: true })); npStart.core.http - .post(`/api/lens/index_stats/${indexPatterns[currentIndexPatternId].title}`, { + .post(`/api/lens/index_stats/${currentIndexPattern.title}`, { body: JSON.stringify({ earliest: dateRange.fromDate, latest: dateRange.toDate, size: 500, - timeFieldName: indexPatterns[currentIndexPatternId].timeFieldName, + timeFieldName: currentIndexPattern.timeFieldName, fields: allFields .filter(field => field.aggregatable) .map(field => ({ @@ -300,7 +302,7 @@ export const InnerIndexPatternDataPanel = function InnerIndexPatternDataPanel({ updateFieldsWithCounts( currentIndexPatternId, - indexPatterns[currentIndexPatternId].fields.map(field => { + allFields.map(field => { const matching = results[field.name]; if (!matching) { return { ...field, exists: false }; @@ -334,9 +336,9 @@ export const InnerIndexPatternDataPanel = function InnerIndexPatternDataPanel({

- {indexPatterns[currentIndexPatternId].title}{' '} + {currentIndexPattern.title}{' '}

( { onToggleEmptyFields(); }} - label={i18n.translate('xpack.lens.datatypes.hiddenFields', { - defaultMessage: 'Only show fields that contain data', + label={i18n.translate('xpack.lens.indexPatterns.toggleEmptyFieldsSwitch', { + defaultMessage: 'Only show fields with data', })} data-test-subj="lnsEmptyFilter" /> @@ -493,7 +496,8 @@ export const InnerIndexPatternDataPanel = function InnerIndexPatternDataPanel({ }} > - - -
} {paginatedFields.map(field => { - const overallField = indexPatterns[currentIndexPatternId].fields.find( - f => f.name === field.name - ); + const overallField = fieldByName[field.name]; return ( )} diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/dimension_panel/field_select.tsx b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/dimension_panel/field_select.tsx index db15407adcb29..16d57dc595117 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/dimension_panel/field_select.tsx +++ b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/dimension_panel/field_select.tsx @@ -103,7 +103,7 @@ export function FieldSelect({ exists: fieldMap[field].exists || false, compatible: isCompatibleWithCurrentOperation(field), })) - .filter(field => (showEmptyFields ? true : field.exists)) + .filter(field => showEmptyFields || field.exists) .sort((a, b) => { if (a.compatible && !b.compatible) { return -1; diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/indexpattern.tsx b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/indexpattern.tsx index c2ac476ebc99c..90e13bff5ed03 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/indexpattern.tsx +++ b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/indexpattern.tsx @@ -37,7 +37,6 @@ export interface IndexPattern { fields: IndexPatternField[]; title: string; timeFieldName?: string | null; - hasExistence?: boolean; fieldFormatMap?: Record< string, { @@ -45,6 +44,9 @@ export interface IndexPattern { params: unknown; } >; + + // TODO: Load index patterns and existence data in one API call + hasExistence?: boolean; } export interface IndexPatternField { @@ -67,7 +69,7 @@ export interface IndexPatternField { > >; - // Loaded separately + // TODO: This is loaded separately, but should be combined into one API exists?: boolean; cardinality?: number; count?: number; @@ -156,6 +158,8 @@ export function getIndexPatternDatasource({ // Not stateful. State is persisted to the frame const indexPatternDatasource: Datasource = { async initialize(state?: IndexPatternPersistedState) { + // TODO: The initial request should only load index pattern names because each saved object is large + // Followup requests should load a single index pattern + existence information const indexPatternObjects = await getIndexPatterns(chrome, toastNotifications); const indexPatterns: Record = {}; diff --git a/x-pack/legacy/plugins/lens/public/register_vis_type_alias.ts b/x-pack/legacy/plugins/lens/public/register_vis_type_alias.ts index 595eb4d0e350b..b12c2e186d4db 100644 --- a/x-pack/legacy/plugins/lens/public/register_vis_type_alias.ts +++ b/x-pack/legacy/plugins/lens/public/register_vis_type_alias.ts @@ -6,11 +6,12 @@ import { i18n } from '@kbn/i18n'; import { visualizations } from '../../../../../src/legacy/core_plugins/visualizations/public'; +import { BASE_APP_URL } from '../common'; const NOT_INTERNATIONALIZED_PRODUCT_NAME = 'Lens Visualizations'; visualizations.types.visTypeAliasRegistry.add({ - aliasUrl: '/app/lens/', + aliasUrl: BASE_APP_URL, name: NOT_INTERNATIONALIZED_PRODUCT_NAME, title: i18n.translate('xpack.lens.visTypeAlias.title', { defaultMessage: 'Lens Visualizations', diff --git a/x-pack/legacy/plugins/lens/public/types.ts b/x-pack/legacy/plugins/lens/public/types.ts index 8ce2164eddc2a..007550263f441 100644 --- a/x-pack/legacy/plugins/lens/public/types.ts +++ b/x-pack/legacy/plugins/lens/public/types.ts @@ -176,7 +176,7 @@ export interface VisualizationProps { dragDropContext: DragContextState; frame: FramePublicAPI; state: T; - setState: StateSetter; + setState: (newState: T) => void; } export interface SuggestionRequest { diff --git a/x-pack/legacy/plugins/lens/server/plugin.tsx b/x-pack/legacy/plugins/lens/server/plugin.tsx index 6ccfa2399b253..9c33889a514a4 100644 --- a/x-pack/legacy/plugins/lens/server/plugin.tsx +++ b/x-pack/legacy/plugins/lens/server/plugin.tsx @@ -10,7 +10,7 @@ import { setupRoutes } from './routes'; export class LensServer implements Plugin<{}, {}, {}, {}> { constructor() {} - async setup(core: CoreSetup) { + setup(core: CoreSetup) { setupRoutes(core); return {}; @@ -20,5 +20,5 @@ export class LensServer implements Plugin<{}, {}, {}, {}> { return {}; } - async stop() {} + stop() {} } diff --git a/x-pack/legacy/plugins/lens/server/routes/index_stats.test.ts b/x-pack/legacy/plugins/lens/server/routes/index_stats.test.ts index 1afa7ea363c84..4116db05a5f60 100644 --- a/x-pack/legacy/plugins/lens/server/routes/index_stats.test.ts +++ b/x-pack/legacy/plugins/lens/server/routes/index_stats.test.ts @@ -11,7 +11,7 @@ import stubbedLogstashFields from '../../../../../../src/fixtures/logstash_field import { recursiveFlatten } from './index_stats'; describe('Index Stats route', () => { - it('should ignore falsy fields', () => { + it('should ignore empty fields, but not falsy ones', () => { const results = recursiveFlatten( [{ _source: {} }, { _source: { bytes: false } }], stubbedLogstashFields(), @@ -24,10 +24,19 @@ describe('Index Stats route', () => { name: 'bytes', type: 'number', }, + { + name: 'geo.src', + type: 'string', + }, ] ); - expect(results).toEqual({}); + expect(results).toEqual({ + bytes: { + cardinality: 1, + count: 1, + }, + }); }); it('should find existing fields based on mapping', () => { @@ -53,4 +62,44 @@ describe('Index Stats route', () => { }, }); }); + + // TODO: Alias information is not persisted in the index pattern, so we don't have access + it('fails to map alias fields', () => { + const results = recursiveFlatten(realHits, stubbedLogstashFields(), [ + { + name: '@timestamp', + type: 'date', + }, + ]); + + expect(results).toEqual({}); + }); + + // TODO: Scripts are not currently run in the _search query + it('should fail to map scripted fields', () => { + const scriptedField = { + name: 'hour_of_day', + type: 'number', + count: 0, + scripted: true, + script: "doc['timestamp'].value.hourOfDay", + lang: 'painless', + searchable: true, + aggregatable: true, + readFromDocValues: false, + }; + + const results = recursiveFlatten( + realHits, + [...stubbedLogstashFields(), scriptedField], + [ + { + name: 'hour_of_day', + type: 'number', + }, + ] + ); + + expect(results).toEqual({}); + }); }); diff --git a/x-pack/legacy/plugins/lens/server/routes/index_stats.ts b/x-pack/legacy/plugins/lens/server/routes/index_stats.ts index 0ce566f30c597..412fb2690ea78 100644 --- a/x-pack/legacy/plugins/lens/server/routes/index_stats.ts +++ b/x-pack/legacy/plugins/lens/server/routes/index_stats.ts @@ -114,47 +114,36 @@ export function recursiveFlatten( string, { count: number; - samples: unknown[]; + samples: Set; } > = {}; - const expectedKeys: Record = {}; - fields.forEach(field => { - expectedKeys[field.name] = true; - }); + const expectedKeys = new Set(fields.map(f => f.name)); - // TODO: Alias types indexPattern.forEach(field => { - if (!expectedKeys[field.name]) { + if (!expectedKeys.has(field.name)) { return; } - let matches; - if (field.parent) { - matches = docs.map(doc => { - if (!doc) { - return; - } - return get(doc._source, field.parent!); - }); - } else { - matches = docs.map(doc => { - if (!doc) { - return; - } - return get(doc._source, field.name); - }); - } + docs.forEach(doc => { + if (!doc) { + return; + } + + const match = get(doc._source, field.parent || field.name); + if (typeof match === 'undefined') { + return; + } - matches.forEach(match => { const record = overallKeys[field.name]; if (record) { record.count += 1; - record.samples.push(match); - } else if (match) { + record.samples.add(match); + } else { overallKeys[field.name] = { count: 1, - samples: [match], + // Using a set here makes the most sense and avoids the later uniq computation + samples: new Set([match]), }; } }); @@ -170,7 +159,7 @@ export function recursiveFlatten( Object.entries(overallKeys).forEach(([key, value]) => { returnTypes[key] = { count: value.count, - cardinality: uniq(value.samples).length, + cardinality: value.samples.size, }; }); return returnTypes; diff --git a/x-pack/test/api_integration/apis/lens/index.ts b/x-pack/test/api_integration/apis/lens/index.ts index 8b890c99a77b9..9827eadb1278b 100644 --- a/x-pack/test/api_integration/apis/lens/index.ts +++ b/x-pack/test/api_integration/apis/lens/index.ts @@ -6,7 +6,7 @@ import { FtrProviderContext } from '../../ftr_provider_context'; -export default function consoleApiIntegrationTests({ loadTestFile }: FtrProviderContext) { +export default function lensApiIntegrationTests({ loadTestFile }: FtrProviderContext) { describe('Lens', () => { loadTestFile(require.resolve('./index_stats')); }); diff --git a/x-pack/test/functional/page_objects/lens.ts b/x-pack/test/functional/page_objects/lens.ts index 994b68d9f36f2..fa762dc86fd8d 100644 --- a/x-pack/test/functional/page_objects/lens.ts +++ b/x-pack/test/functional/page_objects/lens.ts @@ -11,7 +11,7 @@ export function LensPageProvider({ getService }: FtrProviderContext) { return { async openIndexPatternFiltersPopover() { - await testSubjects.click('lnsIndexPatternTypeFilterButton'); + await testSubjects.click('lnsIndexPatternFiltersToggle'); }, async toggleExistenceFilter() { From 2cbf00b2455646ea4aff36e21296041b782f8437 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Tue, 3 Sep 2019 12:47:53 -0400 Subject: [PATCH 6/9] Update class names --- .../plugins/lens/public/indexpattern_plugin/field_item.tsx | 2 +- .../plugins/lens/public/indexpattern_plugin/indexpattern.scss | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/field_item.tsx b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/field_item.tsx index f0d9906974525..403b76ed1c49c 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/field_item.tsx +++ b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/field_item.tsx @@ -48,7 +48,7 @@ export function FieldItem({ field, indexPattern, highlight, exists }: FieldItemP draggable className={`lnsFieldListPanel__field lnsFieldListPanel__field-btn-${ field.type - } lnsFieldListPanel__field-${exists ? 'exists' : 'missing'}`} + } lnsFieldListPanel__field--${exists ? 'exists' : 'missing'}`} >
diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/indexpattern.scss b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/indexpattern.scss index 3dfd3c2c90756..dcc579dd05ec6 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/indexpattern.scss +++ b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/indexpattern.scss @@ -63,7 +63,7 @@ } } -.lnsFieldListPanel__field-missing { +.lnsFieldListPanel__field--missing { background: $euiColorLightestShade; } From ed662c48581cb0d65379e15ac0c77bb688bc8261 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Tue, 3 Sep 2019 12:51:09 -0400 Subject: [PATCH 7/9] Use new URL constant --- x-pack/legacy/plugins/lens/public/register_vis_type_alias.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/legacy/plugins/lens/public/register_vis_type_alias.ts b/x-pack/legacy/plugins/lens/public/register_vis_type_alias.ts index e1cf1e6b552dd..8e01c198bc54b 100644 --- a/x-pack/legacy/plugins/lens/public/register_vis_type_alias.ts +++ b/x-pack/legacy/plugins/lens/public/register_vis_type_alias.ts @@ -30,7 +30,7 @@ visualizations.types.visTypeAliasRegistry.add({ return { id, title, - editUrl: `/app/lens#/edit/${id}`, + editUrl: `${BASE_APP_URL}#/edit/${id}`, icon: 'faceHappy', isExperimental: true, savedObjectType: type, From ed3e7342e9d4e282daa2f9a46b86868dde8a7bf0 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Tue, 3 Sep 2019 12:57:41 -0400 Subject: [PATCH 8/9] Fix usage of base path --- x-pack/legacy/plugins/lens/common/constants.ts | 4 ++-- x-pack/legacy/plugins/lens/public/register_vis_type_alias.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/legacy/plugins/lens/common/constants.ts b/x-pack/legacy/plugins/lens/common/constants.ts index a7b59b5142d83..787a348a788b8 100644 --- a/x-pack/legacy/plugins/lens/common/constants.ts +++ b/x-pack/legacy/plugins/lens/common/constants.ts @@ -6,9 +6,9 @@ export const PLUGIN_ID = 'lens'; -export const BASE_APP_URL = 'app/lens'; +export const BASE_APP_URL = '/app/lens'; export const BASE_API_URL = '/api/lens'; export function getEditPath(id: string) { - return `/${BASE_APP_URL}#/edit/${encodeURIComponent(id)}`; + return `${BASE_APP_URL}#/edit/${encodeURIComponent(id)}`; } diff --git a/x-pack/legacy/plugins/lens/public/register_vis_type_alias.ts b/x-pack/legacy/plugins/lens/public/register_vis_type_alias.ts index 8e01c198bc54b..8f49f6f12ee16 100644 --- a/x-pack/legacy/plugins/lens/public/register_vis_type_alias.ts +++ b/x-pack/legacy/plugins/lens/public/register_vis_type_alias.ts @@ -6,7 +6,7 @@ import { i18n } from '@kbn/i18n'; import { visualizations } from '../../../../../src/legacy/core_plugins/visualizations/public'; -import { BASE_APP_URL } from '../common'; +import { BASE_APP_URL, getEditPath } from '../common'; const NOT_INTERNATIONALIZED_PRODUCT_NAME = 'Lens Visualizations'; @@ -30,7 +30,7 @@ visualizations.types.visTypeAliasRegistry.add({ return { id, title, - editUrl: `${BASE_APP_URL}#/edit/${id}`, + editUrl: getEditPath(id), icon: 'faceHappy', isExperimental: true, savedObjectType: type, From 2466788a188cf73e18064598ca2de66f4d2d800e Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Tue, 3 Sep 2019 13:49:51 -0400 Subject: [PATCH 9/9] Fix lint errors --- .../lens/public/indexpattern_plugin/datapanel.test.tsx | 4 ++-- .../indexpattern_plugin/indexpattern_suggestions.test.tsx | 3 +++ x-pack/legacy/plugins/lens/server/routes/index_stats.ts | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/datapanel.test.tsx b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/datapanel.test.tsx index 4609fc64619c0..d6e7337a32d82 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/datapanel.test.tsx +++ b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/datapanel.test.tsx @@ -5,8 +5,8 @@ */ import { shallow, mount } from 'enzyme'; -import React, { ChangeEvent, ReactElement } from 'react'; -import { EuiComboBox, EuiContextMenuPanel } from '@elastic/eui'; +import React, { ChangeEvent } from 'react'; +import { EuiComboBox } from '@elastic/eui'; import { IndexPatternPrivateState, IndexPatternColumn } from './indexpattern'; import { createMockedDragDropContext } from './mocks'; import { InnerIndexPatternDataPanel, IndexPatternDataPanel, MemoizedDataPanel } from './datapanel'; diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/indexpattern_suggestions.test.tsx b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/indexpattern_suggestions.test.tsx index a5b34bf6fe22c..ae26633a848ea 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/indexpattern_suggestions.test.tsx +++ b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/indexpattern_suggestions.test.tsx @@ -1171,6 +1171,7 @@ describe('IndexPattern Data Source suggestions', () => { ], }, }, + showEmptyFields: true, layers: { first: { ...persistedState.layers.first, @@ -1287,6 +1288,7 @@ describe('IndexPattern Data Source suggestions', () => { ], }, }, + showEmptyFields: true, layers: { first: { ...persistedState.layers.first, @@ -1337,6 +1339,7 @@ describe('IndexPattern Data Source suggestions', () => { ], }, }, + showEmptyFields: true, layers: { first: { ...persistedState.layers.first, diff --git a/x-pack/legacy/plugins/lens/server/routes/index_stats.ts b/x-pack/legacy/plugins/lens/server/routes/index_stats.ts index 412fb2690ea78..6918579428dc6 100644 --- a/x-pack/legacy/plugins/lens/server/routes/index_stats.ts +++ b/x-pack/legacy/plugins/lens/server/routes/index_stats.ts @@ -5,7 +5,7 @@ */ import Boom from 'boom'; -import { get, uniq } from 'lodash'; +import { get } from 'lodash'; import { schema } from '@kbn/config-schema'; import { SearchResponse } from 'elasticsearch'; import { CoreSetup } from 'src/core/server';