diff --git a/src/fixtures/logstash_fields.js b/src/fixtures/logstash_fields.js index 1bd2c050b4563..5771a01047c2e 100644 --- a/src/fixtures/logstash_fields.js +++ b/src/fixtures/logstash_fields.js @@ -39,7 +39,8 @@ function stubbedLogstashFields() { ['area', 'geo_shape', true, true ], ['hashed', 'murmur3', false, true ], ['geo.coordinates', 'geo_point', true, true ], - ['extension', 'keyword', true, true ], + ['extension', 'text', true, true], + ['extension.keyword', 'keyword', true, true, {}, 'extension', 'multi' ], ['machine.os', 'text', true, true ], ['machine.os.raw', 'keyword', true, true, {}, 'machine.os', 'multi' ], ['geo.src', 'keyword', true, true ], diff --git a/x-pack/legacy/plugins/lens/common/constants.ts b/x-pack/legacy/plugins/lens/common/constants.ts index edcc34c58881a..787a348a788b8 100644 --- a/x-pack/legacy/plugins/lens/common/constants.ts +++ b/x-pack/legacy/plugins/lens/common/constants.ts @@ -6,8 +6,9 @@ export const PLUGIN_ID = 'lens'; -export const BASE_URL = 'app/lens'; +export const BASE_APP_URL = '/app/lens'; +export const BASE_API_URL = '/api/lens'; export function getEditPath(id: string) { - return `/${BASE_URL}#/edit/${encodeURIComponent(id)}`; + return `${BASE_APP_URL}#/edit/${encodeURIComponent(id)}`; } diff --git a/x-pack/legacy/plugins/lens/index.ts b/x-pack/legacy/plugins/lens/index.ts index f205b14af9dab..399a65041b664 100644 --- a/x-pack/legacy/plugins/lens/index.ts +++ b/x-pack/legacy/plugins/lens/index.ts @@ -5,11 +5,13 @@ */ import * as Joi from 'joi'; -import { Server } from 'hapi'; import { resolve } from 'path'; import { LegacyPluginInitializer } from 'src/legacy/types'; +import KbnServer, { Server } from 'src/legacy/server/kbn_server'; +import { CoreSetup } from 'src/core/server'; import mappings from './mappings.json'; -import { PLUGIN_ID, getEditPath } from './common'; +import { PLUGIN_ID, getEditPath, BASE_API_URL } from './common'; +import { lensServerPlugin } from './server'; const NOT_INTERNATIONALIZED_PRODUCT_NAME = 'Lens Visualizations'; @@ -51,6 +53,8 @@ export const lens: LegacyPluginInitializer = kibana => { }, init(server: Server) { + const kbnServer = (server as unknown) as KbnServer; + server.plugins.xpack_main.registerFeature({ id: PLUGIN_ID, name: NOT_INTERNATIONALIZED_PRODUCT_NAME, @@ -77,6 +81,19 @@ export const lens: LegacyPluginInitializer = kibana => { }, }, }); + + // Set up with the new platform plugin lifecycle API. + const plugin = lensServerPlugin(); + plugin.setup(({ + http: { + ...kbnServer.newPlatform.setup.core.http, + createRouter: () => kbnServer.newPlatform.setup.core.http.createRouter(BASE_API_URL), + }, + } as unknown) as CoreSetup); + + server.events.on('stop', () => { + plugin.stop(); + }); }, }); }; diff --git a/x-pack/legacy/plugins/lens/public/editor_frame_plugin/editor_frame/data_panel_wrapper.tsx b/x-pack/legacy/plugins/lens/public/editor_frame_plugin/editor_frame/data_panel_wrapper.tsx index a013d9b1bceae..d3beddc0689c1 100644 --- a/x-pack/legacy/plugins/lens/public/editor_frame_plugin/editor_frame/data_panel_wrapper.tsx +++ b/x-pack/legacy/plugins/lens/public/editor_frame_plugin/editor_frame/data_panel_wrapper.tsx @@ -7,11 +7,12 @@ import React, { useMemo, memo, useContext, useState } from 'react'; import { i18n } from '@kbn/i18n'; import { EuiPopover, EuiButtonIcon, EuiContextMenuPanel, EuiContextMenuItem } from '@elastic/eui'; +import { Query } from 'src/plugins/data/common'; import { DatasourceDataPanelProps, Datasource } from '../../../public'; import { NativeRenderer } from '../../native_renderer'; import { Action } from './state_management'; import { DragContext } from '../../drag_drop'; -import { StateSetter } from '../../types'; +import { StateSetter, FramePublicAPI } from '../../types'; interface DataPanelWrapperProps { datasourceState: unknown; @@ -19,6 +20,8 @@ interface DataPanelWrapperProps { activeDatasource: string | null; datasourceIsLoading: boolean; dispatch: (action: Action) => void; + query: Query; + dateRange: FramePublicAPI['dateRange']; } export const DataPanelWrapper = memo((props: DataPanelWrapperProps) => { @@ -37,6 +40,8 @@ export const DataPanelWrapper = memo((props: DataPanelWrapperProps) => { dragDropContext: useContext(DragContext), state: props.datasourceState, setState: setDatasourceState, + query: props.query, + dateRange: props.dateRange, }; const [showDatasourceSwitcher, setDatasourceSwitcher] = useState(false); diff --git a/x-pack/legacy/plugins/lens/public/editor_frame_plugin/editor_frame/editor_frame.tsx b/x-pack/legacy/plugins/lens/public/editor_frame_plugin/editor_frame/editor_frame.tsx index 927a1f1abf3dc..f7837bc02b9be 100644 --- a/x-pack/legacy/plugins/lens/public/editor_frame_plugin/editor_frame/editor_frame.tsx +++ b/x-pack/legacy/plugins/lens/public/editor_frame_plugin/editor_frame/editor_frame.tsx @@ -219,6 +219,8 @@ export function EditorFrame(props: EditorFrameProps) { : true } dispatch={dispatch} + query={props.query} + dateRange={props.dateRange} /> } configPanel={ diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/datapanel.test.tsx b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/datapanel.test.tsx index dfd4adde48560..d6e7337a32d82 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/datapanel.test.tsx +++ b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/datapanel.test.tsx @@ -4,20 +4,24 @@ * you may not use this file except in compliance with the Elastic License. */ -import { shallow } from 'enzyme'; -import React, { ChangeEvent, ReactElement } from 'react'; -import { EuiComboBox, EuiFieldSearch, EuiContextMenuPanel } from '@elastic/eui'; +import { shallow, mount } from 'enzyme'; +import React, { ChangeEvent } from 'react'; +import { EuiComboBox } from '@elastic/eui'; import { IndexPatternPrivateState, IndexPatternColumn } from './indexpattern'; import { createMockedDragDropContext } from './mocks'; import { InnerIndexPatternDataPanel, IndexPatternDataPanel, MemoizedDataPanel } from './datapanel'; import { FieldItem } from './field_item'; import { act } from 'react-dom/test-utils'; +import { npStart as npStartMock } from 'ui/new_platform'; jest.mock('ui/new_platform'); jest.mock('./loader'); +const waitForPromises = () => new Promise(resolve => setTimeout(resolve)); + const initialState: IndexPatternPrivateState = { currentIndexPatternId: '1', + showEmptyFields: false, layers: { first: { indexPatternId: '1', @@ -203,6 +207,13 @@ describe('IndexPattern Data Panel', () => { showIndexPatternSwitcher: false, setShowIndexPatternSwitcher: jest.fn(), onChangeIndexPattern: jest.fn(), + dateRange: { + fromDate: 'now-7d', + toDate: 'now', + }, + query: { query: '', language: 'lucene' }, + showEmptyFields: false, + onToggleEmptyFields: jest.fn(), }; }); @@ -210,6 +221,7 @@ describe('IndexPattern Data Panel', () => { const setStateSpy = jest.fn(); const wrapper = shallow( { }; const wrapper = shallow( {} }} @@ -270,6 +283,7 @@ describe('IndexPattern Data Panel', () => { }; const wrapper = shallow( {} }} @@ -312,98 +326,259 @@ describe('IndexPattern Data Panel', () => { expect(defaultProps.onChangeIndexPattern).toHaveBeenCalledWith('2'); }); - it('should list all supported fields in the pattern sorted alphabetically', async () => { - const wrapper = shallow(); + describe('loading existence data', () => { + beforeEach(() => { + (npStartMock.core.http.post as jest.Mock).mockClear(); + }); - expect(wrapper.find(FieldItem).map(fieldItem => fieldItem.prop('field').name)).toEqual([ - 'bytes', - 'memory', - 'source', - 'timestamp', - ]); - }); + it('loads existence data and updates the index pattern', async () => { + (npStartMock.core.http.post as jest.Mock).mockResolvedValue({ + timestamp: { + exists: true, + cardinality: 500, + count: 500, + }, + }); + const updateFields = jest.fn(); + mount(); - it('should filter down by name', async () => { - const wrapper = shallow(); + await waitForPromises(); - act(() => { - wrapper.find(EuiFieldSearch).prop('onChange')!({ target: { value: 'mem' } } as ChangeEvent< - HTMLInputElement - >); + expect(npStartMock.core.http.post as jest.Mock).toHaveBeenCalledWith( + `/api/lens/index_stats/my-fake-index-pattern`, + { + body: JSON.stringify({ + earliest: 'now-7d', + latest: 'now', + size: 500, + timeFieldName: 'timestamp', + fields: [ + { + name: 'timestamp', + type: 'date', + }, + { + name: 'bytes', + type: 'number', + }, + { + name: 'memory', + type: 'number', + }, + { + name: 'unsupported', + type: 'geo', + }, + { + name: 'source', + type: 'string', + }, + ], + }), + } + ); + + expect(updateFields).toHaveBeenCalledWith('1', [ + { + name: 'timestamp', + type: 'date', + exists: true, + cardinality: 500, + count: 500, + aggregatable: true, + searchable: true, + }, + ...defaultProps.indexPatterns['1'].fields + .slice(1) + .map(field => ({ ...field, exists: false })), + ]); }); - expect(wrapper.find(FieldItem).map(fieldItem => fieldItem.prop('field').name)).toEqual([ - 'memory', - ]); - }); + it('does not attempt to load existence data if the index pattern has it', async () => { + const updateFields = jest.fn(); + const newIndexPatterns = { + ...defaultProps.indexPatterns, + '1': { + ...defaultProps.indexPatterns['1'], + hasExistence: true, + }, + }; - it('should filter down by type', async () => { - const wrapper = shallow(); + const props = { ...defaultProps, indexPatterns: newIndexPatterns }; - act(() => { - (wrapper - .find(EuiContextMenuPanel) - .prop('items')! - .find( - item => (item as ReactElement).props['data-test-subj'] === 'typeFilter-number' - )! as ReactElement).props.onClick(); - }); + mount(); - expect(wrapper.find(FieldItem).map(fieldItem => fieldItem.prop('field').name)).toEqual([ - 'bytes', - 'memory', - ]); + await waitForPromises(); + + expect(npStartMock.core.http.post as jest.Mock).not.toHaveBeenCalled(); + }); }); - it('should toggle type if clicked again', async () => { - const wrapper = shallow(); + describe('while showing empty fields', () => { + it('should list all supported fields in the pattern sorted alphabetically', async () => { + const wrapper = shallow( + + ); + + expect(wrapper.find(FieldItem).map(fieldItem => fieldItem.prop('field').name)).toEqual([ + 'bytes', + 'memory', + 'source', + 'timestamp', + ]); + }); - act(() => { - (wrapper - .find(EuiContextMenuPanel) - .prop('items')! - .find( - item => (item as ReactElement).props['data-test-subj'] === 'typeFilter-number' - )! as ReactElement).props.onClick(); + it('should filter down by name', () => { + const wrapper = shallow( + + ); + + act(() => { + wrapper.find('[data-test-subj="lnsIndexPatternFieldSearch"]').prop('onChange')!({ + target: { value: 'mem' }, + } as ChangeEvent); + }); + + expect(wrapper.find(FieldItem).map(fieldItem => fieldItem.prop('field').name)).toEqual([ + 'memory', + ]); }); - act(() => { - (wrapper - .find(EuiContextMenuPanel) - .prop('items')! - .find( - item => (item as ReactElement).props['data-test-subj'] === 'typeFilter-number' - )! as ReactElement).props.onClick(); + it('should filter down by type', () => { + const wrapper = mount( + + ); + + wrapper + .find('[data-test-subj="lnsIndexPatternFiltersToggle"]') + .first() + .simulate('click'); + + wrapper + .find('[data-test-subj="typeFilter-number"]') + .first() + .simulate('click'); + + expect(wrapper.find(FieldItem).map(fieldItem => fieldItem.prop('field').name)).toEqual([ + 'bytes', + 'memory', + ]); }); - expect(wrapper.find(FieldItem).map(fieldItem => fieldItem.prop('field').name)).toEqual([ - 'bytes', - 'memory', - 'source', - 'timestamp', - ]); + it('should toggle type if clicked again', () => { + const wrapper = mount( + + ); + + wrapper + .find('[data-test-subj="lnsIndexPatternFiltersToggle"]') + .first() + .simulate('click'); + + wrapper + .find('[data-test-subj="typeFilter-number"]') + .first() + .simulate('click'); + wrapper + .find('[data-test-subj="typeFilter-number"]') + .first() + .simulate('click'); + + expect(wrapper.find(FieldItem).map(fieldItem => fieldItem.prop('field').name)).toEqual([ + 'bytes', + 'memory', + 'source', + 'timestamp', + ]); + }); + + it('should filter down by type and by name', () => { + const wrapper = mount( + + ); + + act(() => { + wrapper.find('[data-test-subj="lnsIndexPatternFieldSearch"]').prop('onChange')!({ + target: { value: 'mem' }, + } as ChangeEvent); + }); + + wrapper + .find('[data-test-subj="lnsIndexPatternFiltersToggle"]') + .first() + .simulate('click'); + + wrapper + .find('[data-test-subj="typeFilter-number"]') + .first() + .simulate('click'); + + expect(wrapper.find(FieldItem).map(fieldItem => fieldItem.prop('field').name)).toEqual([ + 'memory', + ]); + }); }); - it('should filter down by type and by name', async () => { - const wrapper = shallow(); + describe('filtering out empty fields', () => { + let emptyFieldsTestProps: typeof defaultProps; + + beforeEach(() => { + emptyFieldsTestProps = { + ...defaultProps, + indexPatterns: { + ...defaultProps.indexPatterns, + '1': { + ...defaultProps.indexPatterns['1'], + hasExistence: true, + fields: defaultProps.indexPatterns['1'].fields.map(field => ({ + ...field, + exists: field.type === 'number', + })), + }, + }, + onToggleEmptyFields: jest.fn(), + }; + }); - act(() => { - wrapper.find(EuiFieldSearch).prop('onChange')!({ target: { value: 'mem' } } as ChangeEvent< - HTMLInputElement - >); + it('should list all supported fields in the pattern sorted alphabetically', async () => { + const wrapper = shallow(); + + expect(wrapper.find(FieldItem).map(fieldItem => fieldItem.prop('field').name)).toEqual([ + 'bytes', + 'memory', + ]); }); - act(() => { - (wrapper - .find(EuiContextMenuPanel) - .prop('items')! - .find( - item => (item as ReactElement).props['data-test-subj'] === 'typeFilter-number' - )! as ReactElement).props.onClick(); + it('should filter down by name', () => { + const wrapper = shallow( + + ); + + act(() => { + wrapper.find('[data-test-subj="lnsIndexPatternFieldSearch"]').prop('onChange')!({ + target: { value: 'mem' }, + } as ChangeEvent); + }); + + expect(wrapper.find(FieldItem).map(fieldItem => fieldItem.prop('field').name)).toEqual([ + 'memory', + ]); }); - expect(wrapper.find(FieldItem).map(fieldItem => fieldItem.prop('field').name)).toEqual([ - 'memory', - ]); + it('should allow removing the filter for data', () => { + const wrapper = mount(); + + wrapper + .find('[data-test-subj="lnsIndexPatternFiltersToggle"]') + .first() + .simulate('click'); + + wrapper + .find('[data-test-subj="lnsEmptyFilter"]') + .first() + .prop('onChange')!({} as ChangeEvent); + + expect(emptyFieldsTestProps.onToggleEmptyFields).toHaveBeenCalled(); + }); }); }); diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/datapanel.tsx b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/datapanel.tsx index 74c19ad2cf0e3..832dfaf47a950 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/datapanel.tsx +++ b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/datapanel.tsx @@ -4,27 +4,33 @@ * you may not use this file except in compliance with the Elastic License. */ -import { mapValues, uniq } from 'lodash'; +import { mapValues, uniq, indexBy } from 'lodash'; import React, { useState, useEffect, memo, useCallback } from 'react'; import { EuiComboBox, - EuiFieldSearch, + EuiLoadingSpinner, // @ts-ignore EuiHighlight, EuiFlexGroup, EuiFlexItem, EuiTitle, EuiButtonEmpty, - EuiFilterGroup, - EuiFilterButton, EuiContextMenuPanel, EuiContextMenuItem, EuiContextMenuPanelProps, EuiPopover, + EuiPopoverTitle, + EuiPopoverFooter, EuiCallOut, + EuiText, + EuiFormControlLayout, + EuiSwitch, + EuiIcon, } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; +import { npStart } from 'ui/new_platform'; +import { Query } from 'src/plugins/data/common'; import { DatasourceDataPanelProps, DataType } from '../types'; import { IndexPatternPrivateState, IndexPatternField, IndexPattern } from './indexpattern'; import { ChildDragDropProvider, DragContextState } from '../drag_drop'; @@ -60,6 +66,8 @@ export function IndexPatternDataPanel({ setState, state, dragDropContext, + query, + dateRange, }: DatasourceDataPanelProps) { const { indexPatterns, currentIndexPatternId } = state; const [showIndexPatternSwitcher, setShowIndexPatternSwitcher] = useState(false); @@ -79,64 +87,93 @@ export function IndexPatternDataPanel({ [state, setState] ); + const updateFieldsWithCounts = useCallback( + (indexPatternId: string, allFields: IndexPattern['fields']) => { + setState(prevState => { + return { + ...prevState, + indexPatterns: { + ...prevState.indexPatterns, + [indexPatternId]: { + ...prevState.indexPatterns[indexPatternId], + hasExistence: true, + fields: allFields, + }, + }, + }; + }); + }, + [currentIndexPatternId, indexPatterns[currentIndexPatternId]] + ); + + const onToggleEmptyFields = useCallback(() => { + setState(prevState => ({ ...prevState, showEmptyFields: !prevState.showEmptyFields })); + }, [state, setState]); + return ( ); } +type OverallFields = Record< + string, + { + count: number; + cardinality: number; + } +>; + +interface DataPanelState { + isLoading: boolean; + nameFilter: string; + typeFilter: DataType[]; + isTypeFilterOpen: boolean; +} + export const InnerIndexPatternDataPanel = function InnerIndexPatternDataPanel({ currentIndexPatternId, indexPatterns, + query, + dateRange, dragDropContext, showIndexPatternSwitcher, setShowIndexPatternSwitcher, onChangeIndexPattern, -}: { + updateFieldsWithCounts, + showEmptyFields, + onToggleEmptyFields, +}: Partial & { currentIndexPatternId: string; indexPatterns: Record; + dateRange: DatasourceDataPanelProps['dateRange']; + query: Query; dragDropContext: DragContextState; showIndexPatternSwitcher: boolean; setShowIndexPatternSwitcher: (show: boolean) => void; + showEmptyFields: boolean; + onToggleEmptyFields: () => void; onChangeIndexPattern?: (newId: string) => void; + updateFieldsWithCounts?: (indexPatternId: string, fields: IndexPattern['fields']) => void; }) { - const [state, setState] = useState({ - nameFilter: '', - typeFilter: [] as DataType[], - isTypeFilterOpen: false, - }); - const [pageSize, setPageSize] = useState(PAGINATION_SIZE); - const [scrollContainer, setScrollContainer] = useState(undefined); - const lazyScroll = () => { - if (scrollContainer) { - const nearBottom = - scrollContainer.scrollTop + scrollContainer.clientHeight > - scrollContainer.scrollHeight * 0.9; - if (nearBottom) { - setPageSize(Math.min(pageSize * 1.5, allFields.length)); - } - } - }; - - useEffect(() => { - if (scrollContainer) { - scrollContainer.scrollTop = 0; - setPageSize(PAGINATION_SIZE); - lazyScroll(); - } - }, [state.nameFilter, state.typeFilter, currentIndexPatternId]); - if (Object.keys(indexPatterns).length === 0) { return ( - field.name.toLowerCase().includes(state.nameFilter.toLowerCase()) && - supportedFieldTypes.includes(field.type) - ) - .slice(0, pageSize); - - const availableFieldTypes = uniq(filteredFields.map(({ type }) => type)); - const availableFilteredTypes = state.typeFilter.filter(type => - availableFieldTypes.includes(type) + const [localState, setLocalState] = useState({ + isLoading: false, + nameFilter: '', + typeFilter: [], + isTypeFilterOpen: false, + }); + const [pageSize, setPageSize] = useState(PAGINATION_SIZE); + const [scrollContainer, setScrollContainer] = useState(undefined); + + const currentIndexPattern = indexPatterns[currentIndexPatternId]; + const allFields = currentIndexPattern.fields; + const fieldByName = indexBy(allFields, 'name'); + + const lazyScroll = () => { + if (scrollContainer) { + const nearBottom = + scrollContainer.scrollTop + scrollContainer.clientHeight > + scrollContainer.scrollHeight * 0.9; + if (nearBottom) { + setPageSize(Math.min(pageSize * 1.5, allFields.length)); + } + } + }; + + useEffect(() => { + if (scrollContainer) { + scrollContainer.scrollTop = 0; + setPageSize(PAGINATION_SIZE); + lazyScroll(); + } + }, [localState.nameFilter, localState.typeFilter, currentIndexPatternId, showEmptyFields]); + + const availableFieldTypes = uniq(allFields.map(({ type }) => type)).filter( + type => type in fieldTypeNames ); + const displayedFields = allFields.filter(field => { + if (!supportedFieldTypes.includes(field.type)) { + return false; + } + + if ( + localState.nameFilter.length && + !field.name.toLowerCase().includes(localState.nameFilter.toLowerCase()) + ) { + return false; + } + + if (!showEmptyFields) { + const indexField = + currentIndexPattern && currentIndexPattern.hasExistence && fieldByName[field.name]; + if (localState.typeFilter.length > 0) { + return ( + indexField && indexField.exists && localState.typeFilter.includes(field.type as DataType) + ); + } + return indexField && indexField.exists; + } + + if (localState.typeFilter.length > 0) { + return localState.typeFilter.includes(field.type as DataType); + } + + return true; + }); + + const paginatedFields = displayedFields.sort(sortFields).slice(0, pageSize); + + // Side effect: Fetch field existence data when the index pattern is switched + useEffect(() => { + if (localState.isLoading || currentIndexPattern.hasExistence || !updateFieldsWithCounts) { + return; + } + + setLocalState(s => ({ ...s, isLoading: true })); + + npStart.core.http + .post(`/api/lens/index_stats/${currentIndexPattern.title}`, { + body: JSON.stringify({ + earliest: dateRange.fromDate, + latest: dateRange.toDate, + size: 500, + timeFieldName: currentIndexPattern.timeFieldName, + fields: allFields + .filter(field => field.aggregatable) + .map(field => ({ + name: field.name, + type: field.type, + })), + }), + }) + .then((results: OverallFields) => { + setLocalState(s => ({ + ...s, + isLoading: false, + })); + + if (!updateFieldsWithCounts) { + return; + } + + updateFieldsWithCounts( + currentIndexPatternId, + allFields.map(field => { + const matching = results[field.name]; + if (!matching) { + return { ...field, exists: false }; + } + return { + ...field, + exists: true, + cardinality: matching.cardinality, + count: matching.count, + }; + }) + ); + }) + .catch(() => { + setLocalState(s => ({ ...s, isLoading: false })); + }); + }, [currentIndexPatternId]); + return (

- {indexPatterns[currentIndexPatternId].title}{' '} + {currentIndexPattern.title}{' '}

{ onChangeIndexPattern!(choices[0].value as string); - setState({ - ...state, + setLocalState(s => ({ + ...s, nameFilter: '', typeFilter: [], - }); + })); setShowIndexPatternSwitcher(false); }} @@ -259,69 +404,114 @@ export const InnerIndexPatternDataPanel = function InnerIndexPatternDataPanel({ responsive={false} > - { - setState({ ...state, nameFilter: e.target.value }); - }} - aria-label={i18n.translate('xpack.lens.indexPatterns.filterByNameAriaLabel', { - defaultMessage: 'Search fields', - })} - /> - - - - setState({ ...state, isTypeFilterOpen: false })} - button={ - - setState({ ...state, isTypeFilterOpen: !state.isTypeFilterOpen }) - } - iconType="arrowDown" - data-test-subj="indexPatternTypeFilterButton" - isSelected={state.isTypeFilterOpen} - numFilters={availableFieldTypes.length} - hasActiveFilters={availableFilteredTypes.length > 0} - numActiveFilters={availableFilteredTypes.length} - > - {i18n.translate('xpack.lens.indexPatterns.typeFilterLabel', { - defaultMessage: 'Types', - })} - - } - > - ( - - setState({ - ...state, - typeFilter: state.typeFilter.includes(type) - ? state.typeFilter.filter(t => t !== type) - : [...state.typeFilter, type], - }) - } + + setLocalState(s => ({ ...localState, isTypeFilterOpen: false })) + } + button={ + { + setLocalState(s => ({ + ...s, + isTypeFilterOpen: !localState.isTypeFilterOpen, + })); + }} + data-test-subj="lnsIndexPatternFiltersToggle" + title={i18n.translate('xpack.lens.indexPatterns.toggleFiltersPopover', { + defaultMessage: 'Toggle filters for index pattern', + })} + aria-label={i18n.translate( + 'xpack.lens.indexPatterns.toggleFiltersPopover', + { + defaultMessage: 'Toggle filters for index pattern', + } + )} > - {fieldTypeNames[type]} - - ))} - /> - - + + + } + > + + {i18n.translate('xpack.lens.indexPatterns.filterByTypeLabel', { + defaultMessage: 'Filter by type', + })} + + ( + + setLocalState(s => ({ + ...s, + typeFilter: localState.typeFilter.includes(type) + ? localState.typeFilter.filter(t => t !== type) + : [...localState.typeFilter, type], + })) + } + > + {fieldTypeNames[type]} + + ))} + /> + + { + onToggleEmptyFields(); + }} + label={i18n.translate('xpack.lens.indexPatterns.toggleEmptyFieldsSwitch', { + defaultMessage: 'Only show fields with data', + })} + data-test-subj="lnsEmptyFilter" + /> + + + } + clear={{ + title: i18n.translate('xpack.lens.indexPatterns.clearFiltersLabel', { + defaultMessage: 'Clear name and type filters', + }), + 'aria-label': i18n.translate('xpack.lens.indexPatterns.clearFiltersLabel', { + defaultMessage: 'Clear name and type filters', + }), + onClick: () => { + setLocalState(s => ({ + ...s, + nameFilter: '', + typeFilter: [], + })); + }, + }} + > + { + setLocalState({ ...localState, nameFilter: e.target.value }); + }} + aria-label={i18n.translate('xpack.lens.indexPatterns.filterByNameAriaLabel', { + defaultMessage: 'Search fields', + })} + /> +
- {filteredFields - .filter( - field => - state.typeFilter.length === 0 || - state.typeFilter.includes(field.type as DataType) - ) - .sort(sortFields) - .map(field => ( + {localState.isLoading && } + + {paginatedFields.map(field => { + const overallField = fieldByName[field.name]; + return ( - ))} + ); + })} + + {!localState.isLoading && paginatedFields.length === 0 && ( + + {showEmptyFields + ? i18n.translate('xpack.lens.indexPatterns.hiddenFieldsLabel', { + defaultMessage: + 'No fields have data with the current filters. You can show fields without data using the filters above.', + }) + : i18n.translate('xpack.lens.indexPatterns.noFieldsLabel', { + defaultMessage: 'No fields can be visualized from {title}', + values: { title: currentIndexPattern.title }, + })} + + )}
diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/dimension_panel/_popover.scss b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/dimension_panel/_popover.scss index b5701daf31d7e..42f9366bd1f1e 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/dimension_panel/_popover.scss +++ b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/dimension_panel/_popover.scss @@ -16,6 +16,10 @@ color: $euiColorLightShade; } +.lnsConfigPanel__fieldOption--nonExistant { + background-color: $euiColorLightestShade; +} + .lnsConfigPanel__operation { padding: $euiSizeXS; font-size: 0.875rem; diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/dimension_panel/dimension_panel.test.tsx b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/dimension_panel/dimension_panel.test.tsx index 2ddfce6b7e0a5..1e801467f3f94 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/dimension_panel/dimension_panel.test.tsx +++ b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/dimension_panel/dimension_panel.test.tsx @@ -33,30 +33,35 @@ const expectedIndexPatterns = { id: '1', title: 'my-fake-index-pattern', timeFieldName: 'timestamp', + hasExistence: true, fields: [ { name: 'timestamp', type: 'date', aggregatable: true, searchable: true, + exists: true, }, { name: 'bytes', type: 'number', aggregatable: true, searchable: true, + exists: true, }, { name: 'memory', type: 'number', aggregatable: true, searchable: true, + exists: true, }, { name: 'source', type: 'string', aggregatable: true, searchable: true, + exists: true, }, ], }, @@ -80,6 +85,7 @@ describe('IndexPatternDimensionPanel', () => { state = { indexPatterns: expectedIndexPatterns, currentIndexPatternId: '1', + showEmptyFields: false, layers: { first: { indexPatternId: '1', @@ -947,6 +953,7 @@ describe('IndexPatternDimensionPanel', () => { }, }, currentIndexPatternId: '1', + showEmptyFields: false, layers: { myLayer: { indexPatternId: 'foo', diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/dimension_panel/field_select.tsx b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/dimension_panel/field_select.tsx index 2874449bcb4ff..16d57dc595117 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/dimension_panel/field_select.tsx +++ b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/dimension_panel/field_select.tsx @@ -24,6 +24,7 @@ export type FieldChoice = export interface FieldSelectProps { currentIndexPattern: IndexPattern; + showEmptyFields: boolean; fieldMap: Record; incompatibleSelectedOperationType: OperationType | null; selectedColumnOperationType?: OperationType; @@ -34,16 +35,18 @@ export interface FieldSelectProps { } export function FieldSelect({ + currentIndexPattern, + showEmptyFields, + fieldMap, incompatibleSelectedOperationType, selectedColumnOperationType, selectedColumnSourceField, operationFieldSupportMatrix, - currentIndexPattern, - fieldMap, onChoose, onDeleteColumn, }: FieldSelectProps) { const { operationByDocument, operationByField } = operationFieldSupportMatrix; + const memoizedFieldOptions = useMemo(() => { const fields = Object.keys(operationByField).sort(); @@ -97,25 +100,31 @@ export function FieldSelect({ ? selectedColumnOperationType : undefined, }, + exists: fieldMap[field].exists || false, compatible: isCompatibleWithCurrentOperation(field), })) - .sort(({ compatible: a }, { compatible: b }) => { - if (a && !b) { + .filter(field => showEmptyFields || field.exists) + .sort((a, b) => { + if (a.compatible && !b.compatible) { return -1; } - if (!a && b) { + if (!a.compatible && b.compatible) { return 1; } return 0; }) - .map(({ label, value, compatible }) => ({ + .map(({ label, value, compatible, exists }) => ({ label, value, - className: classNames({ 'lnsConfigPanel__fieldOption--incompatible': !compatible }), + className: classNames({ + 'lnsConfigPanel__fieldOption--incompatible': !compatible, + 'lnsConfigPanel__fieldOption--nonExistant': !exists, + }), 'data-test-subj': `lns-fieldOption${compatible ? '' : 'Incompatible'}-${label}`, })), }); } + return fieldOptions; }, [ incompatibleSelectedOperationType, @@ -124,6 +133,7 @@ export function FieldSelect({ operationFieldSupportMatrix, currentIndexPattern, fieldMap, + showEmptyFields, ]); return ( diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/dimension_panel/popover_editor.tsx b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/dimension_panel/popover_editor.tsx index 960e81b98699b..7eb03152b341f 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/dimension_panel/popover_editor.tsx +++ b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/dimension_panel/popover_editor.tsx @@ -258,8 +258,9 @@ export function PopoverEditor(props: PopoverEditorProps) { acc + ch.charCodeAt(0), 1); } -export type UnwrapArray = T extends Array ? P : T; - -export function FieldIcon({ type }: { type: DataType }) { - const icons: Partial>> = { +function getIconForDataType(dataType: string) { + const icons: Partial>> = { boolean: 'invert', date: 'calendar', }; + return icons[dataType] || ICON_TYPES.find(t => t === dataType) || 'empty'; +} - const iconType = icons[type] || ICON_TYPES.find(t => t === type) || 'empty'; +export function getColorForDataType(type: string) { + const iconType = getIconForDataType(type); const { colors } = palettes.euiPaletteColorBlind; const colorIndex = stringToNum(iconType) % colors.length; + return colors[colorIndex]; +} + +export type UnwrapArray = T extends Array ? P : T; + +export function FieldIcon({ type }: { type: DataType }) { + const iconType = getIconForDataType(type); const classes = classNames( 'lnsFieldListPanel__fieldIcon', `lnsFieldListPanel__fieldIcon--${type}` ); - return ; + return ; } diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/field_item.tsx b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/field_item.tsx index 51d519da21693..403b76ed1c49c 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/field_item.tsx +++ b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/field_item.tsx @@ -5,15 +5,16 @@ */ import React from 'react'; -import { IndexPatternField, DraggedField } from './indexpattern'; +import { IndexPattern, IndexPatternField, DraggedField } from './indexpattern'; import { DragDrop } from '../drag_drop'; import { FieldIcon } from './field_icon'; import { DataType } from '..'; export interface FieldItemProps { field: IndexPatternField; - indexPatternId: string; + indexPattern: IndexPattern; highlight?: string; + exists: boolean; } function wrapOnDot(str?: string) { @@ -23,8 +24,8 @@ function wrapOnDot(str?: string) { return str ? str.replace(/\./g, '.\u200B') : ''; } -export function FieldItem({ field, indexPatternId, highlight }: FieldItemProps) { - const wrappableName = wrapOnDot(field.name); +export function FieldItem({ field, indexPattern, highlight, exists }: FieldItemProps) { + const wrappableName = wrapOnDot(field.name)!; const wrappableHighlight = wrapOnDot(highlight); const highlightIndex = wrappableHighlight ? wrappableName.toLowerCase().indexOf(wrappableHighlight.toLowerCase()) @@ -42,15 +43,20 @@ export function FieldItem({ field, indexPatternId, highlight }: FieldItemProps) return ( - - - {wrappableHighlightableFieldName} - +
+ + + + {wrappableHighlightableFieldName} + +
); } diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/indexpattern.scss b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/indexpattern.scss index adbbcacaa8f30..dcc579dd05ec6 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/indexpattern.scss +++ b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/indexpattern.scss @@ -53,10 +53,7 @@ background: $euiColorEmptyShade; border-radius: $euiBorderRadius; padding: $euiSizeS; - display: flex; - align-items: center; margin-bottom: $euiSizeXS; - font-weight: $euiFontWeightMedium; transition: box-shadow $euiAnimSpeedFast $euiAnimSlightResistance; &:hover { @@ -66,6 +63,10 @@ } } +.lnsFieldListPanel__field--missing { + background: $euiColorLightestShade; +} + .lnsFieldListPanel__fieldName { margin-left: $euiSizeXS; } diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/indexpattern.test.ts b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/indexpattern.test.ts index 336deef6147a3..d69a3827a43c9 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/indexpattern.test.ts +++ b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/indexpattern.test.ts @@ -176,6 +176,7 @@ describe('IndexPattern Data Source', () => { currentIndexPatternId: '1', indexPatterns: expectedIndexPatterns, layers: {}, + showEmptyFields: false, }); }); @@ -184,6 +185,7 @@ describe('IndexPattern Data Source', () => { expect(state).toEqual({ ...persistedState, indexPatterns: expectedIndexPatterns, + showEmptyFields: false, }); }); }); @@ -263,6 +265,7 @@ describe('IndexPattern Data Source', () => { }, }, currentIndexPatternId: '1', + showEmptyFields: false, }; expect(indexPatternDatasource.insertLayer(state, 'newLayer')).toEqual({ ...state, @@ -281,6 +284,7 @@ describe('IndexPattern Data Source', () => { describe('#removeLayer', () => { it('should remove a layer', () => { const state = { + showEmptyFields: false, indexPatterns: expectedIndexPatterns, layers: { first: { @@ -313,6 +317,7 @@ describe('IndexPattern Data Source', () => { it('should list the current layers', () => { expect( indexPatternDatasource.getLayers({ + showEmptyFields: false, indexPatterns: expectedIndexPatterns, layers: { first: { @@ -336,6 +341,7 @@ describe('IndexPattern Data Source', () => { it('should return the title of the index patterns', () => { expect( indexPatternDatasource.getMetaData({ + showEmptyFields: false, indexPatterns: expectedIndexPatterns, layers: { first: { diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/indexpattern.tsx b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/indexpattern.tsx index 874ee486b3c6b..c21f6a68c0416 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/indexpattern.tsx +++ b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/indexpattern.tsx @@ -28,7 +28,7 @@ import { import { isDraggedField } from './utils'; import { LayerPanel } from './layerpanel'; import { IndexPatternColumn } from './operations'; -import { Datasource } from '..'; +import { Datasource, StateSetter } from '..'; export { OperationType, IndexPatternColumn } from './operations'; @@ -37,6 +37,16 @@ export interface IndexPattern { fields: IndexPatternField[]; title: string; timeFieldName?: string | null; + fieldFormatMap?: Record< + string, + { + id: string; + params: unknown; + } + >; + + // TODO: Load index patterns and existence data in one API call + hasExistence?: boolean; } export interface IndexPatternField { @@ -58,6 +68,11 @@ export interface IndexPatternField { } > >; + + // TODO: This is loaded separately, but should be combined into one API + exists?: boolean; + cardinality?: number; + count?: number; } export interface DraggedField { @@ -78,6 +93,7 @@ export interface IndexPatternPersistedState { export type IndexPatternPrivateState = IndexPatternPersistedState & { indexPatterns: Record; + showEmptyFields: boolean; }; export function columnToOperation(column: IndexPatternColumn): Operation { @@ -141,6 +157,8 @@ export function getIndexPatternDatasource({ // Not stateful. State is persisted to the frame const indexPatternDatasource: Datasource = { async initialize(state?: IndexPatternPersistedState) { + // TODO: The initial request should only load index pattern names because each saved object is large + // Followup requests should load a single index pattern + existence information const indexPatternObjects = await getIndexPatterns(chrome, toastNotifications); const indexPatterns: Record = {}; @@ -154,12 +172,14 @@ export function getIndexPatternDatasource({ return { ...state, indexPatterns, + showEmptyFields: false, }; } return { currentIndexPatternId: indexPatternObjects ? indexPatternObjects[0].id : '', indexPatterns, layers: {}, + showEmptyFields: false, }; }, @@ -222,7 +242,11 @@ export function getIndexPatternDatasource({ ); }, - getPublicAPI(state, setState, layerId) { + getPublicAPI( + state: IndexPatternPrivateState, + setState: StateSetter, + layerId: string + ) { return { getTableSpec: () => { return state.layers[layerId].columnOrder.map(colId => ({ columnId: colId })); diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/indexpattern_suggestions.test.tsx b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/indexpattern_suggestions.test.tsx index d7c61e1f1c73d..ae26633a848ea 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/indexpattern_suggestions.test.tsx +++ b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/indexpattern_suggestions.test.tsx @@ -313,6 +313,7 @@ describe('IndexPattern Data Source suggestions', () => { it('should not make any suggestions for a number without a time field', async () => { const state: IndexPatternPrivateState = { currentIndexPatternId: '1', + showEmptyFields: false, indexPatterns: { 1: { id: '1', @@ -494,6 +495,7 @@ describe('IndexPattern Data Source suggestions', () => { it('should not make any suggestions for a number without a time field', async () => { const state: IndexPatternPrivateState = { currentIndexPatternId: '1', + showEmptyFields: false, indexPatterns: { 1: { id: '1', @@ -883,6 +885,7 @@ describe('IndexPattern Data Source suggestions', () => { it('returns no suggestions if there are no columns', () => { expect( indexPatternDatasource.getDatasourceSuggestionsFromCurrentState({ + showEmptyFields: false, indexPatterns: expectedIndexPatterns, layers: { first: { @@ -1168,6 +1171,7 @@ describe('IndexPattern Data Source suggestions', () => { ], }, }, + showEmptyFields: true, layers: { first: { ...persistedState.layers.first, @@ -1284,6 +1288,7 @@ describe('IndexPattern Data Source suggestions', () => { ], }, }, + showEmptyFields: true, layers: { first: { ...persistedState.layers.first, @@ -1334,6 +1339,7 @@ describe('IndexPattern Data Source suggestions', () => { ], }, }, + showEmptyFields: true, layers: { first: { ...persistedState.layers.first, diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/layerpanel.test.tsx b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/layerpanel.test.tsx index 46e381d69741b..755ed02904e31 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/layerpanel.test.tsx +++ b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/layerpanel.test.tsx @@ -18,6 +18,7 @@ jest.mock('./state_helpers'); const initialState: IndexPatternPrivateState = { currentIndexPatternId: '1', + showEmptyFields: false, layers: { first: { indexPatternId: '1', diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/loader.ts b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/loader.ts index 8ed19d098134e..79fa208e16cf9 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/loader.ts +++ b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/loader.ts @@ -57,6 +57,9 @@ export const getIndexPatterns = (chrome: Chrome, toastNotifications: ToastNotifi typeMeta: attributes.typeMeta ? (JSON.parse(attributes.typeMeta) as SavedRestrictionsInfo) : undefined, + fieldFormatMap: attributes.fieldFormatMap + ? JSON.parse(attributes.fieldFormatMap) + : undefined, }; }); }) diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/operations/definitions/date_histogram.test.tsx b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/operations/definitions/date_histogram.test.tsx index d77f9343fd124..4b8b556927052 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/operations/definitions/date_histogram.test.tsx +++ b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/operations/definitions/date_histogram.test.tsx @@ -23,6 +23,7 @@ describe('date_histogram', () => { beforeEach(() => { state = { currentIndexPatternId: '1', + showEmptyFields: false, indexPatterns: { 1: { id: '1', diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/operations/definitions/filter_ratio.test.tsx b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/operations/definitions/filter_ratio.test.tsx index 8dd2f3944eee6..8864e959977a8 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/operations/definitions/filter_ratio.test.tsx +++ b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/operations/definitions/filter_ratio.test.tsx @@ -32,6 +32,7 @@ describe('filter_ratio', () => { }, }, currentIndexPatternId: '1', + showEmptyFields: false, layers: { first: { indexPatternId: '1', diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/operations/definitions/terms.test.tsx b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/operations/definitions/terms.test.tsx index 18b4e5e754f19..56b15eaaa47db 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/operations/definitions/terms.test.tsx +++ b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/operations/definitions/terms.test.tsx @@ -24,6 +24,7 @@ describe('terms', () => { state = { indexPatterns: {}, currentIndexPatternId: '1', + showEmptyFields: false, layers: { first: { indexPatternId: '1', diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/operations/operations.test.ts b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/operations/operations.test.ts index 878b1dda7b1ce..0a8e4b57521fe 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/operations/operations.test.ts +++ b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/operations/operations.test.ts @@ -146,6 +146,7 @@ describe('getOperationTypesForField', () => { describe('buildColumn', () => { const state: IndexPatternPrivateState = { currentIndexPatternId: '1', + showEmptyFields: false, indexPatterns: expectedIndexPatterns, layers: { first: { diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/state_helpers.test.ts b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/state_helpers.test.ts index d093f50c1bea1..9023173ab95df 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/state_helpers.test.ts +++ b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/state_helpers.test.ts @@ -41,6 +41,7 @@ describe('state_helpers', () => { const state: IndexPatternPrivateState = { indexPatterns: {}, currentIndexPatternId: '1', + showEmptyFields: false, layers: { first: { indexPatternId: '1', @@ -86,6 +87,7 @@ describe('state_helpers', () => { const state: IndexPatternPrivateState = { indexPatterns: {}, currentIndexPatternId: '1', + showEmptyFields: false, layers: { first: { indexPatternId: '1', @@ -135,6 +137,7 @@ describe('state_helpers', () => { const state: IndexPatternPrivateState = { indexPatterns: {}, currentIndexPatternId: '1', + showEmptyFields: false, layers: { first: { indexPatternId: '1', @@ -166,6 +169,7 @@ describe('state_helpers', () => { const state: IndexPatternPrivateState = { indexPatterns: {}, currentIndexPatternId: '1', + showEmptyFields: false, layers: { first: { indexPatternId: '1', @@ -225,6 +229,7 @@ describe('state_helpers', () => { const state: IndexPatternPrivateState = { indexPatterns: {}, currentIndexPatternId: '1', + showEmptyFields: false, layers: { first: { indexPatternId: '1', @@ -300,6 +305,7 @@ describe('state_helpers', () => { const state: IndexPatternPrivateState = { indexPatterns: {}, currentIndexPatternId: '1', + showEmptyFields: false, layers: { first: { indexPatternId: '1', diff --git a/x-pack/legacy/plugins/lens/public/register_vis_type_alias.ts b/x-pack/legacy/plugins/lens/public/register_vis_type_alias.ts index 78b22e0974a09..8f49f6f12ee16 100644 --- a/x-pack/legacy/plugins/lens/public/register_vis_type_alias.ts +++ b/x-pack/legacy/plugins/lens/public/register_vis_type_alias.ts @@ -6,11 +6,12 @@ import { i18n } from '@kbn/i18n'; import { visualizations } from '../../../../../src/legacy/core_plugins/visualizations/public'; +import { BASE_APP_URL, getEditPath } from '../common'; const NOT_INTERNATIONALIZED_PRODUCT_NAME = 'Lens Visualizations'; visualizations.types.visTypeAliasRegistry.add({ - aliasUrl: '/app/lens', + aliasUrl: BASE_APP_URL, name: NOT_INTERNATIONALIZED_PRODUCT_NAME, title: i18n.translate('xpack.lens.visTypeAlias.title', { defaultMessage: 'Lens Visualizations', @@ -29,7 +30,7 @@ visualizations.types.visTypeAliasRegistry.add({ return { id, title, - editUrl: `/app/lens#/edit/${id}`, + editUrl: getEditPath(id), icon: 'faceHappy', isExperimental: true, savedObjectType: type, diff --git a/x-pack/legacy/plugins/lens/public/types.ts b/x-pack/legacy/plugins/lens/public/types.ts index 00bb670839f81..affe077909947 100644 --- a/x-pack/legacy/plugins/lens/public/types.ts +++ b/x-pack/legacy/plugins/lens/public/types.ts @@ -162,6 +162,8 @@ export interface DatasourceDataPanelProps { state: T; dragDropContext: DragContextState; setState: StateSetter; + query: Query; + dateRange: FramePublicAPI['dateRange']; } // The only way a visualization has to restrict the query building diff --git a/x-pack/legacy/plugins/lens/readme.md b/x-pack/legacy/plugins/lens/readme.md index 2bc027121fed5..0ea0778dd17ef 100644 --- a/x-pack/legacy/plugins/lens/readme.md +++ b/x-pack/legacy/plugins/lens/readme.md @@ -2,8 +2,13 @@ ## Testing +Run all tests from the `x-pack` root directory + - Unit tests: `node scripts/jest --watch lens` - Functional tests: - Run `node scripts/functional_tests_server` - Run `node ../scripts/functional_test_runner.js --config ./test/functional/config.js` - You may want to comment out all imports except for Lens in the config file. +- API Functional tests: + - Run `node scripts/functional_tests_server` + - Run `node ../scripts/functional_test_runner.js --config ./test/api_integration/config.js --grep=Lens` diff --git a/x-pack/legacy/plugins/lens/server/index.ts b/x-pack/legacy/plugins/lens/server/index.ts new file mode 100644 index 0000000000000..ae14d1c5a0052 --- /dev/null +++ b/x-pack/legacy/plugins/lens/server/index.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { LensServer } from './plugin'; + +export * from './plugin'; + +export const lensServerPlugin = () => new LensServer(); diff --git a/x-pack/legacy/plugins/lens/server/plugin.tsx b/x-pack/legacy/plugins/lens/server/plugin.tsx new file mode 100644 index 0000000000000..9c33889a514a4 --- /dev/null +++ b/x-pack/legacy/plugins/lens/server/plugin.tsx @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Plugin, CoreSetup } from 'src/core/server'; +import { setupRoutes } from './routes'; + +export class LensServer implements Plugin<{}, {}, {}, {}> { + constructor() {} + + setup(core: CoreSetup) { + setupRoutes(core); + + return {}; + } + + start() { + return {}; + } + + stop() {} +} diff --git a/x-pack/legacy/plugins/lens/server/routes/index.ts b/x-pack/legacy/plugins/lens/server/routes/index.ts new file mode 100644 index 0000000000000..9a957765cc87d --- /dev/null +++ b/x-pack/legacy/plugins/lens/server/routes/index.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { CoreSetup } from 'src/core/server'; +import { initStatsRoute } from './index_stats'; + +export function setupRoutes(setup: CoreSetup) { + initStatsRoute(setup); +} diff --git a/x-pack/legacy/plugins/lens/server/routes/index_stats.test.ts b/x-pack/legacy/plugins/lens/server/routes/index_stats.test.ts new file mode 100644 index 0000000000000..4116db05a5f60 --- /dev/null +++ b/x-pack/legacy/plugins/lens/server/routes/index_stats.test.ts @@ -0,0 +1,105 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +// @ts-ignore +import realHits from '../../../../../../src/fixtures/real_hits'; +// @ts-ignore +import stubbedLogstashFields from '../../../../../../src/fixtures/logstash_fields'; +import { recursiveFlatten } from './index_stats'; + +describe('Index Stats route', () => { + it('should ignore empty fields, but not falsy ones', () => { + const results = recursiveFlatten( + [{ _source: {} }, { _source: { bytes: false } }], + stubbedLogstashFields(), + [ + { + name: 'extension.keyword', + type: 'string', + }, + { + name: 'bytes', + type: 'number', + }, + { + name: 'geo.src', + type: 'string', + }, + ] + ); + + expect(results).toEqual({ + bytes: { + cardinality: 1, + count: 1, + }, + }); + }); + + it('should find existing fields based on mapping', () => { + const results = recursiveFlatten(realHits, stubbedLogstashFields(), [ + { + name: 'extension.keyword', + type: 'string', + }, + { + name: 'bytes', + type: 'number', + }, + ]); + + expect(results).toEqual({ + bytes: { + count: 20, + cardinality: 16, + }, + 'extension.keyword': { + count: 20, + cardinality: 4, + }, + }); + }); + + // TODO: Alias information is not persisted in the index pattern, so we don't have access + it('fails to map alias fields', () => { + const results = recursiveFlatten(realHits, stubbedLogstashFields(), [ + { + name: '@timestamp', + type: 'date', + }, + ]); + + expect(results).toEqual({}); + }); + + // TODO: Scripts are not currently run in the _search query + it('should fail to map scripted fields', () => { + const scriptedField = { + name: 'hour_of_day', + type: 'number', + count: 0, + scripted: true, + script: "doc['timestamp'].value.hourOfDay", + lang: 'painless', + searchable: true, + aggregatable: true, + readFromDocValues: false, + }; + + const results = recursiveFlatten( + realHits, + [...stubbedLogstashFields(), scriptedField], + [ + { + name: 'hour_of_day', + type: 'number', + }, + ] + ); + + expect(results).toEqual({}); + }); +}); diff --git a/x-pack/legacy/plugins/lens/server/routes/index_stats.ts b/x-pack/legacy/plugins/lens/server/routes/index_stats.ts new file mode 100644 index 0000000000000..6918579428dc6 --- /dev/null +++ b/x-pack/legacy/plugins/lens/server/routes/index_stats.ts @@ -0,0 +1,166 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import Boom from 'boom'; +import { get } from 'lodash'; +import { schema } from '@kbn/config-schema'; +import { SearchResponse } from 'elasticsearch'; +import { CoreSetup } from 'src/core/server'; +import { + IndexPatternsService, + FieldDescriptor, +} from '../../../../../../src/legacy/server/index_patterns/service'; + +type Document = Record; + +type Fields = Array<{ name: string; type: string; esTypes?: string[] }>; + +export async function initStatsRoute(setup: CoreSetup) { + const router = setup.http.createRouter(); + router.post( + { + path: '/index_stats/{indexPatternTitle}', + validate: { + params: schema.object({ + indexPatternTitle: schema.string(), + }), + body: schema.object({ + earliest: schema.string(), + latest: schema.string(), + timeZone: schema.maybe(schema.string()), + timeFieldName: schema.string(), + size: schema.number(), + fields: schema.arrayOf( + schema.object({ + name: schema.string(), + type: schema.string(), + }) + ), + }), + }, + }, + async (context, req, res) => { + const requestClient = context.core.elasticsearch.dataClient; + + const indexPatternsService = new IndexPatternsService(requestClient.callAsCurrentUser); + + const { earliest, latest, timeZone, timeFieldName, fields, size } = req.body; + + try { + const indexPattern = await indexPatternsService.getFieldsForWildcard({ + pattern: req.params.indexPatternTitle, + // TODO: Pull this from kibana advanced settings + metaFields: ['_source', '_id', '_type', '_index', '_score'], + }); + + const results = (await requestClient.callAsCurrentUser('search', { + index: req.params.indexPatternTitle, + body: { + query: { + bool: { + filter: [ + { + range: { + [timeFieldName]: { + gte: earliest, + lte: latest, + time_zone: timeZone, + }, + }, + }, + ], + }, + }, + size, + }, + })) as SearchResponse; + + if (results.hits.hits.length) { + return res.ok({ + body: recursiveFlatten(results.hits.hits, indexPattern, fields), + }); + } + return res.ok({ body: {} }); + } catch (e) { + if (e.isBoom) { + return res.internalError(e); + } else { + return res.internalError({ + body: Boom.internal(e.message || e.name), + }); + } + } + } + ); +} + +export function recursiveFlatten( + docs: Array<{ + _source: Document; + }>, + indexPattern: FieldDescriptor[], + fields: Fields +): Record< + string, + { + count: number; + cardinality: number; + } +> { + const overallKeys: Record< + string, + { + count: number; + samples: Set; + } + > = {}; + + const expectedKeys = new Set(fields.map(f => f.name)); + + indexPattern.forEach(field => { + if (!expectedKeys.has(field.name)) { + return; + } + + docs.forEach(doc => { + if (!doc) { + return; + } + + const match = get(doc._source, field.parent || field.name); + if (typeof match === 'undefined') { + return; + } + + const record = overallKeys[field.name]; + if (record) { + record.count += 1; + record.samples.add(match); + } else { + overallKeys[field.name] = { + count: 1, + // Using a set here makes the most sense and avoids the later uniq computation + samples: new Set([match]), + }; + } + }); + }); + + const returnTypes: Record< + string, + { + count: number; + cardinality: number; + } + > = {}; + Object.entries(overallKeys).forEach(([key, value]) => { + returnTypes[key] = { + count: value.count, + cardinality: value.samples.size, + }; + }); + return returnTypes; +} diff --git a/x-pack/test/api_integration/apis/index.js b/x-pack/test/api_integration/apis/index.js index 99694c250ef3f..87d64126571b3 100644 --- a/x-pack/test/api_integration/apis/index.js +++ b/x-pack/test/api_integration/apis/index.js @@ -26,5 +26,6 @@ export default function ({ loadTestFile }) { loadTestFile(require.resolve('./siem')); loadTestFile(require.resolve('./code')); loadTestFile(require.resolve('./short_urls')); + loadTestFile(require.resolve('./lens')); }); } diff --git a/x-pack/test/api_integration/apis/lens/index.ts b/x-pack/test/api_integration/apis/lens/index.ts new file mode 100644 index 0000000000000..9827eadb1278b --- /dev/null +++ b/x-pack/test/api_integration/apis/lens/index.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { FtrProviderContext } from '../../ftr_provider_context'; + +export default function lensApiIntegrationTests({ loadTestFile }: FtrProviderContext) { + describe('Lens', () => { + loadTestFile(require.resolve('./index_stats')); + }); +} diff --git a/x-pack/test/api_integration/apis/lens/index_stats.ts b/x-pack/test/api_integration/apis/lens/index_stats.ts new file mode 100644 index 0000000000000..286fe2f971621 --- /dev/null +++ b/x-pack/test/api_integration/apis/lens/index_stats.ts @@ -0,0 +1,110 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from '@kbn/expect'; + +import { FtrProviderContext } from '../../ftr_provider_context'; + +const TEST_START_TIME = '2015-09-19T06:31:44.000'; +const TEST_END_TIME = '2015-09-23T18:31:44.000'; +const COMMON_HEADERS = { + 'kbn-xsrf': 'some-xsrf-token', +}; + +const fieldsNotInPattern = [ + { name: 'geo', type: 'object' }, + { name: 'id', type: 'string' }, + { name: 'machine', type: 'object' }, +]; + +const fieldsNotInDocuments = [ + { name: 'meta', type: 'object' }, + { name: 'meta.char', type: 'string' }, + { name: 'meta.related', type: 'string' }, + { name: 'meta.user', type: 'object' }, + { name: 'meta.user.firstname', type: 'string' }, + { name: 'meta.user.lastname', type: 'string' }, +]; + +const fieldsWithData = [ + { name: '@message', type: 'string' }, + { name: '@message.raw', type: 'string' }, + { name: '@tags', type: 'string' }, + { name: '@tags.raw', type: 'string' }, + { name: '@timestamp', type: 'date' }, + { name: 'agent', type: 'string' }, + { name: 'agent.raw', type: 'string' }, + { name: 'bytes', type: 'number' }, + { name: 'clientip', type: 'ip' }, + { name: 'extension', type: 'string' }, + { name: 'extension.raw', type: 'string' }, + { name: 'geo.coordinates', type: 'geo_point' }, + { name: 'geo.dest', type: 'string' }, + { name: 'geo.src', type: 'string' }, + { name: 'geo.srcdest', type: 'string' }, + { name: 'headings', type: 'string' }, + { name: 'headings.raw', type: 'string' }, + { name: 'host', type: 'string' }, + { name: 'host.raw', type: 'string' }, + { name: 'index', type: 'string' }, + { name: 'index.raw', type: 'string' }, + { name: 'ip', type: 'ip' }, + { name: 'links', type: 'string' }, + { name: 'links.raw', type: 'string' }, + { name: 'machine.os', type: 'string' }, + { name: 'machine.os.raw', type: 'string' }, + { name: 'machine.ram', type: 'string' }, + { name: 'memory', type: 'string' }, + { name: 'phpmemory', type: 'string' }, + { name: 'referer', type: 'string' }, + { name: 'request', type: 'string' }, + { name: 'request.raw', type: 'string' }, + { name: 'response', type: 'string' }, + { name: 'response.raw', type: 'string' }, + { name: 'spaces', type: 'string' }, + { name: 'spaces.raw', type: 'string' }, + { name: 'type', type: 'string' }, + { name: 'url', type: 'string' }, + { name: 'url.raw', type: 'string' }, + { name: 'utc_time', type: 'string' }, + { name: 'xss', type: 'string' }, + { name: 'xss.raw', type: 'string' }, +]; + +// eslint-disable-next-line import/no-default-export +export default ({ getService }: FtrProviderContext) => { + const esArchiver = getService('esArchiver'); + const supertest = getService('supertest'); + + describe('index stats apis', () => { + before(async () => { + await esArchiver.loadIfNeeded('logstash_functional'); + await esArchiver.loadIfNeeded('visualize/default'); + }); + after(async () => { + await esArchiver.unload('logstash_functional'); + await esArchiver.unload('visualize/default'); + }); + + describe('existence', () => { + it('should find which fields exist in the sample documents', async () => { + const { body } = await supertest + .post('/api/lens/index_stats/logstash-2015.09.22') + .set(COMMON_HEADERS) + .send({ + earliest: TEST_START_TIME, + latest: TEST_END_TIME, + timeFieldName: '@timestamp', + size: 500, + fields: fieldsWithData.concat(fieldsNotInDocuments, fieldsNotInPattern), + }) + .expect(200); + + expect(Object.keys(body)).to.eql(fieldsWithData.map(field => field.name)); + }); + }); + }); +}; diff --git a/x-pack/test/functional/apps/lens/index.ts b/x-pack/test/functional/apps/lens/index.ts index 085950b9f5f6b..352e7b2d5de80 100644 --- a/x-pack/test/functional/apps/lens/index.ts +++ b/x-pack/test/functional/apps/lens/index.ts @@ -20,6 +20,11 @@ export default function({ getService, loadTestFile }: FtrProviderContext) { await esArchiver.loadIfNeeded('visualize/default'); }); + after(async () => { + await esArchiver.unload('logstash_functional'); + await esArchiver.unload('visualize/default'); + }); + describe('', function() { this.tags(['ciGroup4', 'skipFirefox']); diff --git a/x-pack/test/functional/apps/lens/indexpattern_datapanel.ts b/x-pack/test/functional/apps/lens/indexpattern_datapanel.ts index f0fc37a2c18d9..4c89ef4d51cc9 100644 --- a/x-pack/test/functional/apps/lens/indexpattern_datapanel.ts +++ b/x-pack/test/functional/apps/lens/indexpattern_datapanel.ts @@ -8,9 +8,8 @@ import expect from '@kbn/expect'; import { FtrProviderContext } from '../../ftr_provider_context'; // eslint-disable-next-line import/no-default-export -export default function({ getService, getPageObjects }: FtrProviderContext) { - const PageObjects = getPageObjects(['header', 'common']); - const find = getService('find'); +export default function({ getPageObjects }: FtrProviderContext) { + const PageObjects = getPageObjects(['header', 'common', 'lens']); describe('indexpattern_datapanel', () => { beforeEach(async () => { @@ -18,7 +17,10 @@ export default function({ getService, getPageObjects }: FtrProviderContext) { }); it('should list the index pattern fields', async () => { - const fields = await find.allByCssSelector('[data-test-subj="lnsFieldListPanelField"]'); + await PageObjects.lens.openIndexPatternFiltersPopover(); + await PageObjects.lens.toggleExistenceFilter(); + + const fields = await PageObjects.lens.findAllFields(); const fieldText = await Promise.all(fields.map(field => field.getVisibleText())); expect(fieldText).to.eql([ '_score', diff --git a/x-pack/test/functional/page_objects/index.ts b/x-pack/test/functional/page_objects/index.ts index 690a77ff00aa1..e22e36fcd73aa 100644 --- a/x-pack/test/functional/page_objects/index.ts +++ b/x-pack/test/functional/page_objects/index.ts @@ -44,6 +44,7 @@ import { SnapshotRestorePageProvider } from './snapshot_restore_page'; import { CrossClusterReplicationPageProvider } from './cross_cluster_replication_page'; import { RemoteClustersPageProvider } from './remote_clusters_page'; import { CopySavedObjectsToSpacePageProvider } from './copy_saved_objects_to_space_page'; +import { LensPageProvider } from './lens'; // just like services, PageObjects are defined as a map of // names to Providers. Merge in Kibana's or pick specific ones @@ -74,4 +75,5 @@ export const pageObjects = { crossClusterReplication: CrossClusterReplicationPageProvider, remoteClusters: RemoteClustersPageProvider, copySavedObjectsToSpace: CopySavedObjectsToSpacePageProvider, + lens: LensPageProvider, }; diff --git a/x-pack/test/functional/page_objects/lens.ts b/x-pack/test/functional/page_objects/lens.ts new file mode 100644 index 0000000000000..fa762dc86fd8d --- /dev/null +++ b/x-pack/test/functional/page_objects/lens.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { FtrProviderContext } from '../ftr_provider_context'; + +export function LensPageProvider({ getService }: FtrProviderContext) { + const testSubjects = getService('testSubjects'); + + return { + async openIndexPatternFiltersPopover() { + await testSubjects.click('lnsIndexPatternFiltersToggle'); + }, + + async toggleExistenceFilter() { + await testSubjects.click('lnsEmptyFilter'); + }, + + async findAllFields() { + return await testSubjects.findAll('lnsFieldListPanelField'); + }, + }; +}