From 880513680cc548e688c2e84a28be5d84bf6f37b7 Mon Sep 17 00:00:00 2001 From: dej611 Date: Tue, 15 Feb 2022 09:54:37 +0100 Subject: [PATCH 01/31] :alembic: First steps --- x-pack/plugins/lens/kibana.json | 3 +- .../lens/public/app_plugin/lens_top_nav.tsx | 13 +++++ .../plugins/lens/public/app_plugin/types.ts | 1 + x-pack/plugins/lens/public/plugin.ts | 53 +++++++++++-------- 4 files changed, 46 insertions(+), 24 deletions(-) diff --git a/x-pack/plugins/lens/kibana.json b/x-pack/plugins/lens/kibana.json index 1debe6e6141b2..17a58a0f96770 100644 --- a/x-pack/plugins/lens/kibana.json +++ b/x-pack/plugins/lens/kibana.json @@ -28,7 +28,8 @@ "taskManager", "globalSearch", "savedObjectsTagging", - "spaces" + "spaces", + "discover" ], "configPath": [ "xpack", diff --git a/x-pack/plugins/lens/public/app_plugin/lens_top_nav.tsx b/x-pack/plugins/lens/public/app_plugin/lens_top_nav.tsx index 90e924134d27b..11c5ee16174be 100644 --- a/x-pack/plugins/lens/public/app_plugin/lens_top_nav.tsx +++ b/x-pack/plugins/lens/public/app_plugin/lens_top_nav.tsx @@ -359,6 +359,19 @@ export const LensTopNavMenu = ({ redirectToOrigin(); } }, + showUnderlyingData: () => { + // If Discover is not available, return + // If there's no data, return + if (!activeData) { + return; + } + // If Multiple tables, return + // If there are time shifts, return + const [datatable, ...otherTables] = Object.values(activeData); + if (otherTables.length || datatable) { + return; + } + }, }, }); return [...(additionalMenuEntries || []), ...baseMenuEntries]; diff --git a/x-pack/plugins/lens/public/app_plugin/types.ts b/x-pack/plugins/lens/public/app_plugin/types.ts index 3181df8b3256d..2dddb0dd5ac53 100644 --- a/x-pack/plugins/lens/public/app_plugin/types.ts +++ b/x-pack/plugins/lens/public/app_plugin/types.ts @@ -142,4 +142,5 @@ export interface LensTopNavActions { showSaveModal: () => void; cancel: () => void; exportToCSV: () => void; + showUnderlyingData: () => void; } diff --git a/x-pack/plugins/lens/public/plugin.ts b/x-pack/plugins/lens/public/plugin.ts index bba54c85a67c6..7c2b17a1e08ae 100644 --- a/x-pack/plugins/lens/public/plugin.ts +++ b/x-pack/plugins/lens/public/plugin.ts @@ -91,6 +91,7 @@ import type { SaveModalContainerProps } from './app_plugin/save_modal_container' import { createStartServicesGetter } from '../../../../src/plugins/kibana_utils/public'; import { setupExpressions } from './expressions'; import { getSearchProvider } from './search_provider'; +import type { DiscoverSetup, DiscoverStart } from '../../../../src/plugins/discover/public'; export interface LensPluginSetupDependencies { urlForwarding: UrlForwardingSetup; @@ -102,6 +103,7 @@ export interface LensPluginSetupDependencies { charts: ChartsPluginSetup; globalSearch?: GlobalSearchPluginSetup; usageCollection?: UsageCollectionSetup; + discover?: DiscoverSetup; } export interface LensPluginStartDependencies { @@ -120,6 +122,7 @@ export interface LensPluginStartDependencies { inspector: InspectorStartContract; spaces: SpacesPluginStart; usageCollection?: UsageCollectionStart; + discover?: DiscoverStart; } export interface LensPublicSetup { @@ -230,6 +233,7 @@ export class LensPlugin { charts, globalSearch, usageCollection, + discover, }: LensPluginSetupDependencies ) { const startServices = createStartServicesGetter(core.getStartServices); @@ -238,16 +242,17 @@ export class LensPlugin { const { getLensAttributeService } = await import('./async_services'); const { core: coreStart, plugins } = startServices(); - await this.initParts( - core, - data, - charts, - expressions, - fieldFormats, - plugins.fieldFormats.deserialize - ); - - const visualizationMap = await this.editorFrameService!.loadVisualizations(); + const [, visualizationMap] = await Promise.all([ + this.initParts( + core, + data, + charts, + expressions, + fieldFormats, + plugins.fieldFormats.deserialize + ), + this.editorFrameService!.loadVisualizations(), + ]); return { attributeService: getLensAttributeService(coreStart, plugins), @@ -285,10 +290,10 @@ export class LensPlugin { const getPresentationUtilContext = () => startServices().plugins.presentationUtil.ContextProvider; - const ensureDefaultDataView = async () => { + const ensureDefaultDataView = () => { // make sure a default index pattern exists // if not, the page will be redirected to management and visualize won't be rendered - await startServices().plugins.data.indexPatterns.ensureDefaultDataView(); + startServices().plugins.data.indexPatterns.ensureDefaultDataView(); }; core.application.register({ @@ -298,22 +303,24 @@ export class LensPlugin { mount: async (params: AppMountParameters) => { const { core: coreStart, plugins: deps } = startServices(); - await this.initParts( - core, - data, - charts, - expressions, - fieldFormats, - deps.fieldFormats.deserialize - ); + await Promise.all([ + this.initParts( + core, + data, + charts, + expressions, + fieldFormats, + deps.fieldFormats.deserialize + ), + ensureDefaultDataView(), + ]); const { mountApp, stopReportManager, getLensAttributeService } = await import( './async_services' ); - const frameStart = this.editorFrameService!.start(coreStart, deps); - this.stopReportManager = stopReportManager; - await ensureDefaultDataView(); + + const frameStart = this.editorFrameService!.start(coreStart, deps); return mountApp(core, params, { createEditorFrame: frameStart.createInstance, attributeService: getLensAttributeService(coreStart, deps), From fa64071740f2f87f00d63a756f3697cd2e14a533 Mon Sep 17 00:00:00 2001 From: dej611 Date: Thu, 17 Feb 2022 14:30:38 +0100 Subject: [PATCH 02/31] :tada: Initial implementation for button insider editor --- .../lens/public/app_plugin/lens_top_nav.tsx | 70 +++++++- .../lens/public/app_plugin/mounter.tsx | 9 +- .../public/app_plugin/show_underlying_data.ts | 161 ++++++++++++++++++ .../plugins/lens/public/app_plugin/types.ts | 3 + .../indexpattern_datasource/indexpattern.tsx | 37 ++-- .../public/indexpattern_datasource/utils.tsx | 65 ++++++- x-pack/plugins/lens/public/types.ts | 23 ++- 7 files changed, 340 insertions(+), 28 deletions(-) create mode 100644 x-pack/plugins/lens/public/app_plugin/show_underlying_data.ts diff --git a/x-pack/plugins/lens/public/app_plugin/lens_top_nav.tsx b/x-pack/plugins/lens/public/app_plugin/lens_top_nav.tsx index bdd03c4569a5f..97a0ed25735cc 100644 --- a/x-pack/plugins/lens/public/app_plugin/lens_top_nav.tsx +++ b/x-pack/plugins/lens/public/app_plugin/lens_top_nav.tsx @@ -28,10 +28,16 @@ import { DispatchSetState, } from '../state_management'; import { getIndexPatternsObjects, getIndexPatternsIds, getResolvedDateRange } from '../utils'; +import { + combineQueryAndFilters, + getLayerMetaInfo, + getShowUnderlyingDataLabel, +} from './show_underlying_data'; function getLensTopNavConfig(options: { showSaveAndReturn: boolean; enableExportToCSV: boolean; + showOpenInDiscover?: boolean; showCancel: boolean; isByValueMode: boolean; allowByValue: boolean; @@ -46,6 +52,7 @@ function getLensTopNavConfig(options: { showCancel, allowByValue, enableExportToCSV, + showOpenInDiscover, showSaveAndReturn, savingToLibraryPermitted, savingToDashboardPermitted, @@ -90,6 +97,19 @@ function getLensTopNavConfig(options: { }); } + if (showOpenInDiscover) { + topNavMenu.push({ + label: getShowUnderlyingDataLabel(), + run: actions.showUnderlyingData, + testId: 'lnsApp_openInDiscover', + description: i18n.translate('xpack.lens.app.openInDiscoverAriaLabel', { + defaultMessage: 'Open underlying data in Discover', + }), + disableButton: Boolean(tooltips.showUnderlyingDataWarning()), + tooltip: tooltips.showUnderlyingDataWarning, + }); + } + topNavMenu.push({ label: i18n.translate('xpack.lens.app.inspect', { defaultMessage: 'Inspect', @@ -183,6 +203,7 @@ export const LensTopNavMenu = ({ uiSettings, application, attributeService, + discover, dashboardFeatureFlag, } = useKibana().services; @@ -290,6 +311,19 @@ export const LensTopNavMenu = ({ filters, initialContext, ]); + + const canShowUnderlyingData = useMemo(() => { + if (!activeDatasourceId || !discover) { + return; + } + return getLayerMetaInfo( + datasourceMap[activeDatasourceId], + datasourceStates[activeDatasourceId].state, + activeData, + discover + ); + }, [activeData, activeDatasourceId, datasourceMap, datasourceStates, discover]); + const topNavConfig = useMemo(() => { const baseMenuEntries = getLensTopNavConfig({ showSaveAndReturn: @@ -299,6 +333,7 @@ export const LensTopNavMenu = ({ (dashboardFeatureFlag.allowByValueEmbeddables || Boolean(initialInput)) ) || Boolean(initialContextIsEmbedded), enableExportToCSV: Boolean(isSaveable && activeData && Object.keys(activeData).length), + showOpenInDiscover: Boolean(canShowUnderlyingData?.isVisible), isByValueMode: getIsByValueMode(), allowByValue: dashboardFeatureFlag.allowByValueEmbeddables, showCancel: Boolean(isLinkedToOriginatingApp), @@ -321,6 +356,9 @@ export const LensTopNavMenu = ({ } return undefined; }, + showUnderlyingDataWarning: () => { + return canShowUnderlyingData?.error; + }, }, actions: { inspect: () => lensInspector.inspect({ title }), @@ -389,17 +427,29 @@ export const LensTopNavMenu = ({ } }, showUnderlyingData: () => { - // If Discover is not available, return - // If there's no data, return - if (!activeData) { + if (!canShowUnderlyingData) { return; } - // If Multiple tables, return - // If there are time shifts, return - const [datatable, ...otherTables] = Object.values(activeData); - if (otherTables.length || datatable) { + const { error, meta } = canShowUnderlyingData; + // If Discover is not available, return + // If there's no data, return + if (error || !discover || !meta) { return; } + const { filters: newFilters, query: newQuery } = combineQueryAndFilters( + query, + filters, + meta, + indexPatterns + ); + + discover.locator!.navigate({ + indexPatternId: meta.id, + timeRange: data.query.timefilter.timefilter.getTime(), + filters: newFilters, + query: newQuery, + columns: meta.columns, + }); }, }, }); @@ -411,6 +461,7 @@ export const LensTopNavMenu = ({ initialContextIsEmbedded, isSaveable, activeData, + canShowUnderlyingData, getIsByValueMode, savingToLibraryPermitted, savingToDashboardPermitted, @@ -427,6 +478,11 @@ export const LensTopNavMenu = ({ setIsSaveModalVisible, goBackToOriginatingApp, redirectToOrigin, + discover, + query, + filters, + indexPatterns, + data.query.timefilter.timefilter, ]); const onQuerySubmitWrapped = useCallback( diff --git a/x-pack/plugins/lens/public/app_plugin/mounter.tsx b/x-pack/plugins/lens/public/app_plugin/mounter.tsx index 28db5e9f4c43a..08328b7a505fc 100644 --- a/x-pack/plugins/lens/public/app_plugin/mounter.tsx +++ b/x-pack/plugins/lens/public/app_plugin/mounter.tsx @@ -61,6 +61,7 @@ export async function getLensServices( usageCollection, fieldFormats, spaces, + discover, } = startDependencies; const storage = new Storage(localStorage); @@ -94,6 +95,7 @@ export async function getLensServices( // Temporarily required until the 'by value' paradigm is default. dashboardFeatureFlag: startDependencies.dashboard.dashboardFeatureFlagConfig, spaces, + discover, }; } @@ -114,10 +116,13 @@ export async function mountApp( topNavMenuEntryGenerators, } = mountProps; const [coreStart, startDependencies] = await core.getStartServices(); - const instance = await createEditorFrame(); + // const instance = await createEditorFrame(); const historyLocationState = params.history.location.state as HistoryLocationState; - const lensServices = await getLensServices(coreStart, startDependencies, attributeService); + const [lensServices, instance] = await Promise.all([ + getLensServices(coreStart, startDependencies, attributeService), + createEditorFrame(), + ]); const { stateTransfer, data, storage } = lensServices; diff --git a/x-pack/plugins/lens/public/app_plugin/show_underlying_data.ts b/x-pack/plugins/lens/public/app_plugin/show_underlying_data.ts new file mode 100644 index 0000000000000..66e3bebf315a1 --- /dev/null +++ b/x-pack/plugins/lens/public/app_plugin/show_underlying_data.ts @@ -0,0 +1,161 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + Query, + Filter, + DataViewBase, + buildCustomFilter, + buildEsQuery, + FilterStateStore, +} from '@kbn/es-query'; +import { i18n } from '@kbn/i18n'; +import { DiscoverStart } from '../../../../../src/plugins/discover/public'; +import { TableInspectorAdapter } from '../editor_frame_service/types'; +import { Datasource } from '../types'; + +export const getShowUnderlyingDataLabel = () => + i18n.translate('xpack.lens.app.openInDiscover', { + defaultMessage: 'Open in Discover', + }); + +function joinQueries(queries: Query[][] | undefined) { + if (!queries) { + return ''; + } + const expression = queries + .filter((subQueries) => subQueries.length) + .map((subQueries) => + // reduce the amount of round brackets in case of one query + subQueries.length > 1 + ? `( ${subQueries.map(({ query: filterQuery }) => `( ${filterQuery} )`).join(' OR ')} )` + : `( ${subQueries[0].query} )` + ) + .join(' AND '); + return queries.length > 1 ? `( ${expression} )` : expression; +} + +interface LayerMetaInfo { + id: string; + columns: string[]; + filters: { + kuery: Query[][] | undefined; + lucene: Query[][] | undefined; + }; +} + +export function getLayerMetaInfo( + currentDatasource: Datasource, + datasourceState: unknown, + activeData: TableInspectorAdapter | undefined, + discover: DiscoverStart | undefined +): { meta: LayerMetaInfo | undefined; isVisible: boolean; error: string | undefined } { + const isVisible = Boolean(discover); + // If Multiple tables, return + // If there are time shifts, return + const [datatable, ...otherTables] = Object.values(activeData || {}); + if (!datatable || !currentDatasource || !datasourceState) { + return { + meta: undefined, + error: i18n.translate('xpack.lens.app.showUnderlyingDataNoData', { + defaultMessage: 'Visualization has no data available to show', + }), + isVisible, + }; + } + if (otherTables.length) { + return { + meta: undefined, + error: i18n.translate('xpack.lens.app.showUnderlyingDataMultipleLayers', { + defaultMessage: 'Underlying data cannot be shown for visualizations with multiple layers', + }), + isVisible, + }; + } + const [firstLayerId] = currentDatasource.getLayers(datasourceState); + const datasourceAPI = currentDatasource.getPublicAPI({ + layerId: firstLayerId, + state: datasourceState, + }); + // maybe add also datasourceId validation here? + // if (datasourceAPI.datasourceId !== 'indexpattern') { + // return { + // meta: undefined, + // error: i18n.translate('xpack.lens.app.showUnderlyingDataUnsupportedDatasource', { + // defaultMessage: 'Underlying data does not support the current datasource', + // }), + // isVisible, + // }; + // } + const tableSpec = datasourceAPI.getTableSpec(); + + const uniqueFields = [ + ...new Set( + tableSpec + .filter(({ columnId }) => !datasourceAPI.getOperationForColumnId(columnId)?.hasTimeShift) + .map(({ fields }) => fields) + .flat() + ), + ]; + // If no field, return? + // if (!uniqueFields.length) { + // return { + // meta: undefined, + // error: i18n.translate('xpack.lens.app.showUnderlyingDataNoFields', { + // defaultMessage: 'The current visualization has not available fields to show', + // }), + // isVisible, + // }; + // } + const layerFilters = datasourceAPI.getFilters(); + return { + meta: { id: datasourceAPI.getSourceId()!, columns: uniqueFields, filters: layerFilters }, + error: undefined, + isVisible, + }; +} + +export function combineQueryAndFilters( + query: Query, + filters: Filter[], + meta: LayerMetaInfo, + dataViews: DataViewBase[] | undefined +) { + const { queryLanguage, filtersLanguage }: Record = + query?.language === 'lucene' + ? { queryLanguage: 'lucene', filtersLanguage: 'kuery' } + : { queryLanguage: 'kuery', filtersLanguage: 'lucene' }; + + // build here a query extension based on kql filters + const filtersQuery = joinQueries(meta.filters[queryLanguage]); + const newQuery = { + language: filtersQuery, + query: query ? `( ${query.query} ) ${filtersQuery ? `AND ${filtersQuery}` : ''}` : filtersQuery, + }; + + // extends the filters here with the lucene filters + const queryExpression = joinQueries(meta.filters[filtersLanguage]); + const newFilters = [ + ...filters, + buildCustomFilter( + meta.id!, + buildEsQuery( + dataViews?.find(({ id }) => id === meta.id), + { language: filtersLanguage, query: queryExpression }, + [] + ), + false, + false, + i18n.translate('xpack.lens.app.lensContext', { + defaultMessage: 'Lens context ({language})', + values: { language: filtersLanguage }, + }), + FilterStateStore.APP_STATE + ), + ]; + return { filters: newFilters, query: newQuery }; +} diff --git a/x-pack/plugins/lens/public/app_plugin/types.ts b/x-pack/plugins/lens/public/app_plugin/types.ts index 767b3e15126a1..d719a85cf06a4 100644 --- a/x-pack/plugins/lens/public/app_plugin/types.ts +++ b/x-pack/plugins/lens/public/app_plugin/types.ts @@ -7,6 +7,7 @@ import type { History } from 'history'; import type { OnSaveProps } from 'src/plugins/saved_objects/public'; +import { DiscoverStart } from 'src/plugins/discover/public'; import { SpacesApi } from '../../../spaces/public'; import type { ApplicationStart, @@ -133,6 +134,7 @@ export interface LensAppServices { getOriginatingAppName: () => string | undefined; presentationUtil: PresentationUtilPluginStart; spaces: SpacesApi; + discover?: DiscoverStart; // Temporarily required until the 'by value' paradigm is default. dashboardFeatureFlag: DashboardFeatureFlagConfig; @@ -140,6 +142,7 @@ export interface LensAppServices { export interface LensTopNavTooltips { showExportWarning: () => string | undefined; + showUnderlyingDataWarning: () => string | undefined; } export interface LensTopNavActions { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx index 2a44550af2b58..465f7d476c6aa 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx @@ -18,10 +18,10 @@ import type { DatasourceDimensionEditorProps, DatasourceDimensionTriggerProps, DatasourceDataPanelProps, - Operation, DatasourceLayerPanelProps, PublicAPIProps, InitializationOptions, + OperationDescriptor, } from '../types'; import { loadInitialState, @@ -45,7 +45,7 @@ import { getDatasourceSuggestionsForVisualizeCharts, } from './indexpattern_suggestions'; -import { getVisualDefaultsForLayer, isColumnInvalid } from './utils'; +import { getFiltersInLayer, getVisualDefaultsForLayer, isColumnInvalid } from './utils'; import { normalizeOperationDataType, isDraggedField } from './pure_utils'; import { LayerPanel } from './layerpanel'; import { GenericIndexPatternColumn, getErrorMessages, insertNewColumn } from './operations'; @@ -70,6 +70,7 @@ import { GeoFieldWorkspacePanel } from '../editor_frame_service/editor_frame/wor import { DraggingIdentifier } from '../drag_drop'; import { getStateTimeShiftWarningMessages } from './time_shift_utils'; import { getPrecisionErrorWarningMessages } from './utils'; +import { DOCUMENT_FIELD_NAME } from '../../common/constants'; export type { OperationType, GenericIndexPatternColumn } from './operations'; export { deleteColumn } from './operations'; @@ -77,8 +78,8 @@ export function columnToOperation( column: GenericIndexPatternColumn, uniqueLabel?: string, dataView?: IndexPattern -): Operation { - const { dataType, label, isBucketed, scale, operationType } = column; +): OperationDescriptor { + const { dataType, label, isBucketed, scale, operationType, timeShift, filter } = column; const fieldTypes = 'sourceField' in column ? dataView?.getFieldByName(column.sourceField)?.esTypes : undefined; return { @@ -91,6 +92,8 @@ export function columnToOperation( column.dataType === 'string' && fieldTypes?.includes(ES_FIELD_TYPES.VERSION) ? 'version' : undefined, + hasTimeShift: Boolean(timeShift), + hasFilter: Boolean(filter), }; } @@ -445,18 +448,25 @@ export function getIndexPatternDatasource({ getPublicAPI({ state, layerId }: PublicAPIProps) { const columnLabelMap = indexPatternDatasource.uniqueLabels(state); + const layer = state.layers[layerId]; + const visibleColumnIds = layer.columnOrder.filter((colId) => !isReferenced(layer, colId)); return { datasourceId: 'indexpattern', - getTableSpec: () => { - return state.layers[layerId].columnOrder - .filter((colId) => !isReferenced(state.layers[layerId], colId)) - .map((colId) => ({ columnId: colId })); + const fieldsPerColumn = visibleColumnIds.map((colId) => { + const column = layer.columns[colId]; + if ('sourceField' in column) { + // TOOD: Multi-terms support? + return [column.sourceField].filter((field) => field !== DOCUMENT_FIELD_NAME); + } + }); + return visibleColumnIds.map((colId, i) => ({ + columnId: colId, + fields: fieldsPerColumn[i] || [], + })); }, getOperationForColumnId: (columnId: string) => { - const layer = state.layers[layerId]; - if (layer && layer.columns[columnId]) { if (!isReferenced(layer, columnId)) { return columnToOperation( @@ -468,10 +478,9 @@ export function getIndexPatternDatasource({ } return null; }, - getVisualDefaults: () => { - const layer = state.layers[layerId]; - return getVisualDefaultsForLayer(layer); - }, + getSourceId: () => layer.indexPatternId, + getFilters: () => getFiltersInLayer(layer, visibleColumnIds), + getVisualDefaults: () => getVisualDefaultsForLayer(layer), }; }, getDatasourceSuggestionsForField(state, draggedField, filterLayers) { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/utils.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/utils.tsx index cc8a5c322782d..c831076260029 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/utils.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/utils.tsx @@ -12,6 +12,7 @@ import type { DocLinksStart } from 'kibana/public'; import { EuiLink, EuiTextColor, EuiButton, EuiSpacer } from '@elastic/eui'; import { DatatableColumn } from 'src/plugins/expressions'; +import { groupBy } from 'lodash'; import type { FramePublicAPI, StateSetter } from '../types'; import type { IndexPattern, IndexPatternLayer, IndexPatternPrivateState } from './types'; import type { ReferenceBasedIndexPatternColumn } from './operations/definitions/column_types'; @@ -23,11 +24,12 @@ import { CountIndexPatternColumn, updateColumnParam, updateDefaultLabels, + RangeIndexPatternColumn, } from './operations'; import { getInvalidFieldMessage, isColumnOfType } from './operations/definitions/helpers'; -import { isQueryValid } from './operations/definitions/filters'; -import { checkColumnForPrecisionError } from '../../../../../src/plugins/data/common'; +import { FiltersIndexPatternColumn, isQueryValid } from './operations/definitions/filters'; +import { checkColumnForPrecisionError, Query } from '../../../../../src/plugins/data/common'; import { hasField } from './pure_utils'; import { mergeLayer } from './state_helpers'; import { DEFAULT_MAX_DOC_COUNT } from './operations/definitions/terms'; @@ -232,3 +234,62 @@ export function getVisualDefaultsForLayer(layer: IndexPatternLayer) { {} ); } + +export function getFiltersInLayer(layer: IndexPatternLayer, columnIds: string[]) { + // extract filters from filtered metrics + const filteredMetrics = columnIds + .map((colId) => layer.columns[colId].filter) + .filter(Boolean) as Query[]; + + const { kuery: kqlMetricQueries, lucene: luceneMetricQueries } = groupBy( + filteredMetrics, + 'language' + ); + + const filterOperation = columnIds + .map((colId) => { + const column = layer.columns[colId]; + if (isColumnOfType('filters', column)) { + const groupsByLanguage = groupBy( + column.params.filters, + ({ input }) => input.language + ) as Record<'lucene' | 'kuery', FiltersIndexPatternColumn['params']['filters']>; + return { + kuery: groupsByLanguage.kuery?.map(({ input }) => input), + lucene: groupsByLanguage.lucene?.map(({ input }) => input), + }; + } + if (isColumnOfType('range', column) && column.sourceField) { + return { + kuery: column.params.ranges.map(({ from, to }) => ({ + query: `${column.sourceField} >= ${from} AND ${column.sourceField} <= ${to}`, + language: 'kuery', + })), + }; + } + if ( + isColumnOfType('terms', column) && + !(column.params.otherBucket || column.params.missingBucket) + ) { + // TODO: return field -> terms + // TODO: support multi-terms + return { + kuery: [{ query: `${column.sourceField}: *`, language: 'kuery' }].concat( + column.params.secondaryFields?.map((field) => ({ + query: `${field}: *`, + language: 'kuery', + })) || [] + ), + }; + } + }) + .filter(Boolean) as Array<{ kuery?: Query[]; lucene?: Query[] }>; + return { + kuery: [kqlMetricQueries, ...filterOperation.map(({ kuery }) => kuery)].filter( + Boolean + ) as Query[][], + lucene: [luceneMetricQueries, ...filterOperation.map(({ lucene }) => lucene)].filter( + Boolean + ) as Query[][], + }; +} diff --git a/x-pack/plugins/lens/public/types.ts b/x-pack/plugins/lens/public/types.ts index 483da14207516..b93129712c871 100644 --- a/x-pack/plugins/lens/public/types.ts +++ b/x-pack/plugins/lens/public/types.ts @@ -216,6 +216,7 @@ export interface Datasource { removeLayer: (state: T, layerId: string) => T; clearLayer: (state: T, layerId: string) => T; getLayers: (state: T) => string[]; + // getLayerInfo(state: T, layerId: string): { datasourceId: string; fields: string[] }; removeColumn: (props: { prevState: T; layerId: string; columnId: string }) => T; initializeDimension?: ( state: T, @@ -350,12 +351,20 @@ export interface DatasourceFixAction { */ export interface DatasourcePublicAPI { datasourceId: string; - getTableSpec: () => Array<{ columnId: string }>; - getOperationForColumnId: (columnId: string) => Operation | null; + getTableSpec: () => Array<{ columnId: string; fields: string[] }>; + getOperationForColumnId: (columnId: string) => OperationDescriptor | null; /** * Collect all default visual values given the current state */ getVisualDefaults: () => Record>; + /** + * Retrieve the specific source id for the current state + */ + getSourceId: () => string | undefined; + /** + * Collect all defined filters from all the operations in the layer + */ + getFilters: () => { kuery: Query[][]; lucene: Query[][] }; } export interface DatasourceDataPanelProps { @@ -487,8 +496,16 @@ export interface OperationMetadata { // TODO currently it's not possible to differentiate between a field from a raw // document and an aggregated metric which might be handy in some cases. Once we // introduce a raw document datasource, this should be considered here. +} - isStaticValue?: boolean; +/** + * Specific type used to store some meta information on top of the Operation type + * Rather than populate the Operation type with optional types, it can leverage a super type + */ +export interface OperationDescriptor extends Operation { + hasTimeShift: boolean; + hasFilter: boolean; + isStaticValue: boolean; } export interface VisualizationConfigProps { From 32baaa0372588e5bab2eff2ad9338463d6882571 Mon Sep 17 00:00:00 2001 From: dej611 Date: Thu, 17 Feb 2022 15:59:54 +0100 Subject: [PATCH 03/31] :white_check_mark: fix types and tests --- .../components/dimension_editor.test.tsx | 8 ++- .../visualization.test.tsx | 32 +++++++-- .../editor_frame/editor_frame.test.tsx | 2 + .../editor_frame/suggestion_helpers.test.ts | 6 +- .../workspace_panel/chart_switch.test.tsx | 6 +- .../visualization.test.ts | 10 +-- .../indexpattern.test.ts | 10 +-- .../indexpattern_datasource/indexpattern.tsx | 15 +++- .../indexpattern_suggestions.test.tsx | 50 +++++++++++++ .../visualization.test.ts | 3 + .../lens/public/mocks/datasource_mock.ts | 2 + .../public/pie_visualization/to_expression.ts | 5 +- x-pack/plugins/lens/public/types.ts | 2 +- .../gauge/visualization.test.ts | 8 +-- .../xy_visualization/to_expression.test.ts | 12 ++-- .../xy_visualization/visualization.test.ts | 71 +++++++++++-------- 16 files changed, 180 insertions(+), 62 deletions(-) diff --git a/x-pack/plugins/lens/public/datatable_visualization/components/dimension_editor.test.tsx b/x-pack/plugins/lens/public/datatable_visualization/components/dimension_editor.test.tsx index d8dabd81441da..b6c72cc5fe6fb 100644 --- a/x-pack/plugins/lens/public/datatable_visualization/components/dimension_editor.test.tsx +++ b/x-pack/plugins/lens/public/datatable_visualization/components/dimension_editor.test.tsx @@ -7,7 +7,11 @@ import React from 'react'; import { EuiButtonGroup, EuiComboBox, EuiFieldText } from '@elastic/eui'; -import { FramePublicAPI, Operation, VisualizationDimensionEditorProps } from '../../types'; +import { + FramePublicAPI, + OperationDescriptor, + VisualizationDimensionEditorProps, +} from '../../types'; import { DatatableVisualizationState } from '../visualization'; import { createMockDatasource, createMockFramePublicAPI } from '../../mocks'; import { mountWithIntl } from '@kbn/test-jest-helpers'; @@ -218,7 +222,7 @@ describe('data table dimension editor', () => { it('should not show the dynamic coloring option for a bucketed operation', () => { frame.activeData!.first.columns[0].meta.type = 'number'; frame.datasourceLayers.first.getOperationForColumnId = jest.fn( - () => ({ isBucketed: true } as Operation) + () => ({ isBucketed: true } as OperationDescriptor) ); state.columns[0].colorMode = 'cell'; const instance = mountWithIntl(); diff --git a/x-pack/plugins/lens/public/datatable_visualization/visualization.test.tsx b/x-pack/plugins/lens/public/datatable_visualization/visualization.test.tsx index d03263305bb90..74b246d36d920 100644 --- a/x-pack/plugins/lens/public/datatable_visualization/visualization.test.tsx +++ b/x-pack/plugins/lens/public/datatable_visualization/visualization.test.tsx @@ -370,7 +370,10 @@ describe('Datatable Visualization', () => { const datasource = createMockDatasource('test'); const frame = mockFrame(); frame.datasourceLayers = { a: datasource.publicAPIMock }; - datasource.publicAPIMock.getTableSpec.mockReturnValue([{ columnId: 'c' }, { columnId: 'b' }]); + datasource.publicAPIMock.getTableSpec.mockReturnValue([ + { columnId: 'c', fields: [] }, + { columnId: 'b', fields: [] }, + ]); expect( datatableVisualization.getConfiguration({ @@ -501,7 +504,10 @@ describe('Datatable Visualization', () => { beforeEach(() => { datasource = createMockDatasource('test'); - datasource.publicAPIMock.getTableSpec.mockReturnValue([{ columnId: 'c' }, { columnId: 'b' }]); + datasource.publicAPIMock.getTableSpec.mockReturnValue([ + { columnId: 'c', fields: [] }, + { columnId: 'b', fields: [] }, + ]); frame = mockFrame(); frame.datasourceLayers = { a: datasource.publicAPIMock }; @@ -512,6 +518,9 @@ describe('Datatable Visualization', () => { dataType: 'string', isBucketed: false, // <= make them metrics label: 'label', + isStaticValue: false, + hasTimeShift: false, + hasFilter: false, }); const expression = datatableVisualization.toExpression( @@ -559,6 +568,9 @@ describe('Datatable Visualization', () => { dataType: 'string', isBucketed: true, // move it from the metric to the break down by side label: 'label', + isStaticValue: false, + hasTimeShift: false, + hasFilter: false, }); const expression = datatableVisualization.toExpression( @@ -609,11 +621,17 @@ describe('Datatable Visualization', () => { const datasource = createMockDatasource('test'); const frame = mockFrame(); frame.datasourceLayers = { a: datasource.publicAPIMock }; - datasource.publicAPIMock.getTableSpec.mockReturnValue([{ columnId: 'c' }, { columnId: 'b' }]); + datasource.publicAPIMock.getTableSpec.mockReturnValue([ + { columnId: 'c', fields: [] }, + { columnId: 'b', fields: [] }, + ]); datasource.publicAPIMock.getOperationForColumnId.mockReturnValue({ dataType: 'string', isBucketed: true, // move it from the metric to the break down by side label: 'label', + isStaticValue: false, + hasTimeShift: false, + hasFilter: false, }); const error = datatableVisualization.getErrorMessages({ @@ -629,11 +647,17 @@ describe('Datatable Visualization', () => { const datasource = createMockDatasource('test'); const frame = mockFrame(); frame.datasourceLayers = { a: datasource.publicAPIMock }; - datasource.publicAPIMock.getTableSpec.mockReturnValue([{ columnId: 'c' }, { columnId: 'b' }]); + datasource.publicAPIMock.getTableSpec.mockReturnValue([ + { columnId: 'c', fields: [] }, + { columnId: 'b', fields: [] }, + ]); datasource.publicAPIMock.getOperationForColumnId.mockReturnValue({ dataType: 'string', isBucketed: false, // keep it a metric label: 'label', + isStaticValue: false, + hasTimeShift: false, + hasFilter: false, }); const error = datatableVisualization.getErrorMessages({ diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/editor_frame.test.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/editor_frame.test.tsx index 174bb48bc9e41..a54161863ed24 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/editor_frame.test.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/editor_frame.test.tsx @@ -476,6 +476,8 @@ describe('editor_frame', () => { getOperationForColumnId: jest.fn(), getTableSpec: jest.fn(), getVisualDefaults: jest.fn(), + getSourceId: jest.fn(), + getFilters: jest.fn(), }; mockDatasource.getPublicAPI.mockReturnValue(updatedPublicAPI); diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/suggestion_helpers.test.ts b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/suggestion_helpers.test.ts index 48536f8599060..0167f6e4b5a43 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/suggestion_helpers.test.ts +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/suggestion_helpers.test.ts @@ -702,10 +702,12 @@ describe('suggestion helpers', () => { defaultParams = [ { '1': { - getTableSpec: () => [{ columnId: 'col1' }], + getTableSpec: () => [{ columnId: 'col1', fields: [] }], datasourceId: '', getOperationForColumnId: jest.fn(), getVisualDefaults: jest.fn(), + getSourceId: jest.fn(), + getFilters: jest.fn(), }, }, { activeId: 'testVis', state: {} }, @@ -764,6 +766,8 @@ describe('suggestion helpers', () => { datasourceId: '', getOperationForColumnId: jest.fn(), getVisualDefaults: jest.fn(), + getSourceId: jest.fn(), + getFilters: jest.fn(), }, }; mockVisualization1.getSuggestions.mockReturnValue([]); diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/chart_switch.test.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/chart_switch.test.tsx index c325e6d516c8b..ade7e405f70c6 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/chart_switch.test.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/chart_switch.test.tsx @@ -269,9 +269,9 @@ describe('chart_switch', () => { }, ]); datasourceMap.testDatasource.publicAPIMock.getTableSpec.mockReturnValue([ - { columnId: 'col1' }, - { columnId: 'col2' }, - { columnId: 'col3' }, + { columnId: 'col1', fields: [] }, + { columnId: 'col2', fields: [] }, + { columnId: 'col3', fields: [] }, ]); const { instance } = await mountWithProvider( diff --git a/x-pack/plugins/lens/public/heatmap_visualization/visualization.test.ts b/x-pack/plugins/lens/public/heatmap_visualization/visualization.test.ts index a08b12ca9ae6d..a3d97cdda00fb 100644 --- a/x-pack/plugins/lens/public/heatmap_visualization/visualization.test.ts +++ b/x-pack/plugins/lens/public/heatmap_visualization/visualization.test.ts @@ -20,7 +20,7 @@ import { } from './constants'; import { Position } from '@elastic/charts'; import type { HeatmapVisualizationState } from './types'; -import type { DatasourcePublicAPI, Operation } from '../types'; +import type { DatasourcePublicAPI, OperationDescriptor } from '../types'; import { chartPluginMock } from 'src/plugins/charts/public/mocks'; import { layerTypes } from '../../common'; import { themeServiceMock } from '../../../../../src/core/public/mocks'; @@ -99,7 +99,7 @@ describe('heatmap', () => { mockDatasource.publicAPIMock.getOperationForColumnId.mockReturnValue({ dataType: 'string', label: 'MyOperation', - } as Operation); + } as OperationDescriptor); frame.datasourceLayers = { first: mockDatasource.publicAPIMock, @@ -363,7 +363,7 @@ describe('heatmap', () => { mockDatasource.publicAPIMock.getOperationForColumnId.mockReturnValue({ dataType: 'string', label: 'MyOperation', - } as Operation); + } as OperationDescriptor); datasourceLayers = { first: mockDatasource.publicAPIMock, @@ -483,7 +483,7 @@ describe('heatmap', () => { mockDatasource.publicAPIMock.getOperationForColumnId.mockReturnValue({ dataType: 'string', label: 'MyOperation', - } as Operation); + } as OperationDescriptor); datasourceLayers = { first: mockDatasource.publicAPIMock, @@ -612,7 +612,7 @@ describe('heatmap', () => { mockDatasource.publicAPIMock.getOperationForColumnId.mockReturnValue({ dataType: 'string', label: 'MyOperation', - } as Operation); + } as OperationDescriptor); frame.datasourceLayers = { first: mockDatasource.publicAPIMock, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts index 404c31010278b..15e4e3b4699e0 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts @@ -8,7 +8,7 @@ import React from 'react'; import { IStorageWrapper } from 'src/plugins/kibana_utils/public'; import { getIndexPatternDatasource, GenericIndexPatternColumn } from './indexpattern'; -import { DatasourcePublicAPI, Operation, Datasource, FramePublicAPI } from '../types'; +import { DatasourcePublicAPI, Datasource, FramePublicAPI, OperationDescriptor } from '../types'; import { coreMock } from 'src/core/public/mocks'; import { IndexPatternPersistedState, IndexPatternPrivateState } from './types'; import { dataPluginMock } from '../../../../../src/plugins/data/public/mocks'; @@ -1204,7 +1204,7 @@ describe('IndexPattern Data Source', () => { describe('getTableSpec', () => { it('should include col1', () => { - expect(publicAPI.getTableSpec()).toEqual([{ columnId: 'col1' }]); + expect(publicAPI.getTableSpec()).toEqual([{ columnId: 'col1', fields: ['op'] }]); }); it('should skip columns that are being referenced', () => { @@ -1241,7 +1241,7 @@ describe('IndexPattern Data Source', () => { layerId: 'first', }); - expect(publicAPI.getTableSpec()).toEqual([{ columnId: 'col2' }]); + expect(publicAPI.getTableSpec()).toEqual([{ columnId: 'col2', fields: ['test'] }]); }); }); @@ -1252,7 +1252,9 @@ describe('IndexPattern Data Source', () => { dataType: 'string', isBucketed: true, isStaticValue: false, - } as Operation); + hasFilter: false, + hasTimeShift: false, + } as OperationDescriptor); }); it('should return null for non-existant columns', () => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx index 465f7d476c6aa..f2ec536517a09 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx @@ -48,7 +48,12 @@ import { import { getFiltersInLayer, getVisualDefaultsForLayer, isColumnInvalid } from './utils'; import { normalizeOperationDataType, isDraggedField } from './pure_utils'; import { LayerPanel } from './layerpanel'; -import { GenericIndexPatternColumn, getErrorMessages, insertNewColumn } from './operations'; +import { + GenericIndexPatternColumn, + getErrorMessages, + insertNewColumn, + TermsIndexPatternColumn, +} from './operations'; import { IndexPatternField, IndexPatternPrivateState, @@ -71,6 +76,7 @@ import { DraggingIdentifier } from '../drag_drop'; import { getStateTimeShiftWarningMessages } from './time_shift_utils'; import { getPrecisionErrorWarningMessages } from './utils'; import { DOCUMENT_FIELD_NAME } from '../../common/constants'; +import { isColumnOfType } from './operations/definitions/helpers'; export type { OperationType, GenericIndexPatternColumn } from './operations'; export { deleteColumn } from './operations'; @@ -454,10 +460,13 @@ export function getIndexPatternDatasource({ return { datasourceId: 'indexpattern', getTableSpec: () => { - const fieldsPerColumn = visibleColumnIds.map((colId) => { + // consider also referenced columns in this case + const fieldsPerColumn = layer.columnOrder.map((colId) => { const column = layer.columns[colId]; + if (isColumnOfType('terms', column)) { + return [column.sourceField].concat(column.params.secondaryFields ?? []); + } if ('sourceField' in column) { - // TOOD: Multi-terms support? return [column.sourceField].filter((field) => field !== DOCUMENT_FIELD_NAME); } }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern_suggestions.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern_suggestions.test.tsx index c25b8b7264077..600babd937867 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern_suggestions.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern_suggestions.test.tsx @@ -1189,6 +1189,8 @@ describe('IndexPattern Data Source suggestions', () => { label: '', scale: undefined, isStaticValue: false, + hasTimeShift: false, + hasFilter: false, }, }, { @@ -1199,6 +1201,8 @@ describe('IndexPattern Data Source suggestions', () => { label: 'Count of records', scale: 'ratio', isStaticValue: false, + hasTimeShift: false, + hasFilter: false, }, }, ], @@ -1276,6 +1280,8 @@ describe('IndexPattern Data Source suggestions', () => { label: '', scale: undefined, isStaticValue: false, + hasTimeShift: false, + hasFilter: false, }, }, { @@ -1286,6 +1292,8 @@ describe('IndexPattern Data Source suggestions', () => { label: 'Count of records', scale: 'ratio', isStaticValue: false, + hasTimeShift: false, + hasFilter: false, }, }, ], @@ -1977,6 +1985,8 @@ describe('IndexPattern Data Source suggestions', () => { isBucketed: true, scale: undefined, isStaticValue: false, + hasTimeShift: false, + hasFilter: false, }, }, ], @@ -2000,6 +2010,8 @@ describe('IndexPattern Data Source suggestions', () => { isBucketed: true, scale: undefined, isStaticValue: false, + hasTimeShift: false, + hasFilter: false, }, }, ], @@ -2047,6 +2059,8 @@ describe('IndexPattern Data Source suggestions', () => { isBucketed: true, scale: 'interval', isStaticValue: false, + hasTimeShift: false, + hasFilter: false, }, }, { @@ -2057,6 +2071,8 @@ describe('IndexPattern Data Source suggestions', () => { isBucketed: false, scale: 'ratio', isStaticValue: false, + hasTimeShift: false, + hasFilter: false, }, }, ], @@ -2118,6 +2134,8 @@ describe('IndexPattern Data Source suggestions', () => { isBucketed: true, scale: 'ordinal', isStaticValue: false, + hasTimeShift: false, + hasFilter: false, }, }, { @@ -2128,6 +2146,8 @@ describe('IndexPattern Data Source suggestions', () => { isBucketed: true, scale: 'interval', isStaticValue: false, + hasTimeShift: false, + hasFilter: false, }, }, { @@ -2138,6 +2158,8 @@ describe('IndexPattern Data Source suggestions', () => { isBucketed: false, scale: 'ratio', isStaticValue: false, + hasTimeShift: false, + hasFilter: false, }, }, ], @@ -2218,6 +2240,8 @@ describe('IndexPattern Data Source suggestions', () => { isBucketed: true, scale: 'ordinal', isStaticValue: false, + hasTimeShift: false, + hasFilter: false, }, }, { @@ -2228,6 +2252,8 @@ describe('IndexPattern Data Source suggestions', () => { isBucketed: true, scale: 'interval', isStaticValue: false, + hasTimeShift: false, + hasFilter: false, }, }, { @@ -2238,6 +2264,8 @@ describe('IndexPattern Data Source suggestions', () => { isBucketed: false, scale: 'ratio', isStaticValue: false, + hasTimeShift: false, + hasFilter: false, }, }, ], @@ -2341,6 +2369,8 @@ describe('IndexPattern Data Source suggestions', () => { label: 'My Custom Range', scale: 'ordinal', isStaticValue: false, + hasTimeShift: false, + hasFilter: false, }, }, { @@ -2351,6 +2381,8 @@ describe('IndexPattern Data Source suggestions', () => { label: 'timestampLabel', scale: 'interval', isStaticValue: false, + hasTimeShift: false, + hasFilter: false, }, }, { @@ -2361,6 +2393,8 @@ describe('IndexPattern Data Source suggestions', () => { label: 'Unique count of dest', scale: undefined, isStaticValue: false, + hasTimeShift: false, + hasFilter: false, }, }, ], @@ -2873,6 +2907,8 @@ describe('IndexPattern Data Source suggestions', () => { label: 'My Op', scale: undefined, isStaticValue: false, + hasTimeShift: false, + hasFilter: false, }, }, { @@ -2883,6 +2919,8 @@ describe('IndexPattern Data Source suggestions', () => { label: 'Top 5', scale: undefined, isStaticValue: false, + hasTimeShift: false, + hasFilter: false, }, }, ], @@ -2947,6 +2985,8 @@ describe('IndexPattern Data Source suggestions', () => { label: 'timestampLabel', scale: 'interval', isStaticValue: false, + hasTimeShift: false, + hasFilter: false, }, }, { @@ -2957,6 +2997,8 @@ describe('IndexPattern Data Source suggestions', () => { label: 'Cumulative sum of Records label', scale: undefined, isStaticValue: false, + hasTimeShift: false, + hasFilter: false, }, }, { @@ -2967,6 +3009,8 @@ describe('IndexPattern Data Source suggestions', () => { label: 'Cumulative sum of (incomplete)', scale: undefined, isStaticValue: false, + hasTimeShift: false, + hasFilter: false, }, }, ], @@ -3029,6 +3073,8 @@ describe('IndexPattern Data Source suggestions', () => { label: '', scale: undefined, isStaticValue: false, + hasTimeShift: false, + hasFilter: false, }, }, { @@ -3039,6 +3085,8 @@ describe('IndexPattern Data Source suggestions', () => { label: '', scale: undefined, isStaticValue: false, + hasTimeShift: false, + hasFilter: false, }, }, { @@ -3049,6 +3097,8 @@ describe('IndexPattern Data Source suggestions', () => { label: '', scale: undefined, isStaticValue: false, + hasTimeShift: false, + hasFilter: false, }, }, ], diff --git a/x-pack/plugins/lens/public/metric_visualization/visualization.test.ts b/x-pack/plugins/lens/public/metric_visualization/visualization.test.ts index bba08c1aa2442..385b160053138 100644 --- a/x-pack/plugins/lens/public/metric_visualization/visualization.test.ts +++ b/x-pack/plugins/lens/public/metric_visualization/visualization.test.ts @@ -269,6 +269,9 @@ describe('metric_visualization', () => { dataType: 'number', isBucketed: false, label: 'shazm', + isStaticValue: false, + hasTimeShift: false, + hasFilter: false, }; }, }; diff --git a/x-pack/plugins/lens/public/mocks/datasource_mock.ts b/x-pack/plugins/lens/public/mocks/datasource_mock.ts index 67b286b2ef8a2..c30b39476b1ab 100644 --- a/x-pack/plugins/lens/public/mocks/datasource_mock.ts +++ b/x-pack/plugins/lens/public/mocks/datasource_mock.ts @@ -17,6 +17,8 @@ export function createMockDatasource(id: string): DatasourceMock { getTableSpec: jest.fn(() => []), getOperationForColumnId: jest.fn(), getVisualDefaults: jest.fn(), + getSourceId: jest.fn(), + getFilters: jest.fn(), }; return { diff --git a/x-pack/plugins/lens/public/pie_visualization/to_expression.ts b/x-pack/plugins/lens/public/pie_visualization/to_expression.ts index 9ae9f4ac0cae4..fb143bc058e62 100644 --- a/x-pack/plugins/lens/public/pie_visualization/to_expression.ts +++ b/x-pack/plugins/lens/public/pie_visualization/to_expression.ts @@ -254,7 +254,10 @@ function expressionHelper( const groups = getSortedGroups(datasource, layer); const operations = groups - .map((columnId) => ({ columnId, operation: datasource.getOperationForColumnId(columnId) })) + .map((columnId) => ({ + columnId, + operation: datasource.getOperationForColumnId(columnId) as Operation | null, + })) .filter((o): o is { columnId: string; operation: Operation } => !!o.operation); if (!layer.metric || !operations.length) { diff --git a/x-pack/plugins/lens/public/types.ts b/x-pack/plugins/lens/public/types.ts index b93129712c871..162ed30164ecb 100644 --- a/x-pack/plugins/lens/public/types.ts +++ b/x-pack/plugins/lens/public/types.ts @@ -496,6 +496,7 @@ export interface OperationMetadata { // TODO currently it's not possible to differentiate between a field from a raw // document and an aggregated metric which might be handy in some cases. Once we // introduce a raw document datasource, this should be considered here. + isStaticValue?: boolean; } /** @@ -505,7 +506,6 @@ export interface OperationMetadata { export interface OperationDescriptor extends Operation { hasTimeShift: boolean; hasFilter: boolean; - isStaticValue: boolean; } export interface VisualizationConfigProps { diff --git a/x-pack/plugins/lens/public/visualizations/gauge/visualization.test.ts b/x-pack/plugins/lens/public/visualizations/gauge/visualization.test.ts index 20d86e73d6521..35c18e576345e 100644 --- a/x-pack/plugins/lens/public/visualizations/gauge/visualization.test.ts +++ b/x-pack/plugins/lens/public/visualizations/gauge/visualization.test.ts @@ -8,7 +8,7 @@ import { getGaugeVisualization, isNumericDynamicMetric, isNumericMetric } from './visualization'; import { createMockDatasource, createMockFramePublicAPI } from '../../mocks'; import { GROUP_ID } from './constants'; -import type { DatasourcePublicAPI, Operation } from '../../types'; +import type { DatasourcePublicAPI, OperationDescriptor } from '../../types'; import { chartPluginMock } from 'src/plugins/charts/public/mocks'; import { CustomPaletteParams, layerTypes } from '../../../common'; import type { GaugeVisualizationState } from './constants'; @@ -58,7 +58,7 @@ describe('gauge', () => { mockDatasource.publicAPIMock.getOperationForColumnId.mockReturnValue({ dataType: 'string', label: 'MyOperation', - } as Operation); + } as OperationDescriptor); frame.datasourceLayers = { first: mockDatasource.publicAPIMock, @@ -461,7 +461,7 @@ describe('gauge', () => { mockDatasource.publicAPIMock.getOperationForColumnId.mockReturnValue({ dataType: 'string', label: 'MyOperation', - } as Operation); + } as OperationDescriptor); datasourceLayers = { first: mockDatasource.publicAPIMock, }; @@ -532,7 +532,7 @@ describe('gauge', () => { mockDatasource.publicAPIMock.getOperationForColumnId.mockReturnValue({ dataType: 'string', label: 'MyOperation', - } as Operation); + } as OperationDescriptor); frame.datasourceLayers = { first: mockDatasource.publicAPIMock, }; diff --git a/x-pack/plugins/lens/public/xy_visualization/to_expression.test.ts b/x-pack/plugins/lens/public/xy_visualization/to_expression.test.ts index c0dfbd9986087..74c4d7111addc 100644 --- a/x-pack/plugins/lens/public/xy_visualization/to_expression.test.ts +++ b/x-pack/plugins/lens/public/xy_visualization/to_expression.test.ts @@ -9,7 +9,7 @@ import { Ast } from '@kbn/interpreter'; import { Position } from '@elastic/charts'; import { chartPluginMock } from '../../../../../src/plugins/charts/public/mocks'; import { getXyVisualization } from './xy_visualization'; -import { Operation } from '../types'; +import { OperationDescriptor } from '../types'; import { createMockDatasource, createMockFramePublicAPI } from '../mocks'; import { layerTypes } from '../../common'; import { fieldFormatsServiceMock } from '../../../../../src/plugins/field_formats/public/mocks'; @@ -31,14 +31,14 @@ describe('#toExpression', () => { mockDatasource = createMockDatasource('testDatasource'); mockDatasource.publicAPIMock.getTableSpec.mockReturnValue([ - { columnId: 'd' }, - { columnId: 'a' }, - { columnId: 'b' }, - { columnId: 'c' }, + { columnId: 'd', fields: [] }, + { columnId: 'a', fields: [] }, + { columnId: 'b', fields: [] }, + { columnId: 'c', fields: [] }, ]); mockDatasource.publicAPIMock.getOperationForColumnId.mockImplementation((col) => { - return { label: `col_${col}`, dataType: 'number' } as Operation; + return { label: `col_${col}`, dataType: 'number' } as OperationDescriptor; }); frame.datasourceLayers = { diff --git a/x-pack/plugins/lens/public/xy_visualization/visualization.test.ts b/x-pack/plugins/lens/public/xy_visualization/visualization.test.ts index 51cf15c292647..8d13692ddc022 100644 --- a/x-pack/plugins/lens/public/xy_visualization/visualization.test.ts +++ b/x-pack/plugins/lens/public/xy_visualization/visualization.test.ts @@ -7,7 +7,7 @@ import { getXyVisualization } from './visualization'; import { Position } from '@elastic/charts'; -import { Operation, VisualizeEditorContext, Suggestion } from '../types'; +import { Operation, VisualizeEditorContext, Suggestion, OperationDescriptor } from '../types'; import type { State, XYSuggestion } from './types'; import type { SeriesType, XYLayerConfig } from '../../common/expressions'; import { layerTypes } from '../../common'; @@ -246,10 +246,10 @@ describe('xy_visualization', () => { mockDatasource = createMockDatasource('testDatasource'); mockDatasource.publicAPIMock.getTableSpec.mockReturnValue([ - { columnId: 'd' }, - { columnId: 'a' }, - { columnId: 'b' }, - { columnId: 'c' }, + { columnId: 'd', fields: [] }, + { columnId: 'a', fields: [] }, + { columnId: 'b', fields: [] }, + { columnId: 'c', fields: [] }, ]); frame.datasourceLayers = { @@ -367,10 +367,10 @@ describe('xy_visualization', () => { mockDatasource = createMockDatasource('testDatasource'); mockDatasource.publicAPIMock.getTableSpec.mockReturnValue([ - { columnId: 'd' }, - { columnId: 'a' }, - { columnId: 'b' }, - { columnId: 'c' }, + { columnId: 'd', fields: [] }, + { columnId: 'a', fields: [] }, + { columnId: 'b', fields: [] }, + { columnId: 'c', fields: [] }, ]); frame.datasourceLayers = { @@ -603,10 +603,10 @@ describe('xy_visualization', () => { mockDatasource = createMockDatasource('testDatasource'); mockDatasource.publicAPIMock.getTableSpec.mockReturnValue([ - { columnId: 'd' }, - { columnId: 'a' }, - { columnId: 'b' }, - { columnId: 'c' }, + { columnId: 'd', fields: [] }, + { columnId: 'a', fields: [] }, + { columnId: 'b', fields: [] }, + { columnId: 'c', fields: [] }, ]); frame.datasourceLayers = { @@ -660,10 +660,10 @@ describe('xy_visualization', () => { mockDatasource = createMockDatasource('testDatasource'); mockDatasource.publicAPIMock.getTableSpec.mockReturnValue([ - { columnId: 'd' }, - { columnId: 'a' }, - { columnId: 'b' }, - { columnId: 'c' }, + { columnId: 'd', fields: [] }, + { columnId: 'a', fields: [] }, + { columnId: 'b', fields: [] }, + { columnId: 'c', fields: [] }, ]); frame.datasourceLayers = { @@ -1084,6 +1084,9 @@ describe('xy_visualization', () => { isBucketed: true, scale: 'interval', label: 'date_histogram', + isStaticValue: false, + hasTimeShift: false, + hasFilter: false, }; } return null; @@ -1111,6 +1114,9 @@ describe('xy_visualization', () => { isBucketed: true, scale: 'interval', label: 'date_histogram', + isStaticValue: false, + hasTimeShift: false, + hasFilter: false, }; } return null; @@ -1152,6 +1158,9 @@ describe('xy_visualization', () => { isBucketed: true, scale: 'interval', label: 'histogram', + isStaticValue: false, + hasTimeShift: false, + hasFilter: false, }; } return null; @@ -1181,6 +1190,9 @@ describe('xy_visualization', () => { isBucketed: true, scale: 'ordinal', label: 'top values', + isStaticValue: false, + hasTimeShift: false, + hasFilter: false, }; } return null; @@ -1208,6 +1220,9 @@ describe('xy_visualization', () => { isBucketed: true, scale: 'ordinal', label: 'top values', + isStaticValue: false, + hasTimeShift: false, + hasFilter: false, }; } return null; @@ -1435,8 +1450,8 @@ describe('xy_visualization', () => { it('should respect the order of accessors coming from datasource', () => { mockDatasource.publicAPIMock.getTableSpec.mockReturnValue([ - { columnId: 'c' }, - { columnId: 'b' }, + { columnId: 'c', fields: [] }, + { columnId: 'b', fields: [] }, ]); const paletteGetter = jest.spyOn(paletteServiceMock, 'get'); // overrite palette with a palette returning first blue, then green as color @@ -1470,7 +1485,7 @@ describe('xy_visualization', () => { mockDatasource.publicAPIMock.getOperationForColumnId.mockReturnValue({ dataType: 'string', label: 'MyOperation', - } as Operation); + } as OperationDescriptor); frame.datasourceLayers = { first: mockDatasource.publicAPIMock, @@ -1720,7 +1735,7 @@ describe('xy_visualization', () => { ? ({ dataType: 'date', scale: 'interval', - } as unknown as Operation) + } as unknown as OperationDescriptor) : null ); datasourceLayers.second.getOperationForColumnId = jest.fn((id: string) => @@ -1728,7 +1743,7 @@ describe('xy_visualization', () => { ? ({ dataType: 'number', scale: 'interval', - } as unknown as Operation) + } as unknown as OperationDescriptor) : null ); expect( @@ -1776,7 +1791,7 @@ describe('xy_visualization', () => { ? ({ dataType: 'date', scale: 'interval', - } as unknown as Operation) + } as unknown as OperationDescriptor) : null ); datasourceLayers.second.getOperationForColumnId = jest.fn((id: string) => @@ -1784,7 +1799,7 @@ describe('xy_visualization', () => { ? ({ dataType: 'string', scale: 'ordinal', - } as unknown as Operation) + } as unknown as OperationDescriptor) : null ); expect( @@ -1830,10 +1845,10 @@ describe('xy_visualization', () => { mockDatasource = createMockDatasource('testDatasource'); mockDatasource.publicAPIMock.getTableSpec.mockReturnValue([ - { columnId: 'd' }, - { columnId: 'a' }, - { columnId: 'b' }, - { columnId: 'c' }, + { columnId: 'd', fields: [] }, + { columnId: 'a', fields: [] }, + { columnId: 'b', fields: [] }, + { columnId: 'c', fields: [] }, ]); frame.datasourceLayers = { From f659b6bbbfe6e4838d6a7be1ff0c211d3080119f Mon Sep 17 00:00:00 2001 From: dej611 Date: Thu, 17 Feb 2022 18:59:26 +0100 Subject: [PATCH 04/31] :white_check_mark: Add some tests and some test placeholders --- .../app_plugin/show_underlying_data.test.ts | 197 ++++++++++++++++++ .../public/app_plugin/show_underlying_data.ts | 66 +++--- .../indexpattern.test.ts | 110 +++++++++- .../indexpattern_datasource/indexpattern.tsx | 18 +- .../operations/layer_helpers.test.ts | 160 ++++++++++++++ .../operations/layer_helpers.ts | 23 ++ 6 files changed, 539 insertions(+), 35 deletions(-) create mode 100644 x-pack/plugins/lens/public/app_plugin/show_underlying_data.test.ts diff --git a/x-pack/plugins/lens/public/app_plugin/show_underlying_data.test.ts b/x-pack/plugins/lens/public/app_plugin/show_underlying_data.test.ts new file mode 100644 index 0000000000000..6294ef5991293 --- /dev/null +++ b/x-pack/plugins/lens/public/app_plugin/show_underlying_data.test.ts @@ -0,0 +1,197 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createMockDatasource } from '../mocks'; +import { combineQueryAndFilters, getLayerMetaInfo } from './show_underlying_data'; +import { DiscoverStart } from '../../../../../src/plugins/discover/public'; + +describe('getLayerMetaInfo', () => { + it('should return error in case of no data', () => { + expect( + getLayerMetaInfo(createMockDatasource('testDatasource'), {}, undefined, {} as DiscoverStart) + .error + ).toBe('Visualization has no data available to show'); + }); + it('should return error in case of multiple layers', () => { + expect( + getLayerMetaInfo( + createMockDatasource('testDatasource'), + {}, + { + datatable1: { type: 'datatable', columns: [], rows: [] }, + datatable2: { type: 'datatable', columns: [], rows: [] }, + }, + {} as DiscoverStart + ).error + ).toBe('Cannot show underlying data cannot be shown for visualizations with multiple layers'); + }); + it('should return error in case of missing activeDatasource', () => { + expect(getLayerMetaInfo(undefined, {}, undefined, {} as DiscoverStart).error).toBe( + 'Visualization has no data available to show' + ); + }); + it('should return error in case of missing configuration/state', () => { + expect( + getLayerMetaInfo(createMockDatasource('testDatasource'), undefined, {}, {} as DiscoverStart) + .error + ).toBe('Visualization has no data available to show'); + }); + it('should not be visible if discover is not available', () => { + expect( + getLayerMetaInfo( + createMockDatasource('testDatasource'), + {}, + { + datatable1: { type: 'datatable', columns: [], rows: [] }, + }, + undefined + ).isVisible + ).toBeFalsy(); + }); + it.todo('should basically work collecting fields and filters in the visualization'); +}); +describe('combineQueryAndFilters', () => { + it('should just return same query and filters if no fields or filters are in layer meta', () => { + expect( + combineQueryAndFilters( + { language: 'kuery', query: 'myfield: *' }, + [], + { + id: 'testDatasource', + columns: [], + filters: { kuery: [], lucene: [] }, + }, + undefined + ) + ).toEqual({ query: { language: 'kuery', query: 'myfield: *' }, filters: [] }); + }); + + it('should concatenate filters with existing query if languages match (AND it)', () => { + expect( + combineQueryAndFilters( + { language: 'kuery', query: 'myfield: *' }, + [], + { + id: 'testDatasource', + columns: [], + filters: { kuery: [[{ language: 'kuery', query: 'otherField: *' }]], lucene: [] }, + }, + undefined + ) + ).toEqual({ + query: { language: 'kuery', query: '( myfield: * ) AND ( otherField: * )' }, + filters: [], + }); + }); + + it('should build single kuery expression from meta filters and assign it as query for final use', () => { + expect( + combineQueryAndFilters( + undefined, + [], + { + id: 'testDatasource', + columns: [], + filters: { kuery: [[{ language: 'kuery', query: 'otherField: *' }]], lucene: [] }, + }, + undefined + ) + ).toEqual({ query: { language: 'kuery', query: '( otherField: * )' }, filters: [] }); + }); + + it('should build single kuery expression from meta filters and join using OR and AND at the right level', () => { + // OR queries from the same array, AND queries from different arrays + expect( + combineQueryAndFilters( + undefined, + [], + { + id: 'testDatasource', + columns: [], + filters: { + kuery: [ + [ + { language: 'kuery', query: 'myfield: *' }, + { language: 'kuery', query: 'otherField: *' }, + ], + [ + { language: 'kuery', query: 'myfieldCopy: *' }, + { language: 'kuery', query: 'otherFieldCopy: *' }, + ], + ], + lucene: [], + }, + }, + undefined + ) + ).toEqual({ + query: { + language: 'kuery', + query: + '( ( ( myfield: * ) OR ( otherField: * ) ) AND ( ( myfieldCopy: * ) OR ( otherFieldCopy: * ) ) )', + }, + filters: [], + }); + }); + it('should assign kuery meta filters to app filters if existing query is using lucene language', () => { + expect( + combineQueryAndFilters( + { language: 'lucene', query: 'myField' }, + [], + { + id: 'testDatasource', + columns: [], + filters: { + kuery: [[{ language: 'kuery', query: 'myfield: *' }]], + lucene: [], + }, + }, + undefined + ) + ).toEqual({ + query: { language: 'lucene', query: 'myField' }, + filters: [ + { + $state: { + store: 'appState', + }, + bool: { + filter: [ + { + bool: { + minimum_should_match: 1, + should: [ + { + exists: { + field: 'myfield', + }, + }, + ], + }, + }, + ], + must: [], + must_not: [], + should: [], + }, + meta: { + alias: 'Lens context (kuery)', + disabled: false, + index: 'testDatasource', + negate: false, + type: 'custom', + }, + }, + ], + }); + }); + it.todo( + 'should append lucene meta filters to app filters even if existing filters are using kuery' + ); + it.todo('should work for complex cases of nested meta filters'); + it.todo('should append lucene meta filters to an existing lucene query'); +}); diff --git a/x-pack/plugins/lens/public/app_plugin/show_underlying_data.ts b/x-pack/plugins/lens/public/app_plugin/show_underlying_data.ts index 66e3bebf315a1..380b0727fdbff 100644 --- a/x-pack/plugins/lens/public/app_plugin/show_underlying_data.ts +++ b/x-pack/plugins/lens/public/app_plugin/show_underlying_data.ts @@ -49,7 +49,7 @@ interface LayerMetaInfo { } export function getLayerMetaInfo( - currentDatasource: Datasource, + currentDatasource: Datasource | undefined, datasourceState: unknown, activeData: TableInspectorAdapter | undefined, discover: DiscoverStart | undefined @@ -71,7 +71,8 @@ export function getLayerMetaInfo( return { meta: undefined, error: i18n.translate('xpack.lens.app.showUnderlyingDataMultipleLayers', { - defaultMessage: 'Underlying data cannot be shown for visualizations with multiple layers', + defaultMessage: + 'Cannot show underlying data cannot be shown for visualizations with multiple layers', }), isVisible, }; @@ -120,42 +121,49 @@ export function getLayerMetaInfo( } export function combineQueryAndFilters( - query: Query, + query: Query | undefined, filters: Filter[], meta: LayerMetaInfo, dataViews: DataViewBase[] | undefined ) { + // Unless a lucene query is already defined, kuery is assigned to it const { queryLanguage, filtersLanguage }: Record = query?.language === 'lucene' ? { queryLanguage: 'lucene', filtersLanguage: 'kuery' } : { queryLanguage: 'kuery', filtersLanguage: 'lucene' }; - // build here a query extension based on kql filters - const filtersQuery = joinQueries(meta.filters[queryLanguage]); - const newQuery = { - language: filtersQuery, - query: query ? `( ${query.query} ) ${filtersQuery ? `AND ${filtersQuery}` : ''}` : filtersQuery, - }; + let newQuery = query; + if (meta.filters[queryLanguage]?.length) { + const filtersQuery = joinQueries(meta.filters[queryLanguage]); + newQuery = { + language: queryLanguage, + query: query + ? `( ${query.query} ) ${filtersQuery ? `AND ${filtersQuery}` : ''}` + : filtersQuery, + }; + } - // extends the filters here with the lucene filters - const queryExpression = joinQueries(meta.filters[filtersLanguage]); - const newFilters = [ - ...filters, - buildCustomFilter( - meta.id!, - buildEsQuery( - dataViews?.find(({ id }) => id === meta.id), - { language: filtersLanguage, query: queryExpression }, - [] - ), - false, - false, - i18n.translate('xpack.lens.app.lensContext', { - defaultMessage: 'Lens context ({language})', - values: { language: filtersLanguage }, - }), - FilterStateStore.APP_STATE - ), - ]; + const newFilters = [...filters]; + if (meta.filters[filtersLanguage]?.length) { + const queryExpression = joinQueries(meta.filters[filtersLanguage]); + // Append the new filter based on the queryExpression to the existing ones + newFilters.push( + buildCustomFilter( + meta.id!, + buildEsQuery( + dataViews?.find(({ id }) => id === meta.id), + { language: filtersLanguage, query: queryExpression }, + [] + ), + false, + false, + i18n.translate('xpack.lens.app.lensContext', { + defaultMessage: 'Lens context ({language})', + values: { language: filtersLanguage }, + }), + FilterStateStore.APP_STATE + ) + ); + } return { filters: newFilters, query: newQuery }; } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts index 15e4e3b4699e0..a4532128fa4c1 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts @@ -1204,7 +1204,11 @@ describe('IndexPattern Data Source', () => { describe('getTableSpec', () => { it('should include col1', () => { - expect(publicAPI.getTableSpec()).toEqual([{ columnId: 'col1', fields: ['op'] }]); + expect(publicAPI.getTableSpec()).toEqual([expect.objectContaining({ columnId: 'col1' })]); + }); + + it('should include fields prop for each column', () => { + expect(publicAPI.getTableSpec()).toEqual([expect.objectContaining({ fields: ['op'] })]); }); it('should skip columns that are being referenced', () => { @@ -1241,8 +1245,98 @@ describe('IndexPattern Data Source', () => { layerId: 'first', }); + expect(publicAPI.getTableSpec()).toEqual([expect.objectContaining({ columnId: 'col2' })]); + }); + + it('should collect all fields (also from referenced columns)', () => { + publicAPI = indexPatternDatasource.getPublicAPI({ + state: { + ...enrichBaseState(baseState), + layers: { + first: { + indexPatternId: '1', + columnOrder: ['col1', 'col2'], + columns: { + col1: { + label: 'Sum', + dataType: 'number', + isBucketed: false, + + operationType: 'sum', + sourceField: 'test', + params: {}, + } as GenericIndexPatternColumn, + col2: { + label: 'Cumulative sum', + dataType: 'number', + isBucketed: false, + + operationType: 'cumulative_sum', + references: ['col1'], + params: {}, + } as GenericIndexPatternColumn, + }, + }, + }, + }, + layerId: 'first', + }); + expect(publicAPI.getTableSpec()).toEqual([{ columnId: 'col2', fields: ['test'] }]); }); + + it('should collect and organize fields per visible column', () => { + publicAPI = indexPatternDatasource.getPublicAPI({ + state: { + ...enrichBaseState(baseState), + layers: { + first: { + indexPatternId: '1', + columnOrder: ['col1', 'col2', 'col3'], + columns: { + col1: { + label: 'Sum', + dataType: 'number', + isBucketed: false, + + operationType: 'sum', + sourceField: 'test', + params: {}, + } as GenericIndexPatternColumn, + col2: { + label: 'Cumulative sum', + dataType: 'number', + isBucketed: false, + + operationType: 'cumulative_sum', + references: ['col1'], + params: {}, + } as GenericIndexPatternColumn, + col3: { + label: 'My Op', + dataType: 'string', + isBucketed: true, + operationType: 'terms', + sourceField: 'op', + params: { + size: 5, + orderBy: { type: 'alphabetical' }, + orderDirection: 'asc', + }, + } as TermsIndexPatternColumn, + }, + }, + }, + }, + layerId: 'first', + }); + + // col1 is skipped as referenced but its field gets inherited by col2 + expect(publicAPI.getTableSpec()).toEqual([ + { columnId: 'col2', fields: ['test'] }, + { columnId: 'col3', fields: ['op'] }, + ]); + }); }); describe('getOperationForColumnId', () => { @@ -1297,6 +1391,20 @@ describe('IndexPattern Data Source', () => { expect(publicAPI.getOperationForColumnId('col1')).toEqual(null); }); }); + + describe('getSourceId', () => { + it('should basically return the datasource internal id', () => { + expect(publicAPI.getSourceId()).toEqual('1'); + }); + }); + + describe('getFilters', () => { + it.todo('should return all filters in metrics, grouped by language'); + it.todo('shuold collect top values fields as kuery existence filters'); + it.todo('should collect custom ranges as kuery filters'); + it.todo('should collect filters within filters operation grouped by language'); + it.todo('should support complex scenarios'); + }); }); describe('#getErrorMessages', () => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx index f2ec536517a09..1e402728801bf 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx @@ -54,6 +54,7 @@ import { insertNewColumn, TermsIndexPatternColumn, } from './operations'; +import { getTopColumnFromReference } from './operations/layer_helpers'; import { IndexPatternField, IndexPatternPrivateState, @@ -461,18 +462,25 @@ export function getIndexPatternDatasource({ datasourceId: 'indexpattern', getTableSpec: () => { // consider also referenced columns in this case - const fieldsPerColumn = layer.columnOrder.map((colId) => { + // but map fields to the top referencing column + const fieldsPerColumn: Record = {}; + Object.keys(layer.columns).map((colId) => { + const visibleColumnId = getTopColumnFromReference(layer, colId); + fieldsPerColumn[visibleColumnId] = fieldsPerColumn[visibleColumnId] || []; + const column = layer.columns[colId]; if (isColumnOfType('terms', column)) { - return [column.sourceField].concat(column.params.secondaryFields ?? []); + fieldsPerColumn[visibleColumnId].push( + ...[column.sourceField].concat(column.params.secondaryFields ?? []) + ); } - if ('sourceField' in column) { - return [column.sourceField].filter((field) => field !== DOCUMENT_FIELD_NAME); + if ('sourceField' in column && column.sourceField !== DOCUMENT_FIELD_NAME) { + fieldsPerColumn[visibleColumnId].push(column.sourceField); } }); return visibleColumnIds.map((colId, i) => ({ columnId: colId, - fields: fieldsPerColumn[i] || [], + fields: [...new Set(fieldsPerColumn[colId] || [])], })); }, getOperationForColumnId: (columnId: string) => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts index 1b432c4a34add..51dca2acdedd5 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts @@ -16,6 +16,8 @@ import { updateLayerIndexPattern, getErrorMessages, hasTermsWithManyBuckets, + isReferenced, + getTopColumnFromReference, } from './layer_helpers'; import { operationDefinitionMap, OperationType } from '../operations'; import { TermsIndexPatternColumn } from './definitions/terms'; @@ -3179,4 +3181,162 @@ describe('state_helpers', () => { expect(hasTermsWithManyBuckets(layer)).toBeTruthy(); }); }); + + describe('isReferenced', () => { + it('should return false for top column which has references', () => { + const layer: IndexPatternLayer = { + indexPatternId: '1', + columnOrder: [], + columns: { + col1: { + operationType: 'managedReference', + references: ['col2'], + label: '', + dataType: 'number', + isBucketed: false, + }, + col2: { + operationType: 'testReference', + references: [], + label: '', + dataType: 'number', + isBucketed: false, + }, + }, + }; + expect(isReferenced(layer, 'col1')).toBeFalsy(); + }); + + it('should return true for referenced column', () => { + const layer: IndexPatternLayer = { + indexPatternId: '1', + columnOrder: [], + columns: { + col1: { + operationType: 'managedReference', + references: ['col2'], + label: '', + dataType: 'number', + isBucketed: false, + }, + col2: { + operationType: 'testReference', + references: [], + label: '', + dataType: 'number', + isBucketed: false, + }, + }, + }; + expect(isReferenced(layer, 'col2')).toBeTruthy(); + }); + }); + + describe('getTopColumnFromReference', () => { + it("should just return the column id itself if it's not a referenced column", () => { + const layer: IndexPatternLayer = { + indexPatternId: '1', + columnOrder: [], + columns: { + col1: { + operationType: 'managedReference', + references: ['col2'], + label: '', + dataType: 'number', + isBucketed: false, + }, + col2: { + operationType: 'testReference', + references: [], + label: '', + dataType: 'number', + isBucketed: false, + }, + }, + }; + expect(getTopColumnFromReference(layer, 'col1')).toEqual('col1'); + }); + + it('should return the top column if a referenced column is passed', () => { + const layer: IndexPatternLayer = { + indexPatternId: '1', + columnOrder: [], + columns: { + col1: { + operationType: 'managedReference', + references: ['col2'], + label: '', + dataType: 'number', + isBucketed: false, + }, + col2: { + operationType: 'testReference', + references: [], + label: '', + dataType: 'number', + isBucketed: false, + }, + }, + }; + expect(getTopColumnFromReference(layer, 'col2')).toEqual('col1'); + }); + + it('should work for a formula chain', () => { + const math = { + customLabel: true, + dataType: 'number' as const, + isBucketed: false, + label: 'math', + operationType: 'math' as const, + }; + const layer: IndexPatternLayer = { + indexPatternId: '', + columnOrder: [], + columns: { + source: { + dataType: 'number' as const, + isBucketed: false, + label: 'Formula', + operationType: 'formula' as const, + params: { + formula: 'moving_average(sum(bytes), window=5)', + isFormulaBroken: false, + }, + references: ['formulaX3'], + } as FormulaIndexPatternColumn, + formulaX0: { + customLabel: true, + dataType: 'number' as const, + isBucketed: false, + label: 'formulaX0', + operationType: 'sum' as const, + scale: 'ratio' as const, + sourceField: 'bytes', + }, + formulaX1: { + ...math, + label: 'formulaX1', + references: ['formulaX0'], + params: { tinymathAst: 'formulaX0' }, + } as MathIndexPatternColumn, + formulaX2: { + customLabel: true, + dataType: 'number' as const, + isBucketed: false, + label: 'formulaX2', + operationType: 'moving_average' as const, + params: { window: 5 }, + references: ['formulaX1'], + } as MovingAverageIndexPatternColumn, + formulaX3: { + ...math, + label: 'formulaX3', + references: ['formulaX2'], + params: { tinymathAst: 'formulaX2' }, + } as MathIndexPatternColumn, + }, + }; + expect(getTopColumnFromReference(layer, 'formulaX0')).toEqual('source'); + }); + }); }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts index ab7ee8992f2fe..603791970cf89 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts @@ -8,6 +8,7 @@ import { partition, mapValues, pickBy, isArray } from 'lodash'; import { CoreStart } from 'kibana/public'; import { Query } from 'src/plugins/data/common'; +import memoizeOne from 'memoize-one'; import type { VisualizeEditorLayersContext } from '../../../../../../src/plugins/visualizations/public'; import type { DatasourceFixAction, @@ -1413,6 +1414,28 @@ export function isReferenced(layer: IndexPatternLayer, columnId: string): boolea return allReferences.includes(columnId); } +const computeReferenceLookup = memoizeOne((layer: IndexPatternLayer): Record => { + // speed up things for deep chains as in formula + const refLookup: Record = {}; + for (const [parentId, col] of Object.entries(layer.columns)) { + if ('references' in col) { + for (const colId of col.references) { + refLookup[colId] = parentId; + } + } + } + return refLookup; +}); + +export function getTopColumnFromReference(layer: IndexPatternLayer, columnId: string): string { + const refLookup = computeReferenceLookup(layer); + let currentId = columnId; + while (isReferenced(layer, currentId)) { + currentId = refLookup[currentId]; + } + return currentId; +} + export function getReferencedColumnIds(layer: IndexPatternLayer, columnId: string): string[] { const referencedIds: string[] = []; function collect(id: string) { From 54bdbd2ad2468154fd6cdc18bf5aec70118061e2 Mon Sep 17 00:00:00 2001 From: dej611 Date: Mon, 21 Feb 2022 10:13:00 +0100 Subject: [PATCH 05/31] :bug: Fix issues on mount --- .../lens/public/app_plugin/mounter.tsx | 11 +++++----- x-pack/plugins/lens/public/plugin.ts | 20 +++++++++---------- 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/x-pack/plugins/lens/public/app_plugin/mounter.tsx b/x-pack/plugins/lens/public/app_plugin/mounter.tsx index 08328b7a505fc..8b50abb3be5d5 100644 --- a/x-pack/plugins/lens/public/app_plugin/mounter.tsx +++ b/x-pack/plugins/lens/public/app_plugin/mounter.tsx @@ -115,14 +115,13 @@ export async function mountApp( getPresentationUtilContext, topNavMenuEntryGenerators, } = mountProps; - const [coreStart, startDependencies] = await core.getStartServices(); - // const instance = await createEditorFrame(); - const historyLocationState = params.history.location.state as HistoryLocationState; - - const [lensServices, instance] = await Promise.all([ - getLensServices(coreStart, startDependencies, attributeService), + const [[coreStart, startDependencies], instance] = await Promise.all([ + core.getStartServices(), createEditorFrame(), ]); + const historyLocationState = params.history.location.state as HistoryLocationState; + + const lensServices = await getLensServices(coreStart, startDependencies, attributeService); const { stateTransfer, data, storage } = lensServices; diff --git a/x-pack/plugins/lens/public/plugin.ts b/x-pack/plugins/lens/public/plugin.ts index ee824073c9559..a6f37ece5cdb2 100644 --- a/x-pack/plugins/lens/public/plugin.ts +++ b/x-pack/plugins/lens/public/plugin.ts @@ -244,17 +244,15 @@ export class LensPlugin { const { getLensAttributeService } = await import('./async_services'); const { core: coreStart, plugins } = startServices(); - const [, visualizationMap] = await Promise.all([ - this.initParts( - core, - data, - charts, - expressions, - fieldFormats, - plugins.fieldFormats.deserialize - ), - this.editorFrameService!.loadVisualizations(), - ]); + await this.initParts( + core, + data, + charts, + expressions, + fieldFormats, + plugins.fieldFormats.deserialize + ); + const visualizationMap = await this.editorFrameService!.loadVisualizations(); return { attributeService: getLensAttributeService(coreStart, plugins), From 5b7dbe412c5cc1f6bc3403d0c84d04a2fbf3e77c Mon Sep 17 00:00:00 2001 From: dej611 Date: Mon, 21 Feb 2022 16:08:17 +0100 Subject: [PATCH 06/31] :fire: Remove unused attr --- .../visualization.test.tsx | 4 --- .../indexpattern_suggestions.test.tsx | 25 ------------------- .../visualization.test.ts | 1 - x-pack/plugins/lens/public/types.ts | 1 - .../xy_visualization/visualization.test.ts | 5 ---- 5 files changed, 36 deletions(-) diff --git a/x-pack/plugins/lens/public/datatable_visualization/visualization.test.tsx b/x-pack/plugins/lens/public/datatable_visualization/visualization.test.tsx index 74b246d36d920..13b6581e99d2a 100644 --- a/x-pack/plugins/lens/public/datatable_visualization/visualization.test.tsx +++ b/x-pack/plugins/lens/public/datatable_visualization/visualization.test.tsx @@ -520,7 +520,6 @@ describe('Datatable Visualization', () => { label: 'label', isStaticValue: false, hasTimeShift: false, - hasFilter: false, }); const expression = datatableVisualization.toExpression( @@ -570,7 +569,6 @@ describe('Datatable Visualization', () => { label: 'label', isStaticValue: false, hasTimeShift: false, - hasFilter: false, }); const expression = datatableVisualization.toExpression( @@ -631,7 +629,6 @@ describe('Datatable Visualization', () => { label: 'label', isStaticValue: false, hasTimeShift: false, - hasFilter: false, }); const error = datatableVisualization.getErrorMessages({ @@ -657,7 +654,6 @@ describe('Datatable Visualization', () => { label: 'label', isStaticValue: false, hasTimeShift: false, - hasFilter: false, }); const error = datatableVisualization.getErrorMessages({ diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern_suggestions.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern_suggestions.test.tsx index 600babd937867..a556f9e6d1526 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern_suggestions.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern_suggestions.test.tsx @@ -1190,7 +1190,6 @@ describe('IndexPattern Data Source suggestions', () => { scale: undefined, isStaticValue: false, hasTimeShift: false, - hasFilter: false, }, }, { @@ -1202,7 +1201,6 @@ describe('IndexPattern Data Source suggestions', () => { scale: 'ratio', isStaticValue: false, hasTimeShift: false, - hasFilter: false, }, }, ], @@ -1281,7 +1279,6 @@ describe('IndexPattern Data Source suggestions', () => { scale: undefined, isStaticValue: false, hasTimeShift: false, - hasFilter: false, }, }, { @@ -1293,7 +1290,6 @@ describe('IndexPattern Data Source suggestions', () => { scale: 'ratio', isStaticValue: false, hasTimeShift: false, - hasFilter: false, }, }, ], @@ -1986,7 +1982,6 @@ describe('IndexPattern Data Source suggestions', () => { scale: undefined, isStaticValue: false, hasTimeShift: false, - hasFilter: false, }, }, ], @@ -2011,7 +2006,6 @@ describe('IndexPattern Data Source suggestions', () => { scale: undefined, isStaticValue: false, hasTimeShift: false, - hasFilter: false, }, }, ], @@ -2060,7 +2054,6 @@ describe('IndexPattern Data Source suggestions', () => { scale: 'interval', isStaticValue: false, hasTimeShift: false, - hasFilter: false, }, }, { @@ -2072,7 +2065,6 @@ describe('IndexPattern Data Source suggestions', () => { scale: 'ratio', isStaticValue: false, hasTimeShift: false, - hasFilter: false, }, }, ], @@ -2135,7 +2127,6 @@ describe('IndexPattern Data Source suggestions', () => { scale: 'ordinal', isStaticValue: false, hasTimeShift: false, - hasFilter: false, }, }, { @@ -2147,7 +2138,6 @@ describe('IndexPattern Data Source suggestions', () => { scale: 'interval', isStaticValue: false, hasTimeShift: false, - hasFilter: false, }, }, { @@ -2159,7 +2149,6 @@ describe('IndexPattern Data Source suggestions', () => { scale: 'ratio', isStaticValue: false, hasTimeShift: false, - hasFilter: false, }, }, ], @@ -2241,7 +2230,6 @@ describe('IndexPattern Data Source suggestions', () => { scale: 'ordinal', isStaticValue: false, hasTimeShift: false, - hasFilter: false, }, }, { @@ -2253,7 +2241,6 @@ describe('IndexPattern Data Source suggestions', () => { scale: 'interval', isStaticValue: false, hasTimeShift: false, - hasFilter: false, }, }, { @@ -2265,7 +2252,6 @@ describe('IndexPattern Data Source suggestions', () => { scale: 'ratio', isStaticValue: false, hasTimeShift: false, - hasFilter: false, }, }, ], @@ -2370,7 +2356,6 @@ describe('IndexPattern Data Source suggestions', () => { scale: 'ordinal', isStaticValue: false, hasTimeShift: false, - hasFilter: false, }, }, { @@ -2382,7 +2367,6 @@ describe('IndexPattern Data Source suggestions', () => { scale: 'interval', isStaticValue: false, hasTimeShift: false, - hasFilter: false, }, }, { @@ -2394,7 +2378,6 @@ describe('IndexPattern Data Source suggestions', () => { scale: undefined, isStaticValue: false, hasTimeShift: false, - hasFilter: false, }, }, ], @@ -2908,7 +2891,6 @@ describe('IndexPattern Data Source suggestions', () => { scale: undefined, isStaticValue: false, hasTimeShift: false, - hasFilter: false, }, }, { @@ -2920,7 +2902,6 @@ describe('IndexPattern Data Source suggestions', () => { scale: undefined, isStaticValue: false, hasTimeShift: false, - hasFilter: false, }, }, ], @@ -2986,7 +2967,6 @@ describe('IndexPattern Data Source suggestions', () => { scale: 'interval', isStaticValue: false, hasTimeShift: false, - hasFilter: false, }, }, { @@ -2998,7 +2978,6 @@ describe('IndexPattern Data Source suggestions', () => { scale: undefined, isStaticValue: false, hasTimeShift: false, - hasFilter: false, }, }, { @@ -3010,7 +2989,6 @@ describe('IndexPattern Data Source suggestions', () => { scale: undefined, isStaticValue: false, hasTimeShift: false, - hasFilter: false, }, }, ], @@ -3074,7 +3052,6 @@ describe('IndexPattern Data Source suggestions', () => { scale: undefined, isStaticValue: false, hasTimeShift: false, - hasFilter: false, }, }, { @@ -3086,7 +3063,6 @@ describe('IndexPattern Data Source suggestions', () => { scale: undefined, isStaticValue: false, hasTimeShift: false, - hasFilter: false, }, }, { @@ -3098,7 +3074,6 @@ describe('IndexPattern Data Source suggestions', () => { scale: undefined, isStaticValue: false, hasTimeShift: false, - hasFilter: false, }, }, ], diff --git a/x-pack/plugins/lens/public/metric_visualization/visualization.test.ts b/x-pack/plugins/lens/public/metric_visualization/visualization.test.ts index 065178a060a34..83a54e4f1a3cd 100644 --- a/x-pack/plugins/lens/public/metric_visualization/visualization.test.ts +++ b/x-pack/plugins/lens/public/metric_visualization/visualization.test.ts @@ -271,7 +271,6 @@ describe('metric_visualization', () => { label: 'shazm', isStaticValue: false, hasTimeShift: false, - hasFilter: false, }; }, }; diff --git a/x-pack/plugins/lens/public/types.ts b/x-pack/plugins/lens/public/types.ts index e4fe434881588..e376dfda4a24b 100644 --- a/x-pack/plugins/lens/public/types.ts +++ b/x-pack/plugins/lens/public/types.ts @@ -505,7 +505,6 @@ export interface OperationMetadata { */ export interface OperationDescriptor extends Operation { hasTimeShift: boolean; - hasFilter: boolean; } export interface VisualizationConfigProps { diff --git a/x-pack/plugins/lens/public/xy_visualization/visualization.test.ts b/x-pack/plugins/lens/public/xy_visualization/visualization.test.ts index 8d13692ddc022..d32230bf294aa 100644 --- a/x-pack/plugins/lens/public/xy_visualization/visualization.test.ts +++ b/x-pack/plugins/lens/public/xy_visualization/visualization.test.ts @@ -1086,7 +1086,6 @@ describe('xy_visualization', () => { label: 'date_histogram', isStaticValue: false, hasTimeShift: false, - hasFilter: false, }; } return null; @@ -1116,7 +1115,6 @@ describe('xy_visualization', () => { label: 'date_histogram', isStaticValue: false, hasTimeShift: false, - hasFilter: false, }; } return null; @@ -1160,7 +1158,6 @@ describe('xy_visualization', () => { label: 'histogram', isStaticValue: false, hasTimeShift: false, - hasFilter: false, }; } return null; @@ -1192,7 +1189,6 @@ describe('xy_visualization', () => { label: 'top values', isStaticValue: false, hasTimeShift: false, - hasFilter: false, }; } return null; @@ -1222,7 +1218,6 @@ describe('xy_visualization', () => { label: 'top values', isStaticValue: false, hasTimeShift: false, - hasFilter: false, }; } return null; From c938b01d467c2074010b0d4d6bb2203991b8233f Mon Sep 17 00:00:00 2001 From: dej611 Date: Mon, 21 Feb 2022 16:09:20 +0100 Subject: [PATCH 07/31] :white_check_mark: Add more tests for edge cases --- .../app_plugin/show_underlying_data.test.ts | 373 +++++++++++++++- .../public/app_plugin/show_underlying_data.ts | 36 +- .../indexpattern.test.ts | 422 +++++++++++++++++- .../indexpattern_datasource/indexpattern.tsx | 3 +- .../public/indexpattern_datasource/utils.tsx | 46 +- 5 files changed, 839 insertions(+), 41 deletions(-) diff --git a/x-pack/plugins/lens/public/app_plugin/show_underlying_data.test.ts b/x-pack/plugins/lens/public/app_plugin/show_underlying_data.test.ts index 6294ef5991293..4668dcefbd871 100644 --- a/x-pack/plugins/lens/public/app_plugin/show_underlying_data.test.ts +++ b/x-pack/plugins/lens/public/app_plugin/show_underlying_data.test.ts @@ -8,6 +8,8 @@ import { createMockDatasource } from '../mocks'; import { combineQueryAndFilters, getLayerMetaInfo } from './show_underlying_data'; import { DiscoverStart } from '../../../../../src/plugins/discover/public'; +import { Filter } from '@kbn/es-query'; +import { DatasourcePublicAPI } from '../types'; describe('getLayerMetaInfo', () => { it('should return error in case of no data', () => { @@ -16,6 +18,7 @@ describe('getLayerMetaInfo', () => { .error ).toBe('Visualization has no data available to show'); }); + it('should return error in case of multiple layers', () => { expect( getLayerMetaInfo( @@ -27,19 +30,46 @@ describe('getLayerMetaInfo', () => { }, {} as DiscoverStart ).error - ).toBe('Cannot show underlying data cannot be shown for visualizations with multiple layers'); + ).toBe('Cannot show underlying data for visualizations with multiple layers'); }); + it('should return error in case of missing activeDatasource', () => { expect(getLayerMetaInfo(undefined, {}, undefined, {} as DiscoverStart).error).toBe( 'Visualization has no data available to show' ); }); + it('should return error in case of missing configuration/state', () => { expect( getLayerMetaInfo(createMockDatasource('testDatasource'), undefined, {}, {} as DiscoverStart) .error ).toBe('Visualization has no data available to show'); }); + + it('should return error in case of a timeshift declared in a column', () => { + const mockDatasource = createMockDatasource('testDatasource'); + const updatedPublicAPI: DatasourcePublicAPI = { + datasourceId: 'testDatasource', + getOperationForColumnId: jest.fn(() => ({ + dataType: 'number', + isBucketed: false, + scale: 'ratio', + label: 'A field', + isStaticValue: false, + sortingHint: undefined, + hasTimeShift: true, + })), + getTableSpec: jest.fn(), + getVisualDefaults: jest.fn(), + getSourceId: jest.fn(), + getFilters: jest.fn(), + }; + mockDatasource.getPublicAPI.mockReturnValue(updatedPublicAPI); + expect( + getLayerMetaInfo(createMockDatasource('testDatasource'), {}, {}, {} as DiscoverStart).error + ).toBe('Visualization has no data available to show'); + }); + it('should not be visible if discover is not available', () => { expect( getLayerMetaInfo( @@ -52,7 +82,43 @@ describe('getLayerMetaInfo', () => { ).isVisible ).toBeFalsy(); }); - it.todo('should basically work collecting fields and filters in the visualization'); + + it('should basically work collecting fields and filters in the visualization', () => { + const mockDatasource = createMockDatasource('testDatasource'); + const updatedPublicAPI: DatasourcePublicAPI = { + datasourceId: 'testDatasource', + getOperationForColumnId: jest.fn(), + getTableSpec: jest.fn(() => [{ columnId: 'col1', fields: ['bytes'] }]), + getVisualDefaults: jest.fn(), + getSourceId: jest.fn(), + getFilters: jest.fn(() => ({ + kuery: [[{ language: 'kuery', query: 'memory > 40000' }]], + lucene: [], + })), + }; + mockDatasource.getPublicAPI.mockReturnValue(updatedPublicAPI); + const { error, meta } = getLayerMetaInfo( + mockDatasource, + {}, // the publicAPI has been mocked, so no need for a state here + { + datatable1: { type: 'datatable', columns: [], rows: [] }, + }, + {} as DiscoverStart + ); + expect(error).toBeUndefined(); + expect(meta?.columns).toEqual(['bytes']); + expect(meta?.filters).toEqual({ + kuery: [ + [ + { + language: 'kuery', + query: 'memory > 40000', + }, + ], + ], + lucene: [], + }); + }); }); describe('combineQueryAndFilters', () => { it('should just return same query and filters if no fields or filters are in layer meta', () => { @@ -189,9 +255,302 @@ describe('combineQueryAndFilters', () => { ], }); }); - it.todo( - 'should append lucene meta filters to app filters even if existing filters are using kuery' - ); - it.todo('should work for complex cases of nested meta filters'); - it.todo('should append lucene meta filters to an existing lucene query'); + it('should append lucene meta filters to app filters even if existing filters are using kuery', () => { + expect( + combineQueryAndFilters( + { language: 'kuery', query: 'myField: *' }, + [ + { + $state: { + store: 'appState', + }, + bool: { + filter: [ + { + bool: { + minimum_should_match: 1, + should: [ + { + exists: { + field: 'myfield', + }, + }, + ], + }, + }, + ], + must: [], + must_not: [], + should: [], + }, + meta: { + alias: 'Existing kuery filters', + disabled: false, + index: 'testDatasource', + negate: false, + type: 'custom', + }, + } as Filter, + ], + { + id: 'testDatasource', + columns: [], + filters: { + kuery: [], + lucene: [[{ language: 'lucene', query: 'anotherField' }]], + }, + }, + undefined + ) + ).toEqual({ + filters: [ + { + $state: { + store: 'appState', + }, + bool: { + filter: [ + { + bool: { + minimum_should_match: 1, + should: [ + { + exists: { + field: 'myfield', + }, + }, + ], + }, + }, + ], + must: [], + must_not: [], + should: [], + }, + meta: { + alias: 'Existing kuery filters', + disabled: false, + index: 'testDatasource', + negate: false, + type: 'custom', + }, + }, + { + $state: { + store: 'appState', + }, + bool: { + filter: [], + must: [ + { + query_string: { + query: '( anotherField )', + }, + }, + ], + must_not: [], + should: [], + }, + meta: { + alias: 'Lens context (lucene)', + disabled: false, + index: 'testDatasource', + negate: false, + type: 'custom', + }, + }, + ], + query: { + language: 'kuery', + query: 'myField: *', + }, + }); + }); + it('should append lucene meta filters to an existing lucene query', () => { + expect( + combineQueryAndFilters( + { language: 'lucene', query: 'myField' }, + [], + { + id: 'testDatasource', + columns: [], + filters: { + kuery: [[{ language: 'kuery', query: 'myfield: *' }]], + lucene: [[{ language: 'lucene', query: 'anotherField' }]], + }, + }, + undefined + ) + ).toEqual({ + filters: [ + { + $state: { + store: 'appState', + }, + bool: { + filter: [ + { + bool: { + minimum_should_match: 1, + should: [ + { + exists: { + field: 'myfield', + }, + }, + ], + }, + }, + ], + must: [], + must_not: [], + should: [], + }, + meta: { + alias: 'Lens context (kuery)', + disabled: false, + index: 'testDatasource', + negate: false, + type: 'custom', + }, + }, + ], + query: { + language: 'lucene', + query: '( myField ) AND ( anotherField )', + }, + }); + }); + it('should work for complex cases of nested meta filters', () => { + // scenario overview: + // A kuery query + // A kuery filter pill + // 4 kuery table filter groups (1 from filtered column, 2 from filters, 1 from top values, 1 from custom ranges) + // 2 lucene table filter groups (1 from filtered column + 2 from filters ) + expect( + combineQueryAndFilters( + { language: 'kuery', query: 'myField: *' }, + [ + { + $state: { + store: 'appState', + }, + bool: { + filter: [ + { + bool: { + minimum_should_match: 1, + should: [ + { + exists: { + field: 'myfield', + }, + }, + ], + }, + }, + ], + must: [], + must_not: [], + should: [], + }, + meta: { + alias: 'Existing kuery filters', + disabled: false, + index: 'testDatasource', + negate: false, + type: 'custom', + }, + } as Filter, + ], + { + id: 'testDatasource', + columns: [], + filters: { + kuery: [ + [{ language: 'kuery', query: 'bytes > 4000' }], + [ + { language: 'kuery', query: 'memory > 5000' }, + { language: 'kuery', query: 'memory >= 15000' }, + ], + [{ language: 'kuery', query: 'myField: *' }], + [{ language: 'kuery', query: 'otherField >= 15' }], + ], + lucene: [ + [{ language: 'lucene', query: 'filteredField: 400' }], + [ + { language: 'lucene', query: 'aNewField' }, + { language: 'lucene', query: 'anotherNewField: 200' }, + ], + ], + }, + }, + undefined + ) + ).toEqual({ + filters: [ + { + $state: { + store: 'appState', + }, + bool: { + filter: [ + { + bool: { + minimum_should_match: 1, + should: [ + { + exists: { + field: 'myfield', + }, + }, + ], + }, + }, + ], + must: [], + must_not: [], + should: [], + }, + meta: { + alias: 'Existing kuery filters', + disabled: false, + index: 'testDatasource', + negate: false, + type: 'custom', + }, + }, + { + $state: { + store: 'appState', + }, + bool: { + filter: [], + must: [ + { + query_string: { + query: + '( ( filteredField: 400 ) AND ( ( aNewField ) OR ( anotherNewField: 200 ) ) )', + }, + }, + ], + must_not: [], + should: [], + }, + meta: { + alias: 'Lens context (lucene)', + disabled: false, + index: 'testDatasource', + negate: false, + type: 'custom', + }, + }, + ], + query: { + language: 'kuery', + query: + '( myField: * ) AND ( ( bytes > 4000 ) AND ( ( memory > 5000 ) OR ( memory >= 15000 ) ) AND ( myField: * ) AND ( otherField >= 15 ) )', + }, + }); + }); }); diff --git a/x-pack/plugins/lens/public/app_plugin/show_underlying_data.ts b/x-pack/plugins/lens/public/app_plugin/show_underlying_data.ts index 380b0727fdbff..68185585b6c94 100644 --- a/x-pack/plugins/lens/public/app_plugin/show_underlying_data.ts +++ b/x-pack/plugins/lens/public/app_plugin/show_underlying_data.ts @@ -71,8 +71,7 @@ export function getLayerMetaInfo( return { meta: undefined, error: i18n.translate('xpack.lens.app.showUnderlyingDataMultipleLayers', { - defaultMessage: - 'Cannot show underlying data cannot be shown for visualizations with multiple layers', + defaultMessage: 'Cannot show underlying data for visualizations with multiple layers', }), isVisible, }; @@ -94,27 +93,36 @@ export function getLayerMetaInfo( // } const tableSpec = datasourceAPI.getTableSpec(); - const uniqueFields = [ - ...new Set( - tableSpec - .filter(({ columnId }) => !datasourceAPI.getOperationForColumnId(columnId)?.hasTimeShift) - .map(({ fields }) => fields) - .flat() - ), - ]; - // If no field, return? + const columnsWithNoTimeShifts = tableSpec.filter( + ({ columnId }) => !datasourceAPI.getOperationForColumnId(columnId)?.hasTimeShift + ); + if (columnsWithNoTimeShifts.length < tableSpec.length) { + return { + meta: undefined, + error: i18n.translate('xpack.lens.app.showUnderlyingDataTimeShifts', { + defaultMessage: "Cannot show underlying data when there's a time shift configured", + }), + isVisible, + }; + } + + const uniqueFields = [...new Set(columnsWithNoTimeShifts.map(({ fields }) => fields).flat())]; + // If no field, return? Or rather carry on and show the default columns? // if (!uniqueFields.length) { // return { // meta: undefined, // error: i18n.translate('xpack.lens.app.showUnderlyingDataNoFields', { - // defaultMessage: 'The current visualization has not available fields to show', + // defaultMessage: 'The current visualization has no available fields to show', // }), // isVisible, // }; // } - const layerFilters = datasourceAPI.getFilters(); return { - meta: { id: datasourceAPI.getSourceId()!, columns: uniqueFields, filters: layerFilters }, + meta: { + id: datasourceAPI.getSourceId()!, + columns: uniqueFields, + filters: datasourceAPI.getFilters(), + }, error: undefined, isVisible, }; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts index a4532128fa4c1..dfa18d0cf2b76 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts @@ -23,6 +23,8 @@ import { MovingAverageIndexPatternColumn, MathIndexPatternColumn, FormulaIndexPatternColumn, + RangeIndexPatternColumn, + FiltersIndexPatternColumn, } from './operations'; import { createMockedFullReference } from './operations/mocks'; import { indexPatternFieldEditorPluginMock } from 'src/plugins/data_view_field_editor/public/mocks'; @@ -1346,7 +1348,6 @@ describe('IndexPattern Data Source', () => { dataType: 'string', isBucketed: true, isStaticValue: false, - hasFilter: false, hasTimeShift: false, } as OperationDescriptor); }); @@ -1399,11 +1400,420 @@ describe('IndexPattern Data Source', () => { }); describe('getFilters', () => { - it.todo('should return all filters in metrics, grouped by language'); - it.todo('shuold collect top values fields as kuery existence filters'); - it.todo('should collect custom ranges as kuery filters'); - it.todo('should collect filters within filters operation grouped by language'); - it.todo('should support complex scenarios'); + it('should return all filters in metrics, grouped by language', () => { + publicAPI = indexPatternDatasource.getPublicAPI({ + state: { + ...enrichBaseState(baseState), + layers: { + first: { + indexPatternId: '1', + columnOrder: ['col1', 'col2'], + columns: { + col1: { + label: 'Sum', + dataType: 'number', + isBucketed: false, + operationType: 'sum', + sourceField: 'test', + params: {}, + filter: { language: 'kuery', query: 'bytes > 1000' }, + } as GenericIndexPatternColumn, + col2: { + label: 'Sum', + dataType: 'number', + isBucketed: false, + operationType: 'sum', + sourceField: 'test', + params: {}, + filter: { language: 'lucene', query: 'memory' }, + } as GenericIndexPatternColumn, + }, + }, + }, + }, + layerId: 'first', + }); + expect(publicAPI.getFilters()).toEqual({ + kuery: [[{ language: 'kuery', query: 'bytes > 1000' }]], + lucene: [[{ language: 'lucene', query: 'memory' }]], + }); + }); + it('should ignore empty filtered metrics', () => { + publicAPI = indexPatternDatasource.getPublicAPI({ + state: { + ...enrichBaseState(baseState), + layers: { + first: { + indexPatternId: '1', + columnOrder: ['col1'], + columns: { + col1: { + label: 'Sum', + dataType: 'number', + isBucketed: false, + operationType: 'sum', + sourceField: 'test', + params: {}, + filter: { language: 'kuery', query: '' }, + } as GenericIndexPatternColumn, + }, + }, + }, + }, + layerId: 'first', + }); + expect(publicAPI.getFilters()).toEqual({ kuery: [], lucene: [] }); + }); + it('shuold collect top values fields as kuery existence filters', () => { + publicAPI = indexPatternDatasource.getPublicAPI({ + state: { + ...enrichBaseState(baseState), + layers: { + first: { + indexPatternId: '1', + columnOrder: ['col1', 'col2'], + columns: { + col1: { + label: 'Terms', + dataType: 'string', + isBucketed: true, + operationType: 'terms', + sourceField: 'geo.src', + params: { + orderBy: { type: 'alphabetical' }, + orderDirection: 'asc', + size: 10, + }, + } as TermsIndexPatternColumn, + col2: { + label: 'Terms', + dataType: 'string', + isBucketed: true, + operationType: 'terms', + sourceField: 'geo.dest', + params: { + orderBy: { type: 'alphabetical' }, + orderDirection: 'asc', + size: 10, + secondaryFields: ['myField'], + }, + } as TermsIndexPatternColumn, + }, + }, + }, + }, + layerId: 'first', + }); + expect(publicAPI.getFilters()).toEqual({ + kuery: [ + [{ language: 'kuery', query: 'geo.src: *' }], + [ + { language: 'kuery', query: 'geo.dest: *' }, + { language: 'kuery', query: 'myField: *' }, + ], + ], + lucene: [], + }); + }); + it('should ignore top values fields if other/missing option is enabled', () => { + publicAPI = indexPatternDatasource.getPublicAPI({ + state: { + ...enrichBaseState(baseState), + layers: { + first: { + indexPatternId: '1', + columnOrder: ['col1', 'col2'], + columns: { + col1: { + label: 'Terms', + dataType: 'string', + isBucketed: true, + operationType: 'terms', + sourceField: 'geo.src', + params: { + orderBy: { type: 'alphabetical' }, + orderDirection: 'asc', + size: 10, + otherBucket: true, + }, + } as TermsIndexPatternColumn, + col2: { + label: 'Terms', + dataType: 'string', + isBucketed: true, + operationType: 'terms', + sourceField: 'geo.src', + params: { + orderBy: { type: 'alphabetical' }, + orderDirection: 'asc', + size: 10, + missingBucket: true, + }, + } as TermsIndexPatternColumn, + }, + }, + }, + }, + layerId: 'first', + }); + expect(publicAPI.getFilters()).toEqual({ kuery: [], lucene: [] }); + }); + it('should collect custom ranges as kuery filters', () => { + publicAPI = indexPatternDatasource.getPublicAPI({ + state: { + ...enrichBaseState(baseState), + layers: { + first: { + indexPatternId: '1', + columnOrder: ['col1', 'col2'], + columns: { + col1: { + label: 'Single range', + dataType: 'number', + isBucketed: true, + operationType: 'range', + sourceField: 'bytes', + params: { + type: 'range', + ranges: [{ from: 100, to: 150, label: 'Range 1' }], + }, + } as RangeIndexPatternColumn, + col2: { + label: 'Multiple ranges', + dataType: 'number', + isBucketed: true, + operationType: 'range', + sourceField: 'bytes', + params: { + type: 'range', + ranges: [ + { from: 200, to: 300, label: 'Range 2' }, + { from: 300, to: 400, label: 'Range 3' }, + ], + }, + } as RangeIndexPatternColumn, + }, + }, + }, + }, + layerId: 'first', + }); + expect(publicAPI.getFilters()).toEqual({ + kuery: [ + [{ language: 'kuery', query: 'bytes >= 100 AND bytes <= 150' }], + [ + { language: 'kuery', query: 'bytes >= 200 AND bytes <= 300' }, + { language: 'kuery', query: 'bytes >= 300 AND bytes <= 400' }, + ], + ], + lucene: [], + }); + }); + it('should collect custom ranges as kuery filters as partial', () => { + publicAPI = indexPatternDatasource.getPublicAPI({ + state: { + ...enrichBaseState(baseState), + layers: { + first: { + indexPatternId: '1', + columnOrder: ['col1', 'col2', 'col3'], + columns: { + col1: { + label: 'Empty range', + dataType: 'number', + isBucketed: true, + operationType: 'range', + sourceField: 'bytes', + params: { + type: 'range', + ranges: [{ label: 'Empty range' }], + }, + } as RangeIndexPatternColumn, + col2: { + label: 'From range', + dataType: 'number', + isBucketed: true, + operationType: 'range', + sourceField: 'bytes', + params: { + type: 'range', + ranges: [{ from: 100, label: 'Partial range 1' }], + }, + } as RangeIndexPatternColumn, + col3: { + label: 'To ranges', + dataType: 'number', + isBucketed: true, + operationType: 'range', + sourceField: 'bytes', + params: { + type: 'range', + ranges: [{ to: 300, label: 'Partial Range 2' }], + }, + } as RangeIndexPatternColumn, + }, + }, + }, + }, + layerId: 'first', + }); + expect(publicAPI.getFilters()).toEqual({ + kuery: [ + [{ language: 'kuery', query: 'bytes >= 100' }], + [{ language: 'kuery', query: 'bytes <= 300' }], + ], + lucene: [], + }); + }); + it('should collect filters within filters operation grouped by language', () => { + publicAPI = indexPatternDatasource.getPublicAPI({ + state: { + ...enrichBaseState(baseState), + layers: { + first: { + indexPatternId: '1', + columnOrder: ['col1', 'col2', 'col3'], + columns: { + col1: { + label: 'kuery Filter', + dataType: 'string', + isBucketed: true, + operationType: 'filters', + scale: 'ordinal', + params: { + filters: [{ label: '', input: { language: 'kuery', query: 'bytes > 1000' } }], + }, + } as FiltersIndexPatternColumn, + col2: { + label: 'Lucene Filter', + dataType: 'string', + isBucketed: true, + operationType: 'filters', + scale: 'ordinal', + params: { + filters: [{ label: '', input: { language: 'lucene', query: 'memory' } }], + }, + } as FiltersIndexPatternColumn, + col3: { + label: 'Mixed filters', + dataType: 'string', + isBucketed: true, + operationType: 'filters', + scale: 'ordinal', + params: { + filters: [ + { label: '', input: { language: 'kuery', query: 'bytes > 5000' } }, + { label: '', input: { language: 'kuery', query: 'memory > 500000' } }, + { label: '', input: { language: 'lucene', query: 'phpmemory' } }, + { label: '', input: { language: 'lucene', query: 'memory: 5000000' } }, + ], + }, + } as FiltersIndexPatternColumn, + }, + }, + }, + }, + layerId: 'first', + }); + expect(publicAPI.getFilters()).toEqual({ + kuery: [ + [{ language: 'kuery', query: 'bytes > 1000' }], + [ + { language: 'kuery', query: 'bytes > 5000' }, + { language: 'kuery', query: 'memory > 500000' }, + ], + ], + lucene: [ + [{ language: 'lucene', query: 'memory' }], + [ + { language: 'lucene', query: 'phpmemory' }, + { language: 'lucene', query: 'memory: 5000000' }, + ], + ], + }); + }); + it('should support complete scenarios', () => { + publicAPI = indexPatternDatasource.getPublicAPI({ + state: { + ...enrichBaseState(baseState), + layers: { + first: { + indexPatternId: '1', + columnOrder: ['col1', 'col2', 'col3', 'col4'], + columns: { + col1: { + label: 'Mixed filters', + dataType: 'string', + isBucketed: true, + operationType: 'filters', + scale: 'ordinal', + params: { + filters: [ + { label: '', input: { language: 'kuery', query: 'bytes > 5000' } }, + { label: '', input: { language: 'kuery', query: 'memory > 500000' } }, + { label: '', input: { language: 'lucene', query: 'phpmemory' } }, + { label: '', input: { language: 'lucene', query: 'memory: 5000000' } }, + ], + }, + } as FiltersIndexPatternColumn, + col2: { + label: 'Sum', + dataType: 'number', + isBucketed: false, + operationType: 'sum', + sourceField: 'test', + params: {}, + filter: { language: 'kuery', query: 'bytes > 1000' }, + } as GenericIndexPatternColumn, + col3: { + label: 'Sum', + dataType: 'number', + isBucketed: false, + operationType: 'sum', + sourceField: 'test', + params: {}, + filter: { language: 'lucene', query: 'memory' }, + } as GenericIndexPatternColumn, + col4: { + label: 'Terms', + dataType: 'string', + isBucketed: true, + operationType: 'terms', + sourceField: 'geo.src', + params: { + orderBy: { type: 'alphabetical' }, + orderDirection: 'asc', + size: 10, + secondaryFields: ['myField'], + }, + } as TermsIndexPatternColumn, + }, + }, + }, + }, + layerId: 'first', + }); + expect(publicAPI.getFilters()).toEqual({ + kuery: [ + [{ language: 'kuery', query: 'bytes > 1000' }], + [ + { language: 'kuery', query: 'bytes > 5000' }, + { language: 'kuery', query: 'memory > 500000' }, + ], + [ + { language: 'kuery', query: 'geo.src: *' }, + { language: 'kuery', query: 'myField: *' }, + ], + ], + lucene: [ + [{ language: 'lucene', query: 'memory' }], + [ + { language: 'lucene', query: 'phpmemory' }, + { language: 'lucene', query: 'memory: 5000000' }, + ], + ], + }); + }); }); }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx index 1e402728801bf..3d1a463c6dac4 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx @@ -86,7 +86,7 @@ export function columnToOperation( uniqueLabel?: string, dataView?: IndexPattern ): OperationDescriptor { - const { dataType, label, isBucketed, scale, operationType, timeShift, filter } = column; + const { dataType, label, isBucketed, scale, operationType, timeShift } = column; const fieldTypes = 'sourceField' in column ? dataView?.getFieldByName(column.sourceField)?.esTypes : undefined; return { @@ -100,7 +100,6 @@ export function columnToOperation( ? 'version' : undefined, hasTimeShift: Boolean(timeShift), - hasFilter: Boolean(filter), }; } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/utils.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/utils.tsx index 8341e5b8a3cf7..e78549649e9df 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/utils.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/utils.tsx @@ -244,41 +244,63 @@ export function getVisualDefaultsForLayer(layer: IndexPatternLayer) { export function getFiltersInLayer(layer: IndexPatternLayer, columnIds: string[]) { // extract filters from filtered metrics const filteredMetrics = columnIds - .map((colId) => layer.columns[colId].filter) - .filter(Boolean) as Query[]; + .map((colId) => layer.columns[colId]?.filter) + // filter out empty filters as well + .filter((filter) => filter?.query) as Query[]; const { kuery: kqlMetricQueries, lucene: luceneMetricQueries } = groupBy( filteredMetrics, 'language' ); + function extractUsefulQueries( + queries: FiltersIndexPatternColumn['params']['filters'] | undefined + ) { + return queries?.map(({ input }) => input).filter(({ query }) => query && query !== '*'); + } + const filterOperation = columnIds .map((colId) => { const column = layer.columns[colId]; + if (isColumnOfType('filters', column)) { const groupsByLanguage = groupBy( column.params.filters, ({ input }) => input.language ) as Record<'lucene' | 'kuery', FiltersIndexPatternColumn['params']['filters']>; return { - kuery: groupsByLanguage.kuery?.map(({ input }) => input), - lucene: groupsByLanguage.lucene?.map(({ input }) => input), + kuery: extractUsefulQueries(groupsByLanguage.kuery), + lucene: extractUsefulQueries(groupsByLanguage.lucene), }; } if (isColumnOfType('range', column) && column.sourceField) { return { - kuery: column.params.ranges.map(({ from, to }) => ({ - query: `${column.sourceField} >= ${from} AND ${column.sourceField} <= ${to}`, - language: 'kuery', - })), + kuery: column.params.ranges + .map(({ from, to }) => { + let rangeQuery = ''; + if (from != null && isFinite(from)) { + rangeQuery += `${column.sourceField} >= ${from}`; + } + if (to != null && isFinite(to)) { + if (rangeQuery.length) { + rangeQuery += ' AND '; + } + rangeQuery += `${column.sourceField} <= ${to}`; + } + return { + query: rangeQuery, + language: 'kuery', + }; + }) + .filter(({ query }) => query), }; } if ( isColumnOfType('terms', column) && !(column.params.otherBucket || column.params.missingBucket) ) { - // TODO: return field -> terms - // TODO: support multi-terms + // TODO: return field -> terms? + // TODO: support multi-terms? return { kuery: [{ query: `${column.sourceField}: *`, language: 'kuery' }].concat( column.params.secondaryFields?.map((field) => ({ @@ -292,10 +314,10 @@ export function getFiltersInLayer(layer: IndexPatternLayer, columnIds: string[]) .filter(Boolean) as Array<{ kuery?: Query[]; lucene?: Query[] }>; return { kuery: [kqlMetricQueries, ...filterOperation.map(({ kuery }) => kuery)].filter( - Boolean + (filters) => filters?.length ) as Query[][], lucene: [luceneMetricQueries, ...filterOperation.map(({ lucene }) => lucene)].filter( - Boolean + (filters) => filters?.length ) as Query[][], }; } From 05163c83f26d404f751fd09f3a0b234cffcbe589 Mon Sep 17 00:00:00 2001 From: dej611 Date: Mon, 21 Feb 2022 16:19:11 +0100 Subject: [PATCH 08/31] :recycle: First code refactor --- x-pack/plugins/lens/public/app_plugin/show_underlying_data.ts | 2 +- .../lens/public/indexpattern_datasource/indexpattern.tsx | 2 +- x-pack/plugins/lens/public/types.ts | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/lens/public/app_plugin/show_underlying_data.ts b/x-pack/plugins/lens/public/app_plugin/show_underlying_data.ts index 68185585b6c94..813664e6fba2a 100644 --- a/x-pack/plugins/lens/public/app_plugin/show_underlying_data.ts +++ b/x-pack/plugins/lens/public/app_plugin/show_underlying_data.ts @@ -151,7 +151,7 @@ export function combineQueryAndFilters( }; } - const newFilters = [...filters]; + const newFilters = filters; if (meta.filters[filtersLanguage]?.length) { const queryExpression = joinQueries(meta.filters[filtersLanguage]); // Append the new filter based on the queryExpression to the existing ones diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx index 3d1a463c6dac4..8f2e1e3cc5af1 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx @@ -463,7 +463,7 @@ export function getIndexPatternDatasource({ // consider also referenced columns in this case // but map fields to the top referencing column const fieldsPerColumn: Record = {}; - Object.keys(layer.columns).map((colId) => { + Object.keys(layer.columns).forEach((colId) => { const visibleColumnId = getTopColumnFromReference(layer, colId); fieldsPerColumn[visibleColumnId] = fieldsPerColumn[visibleColumnId] || []; diff --git a/x-pack/plugins/lens/public/types.ts b/x-pack/plugins/lens/public/types.ts index e376dfda4a24b..b66ea7a234496 100644 --- a/x-pack/plugins/lens/public/types.ts +++ b/x-pack/plugins/lens/public/types.ts @@ -216,7 +216,6 @@ export interface Datasource { removeLayer: (state: T, layerId: string) => T; clearLayer: (state: T, layerId: string) => T; getLayers: (state: T) => string[]; - // getLayerInfo(state: T, layerId: string): { datasourceId: string; fields: string[] }; removeColumn: (props: { prevState: T; layerId: string; columnId: string }) => T; initializeDimension?: ( state: T, From 6206d2763d9289288f975001af5a3a4356553357 Mon Sep 17 00:00:00 2001 From: dej611 Date: Mon, 21 Feb 2022 17:27:13 +0100 Subject: [PATCH 09/31] :bug: Fix discover capabilities check --- .../lens/public/app_plugin/lens_top_nav.tsx | 11 ++++- .../app_plugin/show_underlying_data.test.ts | 40 ++++++++++++++----- .../public/app_plugin/show_underlying_data.ts | 7 ++-- x-pack/plugins/lens/public/plugin.ts | 4 +- 4 files changed, 44 insertions(+), 18 deletions(-) diff --git a/x-pack/plugins/lens/public/app_plugin/lens_top_nav.tsx b/x-pack/plugins/lens/public/app_plugin/lens_top_nav.tsx index 97a0ed25735cc..9e31873e13803 100644 --- a/x-pack/plugins/lens/public/app_plugin/lens_top_nav.tsx +++ b/x-pack/plugins/lens/public/app_plugin/lens_top_nav.tsx @@ -320,9 +320,16 @@ export const LensTopNavMenu = ({ datasourceMap[activeDatasourceId], datasourceStates[activeDatasourceId].state, activeData, - discover + application.capabilities ); - }, [activeData, activeDatasourceId, datasourceMap, datasourceStates, discover]); + }, [ + activeData, + activeDatasourceId, + datasourceMap, + datasourceStates, + discover, + application.capabilities, + ]); const topNavConfig = useMemo(() => { const baseMenuEntries = getLensTopNavConfig({ diff --git a/x-pack/plugins/lens/public/app_plugin/show_underlying_data.test.ts b/x-pack/plugins/lens/public/app_plugin/show_underlying_data.test.ts index 4668dcefbd871..e253562d2eb5c 100644 --- a/x-pack/plugins/lens/public/app_plugin/show_underlying_data.test.ts +++ b/x-pack/plugins/lens/public/app_plugin/show_underlying_data.test.ts @@ -7,15 +7,19 @@ import { createMockDatasource } from '../mocks'; import { combineQueryAndFilters, getLayerMetaInfo } from './show_underlying_data'; -import { DiscoverStart } from '../../../../../src/plugins/discover/public'; import { Filter } from '@kbn/es-query'; import { DatasourcePublicAPI } from '../types'; +import { RecursiveReadonly } from '@kbn/utility-types'; +import { Capabilities } from 'kibana/public'; describe('getLayerMetaInfo', () => { + const capabilities = { + navLinks: { discover: true }, + discover: { show: true }, + } as unknown as RecursiveReadonly; it('should return error in case of no data', () => { expect( - getLayerMetaInfo(createMockDatasource('testDatasource'), {}, undefined, {} as DiscoverStart) - .error + getLayerMetaInfo(createMockDatasource('testDatasource'), {}, undefined, capabilities).error ).toBe('Visualization has no data available to show'); }); @@ -28,21 +32,20 @@ describe('getLayerMetaInfo', () => { datatable1: { type: 'datatable', columns: [], rows: [] }, datatable2: { type: 'datatable', columns: [], rows: [] }, }, - {} as DiscoverStart + capabilities ).error ).toBe('Cannot show underlying data for visualizations with multiple layers'); }); it('should return error in case of missing activeDatasource', () => { - expect(getLayerMetaInfo(undefined, {}, undefined, {} as DiscoverStart).error).toBe( + expect(getLayerMetaInfo(undefined, {}, undefined, capabilities).error).toBe( 'Visualization has no data available to show' ); }); it('should return error in case of missing configuration/state', () => { expect( - getLayerMetaInfo(createMockDatasource('testDatasource'), undefined, {}, {} as DiscoverStart) - .error + getLayerMetaInfo(createMockDatasource('testDatasource'), undefined, {}, capabilities).error ).toBe('Visualization has no data available to show'); }); @@ -66,11 +69,12 @@ describe('getLayerMetaInfo', () => { }; mockDatasource.getPublicAPI.mockReturnValue(updatedPublicAPI); expect( - getLayerMetaInfo(createMockDatasource('testDatasource'), {}, {}, {} as DiscoverStart).error + getLayerMetaInfo(createMockDatasource('testDatasource'), {}, {}, capabilities).error ).toBe('Visualization has no data available to show'); }); it('should not be visible if discover is not available', () => { + // both capabilities should be enabled to enable discover expect( getLayerMetaInfo( createMockDatasource('testDatasource'), @@ -78,7 +82,23 @@ describe('getLayerMetaInfo', () => { { datatable1: { type: 'datatable', columns: [], rows: [] }, }, - undefined + { + navLinks: { discover: false }, + discover: { show: true }, + } as unknown as RecursiveReadonly + ).isVisible + ).toBeFalsy(); + expect( + getLayerMetaInfo( + createMockDatasource('testDatasource'), + {}, + { + datatable1: { type: 'datatable', columns: [], rows: [] }, + }, + { + navLinks: { discover: true }, + discover: { show: false }, + } as unknown as RecursiveReadonly ).isVisible ).toBeFalsy(); }); @@ -103,7 +123,7 @@ describe('getLayerMetaInfo', () => { { datatable1: { type: 'datatable', columns: [], rows: [] }, }, - {} as DiscoverStart + capabilities ); expect(error).toBeUndefined(); expect(meta?.columns).toEqual(['bytes']); diff --git a/x-pack/plugins/lens/public/app_plugin/show_underlying_data.ts b/x-pack/plugins/lens/public/app_plugin/show_underlying_data.ts index 813664e6fba2a..9dc89fceee593 100644 --- a/x-pack/plugins/lens/public/app_plugin/show_underlying_data.ts +++ b/x-pack/plugins/lens/public/app_plugin/show_underlying_data.ts @@ -14,7 +14,8 @@ import { FilterStateStore, } from '@kbn/es-query'; import { i18n } from '@kbn/i18n'; -import { DiscoverStart } from '../../../../../src/plugins/discover/public'; +import { RecursiveReadonly } from '@kbn/utility-types'; +import { Capabilities } from 'kibana/public'; import { TableInspectorAdapter } from '../editor_frame_service/types'; import { Datasource } from '../types'; @@ -52,9 +53,9 @@ export function getLayerMetaInfo( currentDatasource: Datasource | undefined, datasourceState: unknown, activeData: TableInspectorAdapter | undefined, - discover: DiscoverStart | undefined + capabilities: RecursiveReadonly ): { meta: LayerMetaInfo | undefined; isVisible: boolean; error: string | undefined } { - const isVisible = Boolean(discover); + const isVisible = Boolean(capabilities.navLinks?.discover && capabilities.discover?.show); // If Multiple tables, return // If there are time shifts, return const [datatable, ...otherTables] = Object.values(activeData || {}); diff --git a/x-pack/plugins/lens/public/plugin.ts b/x-pack/plugins/lens/public/plugin.ts index a6f37ece5cdb2..3983fde88e4ff 100644 --- a/x-pack/plugins/lens/public/plugin.ts +++ b/x-pack/plugins/lens/public/plugin.ts @@ -93,7 +93,7 @@ import type { SaveModalContainerProps } from './app_plugin/save_modal_container' import { createStartServicesGetter } from '../../../../src/plugins/kibana_utils/public'; import { setupExpressions } from './expressions'; import { getSearchProvider } from './search_provider'; -import type { DiscoverSetup, DiscoverStart } from '../../../../src/plugins/discover/public'; +import type { DiscoverSetup } from '../../../../src/plugins/discover/public'; export interface LensPluginSetupDependencies { urlForwarding: UrlForwardingSetup; @@ -124,7 +124,6 @@ export interface LensPluginStartDependencies { inspector: InspectorStartContract; spaces: SpacesPluginStart; usageCollection?: UsageCollectionStart; - discover?: DiscoverStart; } export interface LensPublicSetup { @@ -235,7 +234,6 @@ export class LensPlugin { charts, globalSearch, usageCollection, - discover, }: LensPluginSetupDependencies ) { const startServices = createStartServicesGetter(core.getStartServices); From 675b4db8381459c13dedc2cb0627596e79a4114e Mon Sep 17 00:00:00 2001 From: dej611 Date: Tue, 22 Feb 2022 13:03:34 +0100 Subject: [PATCH 10/31] :bug: Fix various issues --- .../public/app_plugin/show_underlying_data.ts | 5 ++-- .../public/indexpattern_datasource/utils.tsx | 25 +++++++++++++++---- x-pack/plugins/lens/public/plugin.ts | 3 ++- 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/lens/public/app_plugin/show_underlying_data.ts b/x-pack/plugins/lens/public/app_plugin/show_underlying_data.ts index 9dc89fceee593..8da5cc2b3dc0f 100644 --- a/x-pack/plugins/lens/public/app_plugin/show_underlying_data.ts +++ b/x-pack/plugins/lens/public/app_plugin/show_underlying_data.ts @@ -146,13 +146,14 @@ export function combineQueryAndFilters( const filtersQuery = joinQueries(meta.filters[queryLanguage]); newQuery = { language: queryLanguage, - query: query + query: query?.query.trim() ? `( ${query.query} ) ${filtersQuery ? `AND ${filtersQuery}` : ''}` : filtersQuery, }; } - const newFilters = filters; + // make a copy as the original filters are readonly + const newFilters = [...filters]; if (meta.filters[filtersLanguage]?.length) { const queryExpression = joinQueries(meta.filters[filtersLanguage]); // Append the new filter based on the queryExpression to the existing ones diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/utils.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/utils.tsx index e78549649e9df..a0c88707b8ab0 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/utils.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/utils.tsx @@ -25,6 +25,7 @@ import { updateColumnParam, updateDefaultLabels, RangeIndexPatternColumn, + FormulaIndexPatternColumn, } from './operations'; import { getInvalidFieldMessage, isColumnOfType } from './operations/definitions/helpers'; @@ -243,10 +244,24 @@ export function getVisualDefaultsForLayer(layer: IndexPatternLayer) { export function getFiltersInLayer(layer: IndexPatternLayer, columnIds: string[]) { // extract filters from filtered metrics - const filteredMetrics = columnIds - .map((colId) => layer.columns[colId]?.filter) + // consider all the columns, included referenced ones to cover also the formula case + const filteredMetrics = Object.keys(layer.columns) + .map((colId) => { + // there's a special case to handle when a formula has a global filter applied + // in this case ignore the filter on the formula column and only use the filter + // applied to the referenced columns. + // This will avoid duplicate filters and issues when a global formula filter is + // combine to specific referenced columns filters + if ( + isColumnOfType('formula', layer.columns[colId]) && + layer.columns[colId]?.filter + ) { + return null; + } + return layer.columns[colId]?.filter; + }) // filter out empty filters as well - .filter((filter) => filter?.query) as Query[]; + .filter((filter) => filter?.query?.trim()) as Query[]; const { kuery: kqlMetricQueries, lucene: luceneMetricQueries } = groupBy( filteredMetrics, @@ -256,7 +271,7 @@ export function getFiltersInLayer(layer: IndexPatternLayer, columnIds: string[]) function extractUsefulQueries( queries: FiltersIndexPatternColumn['params']['filters'] | undefined ) { - return queries?.map(({ input }) => input).filter(({ query }) => query && query !== '*'); + return queries?.map(({ input }) => input).filter(({ query }) => query?.trim() && query !== '*'); } const filterOperation = columnIds @@ -292,7 +307,7 @@ export function getFiltersInLayer(layer: IndexPatternLayer, columnIds: string[]) language: 'kuery', }; }) - .filter(({ query }) => query), + .filter(({ query }) => query?.trim()), }; } if ( diff --git a/x-pack/plugins/lens/public/plugin.ts b/x-pack/plugins/lens/public/plugin.ts index 3983fde88e4ff..cfd0f106fae1c 100644 --- a/x-pack/plugins/lens/public/plugin.ts +++ b/x-pack/plugins/lens/public/plugin.ts @@ -93,7 +93,7 @@ import type { SaveModalContainerProps } from './app_plugin/save_modal_container' import { createStartServicesGetter } from '../../../../src/plugins/kibana_utils/public'; import { setupExpressions } from './expressions'; import { getSearchProvider } from './search_provider'; -import type { DiscoverSetup } from '../../../../src/plugins/discover/public'; +import type { DiscoverSetup, DiscoverStart } from '../../../../src/plugins/discover/public'; export interface LensPluginSetupDependencies { urlForwarding: UrlForwardingSetup; @@ -124,6 +124,7 @@ export interface LensPluginStartDependencies { inspector: InspectorStartContract; spaces: SpacesPluginStart; usageCollection?: UsageCollectionStart; + discover?: DiscoverStart; } export interface LensPublicSetup { From 9dfc9d22e8f9d74e497a49aa8125291a69459c56 Mon Sep 17 00:00:00 2001 From: dej611 Date: Tue, 22 Feb 2022 13:03:57 +0100 Subject: [PATCH 11/31] :white_check_mark: Add functional tests --- test/functional/services/filter_bar.ts | 5 + .../dimension_panel/filtering.tsx | 2 +- .../operations/definitions/terms/index.tsx | 1 + .../indexpattern_datasource/query_input.tsx | 4 +- x-pack/test/functional/apps/lens/formula.ts | 1 - x-pack/test/functional/apps/lens/index.ts | 1 + .../apps/lens/show_underlying_data.ts | 158 ++++++++++++++++++ 7 files changed, 169 insertions(+), 3 deletions(-) create mode 100644 x-pack/test/functional/apps/lens/show_underlying_data.ts diff --git a/test/functional/services/filter_bar.ts b/test/functional/services/filter_bar.ts index 5d189506c314d..ea36e38a48c5a 100644 --- a/test/functional/services/filter_bar.ts +++ b/test/functional/services/filter_bar.ts @@ -97,6 +97,11 @@ export class FilterBarService extends FtrService { return filters.length; } + public async getFiltersLabel(): Promise { + const filters = await this.testSubjects.findAll('~filter'); + return Promise.all(filters.map((filter) => filter.getVisibleText())); + } + /** * Adds a filter to the filter bar. * diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/filtering.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/filtering.tsx index 11e9110171f40..f776e0415f1b6 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/filtering.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/filtering.tsx @@ -166,10 +166,10 @@ export function Filtering({ isInvalid={!isQueryInputValid} error={queryInputError} fullWidth={true} + data-test-subj="indexPattern-filter-by-input" > void; @@ -25,12 +26,13 @@ export const QueryInput = ({ isInvalid: boolean; onSubmit: () => void; disableAutoFocus?: boolean; + 'data-test-subj'?: string; }) => { const { inputValue, handleInputChange } = useDebouncedValue({ value, onChange }); return ( lns-empty-dimension', operation: 'formula', diff --git a/x-pack/test/functional/apps/lens/index.ts b/x-pack/test/functional/apps/lens/index.ts index 45c53ea18a601..d1a8d4092ab78 100644 --- a/x-pack/test/functional/apps/lens/index.ts +++ b/x-pack/test/functional/apps/lens/index.ts @@ -57,6 +57,7 @@ export default function ({ getService, loadTestFile, getPageObjects }: FtrProvid loadTestFile(require.resolve('./dashboard')); loadTestFile(require.resolve('./multi_terms')); loadTestFile(require.resolve('./epoch_millis')); + loadTestFile(require.resolve('./show_underlying_data')); }); describe('', function () { diff --git a/x-pack/test/functional/apps/lens/show_underlying_data.ts b/x-pack/test/functional/apps/lens/show_underlying_data.ts new file mode 100644 index 0000000000000..58ebfab493527 --- /dev/null +++ b/x-pack/test/functional/apps/lens/show_underlying_data.ts @@ -0,0 +1,158 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../ftr_provider_context'; + +export default function ({ getService, getPageObjects }: FtrProviderContext) { + const PageObjects = getPageObjects(['visualize', 'lens', 'common', 'header', 'discover']); + const queryBar = getService('queryBar'); + const filterBar = getService('filterBar'); + const listingTable = getService('listingTable'); + const testSubjects = getService('testSubjects'); + const find = getService('find'); + + describe('show underlying data', () => { + it('should show the open button for a compatible saved visualization', async () => { + await PageObjects.visualize.gotoVisualizationLandingPage(); + await listingTable.searchForItemWithName('lnsXYvis'); + await PageObjects.lens.clickVisualizeListItemTitle('lnsXYvis'); + await PageObjects.lens.goToTimeRange(); + + await PageObjects.lens.waitForVisualization(); + // expect the button is shown and enabled + + await testSubjects.clickWhenNotDisabled(`lnsApp_openInDiscover`); + + // discard the changes and navigate to Discover + await testSubjects.click('confirmModalConfirmButton'); + await PageObjects.header.waitUntilLoadingHasFinished(); + await testSubjects.existOrFail('discoverChart'); + // check the table columns + const columns = await PageObjects.discover.getColumnHeaders(); + expect(columns).to.eql(['ip', '@timestamp', 'bytes']); + }); + + it('should ignore the top values column if other category is enabled', async () => { + await PageObjects.visualize.gotoVisualizationLandingPage(); + await listingTable.searchForItemWithName('lnsXYvis'); + await PageObjects.lens.clickVisualizeListItemTitle('lnsXYvis'); + await PageObjects.lens.goToTimeRange(); + + await PageObjects.lens.openDimensionEditor( + 'lnsXY_splitDimensionPanel > lns-dimensionTrigger' + ); + await testSubjects.click('indexPattern-terms-advanced'); + await testSubjects.click('indexPattern-terms-other-bucket'); + + await PageObjects.lens.closeDimensionEditor(); + + await PageObjects.lens.waitForVisualization(); + // expect the button is shown and enabled + + await testSubjects.clickWhenNotDisabled(`lnsApp_openInDiscover`); + + // discard the changes and navigate to Discover + await testSubjects.click('confirmModalConfirmButton'); + await PageObjects.header.waitUntilLoadingHasFinished(); + await testSubjects.existOrFail('discoverChart'); + expect(await queryBar.getQueryString()).be.eql(''); + }); + + it('should show the open button for a compatible saved visualization with a lucene query', async () => { + await PageObjects.visualize.gotoVisualizationLandingPage(); + await listingTable.searchForItemWithName('lnsXYvis'); + await PageObjects.lens.clickVisualizeListItemTitle('lnsXYvis'); + await PageObjects.lens.goToTimeRange(); + + // add a lucene query to the yDimension + await PageObjects.lens.openDimensionEditor('lnsXY_yDimensionPanel > lns-dimensionTrigger'); + await PageObjects.lens.enableFilter(); + // turn off the KQL switch to change the language to lucene + await testSubjects.click('indexPattern-filter-by-input > switchQueryLanguageButton'); + await testSubjects.click('languageToggle'); + await testSubjects.click('indexPattern-filter-by-input > switchQueryLanguageButton'); + await PageObjects.lens.setFilterBy('memory'); + + await PageObjects.lens.waitForVisualization(); + // expect the button is shown and enabled + // await PageObjects.common.sleep(15000); + + await testSubjects.clickWhenNotDisabled(`lnsApp_openInDiscover`); + + // discard the changes and navigate to Discover + await testSubjects.click('confirmModalConfirmButton'); + await PageObjects.header.waitUntilLoadingHasFinished(); + await testSubjects.existOrFail('discoverChart'); + // check the query + expect(await queryBar.getQueryString()).be.eql('( ip: * )'); + const filterPills = await filterBar.getFiltersLabel(); + expect(filterPills.length).to.be(1); + expect(filterPills[0]).to.be('Lens context (lucene)'); + }); + + it('should show the underlying data extracting all filters and columsn from a formula', async () => { + await PageObjects.visualize.gotoVisualizationLandingPage(); + await listingTable.searchForItemWithName('lnsXYvis'); + await PageObjects.lens.clickVisualizeListItemTitle('lnsXYvis'); + await PageObjects.lens.goToTimeRange(); + + await PageObjects.lens.configureDimension({ + dimension: 'lnsXY_yDimensionPanel > lns-empty-dimension', + operation: 'formula', + formula: `median(bytes) + average(memory, kql=`, + keepOpen: true, + }); + + const input = await find.activeElement(); + await input.type(`bytes > 6000`); + + await PageObjects.lens.closeDimensionEditor(); + await PageObjects.lens.waitForVisualization(); + // expect the button is shown and enabled + await testSubjects.clickWhenNotDisabled(`lnsApp_openInDiscover`); + + // discard the changes and navigate to Discover + await testSubjects.click('confirmModalConfirmButton'); + await PageObjects.header.waitUntilLoadingHasFinished(); + await testSubjects.existOrFail('discoverChart'); + // check the columns + const columns = await PageObjects.discover.getColumnHeaders(); + expect(columns).to.eql(['ip', '@timestamp', 'bytes', 'memory']); + // check the query + expect(await queryBar.getQueryString()).be.eql('( ( bytes > 6000 ) AND ( ip: * ) )'); + }); + + it('should extract a filter from a formula global filter', async () => { + await PageObjects.visualize.gotoVisualizationLandingPage(); + await listingTable.searchForItemWithName('lnsXYvis'); + await PageObjects.lens.clickVisualizeListItemTitle('lnsXYvis'); + await PageObjects.lens.goToTimeRange(); + + await PageObjects.lens.configureDimension({ + dimension: 'lnsXY_yDimensionPanel > lns-empty-dimension', + operation: 'formula', + formula: `count()`, + keepOpen: true, + }); + + await PageObjects.lens.enableFilter(); + await PageObjects.lens.setFilterBy('bytes > 4000'); + + await PageObjects.lens.waitForVisualization(); + // expect the button is shown and enabled + await testSubjects.clickWhenNotDisabled(`lnsApp_openInDiscover`); + + // discard the changes and navigate to Discover + await testSubjects.click('confirmModalConfirmButton'); + await PageObjects.header.waitUntilLoadingHasFinished(); + await testSubjects.existOrFail('discoverChart'); + + // check the query + expect(await queryBar.getQueryString()).be.eql('( ( bytes > 4000 ) AND ( ip: * ) )'); + }); + }); +} From aec1234e08d4220380876af161be0a866cd18a47 Mon Sep 17 00:00:00 2001 From: dej611 Date: Tue, 22 Feb 2022 14:43:42 +0100 Subject: [PATCH 12/31] :white_check_mark: Add more tests --- .../indexpattern.test.ts | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts index 39af4c29e3522..8e8d9876479f2 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts @@ -1825,6 +1825,86 @@ describe('IndexPattern Data Source', () => { ], }); }); + + it('should avoid duplicate filters when formula has a global filter', () => { + publicAPI = indexPatternDatasource.getPublicAPI({ + state: { + ...enrichBaseState(baseState), + layers: { + first: { + indexPatternId: '1', + columnOrder: ['formula'], + columns: { + formula: { + label: 'Formula', + dataType: 'number', + operationType: 'formula', + isBucketed: false, + scale: 'ratio', + filter: { language: 'kuery', query: 'bytes > 4000' }, + params: { + formula: "count(kql='memory > 5000') + count()", + isFormulaBroken: false, + }, + references: ['math'], + } as FormulaIndexPatternColumn, + countX0: { + label: 'countX0', + dataType: 'number', + operationType: 'count', + isBucketed: false, + scale: 'ratio', + sourceField: '___records___', + customLabel: true, + filter: { language: 'kuery', query: 'bytes > 4000 AND memory > 5000' }, + }, + countX1: { + label: 'countX1', + dataType: 'number', + operationType: 'count', + isBucketed: false, + scale: 'ratio', + sourceField: '___records___', + customLabel: true, + filter: { language: 'kuery', query: 'bytes > 4000' }, + }, + math: { + label: 'math', + dataType: 'number', + operationType: 'math', + isBucketed: false, + scale: 'ratio', + params: { + tinymathAst: { + type: 'function', + name: 'add', + args: ['countX0', 'countX1'] as unknown as TinymathAST[], + location: { + min: 0, + max: 17, + }, + text: "count(kql='memory > 5000') + count()", + }, + }, + references: ['countX0', 'countX1'], + customLabel: true, + } as MathIndexPatternColumn, + }, + }, + }, + }, + layerId: 'first', + }); + expect(publicAPI.getFilters()).toEqual({ + kuery: [ + [ + { language: 'kuery', query: 'bytes > 4000 AND memory > 5000' }, + { language: 'kuery', query: 'bytes > 4000' }, + ], + ], + lucene: [], + }); + }); }); }); From 71369da8cc0f2ae69eaf9a35f783b4d7dfa3a9b6 Mon Sep 17 00:00:00 2001 From: dej611 Date: Wed, 23 Feb 2022 18:13:30 +0100 Subject: [PATCH 13/31] :sparkles: Add support for terms + multiterms --- .../public/app_plugin/show_underlying_data.ts | 2 +- .../indexpattern.test.ts | 130 +++++++++++++++++- .../indexpattern_datasource/indexpattern.tsx | 4 +- .../public/indexpattern_datasource/utils.tsx | 44 ++++-- x-pack/plugins/lens/public/types.ts | 5 +- .../apps/lens/show_underlying_data.ts | 12 +- 6 files changed, 181 insertions(+), 16 deletions(-) diff --git a/x-pack/plugins/lens/public/app_plugin/show_underlying_data.ts b/x-pack/plugins/lens/public/app_plugin/show_underlying_data.ts index 8da5cc2b3dc0f..42915d6ef8844 100644 --- a/x-pack/plugins/lens/public/app_plugin/show_underlying_data.ts +++ b/x-pack/plugins/lens/public/app_plugin/show_underlying_data.ts @@ -122,7 +122,7 @@ export function getLayerMetaInfo( meta: { id: datasourceAPI.getSourceId()!, columns: uniqueFields, - filters: datasourceAPI.getFilters(), + filters: datasourceAPI.getFilters(activeData), }, error: undefined, isVisible, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts index 8e8d9876479f2..8ad1ee9f6a1de 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts @@ -1475,7 +1475,7 @@ describe('IndexPattern Data Source', () => { }); expect(publicAPI.getFilters()).toEqual({ kuery: [], lucene: [] }); }); - it('shuold collect top values fields as kuery existence filters', () => { + it('shuold collect top values fields as kuery existence filters if no data is provided', () => { publicAPI = indexPatternDatasource.getPublicAPI({ state: { ...enrichBaseState(baseState), @@ -1526,6 +1526,134 @@ describe('IndexPattern Data Source', () => { lucene: [], }); }); + it('shuold collect top values fields and terms as kuery filters if data is provided', () => { + publicAPI = indexPatternDatasource.getPublicAPI({ + state: { + ...enrichBaseState(baseState), + layers: { + first: { + indexPatternId: '1', + columnOrder: ['col1', 'col2'], + columns: { + col1: { + label: 'Terms', + dataType: 'string', + isBucketed: true, + operationType: 'terms', + sourceField: 'geo.src', + params: { + orderBy: { type: 'alphabetical' }, + orderDirection: 'asc', + size: 10, + }, + } as TermsIndexPatternColumn, + col2: { + label: 'Terms', + dataType: 'string', + isBucketed: true, + operationType: 'terms', + sourceField: 'geo.dest', + params: { + orderBy: { type: 'alphabetical' }, + orderDirection: 'asc', + size: 10, + secondaryFields: ['myField'], + }, + } as TermsIndexPatternColumn, + }, + }, + }, + }, + layerId: 'first', + }); + const data = { + first: { + type: 'datatable' as const, + columns: [], + rows: [ + { col1: 'US', col2: { keys: ['IT', 'MyValue'] } }, + { col1: 'IN', col2: { keys: ['DE', 'MyOtherValue'] } }, + ], + }, + }; + expect(publicAPI.getFilters(data)).toEqual({ + kuery: [ + [ + { language: 'kuery', query: 'geo.src: US' }, + { language: 'kuery', query: 'geo.src: IN' }, + ], + [ + { language: 'kuery', query: 'geo.dest: IT AND myField: MyValue' }, + { language: 'kuery', query: 'geo.dest: DE AND myField: MyOtherValue' }, + ], + ], + lucene: [], + }); + }); + it('shuold collect top values fields and terms and carefully handle empty string values', () => { + publicAPI = indexPatternDatasource.getPublicAPI({ + state: { + ...enrichBaseState(baseState), + layers: { + first: { + indexPatternId: '1', + columnOrder: ['col1', 'col2'], + columns: { + col1: { + label: 'Terms', + dataType: 'string', + isBucketed: true, + operationType: 'terms', + sourceField: 'geo.src', + params: { + orderBy: { type: 'alphabetical' }, + orderDirection: 'asc', + size: 10, + }, + } as TermsIndexPatternColumn, + col2: { + label: 'Terms', + dataType: 'string', + isBucketed: true, + operationType: 'terms', + sourceField: 'geo.dest', + params: { + orderBy: { type: 'alphabetical' }, + orderDirection: 'asc', + size: 10, + secondaryFields: ['myField'], + }, + } as TermsIndexPatternColumn, + }, + }, + }, + }, + layerId: 'first', + }); + const data = { + first: { + type: 'datatable' as const, + columns: [], + rows: [ + { col1: 'US', col2: { keys: ['IT', ''] } }, + { col1: 'IN', col2: { keys: ['DE', 'MyOtherValue'] } }, + ], + }, + }; + expect(publicAPI.getFilters(data)).toEqual({ + kuery: [ + [ + { language: 'kuery', query: 'geo.src: US' }, + { language: 'kuery', query: 'geo.src: IN' }, + ], + [ + { language: 'kuery', query: "geo.dest: IT AND myField: ''" }, + { language: 'kuery', query: 'geo.dest: DE AND myField: MyOtherValue' }, + ], + ], + lucene: [], + }); + }); it('should ignore top values fields if other/missing option is enabled', () => { publicAPI = indexPatternDatasource.getPublicAPI({ state: { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx index 568eb772342bf..7d1b24d2d909c 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx @@ -22,6 +22,7 @@ import type { PublicAPIProps, InitializationOptions, OperationDescriptor, + FramePublicAPI, } from '../types'; import { loadInitialState, @@ -496,7 +497,8 @@ export function getIndexPatternDatasource({ return null; }, getSourceId: () => layer.indexPatternId, - getFilters: () => getFiltersInLayer(layer, visibleColumnIds), + getFilters: (activeData: FramePublicAPI['activeData']) => + getFiltersInLayer(layer, visibleColumnIds, activeData?.[layerId]), getVisualDefaults: () => getVisualDefaultsForLayer(layer), }; }, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/utils.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/utils.tsx index a0c88707b8ab0..2046895e0f7a9 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/utils.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/utils.tsx @@ -242,7 +242,11 @@ export function getVisualDefaultsForLayer(layer: IndexPatternLayer) { ); } -export function getFiltersInLayer(layer: IndexPatternLayer, columnIds: string[]) { +export function getFiltersInLayer( + layer: IndexPatternLayer, + columnIds: string[], + layerData: NonNullable[string] | undefined +) { // extract filters from filtered metrics // consider all the columns, included referenced ones to cover also the formula case const filteredMetrics = Object.keys(layer.columns) @@ -314,15 +318,37 @@ export function getFiltersInLayer(layer: IndexPatternLayer, columnIds: string[]) isColumnOfType('terms', column) && !(column.params.otherBucket || column.params.missingBucket) ) { - // TODO: return field -> terms? - // TODO: support multi-terms? + // Fallback in case of no data: just return the field existence + if (!layerData) { + return { + kuery: [{ query: `${column.sourceField}: *`, language: 'kuery' }].concat( + column.params.secondaryFields?.map((field) => ({ + query: `${field}: *`, + language: 'kuery', + })) || [] + ), + }; + } + const fields = [column.sourceField] + .concat(column.params.secondaryFields || []) + .filter(Boolean) as string[]; + // extract the filters from the columns of the activeData + const queryWithTerms = layerData.rows + .map(({ [colId]: value }) => { + if (typeof value !== 'string' && Array.isArray(value.keys)) { + return { + query: value.keys + .map((term: string, index: number) => `${fields[index]}: ${term || "''"}`) + .join(' AND '), + language: 'kuery', + }; + } + return { query: `${column.sourceField}: ${value}`, language: 'kuery' }; + }) + .filter(Boolean) as Query[]; + return { - kuery: [{ query: `${column.sourceField}: *`, language: 'kuery' }].concat( - column.params.secondaryFields?.map((field) => ({ - query: `${field}: *`, - language: 'kuery', - })) || [] - ), + kuery: queryWithTerms, }; } }) diff --git a/x-pack/plugins/lens/public/types.ts b/x-pack/plugins/lens/public/types.ts index b66ea7a234496..64952b30fbb45 100644 --- a/x-pack/plugins/lens/public/types.ts +++ b/x-pack/plugins/lens/public/types.ts @@ -363,7 +363,10 @@ export interface DatasourcePublicAPI { /** * Collect all defined filters from all the operations in the layer */ - getFilters: () => { kuery: Query[][]; lucene: Query[][] }; + getFilters: (activeData?: FramePublicAPI['activeData']) => { + kuery: Query[][]; + lucene: Query[][]; + }; } export interface DatasourceDataPanelProps { diff --git a/x-pack/test/functional/apps/lens/show_underlying_data.ts b/x-pack/test/functional/apps/lens/show_underlying_data.ts index 58ebfab493527..04c4b33358a79 100644 --- a/x-pack/test/functional/apps/lens/show_underlying_data.ts +++ b/x-pack/test/functional/apps/lens/show_underlying_data.ts @@ -88,7 +88,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.header.waitUntilLoadingHasFinished(); await testSubjects.existOrFail('discoverChart'); // check the query - expect(await queryBar.getQueryString()).be.eql('( ip: * )'); + expect(await queryBar.getQueryString()).be.eql( + '( ( ip: 220.120.146.16 ) OR ( ip: 220.120.146.16 ) OR ( ip: 220.120.146.16 ) OR ( ip: 220.120.146.16 ) OR ( ip: 220.120.146.16 ) OR ( ip: 220.120.146.16 ) OR ( ip: 152.56.56.106 ) OR ( ip: 152.56.56.106 ) OR ( ip: 111.55.80.52 ) OR ( ip: 111.55.80.52 ) OR ( ip: 111.55.80.52 ) OR ( ip: 111.55.80.52 ) OR ( ip: 111.55.80.52 ) OR ( ip: 111.55.80.52 ) )' + ); const filterPills = await filterBar.getFiltersLabel(); expect(filterPills.length).to.be(1); expect(filterPills[0]).to.be('Lens context (lucene)'); @@ -123,7 +125,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const columns = await PageObjects.discover.getColumnHeaders(); expect(columns).to.eql(['ip', '@timestamp', 'bytes', 'memory']); // check the query - expect(await queryBar.getQueryString()).be.eql('( ( bytes > 6000 ) AND ( ip: * ) )'); + expect(await queryBar.getQueryString()).be.eql( + '( ( bytes > 6000 ) AND ( ( ip: 97.220.3.248 ) OR ( ip: 169.228.188.120 ) OR ( ip: 78.83.247.30 ) ) )' + ); }); it('should extract a filter from a formula global filter', async () => { @@ -152,7 +156,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await testSubjects.existOrFail('discoverChart'); // check the query - expect(await queryBar.getQueryString()).be.eql('( ( bytes > 4000 ) AND ( ip: * ) )'); + expect(await queryBar.getQueryString()).be.eql( + '( ( bytes > 4000 ) AND ( ( ip: 97.220.3.248 ) OR ( ip: 169.228.188.120 ) OR ( ip: 78.83.247.30 ) ) )' + ); }); }); } From 83a08d309af9e9e300782d581a0a2b5c6fede9d5 Mon Sep 17 00:00:00 2001 From: dej611 Date: Thu, 24 Feb 2022 09:43:54 +0100 Subject: [PATCH 14/31] :sparkles: Make link open a new window with discover --- .../navigation/public/top_nav_menu/top_nav_menu_data.tsx | 2 ++ .../navigation/public/top_nav_menu/top_nav_menu_item.tsx | 9 ++++++++- x-pack/plugins/lens/public/app_plugin/lens_top_nav.tsx | 8 +++++--- x-pack/plugins/lens/public/app_plugin/types.ts | 2 +- 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/plugins/navigation/public/top_nav_menu/top_nav_menu_data.tsx b/src/plugins/navigation/public/top_nav_menu/top_nav_menu_data.tsx index b74fe5249e66c..1c55519e23256 100644 --- a/src/plugins/navigation/public/top_nav_menu/top_nav_menu_data.tsx +++ b/src/plugins/navigation/public/top_nav_menu/top_nav_menu_data.tsx @@ -24,6 +24,8 @@ export interface TopNavMenuData { isLoading?: boolean; iconType?: string; iconSide?: EuiButtonProps['iconSide']; + target?: string; + href?: string; } export interface RegisteredTopNavMenuData extends TopNavMenuData { diff --git a/src/plugins/navigation/public/top_nav_menu/top_nav_menu_item.tsx b/src/plugins/navigation/public/top_nav_menu/top_nav_menu_item.tsx index 721a0fae0e62f..495e5c4ac9593 100644 --- a/src/plugins/navigation/public/top_nav_menu/top_nav_menu_item.tsx +++ b/src/plugins/navigation/public/top_nav_menu/top_nav_menu_item.tsx @@ -50,12 +50,19 @@ export function TopNavMenuItem(props: TopNavMenuData) { className: props.className, }; + // If the item specified a href, then override the suppress the onClick + // and make it become a regular link + const overrideProps = + props.target && props.href + ? { onClick: undefined, href: props.href, target: props.target } + : {}; + const btn = props.emphasize ? ( {getButtonContainer()} ) : ( - + {getButtonContainer()} ); diff --git a/x-pack/plugins/lens/public/app_plugin/lens_top_nav.tsx b/x-pack/plugins/lens/public/app_plugin/lens_top_nav.tsx index 9e31873e13803..cd8a1ca2ea4a4 100644 --- a/x-pack/plugins/lens/public/app_plugin/lens_top_nav.tsx +++ b/x-pack/plugins/lens/public/app_plugin/lens_top_nav.tsx @@ -100,13 +100,15 @@ function getLensTopNavConfig(options: { if (showOpenInDiscover) { topNavMenu.push({ label: getShowUnderlyingDataLabel(), - run: actions.showUnderlyingData, + run: () => {}, testId: 'lnsApp_openInDiscover', description: i18n.translate('xpack.lens.app.openInDiscoverAriaLabel', { defaultMessage: 'Open underlying data in Discover', }), disableButton: Boolean(tooltips.showUnderlyingDataWarning()), tooltip: tooltips.showUnderlyingDataWarning, + target: '_blank', + href: actions.getUnderlyingDataUrl(), }); } @@ -433,7 +435,7 @@ export const LensTopNavMenu = ({ redirectToOrigin(); } }, - showUnderlyingData: () => { + getUnderlyingDataUrl: () => { if (!canShowUnderlyingData) { return; } @@ -450,7 +452,7 @@ export const LensTopNavMenu = ({ indexPatterns ); - discover.locator!.navigate({ + return discover.locator!.getRedirectUrl({ indexPatternId: meta.id, timeRange: data.query.timefilter.timefilter.getTime(), filters: newFilters, diff --git a/x-pack/plugins/lens/public/app_plugin/types.ts b/x-pack/plugins/lens/public/app_plugin/types.ts index d719a85cf06a4..dee72d4aaffeb 100644 --- a/x-pack/plugins/lens/public/app_plugin/types.ts +++ b/x-pack/plugins/lens/public/app_plugin/types.ts @@ -152,5 +152,5 @@ export interface LensTopNavActions { goBack: () => void; cancel: () => void; exportToCSV: () => void; - showUnderlyingData: () => void; + getUnderlyingDataUrl: () => string | undefined; } From 655a35e221056c96cfacc29968324622977026d9 Mon Sep 17 00:00:00 2001 From: dej611 Date: Thu, 24 Feb 2022 09:44:26 +0100 Subject: [PATCH 15/31] :white_check_make: Update functional tests to deal with new tab --- .../apps/lens/show_underlying_data.ts | 32 +++++++++++++------ 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/x-pack/test/functional/apps/lens/show_underlying_data.ts b/x-pack/test/functional/apps/lens/show_underlying_data.ts index 04c4b33358a79..62eb931bc377c 100644 --- a/x-pack/test/functional/apps/lens/show_underlying_data.ts +++ b/x-pack/test/functional/apps/lens/show_underlying_data.ts @@ -14,6 +14,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const listingTable = getService('listingTable'); const testSubjects = getService('testSubjects'); const find = getService('find'); + const browser = getService('browser'); describe('show underlying data', () => { it('should show the open button for a compatible saved visualization', async () => { @@ -27,13 +28,16 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await testSubjects.clickWhenNotDisabled(`lnsApp_openInDiscover`); - // discard the changes and navigate to Discover - await testSubjects.click('confirmModalConfirmButton'); + const [lensWindowHandler, discoverWindowHandle] = await browser.getAllWindowHandles(); + await browser.switchToWindow(discoverWindowHandle); + await PageObjects.header.waitUntilLoadingHasFinished(); await testSubjects.existOrFail('discoverChart'); // check the table columns const columns = await PageObjects.discover.getColumnHeaders(); expect(columns).to.eql(['ip', '@timestamp', 'bytes']); + await browser.closeCurrentWindow(); + await browser.switchToWindow(lensWindowHandler); }); it('should ignore the top values column if other category is enabled', async () => { @@ -55,11 +59,13 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await testSubjects.clickWhenNotDisabled(`lnsApp_openInDiscover`); - // discard the changes and navigate to Discover - await testSubjects.click('confirmModalConfirmButton'); + const [lensWindowHandler, discoverWindowHandle] = await browser.getAllWindowHandles(); + await browser.switchToWindow(discoverWindowHandle); await PageObjects.header.waitUntilLoadingHasFinished(); await testSubjects.existOrFail('discoverChart'); expect(await queryBar.getQueryString()).be.eql(''); + await browser.closeCurrentWindow(); + await browser.switchToWindow(lensWindowHandler); }); it('should show the open button for a compatible saved visualization with a lucene query', async () => { @@ -83,8 +89,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await testSubjects.clickWhenNotDisabled(`lnsApp_openInDiscover`); - // discard the changes and navigate to Discover - await testSubjects.click('confirmModalConfirmButton'); + const [lensWindowHandler, discoverWindowHandle] = await browser.getAllWindowHandles(); + await browser.switchToWindow(discoverWindowHandle); await PageObjects.header.waitUntilLoadingHasFinished(); await testSubjects.existOrFail('discoverChart'); // check the query @@ -94,6 +100,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const filterPills = await filterBar.getFiltersLabel(); expect(filterPills.length).to.be(1); expect(filterPills[0]).to.be('Lens context (lucene)'); + await browser.closeCurrentWindow(); + await browser.switchToWindow(lensWindowHandler); }); it('should show the underlying data extracting all filters and columsn from a formula', async () => { @@ -117,8 +125,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { // expect the button is shown and enabled await testSubjects.clickWhenNotDisabled(`lnsApp_openInDiscover`); - // discard the changes and navigate to Discover - await testSubjects.click('confirmModalConfirmButton'); + const [lensWindowHandler, discoverWindowHandle] = await browser.getAllWindowHandles(); + await browser.switchToWindow(discoverWindowHandle); await PageObjects.header.waitUntilLoadingHasFinished(); await testSubjects.existOrFail('discoverChart'); // check the columns @@ -128,6 +136,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { expect(await queryBar.getQueryString()).be.eql( '( ( bytes > 6000 ) AND ( ( ip: 97.220.3.248 ) OR ( ip: 169.228.188.120 ) OR ( ip: 78.83.247.30 ) ) )' ); + await browser.closeCurrentWindow(); + await browser.switchToWindow(lensWindowHandler); }); it('should extract a filter from a formula global filter', async () => { @@ -150,8 +160,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { // expect the button is shown and enabled await testSubjects.clickWhenNotDisabled(`lnsApp_openInDiscover`); - // discard the changes and navigate to Discover - await testSubjects.click('confirmModalConfirmButton'); + const [lensWindowHandler, discoverWindowHandle] = await browser.getAllWindowHandles(); + await browser.switchToWindow(discoverWindowHandle); await PageObjects.header.waitUntilLoadingHasFinished(); await testSubjects.existOrFail('discoverChart'); @@ -159,6 +169,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { expect(await queryBar.getQueryString()).be.eql( '( ( bytes > 4000 ) AND ( ( ip: 97.220.3.248 ) OR ( ip: 169.228.188.120 ) OR ( ip: 78.83.247.30 ) ) )' ); + await browser.closeCurrentWindow(); + await browser.switchToWindow(lensWindowHandler); }); }); } From f9f95fda4b79dc89744c6dda09226982ad775fcd Mon Sep 17 00:00:00 2001 From: dej611 Date: Thu, 24 Feb 2022 16:09:35 +0100 Subject: [PATCH 16/31] :bug: Fix transposed table case: make it use fallback --- .../public/indexpattern_datasource/utils.tsx | 38 ++++++++++--------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/utils.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/utils.tsx index 2046895e0f7a9..bf3cbe190da6a 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/utils.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/utils.tsx @@ -34,6 +34,7 @@ import { checkColumnForPrecisionError, Query } from '../../../../../src/plugins/ import { hasField } from './pure_utils'; import { mergeLayer } from './state_helpers'; import { DEFAULT_MAX_DOC_COUNT, supportsRarityRanking } from './operations/definitions/terms'; +import { getOriginalId } from '../../common/expressions'; export function isColumnInvalid( layer: IndexPatternLayer, @@ -318,8 +319,9 @@ export function getFiltersInLayer( isColumnOfType('terms', column) && !(column.params.otherBucket || column.params.missingBucket) ) { - // Fallback in case of no data: just return the field existence - if (!layerData) { + // Fallback in case of no data or transposed data: just return the field existence + const dataId = layerData?.columns.find(({ id }) => getOriginalId(id) === colId)?.id; + if (!layerData || !dataId || dataId !== colId) { return { kuery: [{ query: `${column.sourceField}: *`, language: 'kuery' }].concat( column.params.secondaryFields?.map((field) => ({ @@ -332,23 +334,25 @@ export function getFiltersInLayer( const fields = [column.sourceField] .concat(column.params.secondaryFields || []) .filter(Boolean) as string[]; - // extract the filters from the columns of the activeData - const queryWithTerms = layerData.rows - .map(({ [colId]: value }) => { - if (typeof value !== 'string' && Array.isArray(value.keys)) { - return { - query: value.keys - .map((term: string, index: number) => `${fields[index]}: ${term || "''"}`) - .join(' AND '), - language: 'kuery', - }; - } - return { query: `${column.sourceField}: ${value}`, language: 'kuery' }; - }) - .filter(Boolean) as Query[]; + // extract the filters from the columns of the activeData return { - kuery: queryWithTerms, + kuery: layerData.rows + .map(({ [colId]: value }) => { + if (value == null) { + return; + } + if (typeof value !== 'string' && Array.isArray(value.keys)) { + return { + query: value.keys + .map((term: string, index: number) => `${fields[index]}: ${term || "''"}`) + .join(' AND '), + language: 'kuery', + }; + } + return { query: `${column.sourceField}: ${value}`, language: 'kuery' }; + }) + .filter(Boolean) as Query[], }; } }) From c72219fa4975d0d1fb74b1ef7e62b57700340b98 Mon Sep 17 00:00:00 2001 From: dej611 Date: Thu, 24 Feb 2022 17:46:04 +0100 Subject: [PATCH 17/31] :bug: Fix unit tests --- .../indexpattern_datasource/indexpattern.test.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts index 8ad1ee9f6a1de..1246babc7bc60 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts @@ -1569,7 +1569,10 @@ describe('IndexPattern Data Source', () => { const data = { first: { type: 'datatable' as const, - columns: [], + columns: [ + { id: 'col1', name: 'geo.src', meta: { type: 'string' as const } }, + { id: 'col2', name: 'geo.dest > myField', meta: { type: 'string' as const } }, + ], rows: [ { col1: 'US', col2: { keys: ['IT', 'MyValue'] } }, { col1: 'IN', col2: { keys: ['DE', 'MyOtherValue'] } }, @@ -1633,7 +1636,10 @@ describe('IndexPattern Data Source', () => { const data = { first: { type: 'datatable' as const, - columns: [], + columns: [ + { id: 'col1', name: 'geo.src', meta: { type: 'string' as const } }, + { id: 'col2', name: 'geo.dest > myField', meta: { type: 'string' as const } }, + ], rows: [ { col1: 'US', col2: { keys: ['IT', ''] } }, { col1: 'IN', col2: { keys: ['DE', 'MyOtherValue'] } }, From 6124dbc22f2bf365af47193b111c3308824f845a Mon Sep 17 00:00:00 2001 From: dej611 Date: Tue, 1 Mar 2022 10:43:58 +0100 Subject: [PATCH 18/31] :ok_hand: Address review feedback --- .../public/app_plugin/show_underlying_data.ts | 7 +- .../indexpattern.test.ts | 3 +- .../indexpattern_datasource/indexpattern.tsx | 4 +- .../operations/layer_helpers.test.ts | 10 +- .../operations/layer_helpers.ts | 9 +- .../public/indexpattern_datasource/utils.tsx | 170 ++++++++++++------ 6 files changed, 135 insertions(+), 68 deletions(-) diff --git a/x-pack/plugins/lens/public/app_plugin/show_underlying_data.ts b/x-pack/plugins/lens/public/app_plugin/show_underlying_data.ts index 42915d6ef8844..a8c678167f70f 100644 --- a/x-pack/plugins/lens/public/app_plugin/show_underlying_data.ts +++ b/x-pack/plugins/lens/public/app_plugin/show_underlying_data.ts @@ -129,6 +129,11 @@ export function getLayerMetaInfo( }; } +// This enforces on assignment time that the two props are not the same +type LanguageAssignments = + | { queryLanguage: 'lucene'; filtersLanguage: 'kuery' } + | { queryLanguage: 'kuery'; filtersLanguage: 'lucene' }; + export function combineQueryAndFilters( query: Query | undefined, filters: Filter[], @@ -136,7 +141,7 @@ export function combineQueryAndFilters( dataViews: DataViewBase[] | undefined ) { // Unless a lucene query is already defined, kuery is assigned to it - const { queryLanguage, filtersLanguage }: Record = + const { queryLanguage, filtersLanguage }: LanguageAssignments = query?.language === 'lucene' ? { queryLanguage: 'lucene', filtersLanguage: 'kuery' } : { queryLanguage: 'kuery', filtersLanguage: 'lucene' }; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts index 1246babc7bc60..5c441a56dd8fb 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts @@ -1294,7 +1294,8 @@ describe('IndexPattern Data Source', () => { }, layerId: 'first', }); - + // The cumulative sum column has no field, but it references a sum column (hidden) which has it + // The getTableSpec() should walk the reference tree and assign all fields to the root column expect(publicAPI.getTableSpec()).toEqual([{ columnId: 'col2', fields: ['test'] }]); }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx index 7d1b24d2d909c..b74fc83796dd6 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx @@ -56,7 +56,7 @@ import { insertNewColumn, TermsIndexPatternColumn, } from './operations'; -import { getTopColumnFromReference } from './operations/layer_helpers'; +import { getReferenceRoot } from './operations/layer_helpers'; import { IndexPatternField, IndexPatternPrivateState, @@ -466,7 +466,7 @@ export function getIndexPatternDatasource({ // but map fields to the top referencing column const fieldsPerColumn: Record = {}; Object.keys(layer.columns).forEach((colId) => { - const visibleColumnId = getTopColumnFromReference(layer, colId); + const visibleColumnId = getReferenceRoot(layer, colId); fieldsPerColumn[visibleColumnId] = fieldsPerColumn[visibleColumnId] || []; const column = layer.columns[colId]; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts index 51dca2acdedd5..f7a8df3d5ef1f 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts @@ -17,7 +17,7 @@ import { getErrorMessages, hasTermsWithManyBuckets, isReferenced, - getTopColumnFromReference, + getReferenceRoot, } from './layer_helpers'; import { operationDefinitionMap, OperationType } from '../operations'; import { TermsIndexPatternColumn } from './definitions/terms'; @@ -3232,7 +3232,7 @@ describe('state_helpers', () => { }); }); - describe('getTopColumnFromReference', () => { + describe('getReferenceRoot', () => { it("should just return the column id itself if it's not a referenced column", () => { const layer: IndexPatternLayer = { indexPatternId: '1', @@ -3254,7 +3254,7 @@ describe('state_helpers', () => { }, }, }; - expect(getTopColumnFromReference(layer, 'col1')).toEqual('col1'); + expect(getReferenceRoot(layer, 'col1')).toEqual('col1'); }); it('should return the top column if a referenced column is passed', () => { @@ -3278,7 +3278,7 @@ describe('state_helpers', () => { }, }, }; - expect(getTopColumnFromReference(layer, 'col2')).toEqual('col1'); + expect(getReferenceRoot(layer, 'col2')).toEqual('col1'); }); it('should work for a formula chain', () => { @@ -3336,7 +3336,7 @@ describe('state_helpers', () => { } as MathIndexPatternColumn, }, }; - expect(getTopColumnFromReference(layer, 'formulaX0')).toEqual('source'); + expect(getReferenceRoot(layer, 'formulaX0')).toEqual('source'); }); }); }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts index 603791970cf89..ba0883cee0bcf 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts @@ -1427,7 +1427,14 @@ const computeReferenceLookup = memoizeOne((layer: IndexPatternLayer): Record input).filter(({ query }) => query?.trim() && query !== '*'); +} + +/** + * Given an Interval column in range mode transform the ranges into KQL queries + */ +function extractQueriesFromRanges(column: RangeIndexPatternColumn) { + return column.params.ranges + .map(({ from, to }) => { + let rangeQuery = ''; + if (from != null && isFinite(from)) { + rangeQuery += `${column.sourceField} >= ${from}`; + } + if (to != null && isFinite(to)) { + if (rangeQuery.length) { + rangeQuery += ' AND '; + } + rangeQuery += `${column.sourceField} <= ${to}`; + } + return { + query: rangeQuery, + language: 'kuery', + }; + }) + .filter(({ query }) => query?.trim()); +} + +/** + * Given an Terms/Top values column transform each entry into a "field: term" KQL query + * This works also for multi-terms variant + */ +function extractQueriesFromTerms( + column: TermsIndexPatternColumn, + colId: string, + data: NonNullable[string] +) { + const fields = [column.sourceField] + .concat(column.params.secondaryFields || []) + .filter(Boolean) as string[]; + + // extract the filters from the columns of the activeData + return data.rows + .map(({ [colId]: value }) => { + if (value == null) { + return; + } + if (typeof value !== 'string' && Array.isArray(value.keys)) { + return { + query: value.keys + .map((term: string, index: number) => `${fields[index]}: ${term || "''"}`) + .join(' AND '), + language: 'kuery', + }; + } + return { query: `${column.sourceField}: ${value}`, language: 'kuery' }; + }) + .filter(Boolean) as Query[]; +} + +/** + * Used for a Terms column to decide whether to use a simple existence query (fallback) instead + * of more specific queries. + * The check targets the scenarios where no data is available, or when there's a transposed table + * and it's not yet possible to track it back to the original table + */ +function shouldUseTermsFallback( + data: NonNullable[string] | undefined, + colId: string +) { + const dataId = data?.columns.find(({ id }) => getOriginalId(id) === colId)?.id; + return !dataId || dataId !== colId; +} + +interface GroupedQueries { + kuery?: Query[]; + lucene?: Query[]; +} + +function collectOnlyValidQueries( + filteredQueries: GroupedQueries, + operationQueries: GroupedQueries[], + queryLanguage: 'kuery' | 'lucene' +) { + return [ + filteredQueries[queryLanguage], + ...operationQueries.map(({ [queryLanguage]: filter }) => filter), + ].filter((filters) => filters?.length) as Query[][]; +} + export function getFiltersInLayer( layer: IndexPatternLayer, columnIds: string[], @@ -268,16 +367,10 @@ export function getFiltersInLayer( // filter out empty filters as well .filter((filter) => filter?.query?.trim()) as Query[]; - const { kuery: kqlMetricQueries, lucene: luceneMetricQueries } = groupBy( + const filteredQueriesByLanguage = groupBy( filteredMetrics, 'language' - ); - - function extractUsefulQueries( - queries: FiltersIndexPatternColumn['params']['filters'] | undefined - ) { - return queries?.map(({ input }) => input).filter(({ query }) => query?.trim() && query !== '*'); - } + ) as unknown as GroupedQueries; const filterOperation = columnIds .map((colId) => { @@ -288,40 +381,24 @@ export function getFiltersInLayer( column.params.filters, ({ input }) => input.language ) as Record<'lucene' | 'kuery', FiltersIndexPatternColumn['params']['filters']>; + return { - kuery: extractUsefulQueries(groupsByLanguage.kuery), - lucene: extractUsefulQueries(groupsByLanguage.lucene), + kuery: extractQueriesFromFilters(groupsByLanguage.kuery), + lucene: extractQueriesFromFilters(groupsByLanguage.lucene), }; } + if (isColumnOfType('range', column) && column.sourceField) { return { - kuery: column.params.ranges - .map(({ from, to }) => { - let rangeQuery = ''; - if (from != null && isFinite(from)) { - rangeQuery += `${column.sourceField} >= ${from}`; - } - if (to != null && isFinite(to)) { - if (rangeQuery.length) { - rangeQuery += ' AND '; - } - rangeQuery += `${column.sourceField} <= ${to}`; - } - return { - query: rangeQuery, - language: 'kuery', - }; - }) - .filter(({ query }) => query?.trim()), + kuery: extractQueriesFromRanges(column), }; } + if ( isColumnOfType('terms', column) && !(column.params.otherBucket || column.params.missingBucket) ) { - // Fallback in case of no data or transposed data: just return the field existence - const dataId = layerData?.columns.find(({ id }) => getOriginalId(id) === colId)?.id; - if (!layerData || !dataId || dataId !== colId) { + if (!layerData || shouldUseTermsFallback(layerData, colId)) { return { kuery: [{ query: `${column.sourceField}: *`, language: 'kuery' }].concat( column.params.secondaryFields?.map((field) => ({ @@ -331,38 +408,15 @@ export function getFiltersInLayer( ), }; } - const fields = [column.sourceField] - .concat(column.params.secondaryFields || []) - .filter(Boolean) as string[]; - // extract the filters from the columns of the activeData return { - kuery: layerData.rows - .map(({ [colId]: value }) => { - if (value == null) { - return; - } - if (typeof value !== 'string' && Array.isArray(value.keys)) { - return { - query: value.keys - .map((term: string, index: number) => `${fields[index]}: ${term || "''"}`) - .join(' AND '), - language: 'kuery', - }; - } - return { query: `${column.sourceField}: ${value}`, language: 'kuery' }; - }) - .filter(Boolean) as Query[], + kuery: extractQueriesFromTerms(column, colId, layerData), }; } }) - .filter(Boolean) as Array<{ kuery?: Query[]; lucene?: Query[] }>; + .filter(Boolean) as GroupedQueries[]; return { - kuery: [kqlMetricQueries, ...filterOperation.map(({ kuery }) => kuery)].filter( - (filters) => filters?.length - ) as Query[][], - lucene: [luceneMetricQueries, ...filterOperation.map(({ lucene }) => lucene)].filter( - (filters) => filters?.length - ) as Query[][], + kuery: collectOnlyValidQueries(filteredQueriesByLanguage, filterOperation, 'kuery'), + lucene: collectOnlyValidQueries(filteredQueriesByLanguage, filterOperation, 'lucene'), }; } From b30862ee21332665f7f94dbd1835cee67e907755 Mon Sep 17 00:00:00 2001 From: dej611 Date: Wed, 2 Mar 2022 10:27:52 +0100 Subject: [PATCH 19/31] :bug: Skip filtered metrics if there's at least an unfiltered one --- .../indexpattern.test.ts | 109 ++++++++++++++++++ .../public/indexpattern_datasource/utils.tsx | 70 +++++++---- 2 files changed, 155 insertions(+), 24 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts index 5c441a56dd8fb..3783bcc9b02a5 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts @@ -1878,6 +1878,115 @@ describe('IndexPattern Data Source', () => { ], }); }); + it('should ignore filtered metrics if at least one metric is unfiltered', () => { + publicAPI = indexPatternDatasource.getPublicAPI({ + state: { + ...enrichBaseState(baseState), + layers: { + first: { + indexPatternId: '1', + columnOrder: ['col1', 'col2'], + columns: { + col1: { + label: 'Sum', + dataType: 'number', + isBucketed: false, + operationType: 'sum', + sourceField: 'test', + params: {}, + filter: { language: 'kuery', query: 'bytes > 1000' }, + } as GenericIndexPatternColumn, + col2: { + label: 'Sum', + dataType: 'number', + isBucketed: false, + operationType: 'sum', + sourceField: 'test', + params: {}, + } as GenericIndexPatternColumn, + }, + }, + }, + }, + layerId: 'first', + }); + expect(publicAPI.getFilters()).toEqual({ + kuery: [], + lucene: [], + }); + }); + it('should ignore filtered metrics if at least one metric is unfiltered in formula', () => { + publicAPI = indexPatternDatasource.getPublicAPI({ + state: { + ...enrichBaseState(baseState), + layers: { + first: { + indexPatternId: '1', + columnOrder: ['formula'], + columns: { + formula: { + label: 'Formula', + dataType: 'number', + operationType: 'formula', + isBucketed: false, + scale: 'ratio', + params: { + formula: "count(kql='memory > 5000') + count()", + isFormulaBroken: false, + }, + references: ['math'], + } as FormulaIndexPatternColumn, + countX0: { + label: 'countX0', + dataType: 'number', + operationType: 'count', + isBucketed: false, + scale: 'ratio', + sourceField: '___records___', + customLabel: true, + filter: { language: 'kuery', query: 'memory > 5000' }, + }, + countX1: { + label: 'countX1', + dataType: 'number', + operationType: 'count', + isBucketed: false, + scale: 'ratio', + sourceField: '___records___', + customLabel: true, + }, + math: { + label: 'math', + dataType: 'number', + operationType: 'math', + isBucketed: false, + scale: 'ratio', + params: { + tinymathAst: { + type: 'function', + name: 'add', + args: ['countX0', 'countX1'] as unknown as TinymathAST[], + location: { + min: 0, + max: 17, + }, + text: "count(kql='memory > 5000') + count()", + }, + }, + references: ['countX0', 'countX1'], + customLabel: true, + } as MathIndexPatternColumn, + }, + }, + }, + }, + layerId: 'first', + }); + expect(publicAPI.getFilters()).toEqual({ + kuery: [], + lucene: [], + }); + }); it('should support complete scenarios', () => { publicAPI = indexPatternDatasource.getPublicAPI({ state: { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/utils.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/utils.tsx index 2ef793ec90cd4..577a26137bffd 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/utils.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/utils.tsx @@ -326,6 +326,48 @@ function shouldUseTermsFallback( return !dataId || dataId !== colId; } +/** + * Collect filters from metrics: + * * if there's at least one unfiltered metric, then just return an empty list of filters + * * otherwise get all the filters, with the only exception of those from formula (referenced columns will have it anyway) + */ +function collectFiltersFromMetrics(layer: IndexPatternLayer) { + // Isolate filtered metrics first + // mind to ignore non-filterable columns + const metricColumns = Object.keys(layer.columns).filter((colId) => { + const column = layer.columns[colId]; + const operationDefinition = operationDefinitionMap[column?.operationType]; + return !column?.isBucketed && operationDefinition?.filterable; + }); + const { filtered = [], unfiltered = [] } = groupBy(metricColumns, (colId) => + layer.columns[colId]?.filter ? 'filtered' : 'unfiltered' + ); + + // extract filters from filtered metrics + // consider all the columns, included referenced ones to cover also the formula case + return ( + filtered + // if there are metric columns not filtered, then ignore filtered columns completely + .filter(() => !unfiltered.length) + .map((colId) => { + // there's a special case to handle when a formula has a global filter applied + // in this case ignore the filter on the formula column and only use the filter + // applied to the referenced columns. + // This will avoid duplicate filters and issues when a global formula filter is + // combine to specific referenced columns filters + if ( + isColumnOfType('formula', layer.columns[colId]) && + layer.columns[colId]?.filter + ) { + return null; + } + return layer.columns[colId]?.filter; + }) + // filter out empty filters as well + .filter((filter) => filter?.query?.trim()) as Query[] + ); +} + interface GroupedQueries { kuery?: Query[]; lucene?: Query[]; @@ -347,28 +389,8 @@ export function getFiltersInLayer( columnIds: string[], layerData: NonNullable[string] | undefined ) { - // extract filters from filtered metrics - // consider all the columns, included referenced ones to cover also the formula case - const filteredMetrics = Object.keys(layer.columns) - .map((colId) => { - // there's a special case to handle when a formula has a global filter applied - // in this case ignore the filter on the formula column and only use the filter - // applied to the referenced columns. - // This will avoid duplicate filters and issues when a global formula filter is - // combine to specific referenced columns filters - if ( - isColumnOfType('formula', layer.columns[colId]) && - layer.columns[colId]?.filter - ) { - return null; - } - return layer.columns[colId]?.filter; - }) - // filter out empty filters as well - .filter((filter) => filter?.query?.trim()) as Query[]; - - const filteredQueriesByLanguage = groupBy( - filteredMetrics, + const filtersFromMetricsByLanguage = groupBy( + collectFiltersFromMetrics(layer), 'language' ) as unknown as GroupedQueries; @@ -416,7 +438,7 @@ export function getFiltersInLayer( }) .filter(Boolean) as GroupedQueries[]; return { - kuery: collectOnlyValidQueries(filteredQueriesByLanguage, filterOperation, 'kuery'), - lucene: collectOnlyValidQueries(filteredQueriesByLanguage, filterOperation, 'lucene'), + kuery: collectOnlyValidQueries(filtersFromMetricsByLanguage, filterOperation, 'kuery'), + lucene: collectOnlyValidQueries(filtersFromMetricsByLanguage, filterOperation, 'lucene'), }; } From 1a8153fbbc394582fd85a3280d41ae88218f09aa Mon Sep 17 00:00:00 2001 From: dej611 Date: Wed, 2 Mar 2022 10:59:51 +0100 Subject: [PATCH 20/31] :bug: Improve string escaping strategy --- .../indexpattern.test.ts | 16 ++++++++-------- .../public/indexpattern_datasource/utils.tsx | 17 ++++++++--------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts index 3783bcc9b02a5..42dd23a6d2242 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts @@ -1583,12 +1583,12 @@ describe('IndexPattern Data Source', () => { expect(publicAPI.getFilters(data)).toEqual({ kuery: [ [ - { language: 'kuery', query: 'geo.src: US' }, - { language: 'kuery', query: 'geo.src: IN' }, + { language: 'kuery', query: 'geo.src: "US"' }, + { language: 'kuery', query: 'geo.src: "IN"' }, ], [ - { language: 'kuery', query: 'geo.dest: IT AND myField: MyValue' }, - { language: 'kuery', query: 'geo.dest: DE AND myField: MyOtherValue' }, + { language: 'kuery', query: 'geo.dest: "IT" AND myField: "MyValue"' }, + { language: 'kuery', query: 'geo.dest: "DE" AND myField: "MyOtherValue"' }, ], ], lucene: [], @@ -1650,12 +1650,12 @@ describe('IndexPattern Data Source', () => { expect(publicAPI.getFilters(data)).toEqual({ kuery: [ [ - { language: 'kuery', query: 'geo.src: US' }, - { language: 'kuery', query: 'geo.src: IN' }, + { language: 'kuery', query: 'geo.src: "US"' }, + { language: 'kuery', query: 'geo.src: "IN"' }, ], [ - { language: 'kuery', query: "geo.dest: IT AND myField: ''" }, - { language: 'kuery', query: 'geo.dest: DE AND myField: MyOtherValue' }, + { language: 'kuery', query: `geo.dest: "IT" AND myField: ""` }, + { language: 'kuery', query: `geo.dest: "DE" AND myField: "MyOtherValue"` }, ], ], lucene: [], diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/utils.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/utils.tsx index 577a26137bffd..a7d82e29a4aa0 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/utils.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/utils.tsx @@ -12,7 +12,7 @@ import type { DocLinksStart } from 'kibana/public'; import { EuiLink, EuiTextColor, EuiButton, EuiSpacer } from '@elastic/eui'; import { DatatableColumn } from 'src/plugins/expressions'; -import { groupBy } from 'lodash'; +import { groupBy, escape } from 'lodash'; import type { FramePublicAPI, StateSetter } from '../types'; import type { IndexPattern, IndexPatternLayer, IndexPatternPrivateState } from './types'; import type { ReferenceBasedIndexPatternColumn } from './operations/definitions/column_types'; @@ -302,12 +302,12 @@ function extractQueriesFromTerms( if (typeof value !== 'string' && Array.isArray(value.keys)) { return { query: value.keys - .map((term: string, index: number) => `${fields[index]}: ${term || "''"}`) + .map((term: string, index: number) => `${fields[index]}: ${`"${escape(term)}"`}`) .join(' AND '), language: 'kuery', }; } - return { query: `${column.sourceField}: ${value}`, language: 'kuery' }; + return { query: `${column.sourceField}: ${`"${escape(value)}"`}`, language: 'kuery' }; }) .filter(Boolean) as Query[]; } @@ -421,13 +421,12 @@ export function getFiltersInLayer( !(column.params.otherBucket || column.params.missingBucket) ) { if (!layerData || shouldUseTermsFallback(layerData, colId)) { + const fields = operationDefinitionMap[column.operationType]!.getCurrentFields!(column); return { - kuery: [{ query: `${column.sourceField}: *`, language: 'kuery' }].concat( - column.params.secondaryFields?.map((field) => ({ - query: `${field}: *`, - language: 'kuery', - })) || [] - ), + kuery: fields.map((field) => ({ + query: `${field}: *`, + language: 'kuery', + })), }; } From 0111cca78323bc737a9bbc19fbc7a72203e3b3e7 Mon Sep 17 00:00:00 2001 From: dej611 Date: Wed, 2 Mar 2022 17:56:35 +0100 Subject: [PATCH 21/31] :bug: Fix functional tests and improved filters dedup checks --- .../public/indexpattern_datasource/utils.tsx | 48 ++++++++--------- .../apps/lens/show_underlying_data.ts | 51 ++++++++++--------- 2 files changed, 46 insertions(+), 53 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/utils.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/utils.tsx index a7d82e29a4aa0..401a4d9dcef82 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/utils.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/utils.tsx @@ -288,28 +288,28 @@ function extractQueriesFromTerms( column: TermsIndexPatternColumn, colId: string, data: NonNullable[string] -) { +): Query[] { const fields = [column.sourceField] .concat(column.params.secondaryFields || []) .filter(Boolean) as string[]; // extract the filters from the columns of the activeData - return data.rows + const queries = data.rows .map(({ [colId]: value }) => { if (value == null) { return; } if (typeof value !== 'string' && Array.isArray(value.keys)) { - return { - query: value.keys - .map((term: string, index: number) => `${fields[index]}: ${`"${escape(term)}"`}`) - .join(' AND '), - language: 'kuery', - }; + return value.keys + .map((term: string, index: number) => `${fields[index]}: ${`"${escape(term)}"`}`) + .join(' AND '); } - return { query: `${column.sourceField}: ${`"${escape(value)}"`}`, language: 'kuery' }; + return `${column.sourceField}: ${`"${escape(value)}"`}`; }) - .filter(Boolean) as Query[]; + .filter(Boolean) as string[]; + + // dedup queries before returning + return [...new Set(queries)].map((query) => ({ language: 'kuery', query })); } /** @@ -331,13 +331,18 @@ function shouldUseTermsFallback( * * if there's at least one unfiltered metric, then just return an empty list of filters * * otherwise get all the filters, with the only exception of those from formula (referenced columns will have it anyway) */ -function collectFiltersFromMetrics(layer: IndexPatternLayer) { +function collectFiltersFromMetrics(layer: IndexPatternLayer, columnIds: string[]) { // Isolate filtered metrics first - // mind to ignore non-filterable columns + // mind to ignore non-filterable columns and formula columns const metricColumns = Object.keys(layer.columns).filter((colId) => { const column = layer.columns[colId]; const operationDefinition = operationDefinitionMap[column?.operationType]; - return !column?.isBucketed && operationDefinition?.filterable; + return ( + !column?.isBucketed && + // global filters for formulas are picked up by referenced columns + !isColumnOfType('formula', column) && + operationDefinition?.filterable + ); }); const { filtered = [], unfiltered = [] } = groupBy(metricColumns, (colId) => layer.columns[colId]?.filter ? 'filtered' : 'unfiltered' @@ -349,20 +354,7 @@ function collectFiltersFromMetrics(layer: IndexPatternLayer) { filtered // if there are metric columns not filtered, then ignore filtered columns completely .filter(() => !unfiltered.length) - .map((colId) => { - // there's a special case to handle when a formula has a global filter applied - // in this case ignore the filter on the formula column and only use the filter - // applied to the referenced columns. - // This will avoid duplicate filters and issues when a global formula filter is - // combine to specific referenced columns filters - if ( - isColumnOfType('formula', layer.columns[colId]) && - layer.columns[colId]?.filter - ) { - return null; - } - return layer.columns[colId]?.filter; - }) + .map((colId) => layer.columns[colId]?.filter) // filter out empty filters as well .filter((filter) => filter?.query?.trim()) as Query[] ); @@ -390,7 +382,7 @@ export function getFiltersInLayer( layerData: NonNullable[string] | undefined ) { const filtersFromMetricsByLanguage = groupBy( - collectFiltersFromMetrics(layer), + collectFiltersFromMetrics(layer, columnIds), 'language' ) as unknown as GroupedQueries; diff --git a/x-pack/test/functional/apps/lens/show_underlying_data.ts b/x-pack/test/functional/apps/lens/show_underlying_data.ts index 62eb931bc377c..41fbe6fdec173 100644 --- a/x-pack/test/functional/apps/lens/show_underlying_data.ts +++ b/x-pack/test/functional/apps/lens/show_underlying_data.ts @@ -41,11 +41,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); it('should ignore the top values column if other category is enabled', async () => { - await PageObjects.visualize.gotoVisualizationLandingPage(); - await listingTable.searchForItemWithName('lnsXYvis'); - await PageObjects.lens.clickVisualizeListItemTitle('lnsXYvis'); - await PageObjects.lens.goToTimeRange(); - + // Make the breakdown dimention be ignored await PageObjects.lens.openDimensionEditor( 'lnsXY_splitDimensionPanel > lns-dimensionTrigger' ); @@ -69,10 +65,13 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); it('should show the open button for a compatible saved visualization with a lucene query', async () => { - await PageObjects.visualize.gotoVisualizationLandingPage(); - await listingTable.searchForItemWithName('lnsXYvis'); - await PageObjects.lens.clickVisualizeListItemTitle('lnsXYvis'); - await PageObjects.lens.goToTimeRange(); + // Make the breakdown dimention contribute to filters again + await PageObjects.lens.openDimensionEditor( + 'lnsXY_splitDimensionPanel > lns-dimensionTrigger' + ); + await testSubjects.click('indexPattern-terms-advanced'); + await testSubjects.click('indexPattern-terms-other-bucket'); + await PageObjects.lens.closeDimensionEditor(); // add a lucene query to the yDimension await PageObjects.lens.openDimensionEditor('lnsXY_yDimensionPanel > lns-dimensionTrigger'); @@ -81,11 +80,14 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await testSubjects.click('indexPattern-filter-by-input > switchQueryLanguageButton'); await testSubjects.click('languageToggle'); await testSubjects.click('indexPattern-filter-by-input > switchQueryLanguageButton'); + // apparently setting a filter requires some time before and after typing to work properly + await PageObjects.common.sleep(1000); await PageObjects.lens.setFilterBy('memory'); + await PageObjects.common.sleep(1000); + + await PageObjects.lens.closeDimensionEditor(); await PageObjects.lens.waitForVisualization(); - // expect the button is shown and enabled - // await PageObjects.common.sleep(15000); await testSubjects.clickWhenNotDisabled(`lnsApp_openInDiscover`); @@ -95,7 +97,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await testSubjects.existOrFail('discoverChart'); // check the query expect(await queryBar.getQueryString()).be.eql( - '( ( ip: 220.120.146.16 ) OR ( ip: 220.120.146.16 ) OR ( ip: 220.120.146.16 ) OR ( ip: 220.120.146.16 ) OR ( ip: 220.120.146.16 ) OR ( ip: 220.120.146.16 ) OR ( ip: 152.56.56.106 ) OR ( ip: 152.56.56.106 ) OR ( ip: 111.55.80.52 ) OR ( ip: 111.55.80.52 ) OR ( ip: 111.55.80.52 ) OR ( ip: 111.55.80.52 ) OR ( ip: 111.55.80.52 ) OR ( ip: 111.55.80.52 ) )' + '( ( ip: "220.120.146.16" ) OR ( ip: "152.56.56.106" ) OR ( ip: "111.55.80.52" ) )' ); const filterPills = await filterBar.getFiltersLabel(); expect(filterPills.length).to.be(1); @@ -104,16 +106,13 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await browser.switchToWindow(lensWindowHandler); }); - it('should show the underlying data extracting all filters and columsn from a formula', async () => { - await PageObjects.visualize.gotoVisualizationLandingPage(); - await listingTable.searchForItemWithName('lnsXYvis'); - await PageObjects.lens.clickVisualizeListItemTitle('lnsXYvis'); - await PageObjects.lens.goToTimeRange(); + it('should show the underlying data extracting all filters and columns from a formula', async () => { + await PageObjects.lens.removeDimension('lnsXY_yDimensionPanel'); await PageObjects.lens.configureDimension({ dimension: 'lnsXY_yDimensionPanel > lns-empty-dimension', operation: 'formula', - formula: `median(bytes) + average(memory, kql=`, + formula: `average(memory, kql=`, keepOpen: true, }); @@ -131,20 +130,17 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await testSubjects.existOrFail('discoverChart'); // check the columns const columns = await PageObjects.discover.getColumnHeaders(); - expect(columns).to.eql(['ip', '@timestamp', 'bytes', 'memory']); + expect(columns).to.eql(['ip', '@timestamp', 'memory']); // check the query expect(await queryBar.getQueryString()).be.eql( - '( ( bytes > 6000 ) AND ( ( ip: 97.220.3.248 ) OR ( ip: 169.228.188.120 ) OR ( ip: 78.83.247.30 ) ) )' + '( ( bytes > 6000 ) AND ( ( ip: "0.53.251.53" ) OR ( ip: "0.108.3.2" ) OR ( ip: "0.209.80.244" ) ) )' ); await browser.closeCurrentWindow(); await browser.switchToWindow(lensWindowHandler); }); it('should extract a filter from a formula global filter', async () => { - await PageObjects.visualize.gotoVisualizationLandingPage(); - await listingTable.searchForItemWithName('lnsXYvis'); - await PageObjects.lens.clickVisualizeListItemTitle('lnsXYvis'); - await PageObjects.lens.goToTimeRange(); + await PageObjects.lens.removeDimension('lnsXY_yDimensionPanel'); await PageObjects.lens.configureDimension({ dimension: 'lnsXY_yDimensionPanel > lns-empty-dimension', @@ -154,7 +150,12 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); await PageObjects.lens.enableFilter(); + // apparently setting a filter requires some time before and after typing to work properly + await PageObjects.common.sleep(1000); await PageObjects.lens.setFilterBy('bytes > 4000'); + await PageObjects.common.sleep(1000); + + await PageObjects.lens.closeDimensionEditor(); await PageObjects.lens.waitForVisualization(); // expect the button is shown and enabled @@ -167,7 +168,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { // check the query expect(await queryBar.getQueryString()).be.eql( - '( ( bytes > 4000 ) AND ( ( ip: 97.220.3.248 ) OR ( ip: 169.228.188.120 ) OR ( ip: 78.83.247.30 ) ) )' + '( ( bytes > 4000 ) AND ( ( ip: "0.53.251.53" ) OR ( ip: "0.108.3.2" ) OR ( ip: "0.209.80.244" ) ) )' ); await browser.closeCurrentWindow(); await browser.switchToWindow(lensWindowHandler); From 1f6682a73feacf31b888b13a8775de530f4788c6 Mon Sep 17 00:00:00 2001 From: dej611 Date: Thu, 3 Mar 2022 09:27:48 +0100 Subject: [PATCH 22/31] :white_check_mark: Fix functional tests for formula case --- x-pack/test/functional/apps/lens/show_underlying_data.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/x-pack/test/functional/apps/lens/show_underlying_data.ts b/x-pack/test/functional/apps/lens/show_underlying_data.ts index 41fbe6fdec173..dc8d9063cf374 100644 --- a/x-pack/test/functional/apps/lens/show_underlying_data.ts +++ b/x-pack/test/functional/apps/lens/show_underlying_data.ts @@ -119,7 +119,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const input = await find.activeElement(); await input.type(`bytes > 6000`); - await PageObjects.lens.closeDimensionEditor(); await PageObjects.lens.waitForVisualization(); // expect the button is shown and enabled await testSubjects.clickWhenNotDisabled(`lnsApp_openInDiscover`); @@ -155,8 +154,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.lens.setFilterBy('bytes > 4000'); await PageObjects.common.sleep(1000); - await PageObjects.lens.closeDimensionEditor(); - await PageObjects.lens.waitForVisualization(); // expect the button is shown and enabled await testSubjects.clickWhenNotDisabled(`lnsApp_openInDiscover`); From 972b4324cc312177df5a46a0b9a26a6f796d38b5 Mon Sep 17 00:00:00 2001 From: dej611 Date: Thu, 3 Mar 2022 12:41:20 +0100 Subject: [PATCH 23/31] :wrench: Make close editor panel more robust --- .../editor_frame/config_panel/dimension_container.tsx | 8 +++++++- x-pack/test/functional/apps/lens/show_underlying_data.ts | 2 ++ x-pack/test/functional/page_objects/lens_page.ts | 4 ++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.tsx index f7402e78ebd96..d3a047ce79088 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.tsx @@ -135,7 +135,13 @@ export function DimensionContainer({
{panel}
- + {i18n.translate('xpack.lens.dimensionContainer.close', { defaultMessage: 'Close', })} diff --git a/x-pack/test/functional/apps/lens/show_underlying_data.ts b/x-pack/test/functional/apps/lens/show_underlying_data.ts index dc8d9063cf374..1d3e7a64f659b 100644 --- a/x-pack/test/functional/apps/lens/show_underlying_data.ts +++ b/x-pack/test/functional/apps/lens/show_underlying_data.ts @@ -119,6 +119,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const input = await find.activeElement(); await input.type(`bytes > 6000`); + await PageObjects.lens.closeDimensionEditor(); + await PageObjects.lens.waitForVisualization(); // expect the button is shown and enabled await testSubjects.clickWhenNotDisabled(`lnsApp_openInDiscover`); diff --git a/x-pack/test/functional/page_objects/lens_page.ts b/x-pack/test/functional/page_objects/lens_page.ts index 61b0cd10750b2..c1371765364d9 100644 --- a/x-pack/test/functional/page_objects/lens_page.ts +++ b/x-pack/test/functional/page_objects/lens_page.ts @@ -494,6 +494,10 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont async closeDimensionEditor() { await retry.try(async () => { await testSubjects.click('lns-indexPattern-dimensionContainerBack'); + if (await testSubjects.exists('lns-indexPattern-dimensionContainerBack')) { + // add another option to close: formula tooltips can cover the previous button sometimes + await testSubjects.click('lns-indexPattern-dimensionContainerClose'); + } await testSubjects.missingOrFail('lns-indexPattern-dimensionContainerBack'); }); }, From 6eec900746c9bb9d85df6e1257d6a2a61df1e010 Mon Sep 17 00:00:00 2001 From: dej611 Date: Thu, 3 Mar 2022 16:36:02 +0100 Subject: [PATCH 24/31] :wrench: Try focus approach --- x-pack/test/functional/apps/lens/show_underlying_data.ts | 1 + x-pack/test/functional/page_objects/lens_page.ts | 4 ---- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/x-pack/test/functional/apps/lens/show_underlying_data.ts b/x-pack/test/functional/apps/lens/show_underlying_data.ts index 1d3e7a64f659b..9532570e63c8a 100644 --- a/x-pack/test/functional/apps/lens/show_underlying_data.ts +++ b/x-pack/test/functional/apps/lens/show_underlying_data.ts @@ -118,6 +118,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const input = await find.activeElement(); await input.type(`bytes > 6000`); + await input.focus(); await PageObjects.lens.closeDimensionEditor(); diff --git a/x-pack/test/functional/page_objects/lens_page.ts b/x-pack/test/functional/page_objects/lens_page.ts index c1371765364d9..61b0cd10750b2 100644 --- a/x-pack/test/functional/page_objects/lens_page.ts +++ b/x-pack/test/functional/page_objects/lens_page.ts @@ -494,10 +494,6 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont async closeDimensionEditor() { await retry.try(async () => { await testSubjects.click('lns-indexPattern-dimensionContainerBack'); - if (await testSubjects.exists('lns-indexPattern-dimensionContainerBack')) { - // add another option to close: formula tooltips can cover the previous button sometimes - await testSubjects.click('lns-indexPattern-dimensionContainerClose'); - } await testSubjects.missingOrFail('lns-indexPattern-dimensionContainerBack'); }); }, From 7be3ec7c8988bf1180a29691485d9d1b71a5b513 Mon Sep 17 00:00:00 2001 From: dej611 Date: Thu, 3 Mar 2022 18:21:24 +0100 Subject: [PATCH 25/31] :ok_hand: Rename variable --- .../plugins/lens/public/app_plugin/lens_top_nav.tsx | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/lens/public/app_plugin/lens_top_nav.tsx b/x-pack/plugins/lens/public/app_plugin/lens_top_nav.tsx index cd8a1ca2ea4a4..dcd328ff5c483 100644 --- a/x-pack/plugins/lens/public/app_plugin/lens_top_nav.tsx +++ b/x-pack/plugins/lens/public/app_plugin/lens_top_nav.tsx @@ -314,7 +314,7 @@ export const LensTopNavMenu = ({ initialContext, ]); - const canShowUnderlyingData = useMemo(() => { + const layerMetaInfo = useMemo(() => { if (!activeDatasourceId || !discover) { return; } @@ -342,7 +342,7 @@ export const LensTopNavMenu = ({ (dashboardFeatureFlag.allowByValueEmbeddables || Boolean(initialInput)) ) || Boolean(initialContextIsEmbedded), enableExportToCSV: Boolean(isSaveable && activeData && Object.keys(activeData).length), - showOpenInDiscover: Boolean(canShowUnderlyingData?.isVisible), + showOpenInDiscover: Boolean(layerMetaInfo?.isVisible), isByValueMode: getIsByValueMode(), allowByValue: dashboardFeatureFlag.allowByValueEmbeddables, showCancel: Boolean(isLinkedToOriginatingApp), @@ -366,7 +366,7 @@ export const LensTopNavMenu = ({ return undefined; }, showUnderlyingDataWarning: () => { - return canShowUnderlyingData?.error; + return layerMetaInfo?.error; }, }, actions: { @@ -436,10 +436,10 @@ export const LensTopNavMenu = ({ } }, getUnderlyingDataUrl: () => { - if (!canShowUnderlyingData) { + if (!layerMetaInfo) { return; } - const { error, meta } = canShowUnderlyingData; + const { error, meta } = layerMetaInfo; // If Discover is not available, return // If there's no data, return if (error || !discover || !meta) { @@ -470,7 +470,7 @@ export const LensTopNavMenu = ({ initialContextIsEmbedded, isSaveable, activeData, - canShowUnderlyingData, + layerMetaInfo, getIsByValueMode, savingToLibraryPermitted, savingToDashboardPermitted, From dbf8de932a69486b1a12b997e3975036969c13e2 Mon Sep 17 00:00:00 2001 From: dej611 Date: Fri, 4 Mar 2022 09:06:20 +0100 Subject: [PATCH 26/31] :white_check_mark: Try to click outside --- x-pack/test/functional/apps/lens/show_underlying_data.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/test/functional/apps/lens/show_underlying_data.ts b/x-pack/test/functional/apps/lens/show_underlying_data.ts index 9532570e63c8a..cac66a2f5cd4e 100644 --- a/x-pack/test/functional/apps/lens/show_underlying_data.ts +++ b/x-pack/test/functional/apps/lens/show_underlying_data.ts @@ -118,7 +118,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const input = await find.activeElement(); await input.type(`bytes > 6000`); - await input.focus(); + // focus something else to dismiss formula's tooltip + await testSubjects.click('indexPattern-label-edit'); await PageObjects.lens.closeDimensionEditor(); From 5f727c72237f078adf96884ed30d89268b918114 Mon Sep 17 00:00:00 2001 From: dej611 Date: Fri, 4 Mar 2022 09:15:10 +0100 Subject: [PATCH 27/31] :white_check_mark: Use the keyboard arrow approach --- x-pack/test/functional/apps/lens/show_underlying_data.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/test/functional/apps/lens/show_underlying_data.ts b/x-pack/test/functional/apps/lens/show_underlying_data.ts index cac66a2f5cd4e..d6ae299baceaf 100644 --- a/x-pack/test/functional/apps/lens/show_underlying_data.ts +++ b/x-pack/test/functional/apps/lens/show_underlying_data.ts @@ -118,8 +118,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const input = await find.activeElement(); await input.type(`bytes > 6000`); - // focus something else to dismiss formula's tooltip - await testSubjects.click('indexPattern-label-edit'); + // the tooltip seems to be there as long as the focus is in the query string + await input.pressKeys(browser.keys.RIGHT); await PageObjects.lens.closeDimensionEditor(); From aabddab7169af3a29465524a194f9477b521aef9 Mon Sep 17 00:00:00 2001 From: dej611 Date: Mon, 7 Mar 2022 12:16:44 +0100 Subject: [PATCH 28/31] :sparkles: Add disabled filters for wider filtered metrics --- .../app_plugin/show_underlying_data.test.ts | 313 +++++++++++++++--- .../public/app_plugin/show_underlying_data.ts | 49 ++- .../indexpattern.test.ts | 195 ++++++----- .../public/indexpattern_datasource/utils.tsx | 45 ++- x-pack/plugins/lens/public/types.ts | 11 +- 5 files changed, 458 insertions(+), 155 deletions(-) diff --git a/x-pack/plugins/lens/public/app_plugin/show_underlying_data.test.ts b/x-pack/plugins/lens/public/app_plugin/show_underlying_data.test.ts index e253562d2eb5c..92a3f489541c9 100644 --- a/x-pack/plugins/lens/public/app_plugin/show_underlying_data.test.ts +++ b/x-pack/plugins/lens/public/app_plugin/show_underlying_data.test.ts @@ -112,8 +112,11 @@ describe('getLayerMetaInfo', () => { getVisualDefaults: jest.fn(), getSourceId: jest.fn(), getFilters: jest.fn(() => ({ - kuery: [[{ language: 'kuery', query: 'memory > 40000' }]], - lucene: [], + enabled: { + kuery: [[{ language: 'kuery', query: 'memory > 40000' }]], + lucene: [], + }, + disabled: { kuery: [], lucene: [] }, })), }; mockDatasource.getPublicAPI.mockReturnValue(updatedPublicAPI); @@ -128,15 +131,18 @@ describe('getLayerMetaInfo', () => { expect(error).toBeUndefined(); expect(meta?.columns).toEqual(['bytes']); expect(meta?.filters).toEqual({ - kuery: [ - [ - { - language: 'kuery', - query: 'memory > 40000', - }, + enabled: { + kuery: [ + [ + { + language: 'kuery', + query: 'memory > 40000', + }, + ], ], - ], - lucene: [], + lucene: [], + }, + disabled: { kuery: [], lucene: [] }, }); }); }); @@ -149,7 +155,7 @@ describe('combineQueryAndFilters', () => { { id: 'testDatasource', columns: [], - filters: { kuery: [], lucene: [] }, + filters: { enabled: { kuery: [], lucene: [] }, disabled: { kuery: [], lucene: [] } }, }, undefined ) @@ -164,7 +170,10 @@ describe('combineQueryAndFilters', () => { { id: 'testDatasource', columns: [], - filters: { kuery: [[{ language: 'kuery', query: 'otherField: *' }]], lucene: [] }, + filters: { + enabled: { kuery: [[{ language: 'kuery', query: 'otherField: *' }]], lucene: [] }, + disabled: { kuery: [], lucene: [] }, + }, }, undefined ) @@ -182,7 +191,10 @@ describe('combineQueryAndFilters', () => { { id: 'testDatasource', columns: [], - filters: { kuery: [[{ language: 'kuery', query: 'otherField: *' }]], lucene: [] }, + filters: { + enabled: { kuery: [[{ language: 'kuery', query: 'otherField: *' }]], lucene: [] }, + disabled: { kuery: [], lucene: [] }, + }, }, undefined ) @@ -199,17 +211,20 @@ describe('combineQueryAndFilters', () => { id: 'testDatasource', columns: [], filters: { - kuery: [ - [ - { language: 'kuery', query: 'myfield: *' }, - { language: 'kuery', query: 'otherField: *' }, + enabled: { + kuery: [ + [ + { language: 'kuery', query: 'myfield: *' }, + { language: 'kuery', query: 'otherField: *' }, + ], + [ + { language: 'kuery', query: 'myfieldCopy: *' }, + { language: 'kuery', query: 'otherFieldCopy: *' }, + ], ], - [ - { language: 'kuery', query: 'myfieldCopy: *' }, - { language: 'kuery', query: 'otherFieldCopy: *' }, - ], - ], - lucene: [], + lucene: [], + }, + disabled: { kuery: [], lucene: [] }, }, }, undefined @@ -232,8 +247,11 @@ describe('combineQueryAndFilters', () => { id: 'testDatasource', columns: [], filters: { - kuery: [[{ language: 'kuery', query: 'myfield: *' }]], - lucene: [], + enabled: { + kuery: [[{ language: 'kuery', query: 'myfield: *' }]], + lucene: [], + }, + disabled: { kuery: [], lucene: [] }, }, }, undefined @@ -316,8 +334,11 @@ describe('combineQueryAndFilters', () => { id: 'testDatasource', columns: [], filters: { - kuery: [], - lucene: [[{ language: 'lucene', query: 'anotherField' }]], + enabled: { + kuery: [], + lucene: [[{ language: 'lucene', query: 'anotherField' }]], + }, + disabled: { kuery: [], lucene: [] }, }, }, undefined @@ -395,8 +416,11 @@ describe('combineQueryAndFilters', () => { id: 'testDatasource', columns: [], filters: { - kuery: [[{ language: 'kuery', query: 'myfield: *' }]], - lucene: [[{ language: 'lucene', query: 'anotherField' }]], + enabled: { + kuery: [[{ language: 'kuery', query: 'myfield: *' }]], + lucene: [[{ language: 'lucene', query: 'anotherField' }]], + }, + disabled: { kuery: [], lucene: [] }, }, }, undefined @@ -487,22 +511,25 @@ describe('combineQueryAndFilters', () => { id: 'testDatasource', columns: [], filters: { - kuery: [ - [{ language: 'kuery', query: 'bytes > 4000' }], - [ - { language: 'kuery', query: 'memory > 5000' }, - { language: 'kuery', query: 'memory >= 15000' }, + enabled: { + kuery: [ + [{ language: 'kuery', query: 'bytes > 4000' }], + [ + { language: 'kuery', query: 'memory > 5000' }, + { language: 'kuery', query: 'memory >= 15000' }, + ], + [{ language: 'kuery', query: 'myField: *' }], + [{ language: 'kuery', query: 'otherField >= 15' }], ], - [{ language: 'kuery', query: 'myField: *' }], - [{ language: 'kuery', query: 'otherField >= 15' }], - ], - lucene: [ - [{ language: 'lucene', query: 'filteredField: 400' }], - [ - { language: 'lucene', query: 'aNewField' }, - { language: 'lucene', query: 'anotherNewField: 200' }, + lucene: [ + [{ language: 'lucene', query: 'filteredField: 400' }], + [ + { language: 'lucene', query: 'aNewField' }, + { language: 'lucene', query: 'anotherNewField: 200' }, + ], ], - ], + }, + disabled: { kuery: [], lucene: [] }, }, }, undefined @@ -573,4 +600,204 @@ describe('combineQueryAndFilters', () => { }, }); }); + + it('should add ignored filters as disabled', () => { + expect( + combineQueryAndFilters( + { language: 'lucene', query: 'myField' }, + [], + { + id: 'testDatasource', + columns: [], + filters: { + disabled: { + kuery: [[{ language: 'kuery', query: 'myfield: *' }]], + lucene: [[{ language: 'lucene', query: 'anotherField' }]], + }, + enabled: { kuery: [], lucene: [] }, + }, + }, + undefined + ) + ).toEqual({ + filters: [ + { + $state: { + store: 'appState', + }, + bool: { + filter: [], + must: [ + { + query_string: { + query: 'anotherField', + }, + }, + ], + must_not: [], + should: [], + }, + meta: { + alias: 'anotherField (lucene)', + disabled: true, + index: 'testDatasource', + negate: false, + type: 'custom', + }, + }, + { + $state: { + store: 'appState', + }, + bool: { + filter: [ + { + bool: { + minimum_should_match: 1, + should: [ + { + exists: { + field: 'myfield', + }, + }, + ], + }, + }, + ], + must: [], + must_not: [], + should: [], + }, + meta: { + alias: 'myfield: *', + disabled: true, + index: 'testDatasource', + negate: false, + type: 'custom', + }, + }, + ], + query: { + language: 'lucene', + query: 'myField', + }, + }); + }); + + it('should work together with enabled and disabled filters', () => { + expect( + combineQueryAndFilters( + { language: 'lucene', query: 'myField' }, + [], + { + id: 'testDatasource', + columns: [], + filters: { + disabled: { + kuery: [[{ language: 'kuery', query: 'myfield: abc' }]], + lucene: [[{ language: 'lucene', query: 'anotherField > 5000' }]], + }, + enabled: { + kuery: [[{ language: 'kuery', query: 'myfield: *' }]], + lucene: [[{ language: 'lucene', query: 'anotherField' }]], + }, + }, + }, + undefined + ) + ).toEqual({ + filters: [ + { + bool: { + must: [], + filter: [ + { + bool: { + should: [ + { + exists: { + field: 'myfield', + }, + }, + ], + minimum_should_match: 1, + }, + }, + ], + should: [], + must_not: [], + }, + meta: { + index: 'testDatasource', + type: 'custom', + disabled: false, + negate: false, + alias: 'Lens context (kuery)', + }, + $state: { + store: 'appState', + }, + }, + { + bool: { + must: [ + { + query_string: { + query: 'anotherField > 5000', + }, + }, + ], + filter: [], + should: [], + must_not: [], + }, + meta: { + index: 'testDatasource', + type: 'custom', + disabled: true, + negate: false, + alias: 'anotherField > 5000 (lucene)', + }, + $state: { + store: 'appState', + }, + }, + { + bool: { + must: [], + filter: [ + { + bool: { + should: [ + { + match: { + myfield: 'abc', + }, + }, + ], + minimum_should_match: 1, + }, + }, + ], + should: [], + must_not: [], + }, + meta: { + index: 'testDatasource', + type: 'custom', + disabled: true, + negate: false, + alias: 'myfield: abc', + }, + $state: { + store: 'appState', + }, + }, + ], + query: { + language: 'lucene', + query: '( myField ) AND ( anotherField )', + }, + }); + }); }); diff --git a/x-pack/plugins/lens/public/app_plugin/show_underlying_data.ts b/x-pack/plugins/lens/public/app_plugin/show_underlying_data.ts index a8c678167f70f..9252623c0324b 100644 --- a/x-pack/plugins/lens/public/app_plugin/show_underlying_data.ts +++ b/x-pack/plugins/lens/public/app_plugin/show_underlying_data.ts @@ -43,10 +43,13 @@ function joinQueries(queries: Query[][] | undefined) { interface LayerMetaInfo { id: string; columns: string[]; - filters: { - kuery: Query[][] | undefined; - lucene: Query[][] | undefined; - }; + filters: Record< + 'enabled' | 'disabled', + { + kuery: Query[][] | undefined; + lucene: Query[][] | undefined; + } + >; } export function getLayerMetaInfo( @@ -147,8 +150,9 @@ export function combineQueryAndFilters( : { queryLanguage: 'kuery', filtersLanguage: 'lucene' }; let newQuery = query; - if (meta.filters[queryLanguage]?.length) { - const filtersQuery = joinQueries(meta.filters[queryLanguage]); + const enabledFilters = meta.filters.enabled; + if (enabledFilters[queryLanguage]?.length) { + const filtersQuery = joinQueries(enabledFilters[queryLanguage]); newQuery = { language: queryLanguage, query: query?.query.trim() @@ -159,17 +163,14 @@ export function combineQueryAndFilters( // make a copy as the original filters are readonly const newFilters = [...filters]; - if (meta.filters[filtersLanguage]?.length) { - const queryExpression = joinQueries(meta.filters[filtersLanguage]); + const dataView = dataViews?.find(({ id }) => id === meta.id); + if (enabledFilters[filtersLanguage]?.length) { + const queryExpression = joinQueries(enabledFilters[filtersLanguage]); // Append the new filter based on the queryExpression to the existing ones newFilters.push( buildCustomFilter( meta.id!, - buildEsQuery( - dataViews?.find(({ id }) => id === meta.id), - { language: filtersLanguage, query: queryExpression }, - [] - ), + buildEsQuery(dataView, { language: filtersLanguage, query: queryExpression }, []), false, false, i18n.translate('xpack.lens.app.lensContext', { @@ -180,5 +181,27 @@ export function combineQueryAndFilters( ) ); } + // for each disabled filter create a new custom filter and disable it + // note that both languages go into the filter bar + const disabledFilters = meta.filters.disabled; + for (const language of ['lucene', 'kuery'] as const) { + const [disabledQueries] = disabledFilters[language] || []; + for (const disabledQuery of disabledQueries || []) { + let label = disabledQuery.query as string; + if (language === 'lucene') { + label += ` (${language})`; + } + newFilters.push( + buildCustomFilter( + meta.id!, + buildEsQuery(dataView, disabledQuery, []), + true, + false, + label, + FilterStateStore.APP_STATE + ) + ); + } + } return { filters: newFilters, query: newQuery }; } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts index 42dd23a6d2242..ed5163921472f 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts @@ -1446,8 +1446,11 @@ describe('IndexPattern Data Source', () => { layerId: 'first', }); expect(publicAPI.getFilters()).toEqual({ - kuery: [[{ language: 'kuery', query: 'bytes > 1000' }]], - lucene: [[{ language: 'lucene', query: 'memory' }]], + enabled: { + kuery: [[{ language: 'kuery', query: 'bytes > 1000' }]], + lucene: [[{ language: 'lucene', query: 'memory' }]], + }, + disabled: { kuery: [], lucene: [] }, }); }); it('should ignore empty filtered metrics', () => { @@ -1474,7 +1477,10 @@ describe('IndexPattern Data Source', () => { }, layerId: 'first', }); - expect(publicAPI.getFilters()).toEqual({ kuery: [], lucene: [] }); + expect(publicAPI.getFilters()).toEqual({ + enabled: { kuery: [], lucene: [] }, + disabled: { kuery: [], lucene: [] }, + }); }); it('shuold collect top values fields as kuery existence filters if no data is provided', () => { publicAPI = indexPatternDatasource.getPublicAPI({ @@ -1517,14 +1523,17 @@ describe('IndexPattern Data Source', () => { layerId: 'first', }); expect(publicAPI.getFilters()).toEqual({ - kuery: [ - [{ language: 'kuery', query: 'geo.src: *' }], - [ - { language: 'kuery', query: 'geo.dest: *' }, - { language: 'kuery', query: 'myField: *' }, + enabled: { + kuery: [ + [{ language: 'kuery', query: 'geo.src: *' }], + [ + { language: 'kuery', query: 'geo.dest: *' }, + { language: 'kuery', query: 'myField: *' }, + ], ], - ], - lucene: [], + lucene: [], + }, + disabled: { kuery: [], lucene: [] }, }); }); it('shuold collect top values fields and terms as kuery filters if data is provided', () => { @@ -1581,17 +1590,20 @@ describe('IndexPattern Data Source', () => { }, }; expect(publicAPI.getFilters(data)).toEqual({ - kuery: [ - [ - { language: 'kuery', query: 'geo.src: "US"' }, - { language: 'kuery', query: 'geo.src: "IN"' }, - ], - [ - { language: 'kuery', query: 'geo.dest: "IT" AND myField: "MyValue"' }, - { language: 'kuery', query: 'geo.dest: "DE" AND myField: "MyOtherValue"' }, + enabled: { + kuery: [ + [ + { language: 'kuery', query: 'geo.src: "US"' }, + { language: 'kuery', query: 'geo.src: "IN"' }, + ], + [ + { language: 'kuery', query: 'geo.dest: "IT" AND myField: "MyValue"' }, + { language: 'kuery', query: 'geo.dest: "DE" AND myField: "MyOtherValue"' }, + ], ], - ], - lucene: [], + lucene: [], + }, + disabled: { kuery: [], lucene: [] }, }); }); it('shuold collect top values fields and terms and carefully handle empty string values', () => { @@ -1648,17 +1660,20 @@ describe('IndexPattern Data Source', () => { }, }; expect(publicAPI.getFilters(data)).toEqual({ - kuery: [ - [ - { language: 'kuery', query: 'geo.src: "US"' }, - { language: 'kuery', query: 'geo.src: "IN"' }, - ], - [ - { language: 'kuery', query: `geo.dest: "IT" AND myField: ""` }, - { language: 'kuery', query: `geo.dest: "DE" AND myField: "MyOtherValue"` }, + enabled: { + kuery: [ + [ + { language: 'kuery', query: 'geo.src: "US"' }, + { language: 'kuery', query: 'geo.src: "IN"' }, + ], + [ + { language: 'kuery', query: `geo.dest: "IT" AND myField: ""` }, + { language: 'kuery', query: `geo.dest: "DE" AND myField: "MyOtherValue"` }, + ], ], - ], - lucene: [], + lucene: [], + }, + disabled: { kuery: [], lucene: [] }, }); }); it('should ignore top values fields if other/missing option is enabled', () => { @@ -1702,7 +1717,10 @@ describe('IndexPattern Data Source', () => { }, layerId: 'first', }); - expect(publicAPI.getFilters()).toEqual({ kuery: [], lucene: [] }); + expect(publicAPI.getFilters()).toEqual({ + enabled: { kuery: [], lucene: [] }, + disabled: { kuery: [], lucene: [] }, + }); }); it('should collect custom ranges as kuery filters', () => { publicAPI = indexPatternDatasource.getPublicAPI({ @@ -1745,14 +1763,17 @@ describe('IndexPattern Data Source', () => { layerId: 'first', }); expect(publicAPI.getFilters()).toEqual({ - kuery: [ - [{ language: 'kuery', query: 'bytes >= 100 AND bytes <= 150' }], - [ - { language: 'kuery', query: 'bytes >= 200 AND bytes <= 300' }, - { language: 'kuery', query: 'bytes >= 300 AND bytes <= 400' }, + enabled: { + kuery: [ + [{ language: 'kuery', query: 'bytes >= 100 AND bytes <= 150' }], + [ + { language: 'kuery', query: 'bytes >= 200 AND bytes <= 300' }, + { language: 'kuery', query: 'bytes >= 300 AND bytes <= 400' }, + ], ], - ], - lucene: [], + lucene: [], + }, + disabled: { kuery: [], lucene: [] }, }); }); it('should collect custom ranges as kuery filters as partial', () => { @@ -1804,11 +1825,14 @@ describe('IndexPattern Data Source', () => { layerId: 'first', }); expect(publicAPI.getFilters()).toEqual({ - kuery: [ - [{ language: 'kuery', query: 'bytes >= 100' }], - [{ language: 'kuery', query: 'bytes <= 300' }], - ], - lucene: [], + enabled: { + kuery: [ + [{ language: 'kuery', query: 'bytes >= 100' }], + [{ language: 'kuery', query: 'bytes <= 300' }], + ], + lucene: [], + }, + disabled: { kuery: [], lucene: [] }, }); }); it('should collect filters within filters operation grouped by language', () => { @@ -1862,20 +1886,23 @@ describe('IndexPattern Data Source', () => { layerId: 'first', }); expect(publicAPI.getFilters()).toEqual({ - kuery: [ - [{ language: 'kuery', query: 'bytes > 1000' }], - [ - { language: 'kuery', query: 'bytes > 5000' }, - { language: 'kuery', query: 'memory > 500000' }, + enabled: { + kuery: [ + [{ language: 'kuery', query: 'bytes > 1000' }], + [ + { language: 'kuery', query: 'bytes > 5000' }, + { language: 'kuery', query: 'memory > 500000' }, + ], ], - ], - lucene: [ - [{ language: 'lucene', query: 'memory' }], - [ - { language: 'lucene', query: 'phpmemory' }, - { language: 'lucene', query: 'memory: 5000000' }, + lucene: [ + [{ language: 'lucene', query: 'memory' }], + [ + { language: 'lucene', query: 'phpmemory' }, + { language: 'lucene', query: 'memory: 5000000' }, + ], ], - ], + }, + disabled: { kuery: [], lucene: [] }, }); }); it('should ignore filtered metrics if at least one metric is unfiltered', () => { @@ -1911,8 +1938,8 @@ describe('IndexPattern Data Source', () => { layerId: 'first', }); expect(publicAPI.getFilters()).toEqual({ - kuery: [], - lucene: [], + enabled: { kuery: [], lucene: [] }, + disabled: { kuery: [[{ language: 'kuery', query: 'bytes > 1000' }]], lucene: [] }, }); }); it('should ignore filtered metrics if at least one metric is unfiltered in formula', () => { @@ -1983,8 +2010,8 @@ describe('IndexPattern Data Source', () => { layerId: 'first', }); expect(publicAPI.getFilters()).toEqual({ - kuery: [], - lucene: [], + enabled: { kuery: [], lucene: [] }, + disabled: { kuery: [[{ language: 'kuery', query: 'memory > 5000' }]], lucene: [] }, }); }); it('should support complete scenarios', () => { @@ -2049,24 +2076,27 @@ describe('IndexPattern Data Source', () => { layerId: 'first', }); expect(publicAPI.getFilters()).toEqual({ - kuery: [ - [{ language: 'kuery', query: 'bytes > 1000' }], - [ - { language: 'kuery', query: 'bytes > 5000' }, - { language: 'kuery', query: 'memory > 500000' }, + enabled: { + kuery: [ + [{ language: 'kuery', query: 'bytes > 1000' }], + [ + { language: 'kuery', query: 'bytes > 5000' }, + { language: 'kuery', query: 'memory > 500000' }, + ], + [ + { language: 'kuery', query: 'geo.src: *' }, + { language: 'kuery', query: 'myField: *' }, + ], ], - [ - { language: 'kuery', query: 'geo.src: *' }, - { language: 'kuery', query: 'myField: *' }, + lucene: [ + [{ language: 'lucene', query: 'memory' }], + [ + { language: 'lucene', query: 'phpmemory' }, + { language: 'lucene', query: 'memory: 5000000' }, + ], ], - ], - lucene: [ - [{ language: 'lucene', query: 'memory' }], - [ - { language: 'lucene', query: 'phpmemory' }, - { language: 'lucene', query: 'memory: 5000000' }, - ], - ], + }, + disabled: { kuery: [], lucene: [] }, }); }); @@ -2140,13 +2170,16 @@ describe('IndexPattern Data Source', () => { layerId: 'first', }); expect(publicAPI.getFilters()).toEqual({ - kuery: [ - [ - { language: 'kuery', query: 'bytes > 4000 AND memory > 5000' }, - { language: 'kuery', query: 'bytes > 4000' }, + enabled: { + kuery: [ + [ + { language: 'kuery', query: 'bytes > 4000 AND memory > 5000' }, + { language: 'kuery', query: 'bytes > 4000' }, + ], ], - ], - lucene: [], + lucene: [], + }, + disabled: { kuery: [], lucene: [] }, }); }); }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/utils.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/utils.tsx index 401a4d9dcef82..dd0f1e3e20eaa 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/utils.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/utils.tsx @@ -348,16 +348,15 @@ function collectFiltersFromMetrics(layer: IndexPatternLayer, columnIds: string[] layer.columns[colId]?.filter ? 'filtered' : 'unfiltered' ); - // extract filters from filtered metrics - // consider all the columns, included referenced ones to cover also the formula case - return ( - filtered - // if there are metric columns not filtered, then ignore filtered columns completely - .filter(() => !unfiltered.length) - .map((colId) => layer.columns[colId]?.filter) - // filter out empty filters as well - .filter((filter) => filter?.query?.trim()) as Query[] - ); + const filteredMetrics = filtered + .map((colId) => layer.columns[colId]?.filter) + // filter out empty filters as well + .filter((filter) => filter?.query?.trim()) as Query[]; + + return { + enabled: unfiltered.length ? [] : filteredMetrics, + disabled: unfiltered.length ? filteredMetrics : [], + }; } interface GroupedQueries { @@ -381,8 +380,14 @@ export function getFiltersInLayer( columnIds: string[], layerData: NonNullable[string] | undefined ) { - const filtersFromMetricsByLanguage = groupBy( - collectFiltersFromMetrics(layer, columnIds), + const { enabled: enabledFilteredMetrics, disabled: disabledFilteredMetrics } = + collectFiltersFromMetrics(layer, columnIds); + const enabledFiltersFromMetricsByLanguage = groupBy( + enabledFilteredMetrics, + 'language' + ) as unknown as GroupedQueries; + const disabledFitleredFromMetricsByLanguage = groupBy( + disabledFilteredMetrics, 'language' ) as unknown as GroupedQueries; @@ -429,7 +434,19 @@ export function getFiltersInLayer( }) .filter(Boolean) as GroupedQueries[]; return { - kuery: collectOnlyValidQueries(filtersFromMetricsByLanguage, filterOperation, 'kuery'), - lucene: collectOnlyValidQueries(filtersFromMetricsByLanguage, filterOperation, 'lucene'), + enabled: { + kuery: collectOnlyValidQueries(enabledFiltersFromMetricsByLanguage, filterOperation, 'kuery'), + lucene: collectOnlyValidQueries( + enabledFiltersFromMetricsByLanguage, + filterOperation, + 'lucene' + ), + }, + disabled: { + kuery: [disabledFitleredFromMetricsByLanguage.kuery || []].filter((filter) => filter.length), + lucene: [disabledFitleredFromMetricsByLanguage.lucene || []].filter( + (filter) => filter.length + ), + }, }; } diff --git a/x-pack/plugins/lens/public/types.ts b/x-pack/plugins/lens/public/types.ts index 1895d26ea89f5..c73c91b1d9640 100644 --- a/x-pack/plugins/lens/public/types.ts +++ b/x-pack/plugins/lens/public/types.ts @@ -368,10 +368,13 @@ export interface DatasourcePublicAPI { /** * Collect all defined filters from all the operations in the layer */ - getFilters: (activeData?: FramePublicAPI['activeData']) => { - kuery: Query[][]; - lucene: Query[][]; - }; + getFilters: (activeData?: FramePublicAPI['activeData']) => Record< + 'enabled' | 'disabled', + { + kuery: Query[][]; + lucene: Query[][]; + } + >; } export interface DatasourceDataPanelProps { From bda6a9d9aa9b9602d55d1be2b82be8031e731469 Mon Sep 17 00:00:00 2001 From: dej611 Date: Mon, 7 Mar 2022 12:42:25 +0100 Subject: [PATCH 29/31] :ok_hand: Address some issues raised by review --- .../public/app_plugin/show_underlying_data.ts | 28 ++++++------------- .../public/indexpattern_datasource/utils.tsx | 7 +++-- 2 files changed, 14 insertions(+), 21 deletions(-) diff --git a/x-pack/plugins/lens/public/app_plugin/show_underlying_data.ts b/x-pack/plugins/lens/public/app_plugin/show_underlying_data.ts index a8c678167f70f..e1956542f8def 100644 --- a/x-pack/plugins/lens/public/app_plugin/show_underlying_data.ts +++ b/x-pack/plugins/lens/public/app_plugin/show_underlying_data.ts @@ -83,15 +83,15 @@ export function getLayerMetaInfo( state: datasourceState, }); // maybe add also datasourceId validation here? - // if (datasourceAPI.datasourceId !== 'indexpattern') { - // return { - // meta: undefined, - // error: i18n.translate('xpack.lens.app.showUnderlyingDataUnsupportedDatasource', { - // defaultMessage: 'Underlying data does not support the current datasource', - // }), - // isVisible, - // }; - // } + if (datasourceAPI.datasourceId !== 'indexpattern') { + return { + meta: undefined, + error: i18n.translate('xpack.lens.app.showUnderlyingDataUnsupportedDatasource', { + defaultMessage: 'Underlying data does not support the current datasource', + }), + isVisible, + }; + } const tableSpec = datasourceAPI.getTableSpec(); const columnsWithNoTimeShifts = tableSpec.filter( @@ -108,16 +108,6 @@ export function getLayerMetaInfo( } const uniqueFields = [...new Set(columnsWithNoTimeShifts.map(({ fields }) => fields).flat())]; - // If no field, return? Or rather carry on and show the default columns? - // if (!uniqueFields.length) { - // return { - // meta: undefined, - // error: i18n.translate('xpack.lens.app.showUnderlyingDataNoFields', { - // defaultMessage: 'The current visualization has no available fields to show', - // }), - // isVisible, - // }; - // } return { meta: { id: datasourceAPI.getSourceId()!, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/utils.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/utils.tsx index 401a4d9dcef82..161d8b63bd12d 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/utils.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/utils.tsx @@ -301,10 +301,13 @@ function extractQueriesFromTerms( } if (typeof value !== 'string' && Array.isArray(value.keys)) { return value.keys - .map((term: string, index: number) => `${fields[index]}: ${`"${escape(term)}"`}`) + .map( + (term: string, index: number) => + `${fields[index]}: ${`"${term === '' ? escape(term) : term}"`}` + ) .join(' AND '); } - return `${column.sourceField}: ${`"${escape(value)}"`}`; + return `${column.sourceField}: ${`"${value === '' ? escape(value) : value}"`}`; }) .filter(Boolean) as string[]; From fb377f329788d82a1b9c04d4a15ac6df4837b924 Mon Sep 17 00:00:00 2001 From: Marco Liberati Date: Wed, 9 Mar 2022 11:12:53 +0100 Subject: [PATCH 30/31] Update x-pack/plugins/lens/public/indexpattern_datasource/utils.tsx --- x-pack/plugins/lens/public/indexpattern_datasource/utils.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/utils.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/utils.tsx index c863dadfb72f7..703007c309f61 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/utils.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/utils.tsx @@ -384,10 +384,10 @@ export function getFiltersInLayer( layerData: NonNullable[string] | undefined ) { const filtersGroupedByState = collectFiltersFromMetrics(layer, columnIds); - const [enabledFiltersFromMetricsByLanguage, disabledFitleredFromMetricsByLanguage] = [ + const [enabledFiltersFromMetricsByLanguage, disabledFitleredFromMetricsByLanguage] = ([ 'enabled', 'disabled', - ].map((state) => groupBy(filtersGroupedByState[state], 'language') as unknown as GroupedQueries); + ] as const).map((state) => groupBy(filtersGroupedByState[state], 'language') as unknown as GroupedQueries); const filterOperation = columnIds .map((colId) => { From d6390a8aa1c1a4439dee893cece1893e9f411a7a Mon Sep 17 00:00:00 2001 From: dej611 Date: Wed, 9 Mar 2022 15:43:43 +0100 Subject: [PATCH 31/31] :rotating_light: Fix linting issue --- .../plugins/lens/public/indexpattern_datasource/utils.tsx | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/utils.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/utils.tsx index 703007c309f61..b21df483b342e 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/utils.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/utils.tsx @@ -384,10 +384,9 @@ export function getFiltersInLayer( layerData: NonNullable[string] | undefined ) { const filtersGroupedByState = collectFiltersFromMetrics(layer, columnIds); - const [enabledFiltersFromMetricsByLanguage, disabledFitleredFromMetricsByLanguage] = ([ - 'enabled', - 'disabled', - ] as const).map((state) => groupBy(filtersGroupedByState[state], 'language') as unknown as GroupedQueries); + const [enabledFiltersFromMetricsByLanguage, disabledFitleredFromMetricsByLanguage] = ( + ['enabled', 'disabled'] as const + ).map((state) => groupBy(filtersGroupedByState[state], 'language') as unknown as GroupedQueries); const filterOperation = columnIds .map((colId) => {