From 6c926c77f293f6270de3da0e2fc504f8ad9ac14b Mon Sep 17 00:00:00 2001 From: Matthew Kime Date: Tue, 14 Nov 2023 14:31:40 -0600 Subject: [PATCH 01/10] [data views] Allow data views created on hidden and system indices - second attempt (#168882) ## Summary Previously, the 'Allow hidden and system indices' advanced option when creating a data view was only a UI convenience. It allowed you to see which hidden and system indices you were matching but they would be would be selected just the same once the data view was loaded. At some point something changed and now there are system and hidden indices that require `expandWildcards: hidden` to be passed to field caps in order to see anything. `allowHidden: boolean` is added to the DataView and DataViewSpec and passed through when field caps requests are made. This is primarily a tool for troubleshooting. For instance, instead of hitting a full data stream across a number of data tiers you can select a specific index to compare its performance. NOTE: This is a second attempt. What I learned - the whole `expand_wildcards` param is literal - you can directly query a hidden index without `expandWildcards: hidden` since its not using a wildcard. Tests now use a wildcard. Closes: https://github.com/elastic/kibana/issues/164652 --------- Co-authored-by: Lukas Olson Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../fetch/get_search_params.test.ts | 47 +++++++++++++++++++ .../search_source/fetch/get_search_params.ts | 2 +- .../advanced_params_content.tsx | 4 +- .../advanced_params_section.tsx | 5 +- .../data_view_editor_flyout_content.tsx | 3 ++ .../data_view_flyout_content_container.tsx | 3 +- .../public/data_view_editor_service.ts | 1 + .../content_management/v1/cm_services.ts | 1 + .../__snapshots__/data_view.test.ts.snap | 3 ++ .../__snapshots__/data_views.test.ts.snap | 1 + .../common/data_views/abstract_data_views.ts | 8 ++++ .../data_views/common/data_views/data_view.ts | 1 + .../common/data_views/data_views.ts | 7 +++ src/plugins/data_views/common/types.ts | 9 ++++ .../data_views/data_views_api_client.ts | 2 + .../content_management/data_views_storage.ts | 1 + .../server/fetcher/index_patterns_fetcher.ts | 14 +++++- .../data_views/server/fetcher/lib/es_api.ts | 4 ++ .../field_capabilities.test.js | 1 + .../field_capabilities/field_capabilities.ts | 4 ++ .../rest_api_routes/internal/fields_for.ts | 4 ++ .../server/rest_api_routes/route_types.ts | 1 + .../server/rest_api_routes/schema.ts | 1 + .../data_views/_data_view_create_delete.ts | 37 +++++++++++++++ test/functional/page_objects/settings_page.ts | 14 +++++- .../log_views/log_views_client.test.ts | 3 ++ 26 files changed, 174 insertions(+), 7 deletions(-) diff --git a/src/plugins/data/common/search/search_source/fetch/get_search_params.test.ts b/src/plugins/data/common/search/search_source/fetch/get_search_params.test.ts index 1a517bf64d13b..975ad8d4a4878 100644 --- a/src/plugins/data/common/search/search_source/fetch/get_search_params.test.ts +++ b/src/plugins/data/common/search/search_source/fetch/get_search_params.test.ts @@ -9,6 +9,7 @@ import { UI_SETTINGS } from '../../../constants'; import { GetConfigFn } from '../../../types'; import { getSearchParams, getSearchParamsFromRequest } from './get_search_params'; +import { createStubDataView } from '@kbn/data-views-plugin/common/data_views/data_view.stub'; function getConfigStub(config: any = {}): GetConfigFn { return (key) => config[key]; @@ -46,4 +47,50 @@ describe('getSearchParams', () => { query: 123, }); }); + + test('sets expand_wildcards=all if data view has allowHidden=true', () => { + const getConfig = getConfigStub({ + [UI_SETTINGS.COURIER_SET_REQUEST_PREFERENCE]: 'custom', + [UI_SETTINGS.COURIER_CUSTOM_REQUEST_PREFERENCE]: 'aaa', + }); + const index = createStubDataView({ + spec: { + allowHidden: true, + }, + }); + const searchParams = getSearchParamsFromRequest( + { + index, + body: { + query: 123, + track_total_hits: true, + }, + }, + { getConfig } + ); + expect(searchParams).toHaveProperty('expand_wildcards', 'all'); + }); + + test('does not set expand_wildcards if data view has allowHidden=false', () => { + const getConfig = getConfigStub({ + [UI_SETTINGS.COURIER_SET_REQUEST_PREFERENCE]: 'custom', + [UI_SETTINGS.COURIER_CUSTOM_REQUEST_PREFERENCE]: 'aaa', + }); + const index = createStubDataView({ + spec: { + allowHidden: false, + }, + }); + const searchParams = getSearchParamsFromRequest( + { + index, + body: { + query: 123, + track_total_hits: true, + }, + }, + { getConfig } + ); + expect(searchParams).not.toHaveProperty('expand_wildcards', 'all'); + }); }); diff --git a/src/plugins/data/common/search/search_source/fetch/get_search_params.ts b/src/plugins/data/common/search/search_source/fetch/get_search_params.ts index 5f8ca973b3dfe..db630c85177e3 100644 --- a/src/plugins/data/common/search/search_source/fetch/get_search_params.ts +++ b/src/plugins/data/common/search/search_source/fetch/get_search_params.ts @@ -42,8 +42,8 @@ export function getSearchParamsFromRequest( return { index: searchRequest.index.title || searchRequest.index, body, - // @ts-expect-error `track_total_hits` not allowed at top level for `typesWithBodyKey` track_total_hits, + ...(searchRequest.index?.allowHidden && { expand_wildcards: 'all' }), ...searchParams, }; } diff --git a/src/plugins/data_view_editor/public/components/advanced_params_content/advanced_params_content.tsx b/src/plugins/data_view_editor/public/components/advanced_params_content/advanced_params_content.tsx index dcb8e2c1e1d80..db3746d02dc66 100644 --- a/src/plugins/data_view_editor/public/components/advanced_params_content/advanced_params_content.tsx +++ b/src/plugins/data_view_editor/public/components/advanced_params_content/advanced_params_content.tsx @@ -30,14 +30,16 @@ interface AdvancedParamsContentProps { disableAllowHidden: boolean; disableId: boolean; onAllowHiddenChange?: (value: boolean) => void; + defaultVisible?: boolean; } export const AdvancedParamsContent = ({ disableAllowHidden, disableId, onAllowHiddenChange, + defaultVisible = false, }: AdvancedParamsContentProps) => ( - + diff --git a/src/plugins/data_view_editor/public/components/advanced_params_content/advanced_params_section.tsx b/src/plugins/data_view_editor/public/components/advanced_params_content/advanced_params_section.tsx index b313fddb8ee2d..ce52223413ecd 100644 --- a/src/plugins/data_view_editor/public/components/advanced_params_content/advanced_params_section.tsx +++ b/src/plugins/data_view_editor/public/components/advanced_params_content/advanced_params_section.tsx @@ -13,10 +13,11 @@ import { EuiButtonEmpty, EuiSpacer } from '@elastic/eui'; interface Props { children: React.ReactNode; + defaultVisible: boolean; } -export const AdvancedParamsSection = ({ children }: Props) => { - const [isVisible, setIsVisible] = useState(false); +export const AdvancedParamsSection = ({ children, defaultVisible = false }: Props) => { + const [isVisible, setIsVisible] = useState(defaultVisible); const toggleIsVisible = useCallback(() => { setIsVisible(!isVisible); diff --git a/src/plugins/data_view_editor/public/components/data_view_editor_flyout_content.tsx b/src/plugins/data_view_editor/public/components/data_view_editor_flyout_content.tsx index ad344d482c0cf..73432737f9321 100644 --- a/src/plugins/data_view_editor/public/components/data_view_editor_flyout_content.tsx +++ b/src/plugins/data_view_editor/public/components/data_view_editor_flyout_content.tsx @@ -105,6 +105,7 @@ const IndexPatternEditorFlyoutContentComponent = ({ title: editData.getIndexPattern(), id: editData.id, name: editData.name, + allowHidden: editData.getAllowHidden(), ...(editData.timeFieldName ? { timestampField: { label: editData.timeFieldName, value: editData.timeFieldName }, @@ -124,6 +125,7 @@ const IndexPatternEditorFlyoutContentComponent = ({ timeFieldName: formData.timestampField?.value, id: formData.id, name: formData.name, + allowHidden: formData.allowHidden, }; if (type === INDEX_PATTERN_TYPE.ROLLUP && rollupIndex) { @@ -293,6 +295,7 @@ const IndexPatternEditorFlyoutContentComponent = ({ onAllowHiddenChange={() => { form.getFields().title.validate(); }} + defaultVisible={editData?.getAllowHidden()} />