diff --git a/docs/concepts/data-views.asciidoc b/docs/concepts/data-views.asciidoc index 992741be518a7..2eba42aed3051 100644 --- a/docs/concepts/data-views.asciidoc +++ b/docs/concepts/data-views.asciidoc @@ -172,3 +172,14 @@ WARNING: Deleting a {data-source} breaks all visualizations, saved searches, and . Find the {data-source} that you want to delete, and then click image:management/index-patterns/images/delete.png[Delete icon] in the *Actions* column. + +[float] +[[data-view-field-cache]] +=== {data-source} field cache + +The browser caches {data-source} field lists for increased performance. This is particularly impactful +for {data-sources} with a high field count that span a large number of indices and clusters. The field +list is updated every couple of minutes in typical {kib} usage. Alternatively, use the refresh button on the {data-source} +management detail page to get an updated field list. A force reload of {kib} has the same effect. + +The field list may be impacted by changes in indices and user permissions. \ No newline at end of file diff --git a/packages/core/http/core-http-router-server-internal/src/response.ts b/packages/core/http/core-http-router-server-internal/src/response.ts index 1fc8d310233c7..d96eaa6571bf0 100644 --- a/packages/core/http/core-http-router-server-internal/src/response.ts +++ b/packages/core/http/core-http-router-server-internal/src/response.ts @@ -17,6 +17,7 @@ import type { ErrorHttpResponseOptions, KibanaErrorResponseFactory, KibanaRedirectionResponseFactory, + KibanaNotModifiedResponseFactory, KibanaSuccessResponseFactory, KibanaResponseFactory, LifecycleResponseFactory, @@ -51,6 +52,10 @@ const redirectionResponseFactory: KibanaRedirectionResponseFactory = { redirected: (options: RedirectResponseOptions) => new KibanaResponse(302, options.body, options), }; +const notModifiedResponseFactory: KibanaNotModifiedResponseFactory = { + notModified: (options: HttpResponseOptions = {}) => new KibanaResponse(304, undefined, options), +}; + const errorResponseFactory: KibanaErrorResponseFactory = { badRequest: (options: ErrorHttpResponseOptions = {}) => new KibanaResponse(400, options.body || 'Bad Request', options), @@ -120,6 +125,7 @@ export const fileResponseFactory = { export const kibanaResponseFactory: KibanaResponseFactory = { ...successResponseFactory, ...redirectionResponseFactory, + ...notModifiedResponseFactory, ...errorResponseFactory, ...fileResponseFactory, custom: ( diff --git a/packages/core/http/core-http-router-server-internal/src/response_adapter.ts b/packages/core/http/core-http-router-server-internal/src/response_adapter.ts index f6a00bb7e6b92..a5cc1ca1f8d96 100644 --- a/packages/core/http/core-http-router-server-internal/src/response_adapter.ts +++ b/packages/core/http/core-http-router-server-internal/src/response_adapter.ts @@ -33,7 +33,8 @@ function setHeaders(response: HapiResponseObject, headers: Record code >= 100 && code < 300, - isRedirect: (code: number) => code >= 300 && code < 400, + isNotModified: (code: number) => code === 304, + isRedirect: (code: number) => code >= 300 && code < 400 && code !== 304, isError: (code: number) => code >= 400 && code < 600, }; @@ -76,7 +77,10 @@ export class HapiResponseAdapter { if (statusHelpers.isError(kibanaResponse.status)) { return this.toError(kibanaResponse); } - if (statusHelpers.isSuccess(kibanaResponse.status)) { + if ( + statusHelpers.isSuccess(kibanaResponse.status) || + statusHelpers.isNotModified(kibanaResponse.status) + ) { return this.toSuccess(kibanaResponse); } if (statusHelpers.isRedirect(kibanaResponse.status)) { diff --git a/packages/core/http/core-http-router-server-mocks/src/router.mock.ts b/packages/core/http/core-http-router-server-mocks/src/router.mock.ts index 4272cf130b38e..d5904699b5813 100644 --- a/packages/core/http/core-http-router-server-mocks/src/router.mock.ts +++ b/packages/core/http/core-http-router-server-mocks/src/router.mock.ts @@ -121,6 +121,7 @@ const createResponseFactoryMock = (): jest.Mocked => ({ ok: jest.fn(), accepted: jest.fn(), noContent: jest.fn(), + notModified: jest.fn(), custom: jest.fn(), redirected: jest.fn(), badRequest: jest.fn(), diff --git a/packages/core/http/core-http-server/index.ts b/packages/core/http/core-http-server/index.ts index 9a28a7be5033e..82022ce77d6e3 100644 --- a/packages/core/http/core-http-server/index.ts +++ b/packages/core/http/core-http-server/index.ts @@ -95,6 +95,7 @@ export type { IKibanaSocket, KibanaErrorResponseFactory, KibanaRedirectionResponseFactory, + KibanaNotModifiedResponseFactory, KibanaSuccessResponseFactory, KibanaResponseFactory, LifecycleResponseFactory, diff --git a/packages/core/http/core-http-server/src/router/index.ts b/packages/core/http/core-http-server/src/router/index.ts index c72d7386e867d..628bab27db54f 100644 --- a/packages/core/http/core-http-server/src/router/index.ts +++ b/packages/core/http/core-http-server/src/router/index.ts @@ -66,6 +66,7 @@ export type { IKibanaSocket } from './socket'; export type { KibanaErrorResponseFactory, KibanaRedirectionResponseFactory, + KibanaNotModifiedResponseFactory, KibanaSuccessResponseFactory, KibanaResponseFactory, LifecycleResponseFactory, diff --git a/packages/core/http/core-http-server/src/router/response.ts b/packages/core/http/core-http-server/src/router/response.ts index 07ec226e8c3a9..194333d42f9c9 100644 --- a/packages/core/http/core-http-server/src/router/response.ts +++ b/packages/core/http/core-http-server/src/router/response.ts @@ -95,9 +95,7 @@ export interface FileHttpResponseOptions !dataView.getFieldByName(field.name)); // refresh the data view in case there are new fields if (newFields.length) { - await dataViewsService.refreshFields(dataView, false); + await dataViewsService.refreshFields(dataView, false, true); } const allFields = buildFieldList(dataView, metaFields); return { diff --git a/src/plugins/data_view_management/public/components/edit_index_pattern/edit_index_pattern.tsx b/src/plugins/data_view_management/public/components/edit_index_pattern/edit_index_pattern.tsx index 7ce1c9ccbc66d..f850ef4ec3671 100644 --- a/src/plugins/data_view_management/public/components/edit_index_pattern/edit_index_pattern.tsx +++ b/src/plugins/data_view_management/public/components/edit_index_pattern/edit_index_pattern.tsx @@ -95,6 +95,9 @@ export const EditIndexPattern = withRouter( const [showEditDialog, setShowEditDialog] = useState(false); const [relationships, setRelationships] = useState([]); const [allowedTypes, setAllowedTypes] = useState([]); + const [refreshCount, setRefreshCount] = useState(0); // used for forcing rerender of field list + const [isRefreshing, setIsRefreshing] = React.useState(false); + const conflictFieldsUrl = useMemo(() => { return setStateToKbnUrl( APP_STATE_STORAGE_KEY, @@ -144,7 +147,7 @@ export const EditIndexPattern = withRouter( setConflictedFields( indexPattern.fields.getAll().filter((field) => field.type === 'conflict') ); - }, [indexPattern]); + }, [indexPattern, refreshCount]); useEffect(() => { setTags( @@ -332,6 +335,13 @@ export const EditIndexPattern = withRouter( setFields(indexPattern.getNonScriptedFields()); setCompositeRuntimeFields(getCompositeRuntimeFields(indexPattern)); }} + refreshIndexPatternClick={async () => { + setIsRefreshing(true); + await dataViews.refreshFields(indexPattern, false, true); + setRefreshCount(refreshCount + 1); // rerender field list + setIsRefreshing(false); + }} + isRefreshing={isRefreshing} /> {displayIndexPatternEditor} diff --git a/src/plugins/data_view_management/public/components/edit_index_pattern/index_header/index_header.tsx b/src/plugins/data_view_management/public/components/edit_index_pattern/index_header/index_header.tsx index 83c965f2b9332..8b12ed59ed6fd 100644 --- a/src/plugins/data_view_management/public/components/edit_index_pattern/index_header/index_header.tsx +++ b/src/plugins/data_view_management/public/components/edit_index_pattern/index_header/index_header.tsx @@ -17,7 +17,9 @@ interface IndexHeaderProps { setDefault?: () => void; editIndexPatternClick?: () => void; deleteIndexPatternClick?: () => void; + refreshIndexPatternClick?: () => void; canSave: boolean; + isRefreshing?: boolean; } const setDefaultAriaLabel = i18n.translate('indexPatternManagement.editDataView.setDefaultAria', { @@ -63,6 +65,7 @@ export const IndexHeader: React.FC = ({ iconType="pencil" aria-label={editAriaLabel} data-test-subj="editIndexPatternButton" + color="primary" > {editTooltip} diff --git a/src/plugins/data_view_management/public/components/edit_index_pattern/tabs/tabs.tsx b/src/plugins/data_view_management/public/components/edit_index_pattern/tabs/tabs.tsx index 0128d013af6f5..426442bcfb031 100644 --- a/src/plugins/data_view_management/public/components/edit_index_pattern/tabs/tabs.tsx +++ b/src/plugins/data_view_management/public/components/edit_index_pattern/tabs/tabs.tsx @@ -19,9 +19,10 @@ import { EuiTabbedContentTab, EuiSpacer, EuiFieldSearch, - EuiButton, EuiFilterSelectItem, FilterChecked, + EuiToolTip, + EuiButton, } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { fieldWildcardMatcher } from '@kbn/kibana-utils-plugin/public'; @@ -60,6 +61,8 @@ interface TabsProps extends Pick { relationships: SavedObjectRelation[]; allowedTypes: SavedObjectManagementTypeInfo[]; compositeRuntimeFields: Record; + refreshIndexPatternClick: () => void; + isRefreshing?: boolean; } interface FilterItems { @@ -139,6 +142,14 @@ const addFieldButtonLabel = i18n.translate( } ); +const refreshAriaLabel = i18n.translate('indexPatternManagement.editDataView.refreshAria', { + defaultMessage: 'Refresh', +}); + +const refreshTooltip = i18n.translate('indexPatternManagement.editDataView.refreshTooltip', { + defaultMessage: 'Refresh local copy of data view field list', +}); + const SCHEMA_ITEMS: FilterItems[] = [ { value: 'runtime', @@ -159,6 +170,8 @@ export const Tabs: React.FC = ({ relationships, allowedTypes, compositeRuntimeFields, + refreshIndexPatternClick, + isRefreshing, }) => { const { uiSettings, @@ -284,7 +297,9 @@ export const Tabs: React.FC = ({ setIndexedFieldTypes(convertToEuiFilterOptions(tempIndexedFieldTypes)); setScriptedFieldLanguages(convertToEuiFilterOptions(tempScriptedFieldLanguages)); - }, [indexPattern]); + // need to reset based on changes to fields but indexPattern is the same + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [indexPattern, fields]); const closeFieldEditor = useCallback(() => { if (closeEditorHandler.current) { @@ -321,11 +336,13 @@ export const Tabs: React.FC = ({ [uiSettings] ); + const refreshRef = useRef(null); + const userEditPermission = dataViews.getCanSaveSync(); const getFilterSection = useCallback( (type: string) => { return ( - + = ({ + + {refreshTooltip}

}> + { + refreshIndexPatternClick(); + // clear tooltip focus + if (refreshRef.current) { + refreshRef.current.blur(); + } + }} + iconType="refresh" + aria-label={refreshAriaLabel} + data-test-subj="refreshDataViewButton" + isLoading={isRefreshing} + isDisabled={isRefreshing} + size="m" + color="success" + className="eui-fullWidth" + > + {refreshAriaLabel} + + + {userEditPermission && ( - openFieldEditor()} data-test-subj="addField"> + openFieldEditor()} + data-test-subj="addField" + iconType="plusInCircle" + aria-label={addFieldButtonLabel} + color="primary" + fill + > {addFieldButtonLabel} @@ -503,6 +552,8 @@ export const Tabs: React.FC = ({ updateFieldFilter, updateFieldTypeFilter, updateSchemaFieldTypeFilter, + isRefreshing, + refreshIndexPatternClick, ] ); diff --git a/src/plugins/data_view_management/public/components/index_pattern_table/index_pattern_table.tsx b/src/plugins/data_view_management/public/components/index_pattern_table/index_pattern_table.tsx index 6f6df3e77bf09..3010dd41448ea 100644 --- a/src/plugins/data_view_management/public/components/index_pattern_table/index_pattern_table.tsx +++ b/src/plugins/data_view_management/public/components/index_pattern_table/index_pattern_table.tsx @@ -221,7 +221,7 @@ export const IndexPatternTable = ({
{dataView.getName()} {dataView.name ? ( diff --git a/src/plugins/data_views/common/constants.ts b/src/plugins/data_views/common/constants.ts index dc396f2772aa8..43dc67362bf26 100644 --- a/src/plugins/data_views/common/constants.ts +++ b/src/plugins/data_views/common/constants.ts @@ -58,6 +58,12 @@ export const PLUGIN_NAME = 'DataViews'; */ export const FIELDS_FOR_WILDCARD_PATH = '/internal/data_views/_fields_for_wildcard'; +/** + * Fields path. Like fields for wildcard but GET only + * @public + */ +export const FIELDS_PATH = '/internal/data_views/fields'; + /** * Existing indices path * @public diff --git a/src/plugins/data_views/common/data_views/data_view.ts b/src/plugins/data_views/common/data_views/data_view.ts index d60d0b4c1172c..9767b44b95a72 100644 --- a/src/plugins/data_views/common/data_views/data_view.ts +++ b/src/plugins/data_views/common/data_views/data_view.ts @@ -61,6 +61,8 @@ export class DataView extends AbstractDataView implements DataViewBase { */ public flattenHit: (hit: Record, deep?: boolean) => Record; + private etag: string | undefined; + /** * constructor * @param config - config data and dependencies @@ -77,6 +79,10 @@ export class DataView extends AbstractDataView implements DataViewBase { this.fields.replaceAll(Object.values(spec.fields || {})); } + getEtag = () => this.etag; + + setEtag = (etag: string | undefined) => (this.etag = etag); + /** * Returns scripted fields */ diff --git a/src/plugins/data_views/common/data_views/data_views.test.ts b/src/plugins/data_views/common/data_views/data_views.test.ts index e95e5c6736d17..4d0fa196e9ccc 100644 --- a/src/plugins/data_views/common/data_views/data_views.test.ts +++ b/src/plugins/data_views/common/data_views/data_views.test.ts @@ -158,10 +158,23 @@ describe('IndexPatterns', () => { test('force field refresh', async () => { const id = '1'; + // make sure initial load and subsequent reloads use same params + const args = { + allowHidden: undefined, + allowNoIndex: true, + indexFilter: undefined, + metaFields: false, + pattern: 'something', + rollupIndex: undefined, + type: undefined, + }; + await indexPatterns.get(id); expect(apiClient.getFieldsForWildcard).toBeCalledTimes(1); + expect(apiClient.getFieldsForWildcard).toBeCalledWith(args); await indexPatterns.get(id, undefined, true); expect(apiClient.getFieldsForWildcard).toBeCalledTimes(2); + expect(apiClient.getFieldsForWildcard).toBeCalledWith(args); }); test('getFieldsForWildcard called with allowNoIndex set to true as default ', async () => { diff --git a/src/plugins/data_views/common/data_views/data_views.ts b/src/plugins/data_views/common/data_views/data_views.ts index 6052be19d4f99..a244868c0614d 100644 --- a/src/plugins/data_views/common/data_views/data_views.ts +++ b/src/plugins/data_views/common/data_views/data_views.ts @@ -261,7 +261,11 @@ export interface DataViewsServicePublicMethods { * Refresh fields for data view instance * @params dataView - Data view instance */ - refreshFields: (indexPattern: DataView, displayErrors?: boolean) => Promise; + refreshFields: ( + indexPattern: DataView, + displayErrors?: boolean, + forceRefresh?: boolean + ) => Promise; /** * Converts data view saved object to spec * @params savedObject - Data view saved object @@ -535,7 +539,10 @@ export class DataViewsService { (indexPattern as DataViewSpec).allowHidden || (indexPattern as DataView)?.getAllowHidden(), }); - private getFieldsAndIndicesForDataView = async (dataView: DataView) => { + private getFieldsAndIndicesForDataView = async ( + dataView: DataView, + forceRefresh: boolean = false + ) => { const metaFields = await this.config.get(META_FIELDS); return this.apiClient.getFieldsForWildcard({ type: dataView.type, @@ -543,7 +550,8 @@ export class DataViewsService { allowNoIndex: true, pattern: dataView.getIndexPattern(), metaFields, - allowHidden: dataView.getAllowHidden(), + forceRefresh, + allowHidden: dataView.getAllowHidden() || undefined, }); }; @@ -560,8 +568,18 @@ export class DataViewsService { }); }; - private refreshFieldsFn = async (indexPattern: DataView) => { - const { fields, indices } = await this.getFieldsAndIndicesForDataView(indexPattern); + private refreshFieldsFn = async (indexPattern: DataView, forceRefresh: boolean = false) => { + const { fields, indices, etag } = await this.getFieldsAndIndicesForDataView( + indexPattern, + forceRefresh + ); + + if (indexPattern.getEtag() && etag === indexPattern.getEtag()) { + return; + } else { + indexPattern.setEtag(etag); + } + fields.forEach((field) => (field.isMapped = true)); const scripted = this.scriptedFieldsEnabled ? indexPattern.getScriptedFields().map((field) => field.spec) @@ -587,13 +605,17 @@ export class DataViewsService { * @param dataView * @param displayErrors - If set false, API consumer is responsible for displaying and handling errors. */ - refreshFields = async (dataView: DataView, displayErrors: boolean = true) => { + refreshFields = async ( + dataView: DataView, + displayErrors: boolean = true, + forceRefresh: boolean = false + ) => { if (!displayErrors) { - return this.refreshFieldsFn(dataView); + return this.refreshFieldsFn(dataView, forceRefresh); } try { - await this.refreshFieldsFn(dataView); + await this.refreshFieldsFn(dataView, forceRefresh); } catch (err) { if (err instanceof DataViewMissingIndices) { // not considered an error, check dataView.matchedIndices.length to be 0 @@ -634,7 +656,11 @@ export class DataViewsService { : []; try { let updatedFieldList: FieldSpec[]; - const { fields: newFields, indices } = await this.getFieldsAndIndicesForWildcard(options); + const { + fields: newFields, + indices, + etag, + } = await this.getFieldsAndIndicesForWildcard(options); newFields.forEach((field) => (field.isMapped = true)); // If allowNoIndex, only update field list if field caps finds fields. To support @@ -645,7 +671,7 @@ export class DataViewsService { updatedFieldList = fieldsAsArr; } - return { fields: this.fieldArrayToMap(updatedFieldList, fieldAttrs), indices }; + return { fields: this.fieldArrayToMap(updatedFieldList, fieldAttrs), indices, etag }; } catch (err) { if (err instanceof DataViewMissingIndices) { // not considered an error, check dataView.matchedIndices.length to be 0 @@ -759,7 +785,7 @@ export class DataViewsService { displayErrors?: boolean; }) => { const { title, type, typeMeta, runtimeFieldMap } = spec; - const { fields, indices } = await this.refreshFieldSpecMap( + const { fields, indices, etag } = await this.refreshFieldSpecMap( spec.fields || {}, savedObjectId, spec.title as string, @@ -777,7 +803,7 @@ export class DataViewsService { const runtimeFieldSpecs = this.getRuntimeFields(runtimeFieldMap, spec.fieldAttrs); // mapped fields overwrite runtime fields - return { fields: { ...runtimeFieldSpecs, ...fields }, indices: indices || [] }; + return { fields: { ...runtimeFieldSpecs, ...fields }, indices: indices || [], etag }; }; private initFromSavedObject = async ( @@ -791,6 +817,7 @@ export class DataViewsService { let fields: Record = {}; let indices: string[] = []; + let etag: string | undefined; if (!displayErrors) { const fieldsAndIndices = await this.initFromSavedObjectLoadFields({ @@ -800,6 +827,7 @@ export class DataViewsService { }); fields = fieldsAndIndices.fields; indices = fieldsAndIndices.indices; + etag = fieldsAndIndices.etag; } else { try { const fieldsAndIndices = await this.initFromSavedObjectLoadFields({ @@ -809,6 +837,7 @@ export class DataViewsService { }); fields = fieldsAndIndices.fields; indices = fieldsAndIndices.indices; + etag = fieldsAndIndices.etag; } catch (err) { if (err instanceof DataViewMissingIndices) { // not considered an error, check dataView.matchedIndices.length to be 0 @@ -833,6 +862,7 @@ export class DataViewsService { : {}; const indexPattern = await this.createFromSpec(spec, true, displayErrors); + indexPattern.setEtag(etag); indexPattern.matchedIndices = indices; indexPattern.resetOriginalSavedObjectBody(); return indexPattern; diff --git a/src/plugins/data_views/common/types.ts b/src/plugins/data_views/common/types.ts index dc477d2fb5b83..7fb7ddedbccd4 100644 --- a/src/plugins/data_views/common/types.ts +++ b/src/plugins/data_views/common/types.ts @@ -314,6 +314,7 @@ export interface GetFieldsOptions { includeUnmapped?: boolean; fields?: string[]; allowHidden?: boolean; + forceRefresh?: boolean; } /** @@ -322,6 +323,7 @@ export interface GetFieldsOptions { export interface FieldsForWildcardResponse { fields: FieldSpec[]; indices: string[]; + etag?: string; } /** @@ -542,4 +544,5 @@ export interface HasDataService { export interface ClientConfigType { scriptedFieldsEnabled?: boolean; + fieldListCachingEnabled?: boolean; } diff --git a/src/plugins/data_views/common/utils.test.ts b/src/plugins/data_views/common/utils.test.ts index 3351a15da1a13..e8602def7f712 100644 --- a/src/plugins/data_views/common/utils.test.ts +++ b/src/plugins/data_views/common/utils.test.ts @@ -8,6 +8,7 @@ import { isFilterable } from '.'; import type { DataViewField } from './fields'; +import { unwrapEtag } from './utils'; const mockField = { name: 'foo', @@ -16,38 +17,55 @@ const mockField = { type: 'string', } as DataViewField; -describe('isFilterable', () => { - describe('types', () => { - it('should return true for filterable types', () => { - ['string', 'number', 'date', 'ip', 'boolean'].forEach((type) => { - expect(isFilterable({ ...mockField, type } as DataViewField)).toBe(true); +describe('common utils', () => { + describe('isFilterable', () => { + describe('types', () => { + it('should return true for filterable types', () => { + ['string', 'number', 'date', 'ip', 'boolean'].forEach((type) => { + expect(isFilterable({ ...mockField, type } as DataViewField)).toBe(true); + }); }); - }); - it('should return false for filterable types if the field is not searchable', () => { - ['string', 'number', 'date', 'ip', 'boolean'].forEach((type) => { - expect(isFilterable({ ...mockField, type, searchable: false } as DataViewField)).toBe( - false - ); + it('should return false for filterable types if the field is not searchable', () => { + ['string', 'number', 'date', 'ip', 'boolean'].forEach((type) => { + expect(isFilterable({ ...mockField, type, searchable: false } as DataViewField)).toBe( + false + ); + }); }); - }); - it('should return false for un-filterable types', () => { - ['geo_point', 'geo_shape', 'attachment', 'murmur3', '_source', 'unknown', 'conflict'].forEach( - (type) => { + it('should return false for un-filterable types', () => { + [ + 'geo_point', + 'geo_shape', + 'attachment', + 'murmur3', + '_source', + 'unknown', + 'conflict', + ].forEach((type) => { expect(isFilterable({ ...mockField, type } as DataViewField)).toBe(false); - } - ); + }); + }); }); - }); - it('should return true for scripted fields', () => { - expect(isFilterable({ ...mockField, scripted: true, searchable: false } as DataViewField)).toBe( - true - ); + it('should return true for scripted fields', () => { + expect( + isFilterable({ ...mockField, scripted: true, searchable: false } as DataViewField) + ).toBe(true); + }); + + it('should return true for the _id field', () => { + expect(isFilterable({ ...mockField, name: '_id' } as DataViewField)).toBe(true); + }); }); - it('should return true for the _id field', () => { - expect(isFilterable({ ...mockField, name: '_id' } as DataViewField)).toBe(true); + describe('unwrapEtag', () => { + it('should return the etag without quotes', () => { + expect(unwrapEtag('"foo"')).toBe('foo'); + }); + it('should return the etag without quotes and without gzip', () => { + expect(unwrapEtag('"foo-gzip"')).toBe('foo'); + }); }); }); diff --git a/src/plugins/data_views/common/utils.ts b/src/plugins/data_views/common/utils.ts index e63167ff4d3ff..61de24dfcf0c3 100644 --- a/src/plugins/data_views/common/utils.ts +++ b/src/plugins/data_views/common/utils.ts @@ -27,3 +27,11 @@ export async function findByName(client: PersistenceAPI, name: string) { return savedObjects ? savedObjects[0] : undefined; } } + +export function unwrapEtag(ifNoneMatch: string) { + let requestHash = ifNoneMatch.replace(/^"(.+)"$/, '$1'); + if (requestHash.indexOf('-') > -1) { + requestHash = requestHash.split('-')[0]; + } + return requestHash; +} diff --git a/src/plugins/data_views/kibana.jsonc b/src/plugins/data_views/kibana.jsonc index 7595f8e68b3c4..7789383b48ba4 100644 --- a/src/plugins/data_views/kibana.jsonc +++ b/src/plugins/data_views/kibana.jsonc @@ -18,6 +18,7 @@ "requiredBundles": [ "kibanaUtils" ], + "runtimePluginDependencies" : ["security"], "extraPublicDirs": [ "common" ] diff --git a/src/plugins/data_views/public/data_views/data_views_api_client.test.ts b/src/plugins/data_views/public/data_views/data_views_api_client.test.ts index f36776153ee3a..66f025dc07835 100644 --- a/src/plugins/data_views/public/data_views/data_views_api_client.test.ts +++ b/src/plugins/data_views/public/data_views/data_views_api_client.test.ts @@ -9,7 +9,7 @@ import type { HttpSetup } from '@kbn/core/public'; import { http } from './data_views_api_client.test.mock'; import { DataViewsApiClient } from './data_views_api_client'; -import { FIELDS_FOR_WILDCARD_PATH as expectedPath } from '../../common/constants'; +import { FIELDS_PATH as expectedPath } from '../../common/constants'; describe('IndexPatternsApiClient', () => { let fetchSpy: jest.SpyInstance; @@ -17,12 +17,32 @@ describe('IndexPatternsApiClient', () => { beforeEach(() => { fetchSpy = jest.spyOn(http, 'fetch').mockImplementation(() => Promise.resolve({})); - indexPatternsApiClient = new DataViewsApiClient(http as HttpSetup); + indexPatternsApiClient = new DataViewsApiClient(http as HttpSetup, () => + Promise.resolve(undefined) + ); }); test('uses the right URI to fetch fields for wildcard', async function () { - await indexPatternsApiClient.getFieldsForWildcard({ pattern: 'blah' }); + await indexPatternsApiClient.getFieldsForWildcard({ pattern: 'blah', allowHidden: false }); - expect(fetchSpy).toHaveBeenCalledWith(expectedPath, expect.any(Object)); + expect(fetchSpy).toHaveBeenCalledWith(expectedPath, { + // not sure what asResponse is but the rest of the results are useful + asResponse: true, + headers: { + 'user-hash': '', + }, + query: { + allow_hidden: undefined, + allow_no_index: undefined, + apiVersion: '1', // version passed through query params for caching + fields: undefined, + include_unmapped: undefined, + meta_fields: undefined, + pattern: 'blah', + rollup_index: undefined, + type: undefined, + }, + version: '1', // version header + }); }); }); diff --git a/src/plugins/data_views/public/data_views/data_views_api_client.ts b/src/plugins/data_views/public/data_views/data_views_api_client.ts index ca5782736e3cd..65b418642b34f 100644 --- a/src/plugins/data_views/public/data_views/data_views_api_client.ts +++ b/src/plugins/data_views/public/data_views/data_views_api_client.ts @@ -6,33 +6,61 @@ * Side Public License, v 1. */ -import { HttpSetup } from '@kbn/core/public'; +import { HttpSetup, HttpResponse } from '@kbn/core/public'; import { DataViewMissingIndices } from '../../common/lib'; import { GetFieldsOptions, IDataViewsApiClient } from '../../common'; import { FieldsForWildcardResponse } from '../../common/types'; -import { FIELDS_FOR_WILDCARD_PATH } from '../../common/constants'; +import { FIELDS_FOR_WILDCARD_PATH, FIELDS_PATH } from '../../common/constants'; const API_BASE_URL: string = `/api/index_patterns/`; const version = '1'; +async function sha1(str: string) { + const enc = new TextEncoder(); + const hash = await crypto.subtle.digest('SHA-1', enc.encode(str)); + return Array.from(new Uint8Array(hash)) + .map((v) => v.toString(16).padStart(2, '0')) + .join(''); +} + /** * Data Views API Client - client implementation */ export class DataViewsApiClient implements IDataViewsApiClient { private http: HttpSetup; + private getCurrentUserId: () => Promise; /** * constructor * @param http http dependency */ - constructor(http: HttpSetup) { + constructor(http: HttpSetup, getCurrentUserId: () => Promise) { this.http = http; + this.getCurrentUserId = getCurrentUserId; } - private _request(url: string, query?: {}, body?: string): Promise { + private async _request( + url: string, + query?: {}, + body?: string, + forceRefresh?: boolean + ): Promise | undefined> { + const asResponse = true; + const cacheOptions: { cache?: RequestCache } = forceRefresh ? { cache: 'no-cache' } : {}; + const userId = await this.getCurrentUserId(); + + const userHash = userId ? await sha1(userId) : ''; + const request = body - ? this.http.post(url, { query, body, version }) - : this.http.fetch(url, { query, version }); + ? this.http.post(url, { query, body, version, asResponse }) + : this.http.fetch(url, { + query, + version, + ...cacheOptions, + asResponse, + headers: { 'user-hash': userHash }, + }); + return request.catch((resp) => { if (resp.body.statusCode === 404 && resp.body.attributes?.code === 'no_matching_indices') { throw new DataViewMissingIndices(resp.body.message); @@ -60,10 +88,14 @@ export class DataViewsApiClient implements IDataViewsApiClient { indexFilter, includeUnmapped, fields, + forceRefresh, allowHidden, } = options; + const path = indexFilter ? FIELDS_FOR_WILDCARD_PATH : FIELDS_PATH; + const versionQueryParam = indexFilter ? {} : { apiVersion: version }; + return this._request( - FIELDS_FOR_WILDCARD_PATH, + path, { pattern, meta_fields: metaFields, @@ -72,11 +104,18 @@ export class DataViewsApiClient implements IDataViewsApiClient { allow_no_index: allowNoIndex, include_unmapped: includeUnmapped, fields, - allow_hidden: allowHidden, + // default to undefined to keep value out of URL params and improve caching + allow_hidden: allowHidden || undefined, + ...versionQueryParam, }, - indexFilter ? JSON.stringify({ index_filter: indexFilter }) : undefined + indexFilter ? JSON.stringify({ index_filter: indexFilter }) : undefined, + forceRefresh ).then((response) => { - return response || { fields: [], indices: [] }; + return { + indices: response?.body?.indices || [], + fields: response?.body?.fields || [], + etag: response?.response?.headers?.get('etag') || '', + }; }); } @@ -87,6 +126,7 @@ export class DataViewsApiClient implements IDataViewsApiClient { const response = await this._request<{ result: boolean }>( this._getUrl(['has_user_index_pattern']) ); - return response?.result ?? false; + + return response?.body?.result ?? false; } } diff --git a/src/plugins/data_views/public/index.ts b/src/plugins/data_views/public/index.ts index f690552b5a147..96380665548dd 100644 --- a/src/plugins/data_views/public/index.ts +++ b/src/plugins/data_views/public/index.ts @@ -67,6 +67,7 @@ export type { DataViewsContract, HasDataViewsResponse, IndicesViaSearchResponse, + UserIdGetter, } from './types'; // Export plugin after all other imports diff --git a/src/plugins/data_views/public/plugin.ts b/src/plugins/data_views/public/plugin.ts index d00292c3f4fe8..8ff545d8585e1 100644 --- a/src/plugins/data_views/public/plugin.ts +++ b/src/plugins/data_views/public/plugin.ts @@ -8,6 +8,7 @@ import { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from '@kbn/core/public'; import { i18n } from '@kbn/i18n'; +import type { SecurityPluginStart } from '@kbn/security-plugin-types-public'; import { getIndexPatternLoad } from './expressions'; import type { ClientConfigType } from '../common/types'; import { @@ -15,6 +16,7 @@ import { DataViewsPublicPluginStart, DataViewsPublicSetupDependencies, DataViewsPublicStartDependencies, + UserIdGetter, } from './types'; import { DataViewsApiClient } from '.'; @@ -41,6 +43,7 @@ export class DataViewsPublicPlugin { private readonly hasData = new HasData(); private rollupsEnabled: boolean = false; + private userIdGetter: UserIdGetter = async () => undefined; constructor(private readonly initializerContext: PluginInitializerContext) {} @@ -60,6 +63,18 @@ export class DataViewsPublicPlugin }), }); + core.plugins.onStart<{ security: SecurityPluginStart }>('security').then(({ security }) => { + if (security.found) { + const getUserId = async function getUserId(): Promise { + const currentUser = await security.contract.authc.getCurrentUser(); + return currentUser?.profile_uid; + }; + this.userIdGetter = getUserId; + } else { + throw new Error('Security plugin is not available, but is required for Data Views plugin'); + } + }); + return { enableRollups: () => (this.rollupsEnabled = true), }; @@ -86,7 +101,7 @@ export class DataViewsPublicPlugin hasData: this.hasData.start(core), uiSettings: new UiSettingsPublicToCommon(uiSettings), savedObjectsClient: new ContentMagementWrapper(contentManagement.client), - apiClient: new DataViewsApiClient(http), + apiClient: new DataViewsApiClient(http, () => this.userIdGetter()), fieldFormats, http, onNotification: (toastInputFields, key) => { diff --git a/src/plugins/data_views/public/types.ts b/src/plugins/data_views/public/types.ts index 7d6cbbc6cf533..98db0382f641f 100644 --- a/src/plugins/data_views/public/types.ts +++ b/src/plugins/data_views/public/types.ts @@ -106,6 +106,8 @@ export interface DataViewsPublicStartDependencies { contentManagement: ContentManagementPublicStart; } +export type UserIdGetter = () => Promise; + /** * Data plugin public Setup contract */ diff --git a/src/plugins/data_views/server/constants.ts b/src/plugins/data_views/server/constants.ts index 040a321978e17..67235ae3af16d 100644 --- a/src/plugins/data_views/server/constants.ts +++ b/src/plugins/data_views/server/constants.ts @@ -93,3 +93,8 @@ export const INITIAL_REST_VERSION = '2023-10-31'; */ export const INITIAL_REST_VERSION_INTERNAL = '1'; + +/** + * Default field caps cache max-age in seconds + */ +export const DEFAULT_FIELD_CACHE_FRESHNESS = 5; diff --git a/src/plugins/data_views/server/fetcher/lib/field_capabilities/field_capabilities.ts b/src/plugins/data_views/server/fetcher/lib/field_capabilities/field_capabilities.ts index fdf3f1afe7c44..1ca5a6a734470 100644 --- a/src/plugins/data_views/server/fetcher/lib/field_capabilities/field_capabilities.ts +++ b/src/plugins/data_views/server/fetcher/lib/field_capabilities/field_capabilities.ts @@ -60,7 +60,7 @@ export async function getFieldCapabilities(params: FieldCapabilitiesParams) { const allFieldsUnsorted = Object.keys(fieldsFromFieldCapsByName) // not all meta fields are provided, so remove and manually add .filter((name) => !fieldsFromFieldCapsByName[name].metadata_field) - .concat(fieldCapsArr.length ? metaFields : []) + .concat(fieldCapsArr.length ? metaFields : []) // empty field lists should stay empty .reduce<{ names: string[]; map: Map }>( (agg, value) => { // This is intentionally using a Map to be highly optimized with very large indexes AND be safe for user provided data diff --git a/src/plugins/data_views/server/index.ts b/src/plugins/data_views/server/index.ts index eba0a83a8df49..d8382046183a3 100644 --- a/src/plugins/data_views/server/index.ts +++ b/src/plugins/data_views/server/index.ts @@ -46,6 +46,12 @@ const configSchema = schema.object({ schema.boolean({ defaultValue: false }), schema.never() ), + fieldListCachingEnabled: schema.conditional( + schema.contextRef('serverless'), + true, + schema.boolean({ defaultValue: false }), + schema.boolean({ defaultValue: true }) + ), }); type ConfigType = TypeOf; diff --git a/src/plugins/data_views/server/plugin.ts b/src/plugins/data_views/server/plugin.ts index bbcc5dafc81c2..2da7d128b837a 100644 --- a/src/plugins/data_views/server/plugin.ts +++ b/src/plugins/data_views/server/plugin.ts @@ -23,6 +23,7 @@ import { DataViewsServerPluginStartDependencies, } from './types'; import { DataViewsStorage } from './content_management'; +import { cacheMaxAge } from './ui_settings'; export class DataViewsServerPlugin implements @@ -46,14 +47,21 @@ export class DataViewsServerPlugin ) { core.savedObjects.registerType(dataViewSavedObjectType); core.capabilities.registerProvider(capabilitiesProvider); + + const config = this.initializerContext.config.get(); + + if (config.fieldListCachingEnabled) { + core.uiSettings.register(cacheMaxAge); + } + const dataViewRestCounter = usageCollection?.createUsageCounter('dataViewsRestApi'); - registerRoutes( - core.http, - core.getStartServices, - () => this.rollupsEnabled, - dataViewRestCounter - ); + registerRoutes({ + http: core.http, + getStartServices: core.getStartServices, + isRollupsEnabled: () => this.rollupsEnabled, + dataViewRestCounter, + }); expressions.registerFunction(getIndexPatternLoad({ getStartServices: core.getStartServices })); registerIndexPatternsUsageCollector(core.getStartServices, usageCollection); diff --git a/src/plugins/data_views/server/rest_api_routes/internal/fields.ts b/src/plugins/data_views/server/rest_api_routes/internal/fields.ts new file mode 100644 index 0000000000000..cee8efc8b6d1d --- /dev/null +++ b/src/plugins/data_views/server/rest_api_routes/internal/fields.ts @@ -0,0 +1,146 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { createHash } from 'crypto'; +import { IRouter, RequestHandler, StartServicesAccessor } from '@kbn/core/server'; +import { unwrapEtag } from '../../../common/utils'; +import { IndexPatternsFetcher } from '../../fetcher'; +import type { + DataViewsServerPluginStart, + DataViewsServerPluginStartDependencies, +} from '../../types'; +import type { FieldDescriptorRestResponse } from '../route_types'; +import { FIELDS_PATH as path } from '../../../common/constants'; +import { parseFields, IBody, IQuery, querySchema, validate } from './fields_for'; +import { DEFAULT_FIELD_CACHE_FRESHNESS } from '../../constants'; + +export function calculateHash(srcBuffer: Buffer) { + const hash = createHash('sha1'); + hash.update(srcBuffer); + return hash.digest('hex'); +} + +const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, IBody> = + (isRollupsEnabled) => async (context, request, response) => { + const core = await context.core; + const uiSettings = core.uiSettings.client; + const { asCurrentUser } = core.elasticsearch.client; + const indexPatterns = new IndexPatternsFetcher(asCurrentUser, undefined, isRollupsEnabled()); + const { + pattern, + meta_fields: metaFields, + type, + rollup_index: rollupIndex, + allow_no_index: allowNoIndex, + include_unmapped: includeUnmapped, + } = request.query; + + let parsedFields: string[] = []; + let parsedMetaFields: string[] = []; + try { + parsedMetaFields = parseFields(metaFields); + parsedFields = parseFields(request.query.fields ?? []); + } catch (error) { + return response.badRequest(); + } + + try { + const { fields, indices } = await indexPatterns.getFieldsForWildcard({ + pattern, + metaFields: parsedMetaFields, + type, + rollupIndex, + fieldCapsOptions: { + allow_no_indices: allowNoIndex || false, + includeUnmapped, + }, + ...(parsedFields.length > 0 ? { fields: parsedFields } : {}), + }); + + const body: { fields: FieldDescriptorRestResponse[]; indices: string[] } = { + fields, + indices, + }; + + const bodyAsString = JSON.stringify(body); + + const etag = calculateHash(Buffer.from(bodyAsString)); + + const headers: Record = { + 'content-type': 'application/json', + etag, + vary: 'accept-encoding, user-hash', + }; + + // field cache is configurable in classic environment but not on serverless + let cacheMaxAge = DEFAULT_FIELD_CACHE_FRESHNESS; + const cacheMaxAgeSetting = await uiSettings.get( + 'data_views:cache_max_age' + ); + if (cacheMaxAgeSetting !== undefined) { + cacheMaxAge = cacheMaxAgeSetting; + } + + if (cacheMaxAge && fields.length) { + const stale = 365 * 24 * 60 * 60 - cacheMaxAge; + headers[ + 'cache-control' + ] = `private, max-age=${cacheMaxAge}, stale-while-revalidate=${stale}`; + } else { + headers['cache-control'] = 'private, no-cache'; + } + + const ifNoneMatch = request.headers['if-none-match']; + const ifNoneMatchString = Array.isArray(ifNoneMatch) ? ifNoneMatch[0] : ifNoneMatch; + + if (ifNoneMatchString) { + const requestHash = unwrapEtag(ifNoneMatchString); + if (etag === requestHash) { + return response.notModified({ headers }); + } + } + + return response.ok({ + body: bodyAsString, + headers, + }); + } catch (error) { + if ( + typeof error === 'object' && + !!error?.isBoom && + !!error?.output?.payload && + typeof error?.output?.payload === 'object' + ) { + const payload = error?.output?.payload; + return response.notFound({ + body: { + message: payload.message, + attributes: payload, + }, + }); + } else { + return response.notFound(); + } + } + }; + +export const registerFields = async ( + router: IRouter, + getStartServices: StartServicesAccessor< + DataViewsServerPluginStartDependencies, + DataViewsServerPluginStart + >, + isRollupsEnabled: () => boolean +) => { + router.versioned + .get({ path, access: 'internal', enableQueryVersion: true }) + .addVersion( + { version: '1', validate: { request: { query: querySchema }, response: validate.response } }, + handler(isRollupsEnabled) + ); +}; diff --git a/src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts b/src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts index 7d39b41d5caee..3c89cdef17955 100644 --- a/src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts +++ b/src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts @@ -41,8 +41,8 @@ export const parseFields = (fields: string | string[]): string[] => { const access = 'internal'; -type IBody = { index_filter?: estypes.QueryDslQueryContainer } | undefined; -interface IQuery { +export type IBody = { index_filter?: estypes.QueryDslQueryContainer } | undefined; +export interface IQuery { pattern: string; meta_fields: string | string[]; type?: string; @@ -53,7 +53,7 @@ interface IQuery { allow_hidden?: boolean; } -const querySchema = schema.object({ +export const querySchema = schema.object({ pattern: schema.string(), meta_fields: schema.oneOf([schema.string(), schema.arrayOf(schema.string())], { defaultValue: [], @@ -97,7 +97,7 @@ const FieldDescriptorSchema = schema.object({ ), }); -const validate: FullValidationConfig = { +export const validate: FullValidationConfig = { request: { query: querySchema, // not available to get request diff --git a/src/plugins/data_views/server/routes.ts b/src/plugins/data_views/server/routes.ts index c77dd9a8236cf..d6ee36927ff80 100644 --- a/src/plugins/data_views/server/routes.ts +++ b/src/plugins/data_views/server/routes.ts @@ -14,21 +14,30 @@ import type { DataViewsServerPluginStart, DataViewsServerPluginStartDependencies import { registerExistingIndicesPath } from './rest_api_routes/internal/existing_indices'; import { registerFieldForWildcard } from './rest_api_routes/internal/fields_for'; import { registerHasDataViewsRoute } from './rest_api_routes/internal/has_data_views'; +import { registerFields } from './rest_api_routes/internal/fields'; -export function registerRoutes( - http: HttpServiceSetup, +interface RegisterRoutesArgs { + http: HttpServiceSetup; getStartServices: StartServicesAccessor< DataViewsServerPluginStartDependencies, DataViewsServerPluginStart - >, - isRollupsEnabled: () => boolean, - dataViewRestCounter?: UsageCounter -) { + >; + isRollupsEnabled: () => boolean; + dataViewRestCounter?: UsageCounter; +} + +export function registerRoutes({ + http, + getStartServices, + dataViewRestCounter, + isRollupsEnabled, +}: RegisterRoutesArgs) { const router = http.createRouter(); routes.forEach((route) => route(router, getStartServices, dataViewRestCounter)); registerExistingIndicesPath(router); registerFieldForWildcard(router, getStartServices, isRollupsEnabled); + registerFields(router, getStartServices, isRollupsEnabled); registerHasDataViewsRoute(router); } diff --git a/src/plugins/data_views/server/ui_settings.ts b/src/plugins/data_views/server/ui_settings.ts new file mode 100644 index 0000000000000..3ba489a4b6b28 --- /dev/null +++ b/src/plugins/data_views/server/ui_settings.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { i18n } from '@kbn/i18n'; +import { schema } from '@kbn/config-schema'; +import { DEFAULT_FIELD_CACHE_FRESHNESS } from './constants'; + +export const cacheMaxAge = { + 'data_views:cache_max_age': { + name: i18n.translate('dataViews.advancedSettings.cacheMaxAgeTitle', { + defaultMessage: 'Field cache max age (in seconds)', + }), + value: DEFAULT_FIELD_CACHE_FRESHNESS, + description: i18n.translate('dataViews.advancedSettings.cacheMaxAgeText', { + defaultMessage: + 'Sets how long data view fields API requests are cached in seconds. A value of 0 turns off caching. Modifying this value may not take immediate effect, users need to clear browser cache or wait until the current cache expires. To see immediate changes, try a hard reload of Kibana.', + }), + schema: schema.number(), + }, +}; diff --git a/src/plugins/data_views/tsconfig.json b/src/plugins/data_views/tsconfig.json index e5613323bc222..1d414cb98ee95 100644 --- a/src/plugins/data_views/tsconfig.json +++ b/src/plugins/data_views/tsconfig.json @@ -33,6 +33,7 @@ "@kbn/object-versioning", "@kbn/core-saved-objects-server", "@kbn/logging", + "@kbn/security-plugin-types-public", ], "exclude": [ "target/**/*", diff --git a/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts b/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts index 5e49b09b01d24..32b133f210dad 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts @@ -166,6 +166,13 @@ export const stackManagementSchema: MakeSchemaFrom = { type: 'boolean', _meta: { description: 'Non-default value of setting.' }, }, + 'data_views:cache_max_age': { + type: 'long', + _meta: { + description: + "Sets the 'max-age' cache header value for data view fields API requests. A value of 0 will disable caching.", + }, + }, 'discover:searchOnPageLoad': { type: 'boolean', _meta: { description: 'Non-default value of setting.' }, diff --git a/src/plugins/kibana_usage_collection/server/collectors/management/types.ts b/src/plugins/kibana_usage_collection/server/collectors/management/types.ts index a1125a6d118b8..0832675cd54cf 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/management/types.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/management/types.ts @@ -81,6 +81,7 @@ export interface UsageStats { 'doc_table:highlight': boolean; 'discover:searchOnPageLoad': boolean; 'doc_table:hideTimeColumn': boolean; + 'data_views:cache_max_age': number; 'discover:sampleSize': number; 'discover:sampleRowsPerPage': number; defaultColumns: string[]; diff --git a/src/plugins/telemetry/schema/oss_plugins.json b/src/plugins/telemetry/schema/oss_plugins.json index 7e5a19d41d3d3..7fbf4ad7aeeb3 100644 --- a/src/plugins/telemetry/schema/oss_plugins.json +++ b/src/plugins/telemetry/schema/oss_plugins.json @@ -9434,6 +9434,12 @@ "description": "Non-default value of setting." } }, + "data_views:cache_max_age": { + "type": "long", + "_meta": { + "description": "Sets the 'max-age' cache header value for data view fields API requests. A value of 0 will disable caching." + } + }, "discover:searchOnPageLoad": { "type": "boolean", "_meta": { diff --git a/test/api_integration/apis/data_views/fields_route/cache.ts b/test/api_integration/apis/data_views/fields_route/cache.ts new file mode 100644 index 0000000000000..c97be4b7411f3 --- /dev/null +++ b/test/api_integration/apis/data_views/fields_route/cache.ts @@ -0,0 +1,93 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { INITIAL_REST_VERSION_INTERNAL } from '@kbn/data-views-plugin/server/constants'; +import { FIELDS_PATH } from '@kbn/data-views-plugin/common/constants'; +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../../ftr_provider_context'; + +export default function ({ getService }: FtrProviderContext) { + const esArchiver = getService('esArchiver'); + const supertest = getService('supertest'); + const kibanaServer = getService('kibanaServer'); + + describe('cache headers', () => { + before(() => + esArchiver.load('test/api_integration/fixtures/es_archiver/index_patterns/basic_index') + ); + after(() => + esArchiver.unload('test/api_integration/fixtures/es_archiver/index_patterns/basic_index') + ); + + it('are present', async () => { + const response = await supertest.get(FIELDS_PATH).query({ + pattern: '*', + include_unmapped: true, + apiVersion: INITIAL_REST_VERSION_INTERNAL, + }); + + const cacheControlHeader = response.get('cache-control'); + + expect(cacheControlHeader).to.contain('private'); + expect(cacheControlHeader).to.contain('max-age'); + expect(cacheControlHeader).to.contain('stale-while-revalidate'); + expect(response.get('vary')).to.equal('accept-encoding, user-hash'); + expect(response.get('etag')).to.not.be.empty(); + }); + + it('no-cache when data_views:cache_max_age set to zero', async () => { + await kibanaServer.uiSettings.update({ 'data_views:cache_max_age': 0 }); + + const response = await supertest.get(FIELDS_PATH).query({ + pattern: 'b*', + include_unmapped: true, + apiVersion: INITIAL_REST_VERSION_INTERNAL, + }); + + const cacheControlHeader = response.get('cache-control'); + + expect(cacheControlHeader).to.contain('private'); + expect(cacheControlHeader).to.contain('no-cache'); + expect(cacheControlHeader).to.not.contain('max-age'); + expect(cacheControlHeader).to.not.contain('stale-while-revalidate'); + expect(response.get('vary')).to.equal('accept-encoding, user-hash'); + expect(response.get('etag')).to.not.be.empty(); + + kibanaServer.uiSettings.replace({ 'data_views:cache_max_age': 5 }); + }); + + it('returns 304 on matching etag', async () => { + const response = await supertest.get(FIELDS_PATH).query({ + pattern: '*', + include_unmapped: true, + apiVersion: INITIAL_REST_VERSION_INTERNAL, + }); + + await supertest + .get(FIELDS_PATH) + .set('If-None-Match', response.get('etag')) + .query({ + pattern: '*', + include_unmapped: true, + apiVersion: INITIAL_REST_VERSION_INTERNAL, + }) + .expect(304); + }); + + it('handles empty field lists', async () => { + const response = await supertest.get(FIELDS_PATH).query({ + pattern: 'xyz', + include_unmapped: true, + apiVersion: INITIAL_REST_VERSION_INTERNAL, + allow_no_index: true, + }); + + expect(response.body.fields).to.be.empty(); + }); + }); +} diff --git a/test/api_integration/apis/data_views/fields_route/conflicts.ts b/test/api_integration/apis/data_views/fields_route/conflicts.ts new file mode 100644 index 0000000000000..026ca58b6dba4 --- /dev/null +++ b/test/api_integration/apis/data_views/fields_route/conflicts.ts @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { INITIAL_REST_VERSION_INTERNAL } from '@kbn/data-views-plugin/server/constants'; +import { FIELDS_PATH } from '@kbn/data-views-plugin/common/constants'; +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../../ftr_provider_context'; + +export default function ({ getService }: FtrProviderContext) { + const supertest = getService('supertest'); + const esArchiver = getService('esArchiver'); + + describe('conflicts', () => { + before(() => + esArchiver.load('test/api_integration/fixtures/es_archiver/index_patterns/conflicts') + ); + after(() => + esArchiver.unload('test/api_integration/fixtures/es_archiver/index_patterns/conflicts') + ); + + it('flags fields with mismatched types as conflicting', () => + supertest + .get(FIELDS_PATH) + .query({ pattern: 'logs-*', apiVersion: INITIAL_REST_VERSION_INTERNAL }) + .expect(200) + .then((resp) => { + expect(resp.body).to.eql({ + fields: [ + { + name: '@timestamp', + type: 'date', + esTypes: ['date'], + aggregatable: true, + searchable: true, + readFromDocValues: true, + metadata_field: false, + }, + { + name: 'number_conflict', + type: 'number', + esTypes: ['float', 'integer'], + aggregatable: true, + searchable: true, + readFromDocValues: true, + metadata_field: false, + }, + { + name: 'string_conflict', + type: 'string', + esTypes: ['keyword', 'text'], + aggregatable: true, + searchable: true, + readFromDocValues: true, + metadata_field: false, + }, + { + name: 'success', + type: 'conflict', + esTypes: ['keyword', 'boolean'], + aggregatable: true, + searchable: true, + readFromDocValues: false, + conflictDescriptions: { + boolean: ['logs-2017.01.02'], + keyword: ['logs-2017.01.01'], + }, + metadata_field: false, + }, + ], + indices: ['logs-2017.01.01', 'logs-2017.01.02'], + }); + })); + }); +} diff --git a/test/api_integration/apis/data_views/fields_route/index.ts b/test/api_integration/apis/data_views/fields_route/index.ts new file mode 100644 index 0000000000000..4b03c2b58afbe --- /dev/null +++ b/test/api_integration/apis/data_views/fields_route/index.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { FtrProviderContext } from '../../../ftr_provider_context'; + +export default function ({ loadTestFile }: FtrProviderContext) { + describe('data_views/fields route', () => { + loadTestFile(require.resolve('./params')); + loadTestFile(require.resolve('./conflicts')); + loadTestFile(require.resolve('./response')); + loadTestFile(require.resolve('./cache')); + }); +} diff --git a/test/api_integration/apis/data_views/fields_route/params.ts b/test/api_integration/apis/data_views/fields_route/params.ts new file mode 100644 index 0000000000000..7d111936426bb --- /dev/null +++ b/test/api_integration/apis/data_views/fields_route/params.ts @@ -0,0 +1,148 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { INITIAL_REST_VERSION_INTERNAL } from '@kbn/data-views-plugin/server/constants'; +import { FIELDS_PATH } from '@kbn/data-views-plugin/common/constants'; +import { FtrProviderContext } from '../../../ftr_provider_context'; + +export default function ({ getService }: FtrProviderContext) { + const esArchiver = getService('esArchiver'); + const supertest = getService('supertest'); + const randomness = getService('randomness'); + + describe('params', () => { + before(() => + esArchiver.load('test/api_integration/fixtures/es_archiver/index_patterns/basic_index') + ); + after(() => + esArchiver.unload('test/api_integration/fixtures/es_archiver/index_patterns/basic_index') + ); + + it('requires a pattern query param', () => + supertest + .get(FIELDS_PATH) + .query({ + apiVersion: INITIAL_REST_VERSION_INTERNAL, + }) + .expect(400)); + + it('accepts include_unmapped param', () => + supertest + .get(FIELDS_PATH) + .query({ + pattern: '*', + include_unmapped: true, + apiVersion: INITIAL_REST_VERSION_INTERNAL, + }) + .expect(200)); + + it('rejects unexpected query params', () => + supertest + .get(FIELDS_PATH) + .query({ + pattern: randomness.word(), + [randomness.word()]: randomness.word(), + apiVersion: INITIAL_REST_VERSION_INTERNAL, + }) + .expect(400)); + + describe('fields', () => { + it('accepts a JSON formatted fields query param', () => + supertest + .get(FIELDS_PATH) + .query({ + pattern: '*', + fields: JSON.stringify(['baz']), + apiVersion: INITIAL_REST_VERSION_INTERNAL, + }) + .expect(200)); + + it('accepts meta_fields query param in string array', () => + supertest + .get(FIELDS_PATH) + .query({ + pattern: '*', + fields: ['baz', 'foo'], + apiVersion: INITIAL_REST_VERSION_INTERNAL, + }) + .expect(200)); + + it('accepts single array fields query param', () => + supertest + .get(FIELDS_PATH) + .query({ + pattern: '*', + fields: ['baz'], + apiVersion: INITIAL_REST_VERSION_INTERNAL, + }) + .expect(200)); + + it('accepts single fields query param', () => + supertest + .get(FIELDS_PATH) + .query({ + pattern: '*', + fields: 'baz', + apiVersion: INITIAL_REST_VERSION_INTERNAL, + }) + .expect(200)); + + it('rejects a comma-separated list of fields', () => + supertest + .get(FIELDS_PATH) + .query({ + pattern: '*', + fields: 'foo,bar', + apiVersion: INITIAL_REST_VERSION_INTERNAL, + }) + .expect(400)); + }); + + describe('meta_fields', () => { + it('accepts a JSON formatted meta_fields query param', () => + supertest + .get(FIELDS_PATH) + .query({ + pattern: '*', + meta_fields: JSON.stringify(['meta']), + apiVersion: INITIAL_REST_VERSION_INTERNAL, + }) + .expect(200)); + + it('accepts meta_fields query param in string array', () => + supertest + .get(FIELDS_PATH) + .query({ + pattern: '*', + meta_fields: ['_id', 'meta'], + apiVersion: INITIAL_REST_VERSION_INTERNAL, + }) + .expect(200)); + + it('accepts single meta_fields query param', () => + supertest + .get(FIELDS_PATH) + .query({ + pattern: '*', + meta_fields: ['_id'], + apiVersion: INITIAL_REST_VERSION_INTERNAL, + }) + .expect(200)); + + it('rejects a comma-separated list of meta_fields', () => + supertest + .get(FIELDS_PATH) + .query({ + pattern: '*', + meta_fields: 'foo,bar', + apiVersion: INITIAL_REST_VERSION_INTERNAL, + }) + .expect(400)); + }); + }); +} diff --git a/test/api_integration/apis/data_views/fields_route/response.ts b/test/api_integration/apis/data_views/fields_route/response.ts new file mode 100644 index 0000000000000..880206f6fb498 --- /dev/null +++ b/test/api_integration/apis/data_views/fields_route/response.ts @@ -0,0 +1,230 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { INITIAL_REST_VERSION_INTERNAL } from '@kbn/data-views-plugin/server/constants'; +import { FIELDS_PATH } from '@kbn/data-views-plugin/common/constants'; +import expect from '@kbn/expect'; +import { sortBy } from 'lodash'; +import { FtrProviderContext } from '../../../ftr_provider_context'; + +export default function ({ getService }: FtrProviderContext) { + const esArchiver = getService('esArchiver'); + const supertest = getService('supertest'); + + const ensureFieldsAreSorted = (resp: { body: { fields: { name: string } } }) => { + expect(resp.body.fields).to.eql(sortBy(resp.body.fields, 'name')); + }; + + const testFields = [ + { + type: 'boolean', + esTypes: ['boolean'], + searchable: true, + aggregatable: true, + name: 'bar', + readFromDocValues: true, + metadata_field: false, + }, + { + type: 'string', + esTypes: ['text'], + searchable: true, + aggregatable: false, + name: 'baz', + readFromDocValues: false, + metadata_field: false, + }, + { + type: 'string', + esTypes: ['keyword'], + searchable: true, + aggregatable: true, + name: 'baz.keyword', + readFromDocValues: true, + subType: { multi: { parent: 'baz' } }, + metadata_field: false, + }, + { + type: 'number', + esTypes: ['long'], + searchable: true, + aggregatable: true, + name: 'foo', + readFromDocValues: true, + metadata_field: false, + }, + { + aggregatable: true, + esTypes: ['keyword'], + name: 'nestedField.child', + readFromDocValues: true, + searchable: true, + subType: { + nested: { + path: 'nestedField', + }, + }, + type: 'string', + metadata_field: false, + }, + ]; + + describe('fields route response', () => { + before(() => + esArchiver.load('test/api_integration/fixtures/es_archiver/index_patterns/basic_index') + ); + after(() => + esArchiver.unload('test/api_integration/fixtures/es_archiver/index_patterns/basic_index') + ); + + it('returns a flattened version of the fields in es', async () => { + await supertest + .get(FIELDS_PATH) + .query({ pattern: 'basic_index', apiVersion: INITIAL_REST_VERSION_INTERNAL }) + .expect(200, { + fields: testFields, + indices: ['basic_index'], + }) + .then(ensureFieldsAreSorted); + }); + + it('returns a single field as requested', async () => { + await supertest + .get(FIELDS_PATH) + .query({ + pattern: 'basic_index', + fields: JSON.stringify(['bar']), + apiVersion: INITIAL_REST_VERSION_INTERNAL, + }) + .expect(200, { + fields: [testFields[0]], + indices: ['basic_index'], + }); + }); + + it('always returns a field for all passed meta fields', async () => { + await supertest + .get(FIELDS_PATH) + .query({ + pattern: 'basic_index', + meta_fields: JSON.stringify(['_id', '_source', 'crazy_meta_field']), + apiVersion: INITIAL_REST_VERSION_INTERNAL, + }) + .expect(200, { + fields: [ + { + aggregatable: false, + name: '_id', + esTypes: ['_id'], + readFromDocValues: false, + searchable: true, + type: 'string', + metadata_field: true, + }, + { + aggregatable: false, + name: '_source', + esTypes: ['_source'], + readFromDocValues: false, + searchable: false, + type: '_source', + metadata_field: true, + }, + { + type: 'boolean', + esTypes: ['boolean'], + searchable: true, + aggregatable: true, + name: 'bar', + readFromDocValues: true, + metadata_field: false, + }, + { + aggregatable: false, + name: 'baz', + esTypes: ['text'], + readFromDocValues: false, + searchable: true, + type: 'string', + metadata_field: false, + }, + { + type: 'string', + esTypes: ['keyword'], + searchable: true, + aggregatable: true, + name: 'baz.keyword', + readFromDocValues: true, + subType: { multi: { parent: 'baz' } }, + metadata_field: false, + }, + { + aggregatable: false, + name: 'crazy_meta_field', + readFromDocValues: false, + searchable: false, + type: 'string', + metadata_field: true, + }, + { + type: 'number', + esTypes: ['long'], + searchable: true, + aggregatable: true, + name: 'foo', + readFromDocValues: true, + metadata_field: false, + }, + { + aggregatable: true, + esTypes: ['keyword'], + name: 'nestedField.child', + readFromDocValues: true, + searchable: true, + subType: { + nested: { + path: 'nestedField', + }, + }, + type: 'string', + metadata_field: false, + }, + ], + indices: ['basic_index'], + }) + .then(ensureFieldsAreSorted); + }); + + it('returns fields when one pattern exists and the other does not', async () => { + await supertest + .get(FIELDS_PATH) + .query({ pattern: 'bad_index,basic_index', apiVersion: INITIAL_REST_VERSION_INTERNAL }) + .expect(200, { + fields: testFields, + indices: ['basic_index'], + }); + }); + + it('returns 404 when neither exists', async () => { + await supertest + .get(FIELDS_PATH) + .query({ pattern: 'bad_index,bad_index_2', apiVersion: INITIAL_REST_VERSION_INTERNAL }) + .expect(404); + }); + + it('returns 404 when no patterns exist', async () => { + await supertest + .get(FIELDS_PATH) + .query({ + pattern: 'bad_index', + apiVersion: INITIAL_REST_VERSION_INTERNAL, + }) + .expect(404); + }); + }); +} diff --git a/test/api_integration/apis/data_views/index.ts b/test/api_integration/apis/data_views/index.ts index a6589d5680b3d..654421937f44b 100644 --- a/test/api_integration/apis/data_views/index.ts +++ b/test/api_integration/apis/data_views/index.ts @@ -23,5 +23,6 @@ export default function ({ loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./has_user_index_pattern')); loadTestFile(require.resolve('./swap_references')); loadTestFile(require.resolve('./resolve_index')); + loadTestFile(require.resolve('./fields_route')); }); } diff --git a/test/functional/apps/management/data_views/_cache.ts b/test/functional/apps/management/data_views/_cache.ts new file mode 100644 index 0000000000000..447e44052bb8d --- /dev/null +++ b/test/functional/apps/management/data_views/_cache.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../../ftr_provider_context'; + +export default function ({ getService, getPageObjects }: FtrProviderContext) { + const PageObjects = getPageObjects(['settings', 'common', 'header']); + const find = getService('find'); + + describe('Data view field caps cache advanced setting', async function () { + before(async () => { + await PageObjects.settings.navigateTo(); + await PageObjects.settings.clickKibanaSettings(); + }); + it('should have cache setting', async () => { + const cacheSetting = await find.byCssSelector('#data_views\\:cache_max_age-group'); + expect(cacheSetting).to.not.be(undefined); + }); + }); +} diff --git a/test/functional/apps/management/data_views/_field_formatter.ts b/test/functional/apps/management/data_views/_field_formatter.ts index fb016151cce5f..dfcdc0b877581 100644 --- a/test/functional/apps/management/data_views/_field_formatter.ts +++ b/test/functional/apps/management/data_views/_field_formatter.ts @@ -465,6 +465,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.settings.navigateTo(); await PageObjects.settings.clickKibanaIndexPatterns(); await PageObjects.settings.clickIndexPatternByName(indexTitle); + await PageObjects.settings.refreshDataViewFieldList(); }); afterEach(async () => { diff --git a/test/functional/apps/management/data_views/_index_pattern_filter.ts b/test/functional/apps/management/data_views/_index_pattern_filter.ts index 81ff2b450755d..941451424dfb1 100644 --- a/test/functional/apps/management/data_views/_index_pattern_filter.ts +++ b/test/functional/apps/management/data_views/_index_pattern_filter.ts @@ -170,6 +170,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.settings.clickIndexPatternLogstash(); + await PageObjects.settings.refreshDataViewFieldList(); + await testSubjects.existOrFail('dataViewMappingConflict'); expect(await PageObjects.settings.getFieldTypes()).to.eql([ diff --git a/test/functional/apps/management/index.ts b/test/functional/apps/management/index.ts index 85de9b5de0926..808a4aaccc9c2 100644 --- a/test/functional/apps/management/index.ts +++ b/test/functional/apps/management/index.ts @@ -47,5 +47,6 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./data_views/_edit_field')); loadTestFile(require.resolve('./_files')); loadTestFile(require.resolve('./_data_view_field_filters')); + loadTestFile(require.resolve('./data_views/_cache')); }); } diff --git a/test/functional/page_objects/settings_page.ts b/test/functional/page_objects/settings_page.ts index cd650086e9e1a..9fc802a03cc45 100644 --- a/test/functional/page_objects/settings_page.ts +++ b/test/functional/page_objects/settings_page.ts @@ -494,6 +494,25 @@ export class SettingsPageObject extends FtrService { await customDataViewIdInput.type(value); } + async refreshDataViewFieldList(dataViewName?: string) { + if (dataViewName) { + await this.common.navigateToApp('management/kibana/dataViews'); + await this.header.waitUntilLoadingHasFinished(); + await this.testSubjects.click(`detail-link-${dataViewName}`); + } + await this.testSubjects.click('refreshDataViewButton'); + + // wait for refresh to start + await new Promise((r) => setTimeout(r, 500)); + + // wait for refresh to finish + await this.retry.try(async () => { + const btn = await this.testSubjects.find('refreshDataViewButton'); + const disabled = await btn.getAttribute('disabled'); + expect(disabled).to.be(null); + }); + } + async allowHiddenClick() { await this.testSubjects.click('toggleAdvancedSetting'); const allowHiddenField = await this.testSubjects.find('allowHiddenField'); diff --git a/x-pack/plugins/logs_shared/server/services/log_views/log_views_client.test.ts b/x-pack/plugins/logs_shared/server/services/log_views/log_views_client.test.ts index e50d08c630eb7..e2741bea55931 100644 --- a/x-pack/plugins/logs_shared/server/services/log_views/log_views_client.test.ts +++ b/x-pack/plugins/logs_shared/server/services/log_views/log_views_client.test.ts @@ -253,6 +253,7 @@ describe('LogViewsClient class', () => { "allowNoIndex": false, "deleteFieldFormat": [Function], "deleteScriptedFieldInternal": [Function], + "etag": undefined, "fieldAttrs": Object {}, "fieldFormatMap": Object {}, "fieldFormats": Object { @@ -276,6 +277,7 @@ describe('LogViewsClient class', () => { "fields": FldList [], "flattenHit": [Function], "getAllowHidden": [Function], + "getEtag": [Function], "getFieldAttrs": [Function], "getIndexPattern": [Function], "getName": [Function], @@ -301,6 +303,7 @@ describe('LogViewsClient class', () => { }, "scriptedFields": Array [], "setAllowHidden": [Function], + "setEtag": [Function], "setFieldFormat": [Function], "setIndexPattern": [Function], "shortDotsEnable": false, diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/automated_response_actions/form.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/automated_response_actions/form.cy.ts index fb1285fb89f05..27458ef09e15b 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/automated_response_actions/form.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/automated_response_actions/form.cy.ts @@ -5,7 +5,6 @@ * 2.0. */ -import { FIELDS_FOR_WILDCARD_PATH } from '@kbn/data-views-plugin/common/constants'; import { addEndpointResponseAction, fillUpNewRule, @@ -186,9 +185,7 @@ describe( }); it('All response action controls are disabled', () => { - cy.intercept('GET', `${FIELDS_FOR_WILDCARD_PATH}*`).as('getFieldsForWildcard'); visitRuleActions(ruleId); - cy.wait('@getFieldsForWildcard'); cy.getByTestSubj('edit-rule-actions-tab').click(); cy.getByTestSubj('response-actions-wrapper').within(() => { diff --git a/x-pack/plugins/security_solution/public/management/cypress/tsconfig.json b/x-pack/plugins/security_solution/public/management/cypress/tsconfig.json index 8a38566931281..20cb7534941b3 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/tsconfig.json +++ b/x-pack/plugins/security_solution/public/management/cypress/tsconfig.json @@ -30,7 +30,6 @@ "@kbn/cases-plugin", "@kbn/test", "@kbn/repo-info", - "@kbn/data-views-plugin", "@kbn/tooling-log", ] } diff --git a/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts b/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts index 6b668f75d56d3..bffa564fba774 100644 --- a/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts +++ b/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts @@ -399,6 +399,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); it('should navigate to alert results via link provided in notification', async () => { + await PageObjects.settings.refreshDataViewFieldList(OUTPUT_DATA_VIEW); await openAlertResults(RULE_NAME); await checkInitialRuleParamsState(SOURCE_DATA_VIEW); }); diff --git a/x-pack/test/scalability/apis/api.fields.32_fields.json b/x-pack/test/scalability/apis/api.fields.32_fields.json new file mode 100644 index 0000000000000..03837b238e44e --- /dev/null +++ b/x-pack/test/scalability/apis/api.fields.32_fields.json @@ -0,0 +1,50 @@ +{ + "journeyName": "GET /internal/data_views/fields - 32 fields", + "scalabilitySetup": { + "responseTimeThreshold": { + "threshold1": 5000, + "threshold2": 10000, + "threshold3": 20000 + }, + "warmup": [ + { + "action": "constantUsersPerSec", + "userCount": 10, + "duration": "30s" + } + ], + "test": [ + { + "action": "rampUsersPerSec", + "minUsersCount": 10, + "maxUsersCount": 375, + "duration": "140s" + } + ], + "maxDuration": "5m" + }, + "testData": { + "esArchives": ["test/functional/fixtures/es_archiver/kibana_sample_data_flights"] + }, + "streams": [ + { + "requests": [ + { + "http": { + "method": "GET", + "path": "/internal/data_views/fields", + "query": "?pattern=kibana*&apiVersion=1", + "headers": { + "Cookie": "", + "Kbn-Version": "", + "Accept-Encoding": "gzip, deflate, br", + "Content-Type": "application/json", + "x-elastic-internal-origin": "kibana" + }, + "statusCode": 200 + } + } + ] + } + ] +} diff --git a/x-pack/test/scalability/apis/api.fields.6800_fields.json b/x-pack/test/scalability/apis/api.fields.6800_fields.json new file mode 100644 index 0000000000000..362eaa211acae --- /dev/null +++ b/x-pack/test/scalability/apis/api.fields.6800_fields.json @@ -0,0 +1,50 @@ +{ + "journeyName": "GET /internal/data_views/fields - 6800 fields", + "scalabilitySetup": { + "responseTimeThreshold": { + "threshold1": 5000, + "threshold2": 10000, + "threshold3": 20000 + }, + "warmup": [ + { + "action": "constantUsersPerSec", + "userCount": 1, + "duration": "30s" + } + ], + "test": [ + { + "action": "rampUsersPerSec", + "minUsersCount": 1, + "maxUsersCount": 7, + "duration": "140s" + } + ], + "maxDuration": "5m" + }, + "testData": { + "esArchives": ["test/functional/fixtures/es_archiver/many_fields"] + }, + "streams": [ + { + "requests": [ + { + "http": { + "method": "GET", + "path": "/internal/data_views/fields", + "query": "?pattern=indices*&apiVersion=1", + "headers": { + "Cookie": "", + "Kbn-Version": "", + "Accept-Encoding": "gzip, deflate, br", + "Content-Type": "application/json", + "x-elastic-internal-origin": "kibana" + }, + "statusCode": 200 + } + } + ] + } + ] +} diff --git a/x-pack/test_serverless/functional/test_suites/common/discover_ml_uptime/discover/search_source_alert.ts b/x-pack/test_serverless/functional/test_suites/common/discover_ml_uptime/discover/search_source_alert.ts index f683e02237996..d2e8f863d5fe9 100644 --- a/x-pack/test_serverless/functional/test_suites/common/discover_ml_uptime/discover/search_source_alert.ts +++ b/x-pack/test_serverless/functional/test_suites/common/discover_ml_uptime/discover/search_source_alert.ts @@ -378,9 +378,12 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { // should not have data view selected by default const dataViewSelector = await testSubjects.find('selectDataViewExpression'); - // TODO: Serverless Security has an existing data view by default + // TODO: Serverless Security and Search have an existing data view by default const dataViewSelectorText = await dataViewSelector.getVisibleText(); - if (!dataViewSelectorText.includes('.alerts-security')) { + if ( + !dataViewSelectorText.includes('.alerts-security') && + !dataViewSelectorText.includes('default:all-data') + ) { expect(await dataViewSelector.getVisibleText()).to.eql('DATA VIEW\nSelect a data view'); } @@ -444,6 +447,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); it('should navigate to alert results via link provided in notification', async () => { + await PageObjects.settings.refreshDataViewFieldList(OUTPUT_DATA_VIEW); await openAlertResults(RULE_NAME); await checkInitialRuleParamsState(SOURCE_DATA_VIEW); }); diff --git a/x-pack/test_serverless/functional/test_suites/common/management/data_views/_cache.ts b/x-pack/test_serverless/functional/test_suites/common/management/data_views/_cache.ts new file mode 100644 index 0000000000000..3050ef2456fbf --- /dev/null +++ b/x-pack/test_serverless/functional/test_suites/common/management/data_views/_cache.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrProviderContext } from '../../../../ftr_provider_context'; + +export default function ({ getService, getPageObjects }: FtrProviderContext) { + const PageObjects = getPageObjects(['settings', 'common', 'header']); + const testSubjects = getService('testSubjects'); + + describe('Data view field caps cache advanced setting', async function () { + before(async () => { + await PageObjects.settings.navigateTo(); + }); + + it('should not have cache setting', async () => { + await testSubjects.missingOrFail( + 'advancedSetting-editField-data_views\\:cache_max_age-group' + ); + }); + }); +} diff --git a/x-pack/test_serverless/functional/test_suites/common/management/data_views/_index_pattern_filter.ts b/x-pack/test_serverless/functional/test_suites/common/management/data_views/_index_pattern_filter.ts index e5d51f55aa7c1..9b8a51feac10f 100644 --- a/x-pack/test_serverless/functional/test_suites/common/management/data_views/_index_pattern_filter.ts +++ b/x-pack/test_serverless/functional/test_suites/common/management/data_views/_index_pattern_filter.ts @@ -174,6 +174,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.settings.clickIndexPatternLogstash(); + await PageObjects.settings.refreshDataViewFieldList(); + await testSubjects.existOrFail('dataViewMappingConflict'); expect(await PageObjects.settings.getFieldTypes()).to.eql([ diff --git a/x-pack/test_serverless/functional/test_suites/common/management/data_views/index.ts b/x-pack/test_serverless/functional/test_suites/common/management/data_views/index.ts index 6a512b6d6e273..69fec71b2ad22 100644 --- a/x-pack/test_serverless/functional/test_suites/common/management/data_views/index.ts +++ b/x-pack/test_serverless/functional/test_suites/common/management/data_views/index.ts @@ -30,5 +30,6 @@ export default ({ getService, loadTestFile, getPageObject }: FtrProviderContext) loadTestFile(require.resolve('./_exclude_index_pattern')); loadTestFile(require.resolve('./_index_pattern_filter')); loadTestFile(require.resolve('./_edit_field')); + loadTestFile(require.resolve('./_cache')); }); }; diff --git a/x-pack/test_serverless/functional/test_suites/search/cases/attachment_framework.ts b/x-pack/test_serverless/functional/test_suites/search/cases/attachment_framework.ts index 3cf087eb1ae9b..e74262f60ac3e 100644 --- a/x-pack/test_serverless/functional/test_suites/search/cases/attachment_framework.ts +++ b/x-pack/test_serverless/functional/test_suites/search/cases/attachment_framework.ts @@ -16,6 +16,7 @@ export default ({ getPageObject, getService }: FtrProviderContext) => { const svlSearchNavigation = getService('svlSearchNavigation'); const svlCommonNavigation = getPageObject('svlCommonNavigation'); const svlCommonPage = getPageObject('svlCommonPage'); + const settings = getPageObject('settings'); describe('persistable attachment', () => { before(async () => { @@ -33,6 +34,8 @@ export default ({ getPageObject, getService }: FtrProviderContext) => { 'x-pack/test/functional/fixtures/kbn_archiver/lens/lens_basic.json' ); + await settings.refreshDataViewFieldList('default:all-data'); + await svlSearchNavigation.navigateToLandingPage(); await svlCommonNavigation.sidenav.clickLink({ deepLinkId: 'dashboards' });