From ed19554ac4e61f4f1a6ba158713eb19155ec4a1e Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Sat, 14 Oct 2023 21:44:33 -0500 Subject: [PATCH 001/168] add stale while revalidate --- .../rest_api_routes/internal/fields_for.ts | 32 ++++++++++++++----- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts b/src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts index 7d39b41d5caee..1f6965d060b73 100644 --- a/src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts +++ b/src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts @@ -113,8 +113,12 @@ const validate: FullValidationConfig = { }, }; -const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, IBody> = - (isRollupsEnabled) => async (context, request, response) => { +const handler: ( + isRollupsEnabled: () => boolean +) => (cacheHeader: boolean) => RequestHandler<{}, IQuery, IBody> = + (isRollupsEnabled) => + (cacheHeader = false) => + async (context, request, response) => { const { asCurrentUser } = (await context.core).elasticsearch.client; const indexPatterns = new IndexPatternsFetcher(asCurrentUser, undefined, isRollupsEnabled()); const { @@ -159,11 +163,19 @@ const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, I indices, }; + const headers: Record = { + 'content-type': 'application/json', + // Etag? + // Expires + }; + + if (cacheHeader) { + headers['cache-control'] = 'private, max-age=31536000, stale-while-revalidate=86400'; + } + return response.ok({ body, - headers: { - 'content-type': 'application/json', - }, + headers, }); } catch (error) { if ( @@ -196,12 +208,16 @@ export const registerFieldForWildcard = async ( const configuredHandler = handler(isRollupsEnabled); // handler - router.versioned.put({ path, access }).addVersion({ version, validate }, configuredHandler); - router.versioned.post({ path, access }).addVersion({ version, validate }, configuredHandler); + router.versioned + .put({ path, access, options: {} }) + .addVersion({ version, validate }, configuredHandler(false)); + router.versioned + .post({ path, access }) + .addVersion({ version, validate }, configuredHandler(false)); router.versioned .get({ path, access }) .addVersion( { version, validate: { request: { query: querySchema }, response: validate.response } }, - configuredHandler + configuredHandler(true) ); }; From da2dddc39e6b3c41095674817ade620a583b986c Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Sat, 14 Oct 2023 22:16:33 -0500 Subject: [PATCH 002/168] add cache busting --- src/plugins/data_views/common/types.ts | 1 + .../public/data_views/data_views_api_client.ts | 14 +++++++++++--- .../server/rest_api_routes/internal/fields_for.ts | 1 + 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/plugins/data_views/common/types.ts b/src/plugins/data_views/common/types.ts index 522bec8873534..16190b233717d 100644 --- a/src/plugins/data_views/common/types.ts +++ b/src/plugins/data_views/common/types.ts @@ -314,6 +314,7 @@ export interface GetFieldsOptions { includeUnmapped?: boolean; fields?: string[]; allowHidden?: boolean; + forceRefresh?: boolean; } /** diff --git a/src/plugins/data_views/public/data_views/data_views_api_client.ts b/src/plugins/data_views/public/data_views/data_views_api_client.ts index bb53c68eafaa7..a31ea076e0e88 100644 --- a/src/plugins/data_views/public/data_views/data_views_api_client.ts +++ b/src/plugins/data_views/public/data_views/data_views_api_client.ts @@ -29,10 +29,16 @@ export class DataViewsApiClient implements IDataViewsApiClient { this.http = http; } - private _request(url: string, query?: {}, body?: string): Promise { + private _request( + url: string, + query?: {}, + body?: string, + forceRefresh?: boolean + ): Promise { + const headers = forceRefresh ? { cache: 'reload' } : undefined; const request = body ? this.http.post(url, { query, body, version }) - : this.http.fetch(url, { query, version }); + : this.http.fetch(url, { query, version, headers }); return request.catch((resp) => { if (resp.body.statusCode === 404 && resp.body.attributes?.code === 'no_matching_indices') { throw new DataViewMissingIndices(resp.body.message); @@ -60,6 +66,7 @@ export class DataViewsApiClient implements IDataViewsApiClient { indexFilter, includeUnmapped, fields, + forceRefresh, } = options; return this._request( FIELDS_FOR_WILDCARD_PATH, @@ -72,7 +79,8 @@ export class DataViewsApiClient implements IDataViewsApiClient { include_unmapped: includeUnmapped, fields, }, - indexFilter ? JSON.stringify({ index_filter: indexFilter }) : undefined + indexFilter ? JSON.stringify({ index_filter: indexFilter }) : undefined, + forceRefresh ).then((response) => { return response || { fields: [], indices: [] }; }); diff --git a/src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts b/src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts index 1f6965d060b73..d076894fb972f 100644 --- a/src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts +++ b/src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts @@ -170,6 +170,7 @@ const handler: ( }; if (cacheHeader) { + // revalidates if 5 minutes passed, otherwise caches for a year headers['cache-control'] = 'private, max-age=31536000, stale-while-revalidate=86400'; } From fdcbfb6a591225c3e1ed138c0bec35b7a06f2094 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Mon, 16 Oct 2023 22:17:50 -0500 Subject: [PATCH 003/168] garbage test code mostly to be removed later --- src/plugins/data_views/common/types.ts | 3 --- .../data_views/public/data_views/data_views_api_client.ts | 8 +++++++- .../server/rest_api_routes/internal/fields_for.ts | 5 ++++- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/plugins/data_views/common/types.ts b/src/plugins/data_views/common/types.ts index 8a9ba90a158bb..b29ec67c71d60 100644 --- a/src/plugins/data_views/common/types.ts +++ b/src/plugins/data_views/common/types.ts @@ -309,11 +309,8 @@ export interface GetFieldsOptions { indexFilter?: QueryDslQueryContainer; includeUnmapped?: boolean; fields?: string[]; -<<<<<<< HEAD allowHidden?: boolean; forceRefresh?: boolean; -======= ->>>>>>> main } /** diff --git a/src/plugins/data_views/public/data_views/data_views_api_client.ts b/src/plugins/data_views/public/data_views/data_views_api_client.ts index a31ea076e0e88..9baaf93c91e22 100644 --- a/src/plugins/data_views/public/data_views/data_views_api_client.ts +++ b/src/plugins/data_views/public/data_views/data_views_api_client.ts @@ -35,7 +35,13 @@ export class DataViewsApiClient implements IDataViewsApiClient { body?: string, forceRefresh?: boolean ): Promise { - const headers = forceRefresh ? { cache: 'reload' } : undefined; + console.log('HERE', new Date().getMinutes()); + if (new Date().getMinutes() % 2 === 0) { + forceRefresh = true; + } + // const headers = forceRefresh ? { cache: 'reload' } : undefined; + const headers = forceRefresh ? { 'Cache-Control': 'no-cache' } : undefined; + const request = body ? this.http.post(url, { query, body, version }) : this.http.fetch(url, { query, version, headers }); diff --git a/src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts b/src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts index 52e7214762986..25048e52b85c1 100644 --- a/src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts +++ b/src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts @@ -161,13 +161,16 @@ const handler: ( const headers: Record = { 'content-type': 'application/json', + Etag: '123456', + 'If-None-Match': '123456', // Etag? // Expires }; if (cacheHeader) { // revalidates if 5 minutes passed, otherwise caches for a year - headers['cache-control'] = 'private, max-age=31536000, stale-while-revalidate=86400'; + // headers['cache-control'] = 'private, max-age=31536000, stale-while-revalidate=86400'; + headers['cache-control'] = 'max-age=31536000, stale-while-revalidate=86400'; } return response.ok({ From 7f0609591a0baef95189a8de8cb632e26cf4431a Mon Sep 17 00:00:00 2001 From: Matthew Kime Date: Sun, 29 Oct 2023 23:45:45 -0500 Subject: [PATCH 004/168] Update src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts Co-authored-by: Davis McPhee --- .../data_views/server/rest_api_routes/internal/fields_for.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts b/src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts index d076894fb972f..d5d93163356f2 100644 --- a/src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts +++ b/src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts @@ -171,7 +171,7 @@ const handler: ( if (cacheHeader) { // revalidates if 5 minutes passed, otherwise caches for a year - headers['cache-control'] = 'private, max-age=31536000, stale-while-revalidate=86400'; + headers['cache-control'] = 'private, max-age=300, stale-while-revalidate=2592000'; } return response.ok({ From 671461f5ae65fd35bec09186fb6b223bf788179b Mon Sep 17 00:00:00 2001 From: Matthew Kime Date: Tue, 31 Oct 2023 21:41:59 -0500 Subject: [PATCH 005/168] Update src/plugins/data_views/public/data_views/data_views_api_client.ts Co-authored-by: Davis McPhee --- .../data_views/public/data_views/data_views_api_client.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/data_views/public/data_views/data_views_api_client.ts b/src/plugins/data_views/public/data_views/data_views_api_client.ts index a31ea076e0e88..ec496721b7b94 100644 --- a/src/plugins/data_views/public/data_views/data_views_api_client.ts +++ b/src/plugins/data_views/public/data_views/data_views_api_client.ts @@ -35,7 +35,7 @@ export class DataViewsApiClient implements IDataViewsApiClient { body?: string, forceRefresh?: boolean ): Promise { - const headers = forceRefresh ? { cache: 'reload' } : undefined; + const cacheOptions = forceRefresh ? { cache: 'reload' } : {}; const request = body ? this.http.post(url, { query, body, version }) : this.http.fetch(url, { query, version, headers }); From 2c61688283e1c151a6a113e42c3914ffd465baf9 Mon Sep 17 00:00:00 2001 From: Matthew Kime Date: Tue, 31 Oct 2023 21:42:07 -0500 Subject: [PATCH 006/168] Update src/plugins/data_views/public/data_views/data_views_api_client.ts Co-authored-by: Davis McPhee --- .../data_views/public/data_views/data_views_api_client.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/data_views/public/data_views/data_views_api_client.ts b/src/plugins/data_views/public/data_views/data_views_api_client.ts index ec496721b7b94..cb069d9dff6bf 100644 --- a/src/plugins/data_views/public/data_views/data_views_api_client.ts +++ b/src/plugins/data_views/public/data_views/data_views_api_client.ts @@ -38,7 +38,7 @@ export class DataViewsApiClient implements IDataViewsApiClient { const cacheOptions = forceRefresh ? { cache: 'reload' } : {}; const request = body ? this.http.post(url, { query, body, version }) - : this.http.fetch(url, { query, version, headers }); + : this.http.fetch(url, { query, version, ...cacheOptions }); return request.catch((resp) => { if (resp.body.statusCode === 404 && resp.body.attributes?.code === 'no_matching_indices') { throw new DataViewMissingIndices(resp.body.message); From 691df4fadad29befcd021db88b4690da921bafc2 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Wed, 1 Nov 2023 20:05:58 -0500 Subject: [PATCH 007/168] caching fields requests mostly working --- src/plugins/data_views/common/constants.ts | 6 + .../data_views/data_views_api_client.ts | 14 +- .../field_capabilities/field_capabilities.ts | 2 +- .../server/rest_api_routes/internal/fields.ts | 205 ++++++++++++++++++ src/plugins/data_views/server/routes.ts | 2 + 5 files changed, 224 insertions(+), 5 deletions(-) create mode 100644 src/plugins/data_views/server/rest_api_routes/internal/fields.ts diff --git a/src/plugins/data_views/common/constants.ts b/src/plugins/data_views/common/constants.ts index dc396f2772aa8..43dc67362bf26 100644 --- a/src/plugins/data_views/common/constants.ts +++ b/src/plugins/data_views/common/constants.ts @@ -58,6 +58,12 @@ export const PLUGIN_NAME = 'DataViews'; */ export const FIELDS_FOR_WILDCARD_PATH = '/internal/data_views/_fields_for_wildcard'; +/** + * Fields path. Like fields for wildcard but GET only + * @public + */ +export const FIELDS_PATH = '/internal/data_views/fields'; + /** * Existing indices path * @public diff --git a/src/plugins/data_views/public/data_views/data_views_api_client.ts b/src/plugins/data_views/public/data_views/data_views_api_client.ts index 104443c16b5ff..9f58daf501490 100644 --- a/src/plugins/data_views/public/data_views/data_views_api_client.ts +++ b/src/plugins/data_views/public/data_views/data_views_api_client.ts @@ -10,7 +10,7 @@ import { HttpSetup } from '@kbn/core/public'; import { DataViewMissingIndices } from '../../common/lib'; import { GetFieldsOptions, IDataViewsApiClient } from '../../common'; import { FieldsForWildcardResponse } from '../../common/types'; -import { FIELDS_FOR_WILDCARD_PATH } from '../../common/constants'; +import { FIELDS_FOR_WILDCARD_PATH, FIELDS_PATH } from '../../common/constants'; const API_BASE_URL: string = `/api/index_patterns/`; const version = '1'; @@ -35,15 +35,19 @@ export class DataViewsApiClient implements IDataViewsApiClient { body?: string, forceRefresh?: boolean ): Promise { + /* console.log('HERE', new Date().getMinutes()); if (new Date().getMinutes() % 2 === 0) { forceRefresh = true; } - const cacheOptions = forceRefresh ? { cache: 'reload' as RequestCache } : {}; + */ + // const headers = forceRefresh ? { 'Cache-Control': '' } : undefined; + // const cacheOptions = forceRefresh ? { cache: 'reload' as RequestCache } : {}; const request = body ? this.http.post(url, { query, body, version }) - : this.http.fetch(url, { query, version, ...cacheOptions }); + : this.http.fetch(url, { query, version }); + // : this.http.fetch(url, { query, version, ...cacheOptions }); return request.catch((resp) => { if (resp.body.statusCode === 404 && resp.body.attributes?.code === 'no_matching_indices') { throw new DataViewMissingIndices(resp.body.message); @@ -73,8 +77,10 @@ export class DataViewsApiClient implements IDataViewsApiClient { fields, forceRefresh, } = options; + const path = indexFilter ? FIELDS_FOR_WILDCARD_PATH : FIELDS_PATH; + return this._request( - FIELDS_FOR_WILDCARD_PATH, + path, { pattern, meta_fields: metaFields, diff --git a/src/plugins/data_views/server/fetcher/lib/field_capabilities/field_capabilities.ts b/src/plugins/data_views/server/fetcher/lib/field_capabilities/field_capabilities.ts index 73e550ebd68ce..f3b2f2db2b108 100644 --- a/src/plugins/data_views/server/fetcher/lib/field_capabilities/field_capabilities.ts +++ b/src/plugins/data_views/server/fetcher/lib/field_capabilities/field_capabilities.ts @@ -56,7 +56,7 @@ export async function getFieldCapabilities(params: FieldCapabilitiesParams) { const allFieldsUnsorted = Object.keys(fieldsFromFieldCapsByName) // not all meta fields are provided, so remove and manually add .filter((name) => !fieldsFromFieldCapsByName[name].metadata_field) - .concat(fieldCapsArr.length ? metaFields : []) + .concat(fieldCapsArr.length ? metaFields : []) // empty field lists should stay empty .reduce<{ names: string[]; map: Map }>( (agg, value) => { // This is intentionally using a Map to be highly optimized with very large indexes AND be safe for user provided data diff --git a/src/plugins/data_views/server/rest_api_routes/internal/fields.ts b/src/plugins/data_views/server/rest_api_routes/internal/fields.ts new file mode 100644 index 0000000000000..210ceb48890a0 --- /dev/null +++ b/src/plugins/data_views/server/rest_api_routes/internal/fields.ts @@ -0,0 +1,205 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { estypes } from '@elastic/elasticsearch'; +import { schema } from '@kbn/config-schema'; +import { IRouter, RequestHandler, StartServicesAccessor } from '@kbn/core/server'; +import { FullValidationConfig } from '@kbn/core-http-server'; +import { IndexPatternsFetcher } from '../../fetcher'; +import type { + DataViewsServerPluginStart, + DataViewsServerPluginStartDependencies, +} from '../../types'; +import type { FieldDescriptorRestResponse } from '../route_types'; +import { FIELDS_PATH as path } from '../../../common/constants'; + +/** + * Accepts one of the following: + * 1. An array of field names + * 2. A JSON-stringified array of field names + * 3. A single field name (not comma-separated) + * @returns an array of field names + * @param fields + */ +export const parseFields = (fields: string | string[]): string[] => { + if (Array.isArray(fields)) return fields; + try { + return JSON.parse(fields); + } catch (e) { + if (!fields.includes(',')) return [fields]; + throw new Error( + 'metaFields should be an array of field names, a JSON-stringified array of field names, or a single field name' + ); + } +}; + +const access = 'internal'; + +type IBody = { index_filter?: estypes.QueryDslQueryContainer } | undefined; +interface IQuery { + pattern: string; + meta_fields: string | string[]; + type?: string; + rollup_index?: string; + allow_no_index?: boolean; + include_unmapped?: boolean; + fields?: string[]; +} + +const querySchema = schema.object({ + pattern: schema.string(), + meta_fields: schema.oneOf([schema.string(), schema.arrayOf(schema.string())], { + defaultValue: [], + }), + type: schema.maybe(schema.string()), + rollup_index: schema.maybe(schema.string()), + allow_no_index: schema.maybe(schema.boolean()), + include_unmapped: schema.maybe(schema.boolean()), + fields: schema.maybe(schema.oneOf([schema.string(), schema.arrayOf(schema.string())])), +}); + +const fieldSubTypeSchema = schema.object({ + multi: schema.maybe(schema.object({ parent: schema.string() })), + nested: schema.maybe(schema.object({ path: schema.string() })), +}); + +const FieldDescriptorSchema = schema.object({ + aggregatable: schema.boolean(), + name: schema.string(), + readFromDocValues: schema.boolean(), + searchable: schema.boolean(), + type: schema.string(), + esTypes: schema.maybe(schema.arrayOf(schema.string())), + subType: fieldSubTypeSchema, + metadata_field: schema.maybe(schema.boolean()), + fixedInterval: schema.maybe(schema.arrayOf(schema.string())), + timeZone: schema.maybe(schema.arrayOf(schema.string())), + timeSeriesMetric: schema.maybe( + schema.oneOf([ + schema.literal('histogram'), + schema.literal('summary'), + schema.literal('counter'), + schema.literal('gauge'), + schema.literal('position'), + ]) + ), + timeSeriesDimension: schema.maybe(schema.boolean()), + conflictDescriptions: schema.maybe( + schema.recordOf(schema.string(), schema.arrayOf(schema.string())) + ), +}); + +const validate: FullValidationConfig = { + request: { + query: querySchema, + }, + response: { + 200: { + body: schema.object({ + fields: schema.arrayOf(FieldDescriptorSchema), + indices: schema.arrayOf(schema.string()), + }), + }, + }, +}; + +const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, IBody> = + (isRollupsEnabled) => async (context, request, response) => { + const { asCurrentUser } = (await context.core).elasticsearch.client; + const indexPatterns = new IndexPatternsFetcher(asCurrentUser, undefined, isRollupsEnabled()); + const { + pattern, + meta_fields: metaFields, + type, + rollup_index: rollupIndex, + allow_no_index: allowNoIndex, + include_unmapped: includeUnmapped, + } = request.query; + + let parsedFields: string[] = []; + let parsedMetaFields: string[] = []; + try { + parsedMetaFields = parseFields(metaFields); + parsedFields = parseFields(request.query.fields ?? []); + } catch (error) { + return response.badRequest(); + } + + try { + const { fields, indices } = await indexPatterns.getFieldsForWildcard({ + pattern, + // todo should these be added elsewhere? + metaFields: parsedMetaFields, + type, + rollupIndex, + fieldCapsOptions: { + allow_no_indices: allowNoIndex || false, + includeUnmapped, + }, + ...(parsedFields.length > 0 ? { fields: parsedFields } : {}), + }); + + const body: { fields: FieldDescriptorRestResponse[]; indices: string[] } = { + fields, + indices, + }; + + // todo are these needed? + const headers: Record = { + 'content-type': 'application/json', + Etag: '123456', + 'If-None-Match': '123456', + // Etag? + // Expires + }; + + headers['cache-control'] = 'private, max-age=300, stale-while-revalidate=2592000'; + + return response.ok({ + body, + headers, + }); + } catch (error) { + if ( + typeof error === 'object' && + !!error?.isBoom && + !!error?.output?.payload && + typeof error?.output?.payload === 'object' + ) { + const payload = error?.output?.payload; + return response.notFound({ + body: { + message: payload.message, + attributes: payload, + }, + }); + } else { + return response.notFound(); + } + } + }; + +export const registerFields = async ( + router: IRouter, + getStartServices: StartServicesAccessor< + DataViewsServerPluginStartDependencies, + DataViewsServerPluginStart + >, + isRollupsEnabled: () => boolean +) => { + // handler + /* + router.versioned + .get({ path, access }) + .addVersion( + { version, validate: { request: { query: querySchema }, response: validate.response } }, + handler(isRollupsEnabled) + ); + */ + router.get({ path, validate: { query: querySchema } }, handler(isRollupsEnabled)); +}; diff --git a/src/plugins/data_views/server/routes.ts b/src/plugins/data_views/server/routes.ts index c77dd9a8236cf..51f991449029a 100644 --- a/src/plugins/data_views/server/routes.ts +++ b/src/plugins/data_views/server/routes.ts @@ -14,6 +14,7 @@ import type { DataViewsServerPluginStart, DataViewsServerPluginStartDependencies import { registerExistingIndicesPath } from './rest_api_routes/internal/existing_indices'; import { registerFieldForWildcard } from './rest_api_routes/internal/fields_for'; import { registerHasDataViewsRoute } from './rest_api_routes/internal/has_data_views'; +import { registerFields } from './rest_api_routes/internal/fields'; export function registerRoutes( http: HttpServiceSetup, @@ -30,5 +31,6 @@ export function registerRoutes( registerExistingIndicesPath(router); registerFieldForWildcard(router, getStartServices, isRollupsEnabled); + registerFields(router, getStartServices, isRollupsEnabled); registerHasDataViewsRoute(router); } From 609a96221e911dbacc14572fda294bc6585e67c6 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Wed, 1 Nov 2023 22:08:02 -0500 Subject: [PATCH 008/168] implement advanced setting for cache length AND etag headers --- .../src/response.ts | 7 ++++ packages/core/http/core-http-server/index.ts | 1 + .../http/core-http-server/src/router/index.ts | 1 + .../core-http-server/src/router/response.ts | 4 +- .../src/router/response_factory.ts | 13 ++++++ src/plugins/data_views/server/plugin.ts | 2 + .../server/rest_api_routes/internal/fields.ts | 42 +++++++++++++++---- src/plugins/data_views/server/ui_settings.ts | 24 +++++++++++ 8 files changed, 82 insertions(+), 12 deletions(-) create mode 100644 src/plugins/data_views/server/ui_settings.ts diff --git a/packages/core/http/core-http-router-server-internal/src/response.ts b/packages/core/http/core-http-router-server-internal/src/response.ts index 1fc8d310233c7..0251f3b3ef203 100644 --- a/packages/core/http/core-http-router-server-internal/src/response.ts +++ b/packages/core/http/core-http-router-server-internal/src/response.ts @@ -17,6 +17,7 @@ import type { ErrorHttpResponseOptions, KibanaErrorResponseFactory, KibanaRedirectionResponseFactory, + KibanaNotModifiedResponseFactory, KibanaSuccessResponseFactory, KibanaResponseFactory, LifecycleResponseFactory, @@ -51,6 +52,11 @@ const redirectionResponseFactory: KibanaRedirectionResponseFactory = { redirected: (options: RedirectResponseOptions) => new KibanaResponse(302, options.body, options), }; +const notModifiedResponseFactory: KibanaNotModifiedResponseFactory = { + notModified: (options: HttpResponseOptions = {}) => + new KibanaResponse(304, options.body, options), +}; + const errorResponseFactory: KibanaErrorResponseFactory = { badRequest: (options: ErrorHttpResponseOptions = {}) => new KibanaResponse(400, options.body || 'Bad Request', options), @@ -120,6 +126,7 @@ export const fileResponseFactory = { export const kibanaResponseFactory: KibanaResponseFactory = { ...successResponseFactory, ...redirectionResponseFactory, + ...notModifiedResponseFactory, ...errorResponseFactory, ...fileResponseFactory, custom: ( diff --git a/packages/core/http/core-http-server/index.ts b/packages/core/http/core-http-server/index.ts index 94eefba610a44..cd2ae2fd7cec1 100644 --- a/packages/core/http/core-http-server/index.ts +++ b/packages/core/http/core-http-server/index.ts @@ -95,6 +95,7 @@ export type { IKibanaSocket, KibanaErrorResponseFactory, KibanaRedirectionResponseFactory, + KibanaNotModifiedResponseFactory, KibanaSuccessResponseFactory, KibanaResponseFactory, LifecycleResponseFactory, diff --git a/packages/core/http/core-http-server/src/router/index.ts b/packages/core/http/core-http-server/src/router/index.ts index c72d7386e867d..628bab27db54f 100644 --- a/packages/core/http/core-http-server/src/router/index.ts +++ b/packages/core/http/core-http-server/src/router/index.ts @@ -66,6 +66,7 @@ export type { IKibanaSocket } from './socket'; export type { KibanaErrorResponseFactory, KibanaRedirectionResponseFactory, + KibanaNotModifiedResponseFactory, KibanaSuccessResponseFactory, KibanaResponseFactory, LifecycleResponseFactory, diff --git a/packages/core/http/core-http-server/src/router/response.ts b/packages/core/http/core-http-server/src/router/response.ts index 07ec226e8c3a9..194333d42f9c9 100644 --- a/packages/core/http/core-http-server/src/router/response.ts +++ b/packages/core/http/core-http-server/src/router/response.ts @@ -95,9 +95,7 @@ export interface FileHttpResponseOptions { } }; -const access = 'internal'; - type IBody = { index_filter?: estypes.QueryDslQueryContainer } | undefined; interface IQuery { pattern: string; @@ -63,11 +62,13 @@ const querySchema = schema.object({ fields: schema.maybe(schema.oneOf([schema.string(), schema.arrayOf(schema.string())])), }); +/** const fieldSubTypeSchema = schema.object({ multi: schema.maybe(schema.object({ parent: schema.string() })), nested: schema.maybe(schema.object({ path: schema.string() })), }); +/* const FieldDescriptorSchema = schema.object({ aggregatable: schema.boolean(), name: schema.string(), @@ -94,6 +95,7 @@ const FieldDescriptorSchema = schema.object({ ), }); +/* const validate: FullValidationConfig = { request: { query: querySchema, @@ -107,10 +109,19 @@ const validate: FullValidationConfig = { }, }, }; +*/ + +function calculateHash(srcBuffer: Buffer) { + const hash = createHash('sha1'); + hash.update(srcBuffer); + return hash.digest('hex'); +} const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, IBody> = (isRollupsEnabled) => async (context, request, response) => { - const { asCurrentUser } = (await context.core).elasticsearch.client; + const core = await context.core; + const uiSettings = core.uiSettings.client; + const { asCurrentUser } = core.elasticsearch.client; const indexPatterns = new IndexPatternsFetcher(asCurrentUser, undefined, isRollupsEnabled()); const { pattern, @@ -149,16 +160,29 @@ const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, I indices, }; + const etag = calculateHash(Buffer.from(JSON.stringify(body))); + // todo are these needed? const headers: Record = { 'content-type': 'application/json', - Etag: '123456', - 'If-None-Match': '123456', - // Etag? + // 'If-None-Match': '123456', + etag, // Expires }; - headers['cache-control'] = 'private, max-age=300, stale-while-revalidate=2592000'; + // todo consider running in parallel with the request + const cacheMaxAge = await uiSettings.get('data_views:cache_max_age'); + + if (cacheMaxAge) { + const stale = 365 * 24 * 60 * 60 - cacheMaxAge; + headers[ + 'cache-control' + ] = `private, max-age=${cacheMaxAge}, stale-while-revalidate=${stale}`; + } + + if (etag === request.headers.etag) { + return response.notModified({ headers }); + } return response.ok({ body, @@ -193,7 +217,7 @@ export const registerFields = async ( isRollupsEnabled: () => boolean ) => { // handler - /* + /* This seems to fail due to lack of custom headers on cache requests router.versioned .get({ path, access }) .addVersion( diff --git a/src/plugins/data_views/server/ui_settings.ts b/src/plugins/data_views/server/ui_settings.ts new file mode 100644 index 0000000000000..7e995112fdc1c --- /dev/null +++ b/src/plugins/data_views/server/ui_settings.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { i18n } from '@kbn/i18n'; +import { schema } from '@kbn/config-schema'; + +export const cacheMaxAge = { + 'data_views:cache_max_age': { + name: i18n.translate('dataViews.advancedSettings.cacheMaxAgeTitle', { + defaultMessage: 'Field cache max age', + }), + value: 300, + description: i18n.translate('dataViews.advancedSettings.cacheMaxAgeText', { + defaultMessage: + "Sets the 'max-age' cache header value for data view fields API requests. A value of 0 will disable caching.", + }), + schema: schema.number(), + }, +}; From 2aec419735bb70882fbefd9af9953231891849ce Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Wed, 1 Nov 2023 22:20:56 -0500 Subject: [PATCH 009/168] add mock --- .../core/http/core-http-router-server-mocks/src/router.mock.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/core/http/core-http-router-server-mocks/src/router.mock.ts b/packages/core/http/core-http-router-server-mocks/src/router.mock.ts index 4272cf130b38e..d5904699b5813 100644 --- a/packages/core/http/core-http-router-server-mocks/src/router.mock.ts +++ b/packages/core/http/core-http-router-server-mocks/src/router.mock.ts @@ -121,6 +121,7 @@ const createResponseFactoryMock = (): jest.Mocked => ({ ok: jest.fn(), accepted: jest.fn(), noContent: jest.fn(), + notModified: jest.fn(), custom: jest.fn(), redirected: jest.fn(), badRequest: jest.fn(), From 2abf10eb9c96bbf90144c3cd9dee75229d8cb7fa Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Thu, 2 Nov 2023 00:20:24 -0500 Subject: [PATCH 010/168] attempt to fix discover alerting functional test --- .../public/data_views/data_views_api_client.test.ts | 2 +- .../discover_ml_uptime/discover/search_source_alert.ts | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/plugins/data_views/public/data_views/data_views_api_client.test.ts b/src/plugins/data_views/public/data_views/data_views_api_client.test.ts index 90a208650a965..6320226aea96f 100644 --- a/src/plugins/data_views/public/data_views/data_views_api_client.test.ts +++ b/src/plugins/data_views/public/data_views/data_views_api_client.test.ts @@ -8,7 +8,7 @@ import { http } from './data_views_api_client.test.mock'; import { DataViewsApiClient } from './data_views_api_client'; -import { FIELDS_FOR_WILDCARD_PATH as expectedPath } from '../../common/constants'; +import { FIELDS_PATH as expectedPath } from '../../common/constants'; describe('IndexPatternsApiClient', () => { let fetchSpy: jest.SpyInstance; diff --git a/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts b/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts index 6b668f75d56d3..98eb7bd0bbf90 100644 --- a/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts +++ b/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts @@ -307,6 +307,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { describe('Search source Alert', () => { before(async () => { + await kibanaServer.uiSettings.replace({ + 'data_views:fields_max_cache': '0', + }); await security.testUser.setRoles(['discover_alert']); log.debug('create source indices'); @@ -330,6 +333,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await deleteConnector(connectorId); await security.testUser.restoreDefaults(); await kibanaServer.savedObjects.cleanStandardList(); + await kibanaServer.uiSettings.replace({ + 'data_views:fields_max_cache': '300', + }); }); it('should create an alert when there is no data view', async () => { @@ -398,6 +404,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await checkInitialRuleParamsState(SOURCE_DATA_VIEW, true); }); + // this test fails with caching in place it('should navigate to alert results via link provided in notification', async () => { await openAlertResults(RULE_NAME); await checkInitialRuleParamsState(SOURCE_DATA_VIEW); From 4de45b5b76521dd05a8f2cd041bce44ff8824e63 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Thu, 2 Nov 2023 08:55:10 -0500 Subject: [PATCH 011/168] 304 sohuldn't accept body --- .../core/http/core-http-router-server-internal/src/response.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/http/core-http-router-server-internal/src/response.ts b/packages/core/http/core-http-router-server-internal/src/response.ts index 0251f3b3ef203..71b03761b6da8 100644 --- a/packages/core/http/core-http-router-server-internal/src/response.ts +++ b/packages/core/http/core-http-router-server-internal/src/response.ts @@ -49,7 +49,7 @@ const successResponseFactory: KibanaSuccessResponseFactory = { }; const redirectionResponseFactory: KibanaRedirectionResponseFactory = { - redirected: (options: RedirectResponseOptions) => new KibanaResponse(302, options.body, options), + redirected: (options: RedirectResponseOptions) => new KibanaResponse(302, undefined, options), }; const notModifiedResponseFactory: KibanaNotModifiedResponseFactory = { From 8d8fbb4b7728f05eb3c0ae3cb5dff16716325b08 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Fri, 3 Nov 2023 17:44:53 -0500 Subject: [PATCH 012/168] try setting cache to zero to see if fewer tests fail --- src/plugins/data_views/server/ui_settings.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/data_views/server/ui_settings.ts b/src/plugins/data_views/server/ui_settings.ts index 7e995112fdc1c..a2d0feb05bbe8 100644 --- a/src/plugins/data_views/server/ui_settings.ts +++ b/src/plugins/data_views/server/ui_settings.ts @@ -14,7 +14,7 @@ export const cacheMaxAge = { name: i18n.translate('dataViews.advancedSettings.cacheMaxAgeTitle', { defaultMessage: 'Field cache max age', }), - value: 300, + value: 0, description: i18n.translate('dataViews.advancedSettings.cacheMaxAgeText', { defaultMessage: "Sets the 'max-age' cache header value for data view fields API requests. A value of 0 will disable caching.", From 146ba0bcbb647c3608be69d198f0536ba8bfbf11 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Sat, 4 Nov 2023 17:03:34 -0500 Subject: [PATCH 013/168] remove some comments --- .../public/data_views/data_views_api_client.ts | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/src/plugins/data_views/public/data_views/data_views_api_client.ts b/src/plugins/data_views/public/data_views/data_views_api_client.ts index 9f58daf501490..bda9f40417352 100644 --- a/src/plugins/data_views/public/data_views/data_views_api_client.ts +++ b/src/plugins/data_views/public/data_views/data_views_api_client.ts @@ -35,19 +35,11 @@ export class DataViewsApiClient implements IDataViewsApiClient { body?: string, forceRefresh?: boolean ): Promise { - /* - console.log('HERE', new Date().getMinutes()); - if (new Date().getMinutes() % 2 === 0) { - forceRefresh = true; - } - */ - // const headers = forceRefresh ? { 'Cache-Control': '' } : undefined; - // const cacheOptions = forceRefresh ? { cache: 'reload' as RequestCache } : {}; + const cacheOptions = forceRefresh ? { cache: 'reload' as RequestCache } : {}; const request = body ? this.http.post(url, { query, body, version }) - : this.http.fetch(url, { query, version }); - // : this.http.fetch(url, { query, version, ...cacheOptions }); + : this.http.fetch(url, { query, version, ...cacheOptions }); return request.catch((resp) => { if (resp.body.statusCode === 404 && resp.body.attributes?.code === 'no_matching_indices') { throw new DataViewMissingIndices(resp.body.message); From 830b9591813b05a32e98d41f1e265191ea3ec218 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Sat, 4 Nov 2023 21:41:22 -0500 Subject: [PATCH 014/168] fix tests --- .../http/core-http-router-server-internal/src/response.ts | 5 ++--- .../server/collectors/management/schema.ts | 7 +++++++ .../server/collectors/management/types.ts | 1 + src/plugins/telemetry/schema/oss_plugins.json | 6 ++++++ .../cypress/e2e/automated_response_actions/form.cy.ts | 4 ++-- 5 files changed, 18 insertions(+), 5 deletions(-) diff --git a/packages/core/http/core-http-router-server-internal/src/response.ts b/packages/core/http/core-http-router-server-internal/src/response.ts index 71b03761b6da8..d96eaa6571bf0 100644 --- a/packages/core/http/core-http-router-server-internal/src/response.ts +++ b/packages/core/http/core-http-router-server-internal/src/response.ts @@ -49,12 +49,11 @@ const successResponseFactory: KibanaSuccessResponseFactory = { }; const redirectionResponseFactory: KibanaRedirectionResponseFactory = { - redirected: (options: RedirectResponseOptions) => new KibanaResponse(302, undefined, options), + redirected: (options: RedirectResponseOptions) => new KibanaResponse(302, options.body, options), }; const notModifiedResponseFactory: KibanaNotModifiedResponseFactory = { - notModified: (options: HttpResponseOptions = {}) => - new KibanaResponse(304, options.body, options), + notModified: (options: HttpResponseOptions = {}) => new KibanaResponse(304, undefined, options), }; const errorResponseFactory: KibanaErrorResponseFactory = { diff --git a/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts b/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts index 0862d6ece004a..339610725175f 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts @@ -162,6 +162,13 @@ export const stackManagementSchema: MakeSchemaFrom = { type: 'boolean', _meta: { description: 'Non-default value of setting.' }, }, + 'data_views:cache_max_age': { + type: 'long', + _meta: { + description: + "Sets the 'max-age' cache header value for data view fields API requests. A value of 0 will disable caching.", + }, + }, 'discover:searchOnPageLoad': { type: 'boolean', _meta: { description: 'Non-default value of setting.' }, diff --git a/src/plugins/kibana_usage_collection/server/collectors/management/types.ts b/src/plugins/kibana_usage_collection/server/collectors/management/types.ts index 3499471e0d5a8..f6aabe88236b2 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/management/types.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/management/types.ts @@ -78,6 +78,7 @@ export interface UsageStats { 'doc_table:highlight': boolean; 'discover:searchOnPageLoad': boolean; 'doc_table:hideTimeColumn': boolean; + 'data_views:cache_max_age': number; 'discover:sampleSize': number; 'discover:sampleRowsPerPage': number; defaultColumns: string[]; diff --git a/src/plugins/telemetry/schema/oss_plugins.json b/src/plugins/telemetry/schema/oss_plugins.json index 0854944f39404..63eb184813892 100644 --- a/src/plugins/telemetry/schema/oss_plugins.json +++ b/src/plugins/telemetry/schema/oss_plugins.json @@ -9428,6 +9428,12 @@ "description": "Non-default value of setting." } }, + "data_views:cache_max_age": { + "type": "long", + "_meta": { + "description": "Sets the 'max-age' cache header value for data view fields API requests. A value of 0 will disable caching." + } + }, "discover:searchOnPageLoad": { "type": "boolean", "_meta": { diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/automated_response_actions/form.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/automated_response_actions/form.cy.ts index a370f2a89cb6f..11f2651755bf0 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/automated_response_actions/form.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/automated_response_actions/form.cy.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { FIELDS_FOR_WILDCARD_PATH } from '@kbn/data-views-plugin/common/constants'; +import { FIELDS_PATH } from '@kbn/data-views-plugin/common/constants'; import { addEndpointResponseAction, fillUpNewRule, @@ -186,7 +186,7 @@ describe( }); it('All response action controls are disabled', () => { - cy.intercept('GET', `${FIELDS_FOR_WILDCARD_PATH}*`).as('getFieldsForWildcard'); + cy.intercept('GET', `${FIELDS_PATH}*`).as('getFieldsForWildcard'); visitRuleActions(ruleId); cy.wait('@getFieldsForWildcard'); cy.getByTestSubj('edit-rule-actions-tab').click(); From 9638c362afc5c7a07e31932f642be1242f6e6edc Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Sat, 4 Nov 2023 22:56:50 -0500 Subject: [PATCH 015/168] restore default cache max-age --- src/plugins/data_views/server/ui_settings.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/data_views/server/ui_settings.ts b/src/plugins/data_views/server/ui_settings.ts index a2d0feb05bbe8..7e995112fdc1c 100644 --- a/src/plugins/data_views/server/ui_settings.ts +++ b/src/plugins/data_views/server/ui_settings.ts @@ -14,7 +14,7 @@ export const cacheMaxAge = { name: i18n.translate('dataViews.advancedSettings.cacheMaxAgeTitle', { defaultMessage: 'Field cache max age', }), - value: 0, + value: 300, description: i18n.translate('dataViews.advancedSettings.cacheMaxAgeText', { defaultMessage: "Sets the 'max-age' cache header value for data view fields API requests. A value of 0 will disable caching.", From 442c048159bea7310faebf7632177a38186d08e6 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Mon, 6 Nov 2023 22:42:55 -0600 Subject: [PATCH 016/168] fix 304 responses, implement refresh button, hopefully fix a test --- .../src/response_adapter.ts | 8 ++- .../edit_index_pattern/edit_index_pattern.tsx | 12 ++++- .../index_header/index_header.tsx | 29 ++++++++++- .../index_pattern_table.tsx | 2 +- .../data_views/common/data_views/data_view.ts | 6 +++ .../common/data_views/data_views.ts | 52 +++++++++++++++---- src/plugins/data_views/common/types.ts | 1 + .../data_views/data_views_api_client.ts | 26 +++++++--- .../server/rest_api_routes/internal/fields.ts | 13 ++++- .../apps/discover/group3/_sidebar.ts | 17 ++++++ test/functional/page_objects/settings_page.ts | 8 +++ 11 files changed, 149 insertions(+), 25 deletions(-) diff --git a/packages/core/http/core-http-router-server-internal/src/response_adapter.ts b/packages/core/http/core-http-router-server-internal/src/response_adapter.ts index f6a00bb7e6b92..a5cc1ca1f8d96 100644 --- a/packages/core/http/core-http-router-server-internal/src/response_adapter.ts +++ b/packages/core/http/core-http-router-server-internal/src/response_adapter.ts @@ -33,7 +33,8 @@ function setHeaders(response: HapiResponseObject, headers: Record code >= 100 && code < 300, - isRedirect: (code: number) => code >= 300 && code < 400, + isNotModified: (code: number) => code === 304, + isRedirect: (code: number) => code >= 300 && code < 400 && code !== 304, isError: (code: number) => code >= 400 && code < 600, }; @@ -76,7 +77,10 @@ export class HapiResponseAdapter { if (statusHelpers.isError(kibanaResponse.status)) { return this.toError(kibanaResponse); } - if (statusHelpers.isSuccess(kibanaResponse.status)) { + if ( + statusHelpers.isSuccess(kibanaResponse.status) || + statusHelpers.isNotModified(kibanaResponse.status) + ) { return this.toSuccess(kibanaResponse); } if (statusHelpers.isRedirect(kibanaResponse.status)) { diff --git a/src/plugins/data_view_management/public/components/edit_index_pattern/edit_index_pattern.tsx b/src/plugins/data_view_management/public/components/edit_index_pattern/edit_index_pattern.tsx index 71ff8ce38636c..ba1d8373119cb 100644 --- a/src/plugins/data_view_management/public/components/edit_index_pattern/edit_index_pattern.tsx +++ b/src/plugins/data_view_management/public/components/edit_index_pattern/edit_index_pattern.tsx @@ -93,6 +93,9 @@ export const EditIndexPattern = withRouter( const [showEditDialog, setShowEditDialog] = useState(false); const [relationships, setRelationships] = useState([]); const [allowedTypes, setAllowedTypes] = useState([]); + const [refreshCount, setRefreshCount] = useState(0); + const [isRefreshing, setIsRefreshing] = React.useState(false); + const conflictFieldsUrl = useMemo(() => { return setStateToKbnUrl( APP_STATE_STORAGE_KEY, @@ -142,7 +145,7 @@ export const EditIndexPattern = withRouter( setConflictedFields( indexPattern.fields.getAll().filter((field) => field.type === 'conflict') ); - }, [indexPattern]); + }, [indexPattern, refreshCount]); useEffect(() => { setTags( @@ -244,8 +247,15 @@ export const EditIndexPattern = withRouter( deleteIndexPatternClick={() => removeHandler([indexPattern as RemoveDataViewProps],
{warning}
) } + refreshIndexPatternClick={async () => { + setIsRefreshing(true); + await dataViews.refreshFields(indexPattern, false, true); + setRefreshCount(refreshCount + 1); + setIsRefreshing(false); + }} defaultIndex={defaultIndex} canSave={userEditPermission} + isRefreshing={isRefreshing} > diff --git a/src/plugins/data_view_management/public/components/edit_index_pattern/index_header/index_header.tsx b/src/plugins/data_view_management/public/components/edit_index_pattern/index_header/index_header.tsx index 83c965f2b9332..cf1ce9faedb55 100644 --- a/src/plugins/data_view_management/public/components/edit_index_pattern/index_header/index_header.tsx +++ b/src/plugins/data_view_management/public/components/edit_index_pattern/index_header/index_header.tsx @@ -8,7 +8,7 @@ import React from 'react'; import { i18n } from '@kbn/i18n'; -import { EuiButton, EuiButtonEmpty, EuiPageHeader } from '@elastic/eui'; +import { EuiButton, EuiButtonEmpty, EuiPageHeader, EuiToolTip } from '@elastic/eui'; import { DataView } from '@kbn/data-views-plugin/public'; interface IndexHeaderProps { @@ -17,7 +17,9 @@ interface IndexHeaderProps { setDefault?: () => void; editIndexPatternClick?: () => void; deleteIndexPatternClick?: () => void; + refreshIndexPatternClick?: () => void; canSave: boolean; + isRefreshing?: boolean; } const setDefaultAriaLabel = i18n.translate('indexPatternManagement.editDataView.setDefaultAria', { @@ -44,19 +46,44 @@ const removeTooltip = i18n.translate('indexPatternManagement.editDataView.remove defaultMessage: 'Delete', }); +const refreshAriaLabel = i18n.translate('indexPatternManagement.editDataView.refreshAria', { + defaultMessage: 'Refresh', +}); + +const refreshTooltip = i18n.translate('indexPatternManagement.editDataView.refreshTooltip', { + defaultMessage: 'Refresh local copy of data view field list', +}); + +const refreshLabel = i18n.translate('indexPatternManagement.editDataView.refreshLabel', { + defaultMessage: 'Refresh', +}); + export const IndexHeader: React.FC = ({ defaultIndex, indexPattern, setDefault, editIndexPatternClick, deleteIndexPatternClick, + refreshIndexPatternClick, children, canSave, + isRefreshing, }) => { return ( {indexPattern.getName()}} rightSideItems={[ + {refreshTooltip}

}> + + {refreshLabel} + + , canSave && ( {dataView.getName()} {dataView.name ? ( diff --git a/src/plugins/data_views/common/data_views/data_view.ts b/src/plugins/data_views/common/data_views/data_view.ts index 27681021757ff..71145fb6ac157 100644 --- a/src/plugins/data_views/common/data_views/data_view.ts +++ b/src/plugins/data_views/common/data_views/data_view.ts @@ -61,6 +61,8 @@ export class DataView extends AbstractDataView implements DataViewBase { */ public flattenHit: (hit: Record, deep?: boolean) => Record; + private etag: string | undefined; + /** * constructor * @param config - config data and dependencies @@ -77,6 +79,10 @@ export class DataView extends AbstractDataView implements DataViewBase { this.fields.replaceAll(Object.values(spec.fields || {})); } + getEtag = () => this.etag; + + setEtag = (etag: string) => (this.etag = etag); + /** * Returns scripted fields */ diff --git a/src/plugins/data_views/common/data_views/data_views.ts b/src/plugins/data_views/common/data_views/data_views.ts index 7ec161e689e76..d618f4e6b9784 100644 --- a/src/plugins/data_views/common/data_views/data_views.ts +++ b/src/plugins/data_views/common/data_views/data_views.ts @@ -261,7 +261,11 @@ export interface DataViewsServicePublicMethods { * Refresh fields for data view instance * @params dataView - Data view instance */ - refreshFields: (indexPattern: DataView, displayErrors?: boolean) => Promise; + refreshFields: ( + indexPattern: DataView, + displayErrors?: boolean, + forceRefresh?: boolean + ) => Promise; /** * Converts data view saved object to spec * @params savedObject - Data view saved object @@ -533,7 +537,10 @@ export class DataViewsService { pattern: indexPattern.title as string, }); - private getFieldsAndIndicesForDataView = async (dataView: DataView) => { + private getFieldsAndIndicesForDataView = async ( + dataView: DataView, + forceRefresh: boolean = false + ) => { const metaFields = await this.config.get(META_FIELDS); return this.apiClient.getFieldsForWildcard({ type: dataView.type, @@ -541,6 +548,7 @@ export class DataViewsService { allowNoIndex: true, pattern: dataView.getIndexPattern(), metaFields, + forceRefresh, }); }; @@ -556,8 +564,18 @@ export class DataViewsService { }); }; - private refreshFieldsFn = async (indexPattern: DataView) => { - const { fields, indices } = await this.getFieldsAndIndicesForDataView(indexPattern); + private refreshFieldsFn = async (indexPattern: DataView, forceRefresh: boolean = false) => { + const { fields, indices, etag } = await this.getFieldsAndIndicesForDataView( + indexPattern, + forceRefresh + ); + + if (etag === indexPattern.getEtag()) { + return; + } else { + indexPattern.setEtag(etag); + } + fields.forEach((field) => (field.isMapped = true)); const scripted = this.scriptedFieldsEnabled ? indexPattern.getScriptedFields().map((field) => field.spec) @@ -583,13 +601,17 @@ export class DataViewsService { * @param dataView * @param displayErrors - If set false, API consumer is responsible for displaying and handling errors. */ - refreshFields = async (dataView: DataView, displayErrors: boolean = true) => { + refreshFields = async ( + dataView: DataView, + displayErrors: boolean = true, + forceRefresh: boolean = false + ) => { if (!displayErrors) { - return this.refreshFieldsFn(dataView); + return this.refreshFieldsFn(dataView, forceRefresh); } try { - await this.refreshFieldsFn(dataView); + await this.refreshFieldsFn(dataView, forceRefresh); } catch (err) { if (err instanceof DataViewMissingIndices) { // not considered an error, check dataView.matchedIndices.length to be 0 @@ -630,7 +652,11 @@ export class DataViewsService { : []; try { let updatedFieldList: FieldSpec[]; - const { fields: newFields, indices } = await this.getFieldsAndIndicesForWildcard(options); + const { + fields: newFields, + indices, + etag, + } = await this.getFieldsAndIndicesForWildcard(options); newFields.forEach((field) => (field.isMapped = true)); // If allowNoIndex, only update field list if field caps finds fields. To support @@ -641,7 +667,7 @@ export class DataViewsService { updatedFieldList = fieldsAsArr; } - return { fields: this.fieldArrayToMap(updatedFieldList, fieldAttrs), indices }; + return { fields: this.fieldArrayToMap(updatedFieldList, fieldAttrs), indices, etag }; } catch (err) { if (err instanceof DataViewMissingIndices) { // not considered an error, check dataView.matchedIndices.length to be 0 @@ -753,7 +779,7 @@ export class DataViewsService { displayErrors?: boolean; }) => { const { title, type, typeMeta, runtimeFieldMap } = spec; - const { fields, indices } = await this.refreshFieldSpecMap( + const { fields, indices, etag } = await this.refreshFieldSpecMap( spec.fields || {}, savedObjectId, spec.title as string, @@ -770,7 +796,7 @@ export class DataViewsService { const runtimeFieldSpecs = this.getRuntimeFields(runtimeFieldMap, spec.fieldAttrs); // mapped fields overwrite runtime fields - return { fields: { ...runtimeFieldSpecs, ...fields }, indices: indices || [] }; + return { fields: { ...runtimeFieldSpecs, ...fields }, indices: indices || [], etag }; }; private initFromSavedObject = async ( @@ -784,6 +810,7 @@ export class DataViewsService { let fields: Record = {}; let indices: string[] = []; + let etag: string | undefined; if (!displayErrors) { const fieldsAndIndices = await this.initFromSavedObjectLoadFields({ @@ -793,6 +820,7 @@ export class DataViewsService { }); fields = fieldsAndIndices.fields; indices = fieldsAndIndices.indices; + etag = fieldsAndIndices.etag; } else { try { const fieldsAndIndices = await this.initFromSavedObjectLoadFields({ @@ -802,6 +830,7 @@ export class DataViewsService { }); fields = fieldsAndIndices.fields; indices = fieldsAndIndices.indices; + etag = fieldsAndIndices.etag; } catch (err) { if (err instanceof DataViewMissingIndices) { // not considered an error, check dataView.matchedIndices.length to be 0 @@ -826,6 +855,7 @@ export class DataViewsService { : {}; const indexPattern = await this.createFromSpec(spec, true, displayErrors); + indexPattern.setEtag(etag!); indexPattern.matchedIndices = indices; indexPattern.resetOriginalSavedObjectBody(); return indexPattern; diff --git a/src/plugins/data_views/common/types.ts b/src/plugins/data_views/common/types.ts index c2b79c72f8eeb..7c3cc0cacb618 100644 --- a/src/plugins/data_views/common/types.ts +++ b/src/plugins/data_views/common/types.ts @@ -319,6 +319,7 @@ export interface GetFieldsOptions { export interface FieldsForWildcardResponse { fields: FieldSpec[]; indices: string[]; + etag: string; } /** diff --git a/src/plugins/data_views/public/data_views/data_views_api_client.ts b/src/plugins/data_views/public/data_views/data_views_api_client.ts index bda9f40417352..ff184722a4a3b 100644 --- a/src/plugins/data_views/public/data_views/data_views_api_client.ts +++ b/src/plugins/data_views/public/data_views/data_views_api_client.ts @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -import { HttpSetup } from '@kbn/core/public'; +import { HttpSetup, HttpResponse } from '@kbn/core/public'; import { DataViewMissingIndices } from '../../common/lib'; import { GetFieldsOptions, IDataViewsApiClient } from '../../common'; import { FieldsForWildcardResponse } from '../../common/types'; @@ -34,12 +34,17 @@ export class DataViewsApiClient implements IDataViewsApiClient { query?: {}, body?: string, forceRefresh?: boolean - ): Promise { - const cacheOptions = forceRefresh ? { cache: 'reload' as RequestCache } : {}; + ): Promise | undefined> { + const asResponse = true; + // circle back to this, will likely need changes to any code that loads fields + // const rawResponse = true; + const rawResponse = false; + const cacheOptions = forceRefresh ? { cache: 'no-cache' as RequestCache } : {}; const request = body - ? this.http.post(url, { query, body, version }) - : this.http.fetch(url, { query, version, ...cacheOptions }); + ? this.http.post(url, { query, body, version, asResponse }) + : this.http.fetch(url, { query, version, ...cacheOptions, asResponse, rawResponse }); + return request.catch((resp) => { if (resp.body.statusCode === 404 && resp.body.attributes?.code === 'no_matching_indices') { throw new DataViewMissingIndices(resp.body.message); @@ -85,7 +90,11 @@ export class DataViewsApiClient implements IDataViewsApiClient { indexFilter ? JSON.stringify({ index_filter: indexFilter }) : undefined, forceRefresh ).then((response) => { - return response || { fields: [], indices: [] }; + return { + indices: response?.body?.indices || [], + fields: response?.body?.fields || [], + etag: response?.response?.headers?.get('etag') || '', + }; }); } @@ -96,6 +105,9 @@ export class DataViewsApiClient implements IDataViewsApiClient { const response = await this._request<{ result: boolean }>( this._getUrl(['has_user_index_pattern']) ); - return response?.result ?? false; + + // const body = await response?.response?.json(); + // return body?.result ?? false; + return response?.body?.result ?? false; } } diff --git a/src/plugins/data_views/server/rest_api_routes/internal/fields.ts b/src/plugins/data_views/server/rest_api_routes/internal/fields.ts index c965cf1d63895..0e2dd5d0589db 100644 --- a/src/plugins/data_views/server/rest_api_routes/internal/fields.ts +++ b/src/plugins/data_views/server/rest_api_routes/internal/fields.ts @@ -180,8 +180,17 @@ const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, I ] = `private, max-age=${cacheMaxAge}, stale-while-revalidate=${stale}`; } - if (etag === request.headers.etag) { - return response.notModified({ headers }); + // move to util + const ifNoneMatch = request.headers['if-none-match']; + if (ifNoneMatch) { + let requestHash = (ifNoneMatch as string).slice(1, -1); + if (requestHash.indexOf('-') > -1) { + requestHash = requestHash.split('-')[0]; + } + + if (etag === requestHash) { + return response.notModified({ headers }); + } } return response.ok({ diff --git a/test/functional/apps/discover/group3/_sidebar.ts b/test/functional/apps/discover/group3/_sidebar.ts index b47e71f452818..46664ff900978 100644 --- a/test/functional/apps/discover/group3/_sidebar.ts +++ b/test/functional/apps/discover/group3/_sidebar.ts @@ -17,6 +17,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { 'discover', 'timePicker', 'header', + 'settings', 'unifiedSearch', 'unifiedFieldList', ]); @@ -285,6 +286,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); }); + // describe('renders field groups', function () { it('should show field list groups excluding subfields', async function () { await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded(); @@ -475,6 +477,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { ); }); + // it('should work correctly for a data view for a missing index', async function () { // but we are skipping importing the index itself await kibanaServer.importExport.load( @@ -492,6 +495,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.header.waitUntilLoadingHasFinished(); await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded(); + // console.log('######################## await timeout'); + // await new Promise((r) => setTimeout(r, 10000 * 60)); expect(await PageObjects.unifiedFieldList.getSidebarAriaDescription()).to.be( '0 available fields. 0 empty fields. 0 meta fields.' ); @@ -526,6 +531,10 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { ); await browser.refresh(); + // here + await PageObjects.settings.refreshDataViewFieldList('with-timefield'); + await PageObjects.common.navigateToApp('discover'); + await PageObjects.discover.waitUntilSearchingHasFinished(); await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded(); expect(await PageObjects.unifiedFieldList.getSidebarAriaDescription()).to.be( @@ -546,8 +555,12 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.header.waitUntilLoadingHasFinished(); await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded(); + // console.log('######################## await timeout'); + // await new Promise((r) => setTimeout(r, 1000 * 300)); + // todo - comapre to main. This looks correct in this branch expect(await PageObjects.unifiedFieldList.getSidebarAriaDescription()).to.be( '0 available fields. 7 empty fields. 3 meta fields.' + // '0 available fields. 13 empty fields. 0 empty fields. 3 meta fields.' ); await testSubjects.existOrFail( `${PageObjects.unifiedFieldList.getSidebarSectionSelector( @@ -571,6 +584,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await esArchiver.unload( 'test/functional/fixtures/es_archiver/index_pattern_without_timefield' ); + // console.log('######################## await timeout'); + // await new Promise((r) => setTimeout(r, 1000 * 30)); }); it('should work when filters change', async () => { @@ -755,6 +770,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.header.waitUntilLoadingHasFinished(); await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded(); + // console.log('######################## await timeout'); + // await new Promise((r) => setTimeout(r, 1000 * 30)); expect(await PageObjects.unifiedFieldList.getSidebarAriaDescription()).to.be( '0 available fields. 7 empty fields. 3 meta fields.' ); diff --git a/test/functional/page_objects/settings_page.ts b/test/functional/page_objects/settings_page.ts index 0a2fd76076914..d81c1e09777d0 100644 --- a/test/functional/page_objects/settings_page.ts +++ b/test/functional/page_objects/settings_page.ts @@ -483,6 +483,14 @@ export class SettingsPageObject extends FtrService { await customDataViewIdInput.type(value); } + async refreshDataViewFieldList(dataViewName: string) { + await this.navigateTo(); + await this.clickKibanaIndexPatterns(); + await this.header.waitUntilLoadingHasFinished(); + await this.testSubjects.click(`detail-link-${dataViewName}`); + await this.testSubjects.click('refreshDataViewButton'); + } + async createIndexPattern( indexPatternName: string, // null to bypass default value From 3f533ab5edb5f1ee489c8af5f78df787ff8abc29 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Tue, 7 Nov 2023 00:18:23 -0600 Subject: [PATCH 017/168] fix tests --- packages/kbn-optimizer/limits.yml | 2 +- src/plugins/data_views/common/data_views/data_views.ts | 4 ++-- src/plugins/data_views/common/types.ts | 2 +- .../apps/management/data_views/_index_pattern_results_sort.ts | 2 ++ .../server/services/log_views/log_views_client.test.ts | 3 +++ .../apps/observability_log_explorer/columns_selection.ts | 4 +++- 6 files changed, 12 insertions(+), 5 deletions(-) diff --git a/packages/kbn-optimizer/limits.yml b/packages/kbn-optimizer/limits.yml index de07bccf8bbcb..ce983447cb5da 100644 --- a/packages/kbn-optimizer/limits.yml +++ b/packages/kbn-optimizer/limits.yml @@ -31,7 +31,7 @@ pageLoadAssetSize: dataViewEditor: 28082 dataViewFieldEditor: 27000 dataViewManagement: 5100 - dataViews: 48300 + dataViews: 48800 dataVisualizer: 27530 devTools: 38637 discover: 99999 diff --git a/src/plugins/data_views/common/data_views/data_views.ts b/src/plugins/data_views/common/data_views/data_views.ts index d618f4e6b9784..e6ab814ee7bd0 100644 --- a/src/plugins/data_views/common/data_views/data_views.ts +++ b/src/plugins/data_views/common/data_views/data_views.ts @@ -570,10 +570,10 @@ export class DataViewsService { forceRefresh ); - if (etag === indexPattern.getEtag()) { + if (indexPattern.getEtag() && etag === indexPattern.getEtag()) { return; } else { - indexPattern.setEtag(etag); + indexPattern.setEtag(etag!); } fields.forEach((field) => (field.isMapped = true)); diff --git a/src/plugins/data_views/common/types.ts b/src/plugins/data_views/common/types.ts index 7c3cc0cacb618..dae5599e276a0 100644 --- a/src/plugins/data_views/common/types.ts +++ b/src/plugins/data_views/common/types.ts @@ -319,7 +319,7 @@ export interface GetFieldsOptions { export interface FieldsForWildcardResponse { fields: FieldSpec[]; indices: string[]; - etag: string; + etag?: string; } /** diff --git a/test/functional/apps/management/data_views/_index_pattern_results_sort.ts b/test/functional/apps/management/data_views/_index_pattern_results_sort.ts index 71dc85eed1f0e..2f11ae3cfb2e3 100644 --- a/test/functional/apps/management/data_views/_index_pattern_results_sort.ts +++ b/test/functional/apps/management/data_views/_index_pattern_results_sort.ts @@ -67,6 +67,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { describe('field list pagination', function () { const EXPECTED_FIELD_COUNT = 85; it('makelogs data should have expected number of fields', async function () { + // did this work? + await this.testSubjects.click('refreshDataViewButton'); await retry.try(async function () { const TabCount = await PageObjects.settings.getFieldsTabCount(); expect(TabCount).to.be('' + EXPECTED_FIELD_COUNT); diff --git a/x-pack/plugins/logs_shared/server/services/log_views/log_views_client.test.ts b/x-pack/plugins/logs_shared/server/services/log_views/log_views_client.test.ts index 125167d6deea8..5d9ae8eed5394 100644 --- a/x-pack/plugins/logs_shared/server/services/log_views/log_views_client.test.ts +++ b/x-pack/plugins/logs_shared/server/services/log_views/log_views_client.test.ts @@ -252,6 +252,7 @@ describe('LogViewsClient class', () => { "allowNoIndex": false, "deleteFieldFormat": [Function], "deleteScriptedFieldInternal": [Function], + "etag": undefined, "fieldAttrs": Object {}, "fieldFormatMap": Object {}, "fieldFormats": Object { @@ -274,6 +275,7 @@ describe('LogViewsClient class', () => { }, "fields": FldList [], "flattenHit": [Function], + "getEtag": [Function], "getFieldAttrs": [Function], "getIndexPattern": [Function], "getName": [Function], @@ -298,6 +300,7 @@ describe('LogViewsClient class', () => { }, }, "scriptedFields": Array [], + "setEtag": [Function], "setFieldFormat": [Function], "setIndexPattern": [Function], "shortDotsEnable": false, diff --git a/x-pack/test/functional/apps/observability_log_explorer/columns_selection.ts b/x-pack/test/functional/apps/observability_log_explorer/columns_selection.ts index 7738d3b97b2bc..83b88d3615547 100644 --- a/x-pack/test/functional/apps/observability_log_explorer/columns_selection.ts +++ b/x-pack/test/functional/apps/observability_log_explorer/columns_selection.ts @@ -13,7 +13,7 @@ const defaultLogColumns = ['@timestamp', 'service.name', 'host.name', 'message'] export default function ({ getService, getPageObjects }: FtrProviderContext) { const esArchiver = getService('esArchiver'); const retry = getService('retry'); - const PageObjects = getPageObjects(['discover', 'observabilityLogExplorer']); + const PageObjects = getPageObjects(['discover', 'observabilityLogExplorer', 'settings']); describe('Columns selection initialization and update', () => { before(async () => { @@ -30,6 +30,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { describe('when the log explorer loads', () => { it("should initialize the table columns to logs' default selection", async () => { + // did this work? + PageObjects.settings.refreshDataViewFieldList('All logs'); await PageObjects.observabilityLogExplorer.navigateTo(); await retry.try(async () => { From ffde29ebbb26d4d2b6c19c75ec053cc02a09c302 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Tue, 7 Nov 2023 00:41:38 -0600 Subject: [PATCH 018/168] revert fields_for_wildcard changes --- .../rest_api_routes/internal/fields_for.ts | 33 +++++-------------- 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts b/src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts index 9d9f09d79446d..32197ea5a96d4 100644 --- a/src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts +++ b/src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts @@ -111,12 +111,8 @@ const validate: FullValidationConfig = { }, }; -const handler: ( - isRollupsEnabled: () => boolean -) => (cacheHeader: boolean) => RequestHandler<{}, IQuery, IBody> = - (isRollupsEnabled) => - (cacheHeader = false) => - async (context, request, response) => { +const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, IBody> = + (isRollupsEnabled) => async (context, request, response) => { const { asCurrentUser } = (await context.core).elasticsearch.client; const indexPatterns = new IndexPatternsFetcher(asCurrentUser, undefined, isRollupsEnabled()); const { @@ -159,22 +155,11 @@ const handler: ( indices, }; - const headers: Record = { - 'content-type': 'application/json', - Etag: '123456', - 'If-None-Match': '123456', - // Etag? - // Expires - }; - - if (cacheHeader) { - // revalidates if 5 minutes passed, otherwise caches for a year - headers['cache-control'] = 'private, max-age=300, stale-while-revalidate=2592000'; - } - return response.ok({ body, - headers, + headers: { + 'content-type': 'application/json', + }, }); } catch (error) { if ( @@ -209,14 +194,12 @@ export const registerFieldForWildcard = async ( // handler router.versioned .put({ path, access, options: {} }) - .addVersion({ version, validate }, configuredHandler(false)); - router.versioned - .post({ path, access }) - .addVersion({ version, validate }, configuredHandler(false)); + .addVersion({ version, validate }, configuredHandler); + router.versioned.post({ path, access }).addVersion({ version, validate }, configuredHandler); router.versioned .get({ path, access }) .addVersion( { version, validate: { request: { query: querySchema }, response: validate.response } }, - configuredHandler(true) + configuredHandler ); }; From 40704e754279b297cedfb477ac7b1fd291d9240a Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Tue, 7 Nov 2023 00:42:50 -0600 Subject: [PATCH 019/168] revert fields_for_wildcard changes --- .../data_views/server/rest_api_routes/internal/fields_for.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts b/src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts index 32197ea5a96d4..15d761935c0a7 100644 --- a/src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts +++ b/src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts @@ -192,9 +192,7 @@ export const registerFieldForWildcard = async ( const configuredHandler = handler(isRollupsEnabled); // handler - router.versioned - .put({ path, access, options: {} }) - .addVersion({ version, validate }, configuredHandler); + router.versioned.put({ path, access }).addVersion({ version, validate }, configuredHandler); router.versioned.post({ path, access }).addVersion({ version, validate }, configuredHandler); router.versioned .get({ path, access }) From 4b02f2d0738d7c646b3aef119f40b4a81523c091 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Tue, 7 Nov 2023 07:57:15 -0600 Subject: [PATCH 020/168] functional test fixes --- .../apps/management/data_views/_index_pattern_results_sort.ts | 3 ++- .../apps/discover/feature_controls/discover_security.ts | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/test/functional/apps/management/data_views/_index_pattern_results_sort.ts b/test/functional/apps/management/data_views/_index_pattern_results_sort.ts index 2f11ae3cfb2e3..aef25d91634e5 100644 --- a/test/functional/apps/management/data_views/_index_pattern_results_sort.ts +++ b/test/functional/apps/management/data_views/_index_pattern_results_sort.ts @@ -12,6 +12,7 @@ import { FtrProviderContext } from '../../../ftr_provider_context'; export default function ({ getService, getPageObjects }: FtrProviderContext) { const kibanaServer = getService('kibanaServer'); const retry = getService('retry'); + const testSubjects = getService('testSubjects'); const PageObjects = getPageObjects(['settings', 'common']); describe('index result field sort', function describeIndexTests() { @@ -68,7 +69,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const EXPECTED_FIELD_COUNT = 85; it('makelogs data should have expected number of fields', async function () { // did this work? - await this.testSubjects.click('refreshDataViewButton'); + await testSubjects.click('refreshDataViewButton'); await retry.try(async function () { const TabCount = await PageObjects.settings.getFieldsTabCount(); expect(TabCount).to.be('' + EXPECTED_FIELD_COUNT); diff --git a/x-pack/test/functional/apps/discover/feature_controls/discover_security.ts b/x-pack/test/functional/apps/discover/feature_controls/discover_security.ts index ee810f7ddebfb..ed6e1bc27abcb 100644 --- a/x-pack/test/functional/apps/discover/feature_controls/discover_security.ts +++ b/x-pack/test/functional/apps/discover/feature_controls/discover_security.ts @@ -31,6 +31,7 @@ export default function (ctx: FtrProviderContext) { 'spaceSelector', 'header', 'unifiedFieldList', + 'settings', ]); const testSubjects = getService('testSubjects'); const appsMenu = getService('appsMenu'); @@ -447,6 +448,9 @@ export default function (ctx: FtrProviderContext) { it('allows to access only via a permitted index alias', async () => { await globalNav.badgeExistsOrFail('Read only'); + await PageObjects.settings.refreshDataViewFieldList('logstash-*'); + await PageObjects.common.navigateToApp('discover'); + // can't access logstash index directly await PageObjects.discover.selectIndexPattern('logstash-*'); await PageObjects.header.waitUntilLoadingHasFinished(); From 78bd5379f9794813756f2148937b11f44003979a Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Tue, 7 Nov 2023 09:20:30 -0600 Subject: [PATCH 021/168] disable cache for sanity check --- src/plugins/data_views/server/ui_settings.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/data_views/server/ui_settings.ts b/src/plugins/data_views/server/ui_settings.ts index 7e995112fdc1c..a2d0feb05bbe8 100644 --- a/src/plugins/data_views/server/ui_settings.ts +++ b/src/plugins/data_views/server/ui_settings.ts @@ -14,7 +14,7 @@ export const cacheMaxAge = { name: i18n.translate('dataViews.advancedSettings.cacheMaxAgeTitle', { defaultMessage: 'Field cache max age', }), - value: 300, + value: 0, description: i18n.translate('dataViews.advancedSettings.cacheMaxAgeText', { defaultMessage: "Sets the 'max-age' cache header value for data view fields API requests. A value of 0 will disable caching.", From 86681bee37476a92192f12a147e49395f941eb8f Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Tue, 7 Nov 2023 11:15:52 -0600 Subject: [PATCH 022/168] cleanup --- .../edit_index_pattern/edit_index_pattern.tsx | 4 +- src/plugins/data_views/common/utils.test.ts | 66 ++++++++++++------- src/plugins/data_views/common/utils.ts | 8 +++ .../data_views/data_views_api_client.ts | 1 + .../server/rest_api_routes/internal/fields.ts | 30 ++------- 5 files changed, 59 insertions(+), 50 deletions(-) diff --git a/src/plugins/data_view_management/public/components/edit_index_pattern/edit_index_pattern.tsx b/src/plugins/data_view_management/public/components/edit_index_pattern/edit_index_pattern.tsx index ba1d8373119cb..17835dbdf8aef 100644 --- a/src/plugins/data_view_management/public/components/edit_index_pattern/edit_index_pattern.tsx +++ b/src/plugins/data_view_management/public/components/edit_index_pattern/edit_index_pattern.tsx @@ -93,7 +93,7 @@ export const EditIndexPattern = withRouter( const [showEditDialog, setShowEditDialog] = useState(false); const [relationships, setRelationships] = useState([]); const [allowedTypes, setAllowedTypes] = useState([]); - const [refreshCount, setRefreshCount] = useState(0); + const [refreshCount, setRefreshCount] = useState(0); // used for forcing rerender of field list const [isRefreshing, setIsRefreshing] = React.useState(false); const conflictFieldsUrl = useMemo(() => { @@ -250,7 +250,7 @@ export const EditIndexPattern = withRouter( refreshIndexPatternClick={async () => { setIsRefreshing(true); await dataViews.refreshFields(indexPattern, false, true); - setRefreshCount(refreshCount + 1); + setRefreshCount(refreshCount + 1); // rerender field list setIsRefreshing(false); }} defaultIndex={defaultIndex} diff --git a/src/plugins/data_views/common/utils.test.ts b/src/plugins/data_views/common/utils.test.ts index 3351a15da1a13..e8602def7f712 100644 --- a/src/plugins/data_views/common/utils.test.ts +++ b/src/plugins/data_views/common/utils.test.ts @@ -8,6 +8,7 @@ import { isFilterable } from '.'; import type { DataViewField } from './fields'; +import { unwrapEtag } from './utils'; const mockField = { name: 'foo', @@ -16,38 +17,55 @@ const mockField = { type: 'string', } as DataViewField; -describe('isFilterable', () => { - describe('types', () => { - it('should return true for filterable types', () => { - ['string', 'number', 'date', 'ip', 'boolean'].forEach((type) => { - expect(isFilterable({ ...mockField, type } as DataViewField)).toBe(true); +describe('common utils', () => { + describe('isFilterable', () => { + describe('types', () => { + it('should return true for filterable types', () => { + ['string', 'number', 'date', 'ip', 'boolean'].forEach((type) => { + expect(isFilterable({ ...mockField, type } as DataViewField)).toBe(true); + }); }); - }); - it('should return false for filterable types if the field is not searchable', () => { - ['string', 'number', 'date', 'ip', 'boolean'].forEach((type) => { - expect(isFilterable({ ...mockField, type, searchable: false } as DataViewField)).toBe( - false - ); + it('should return false for filterable types if the field is not searchable', () => { + ['string', 'number', 'date', 'ip', 'boolean'].forEach((type) => { + expect(isFilterable({ ...mockField, type, searchable: false } as DataViewField)).toBe( + false + ); + }); }); - }); - it('should return false for un-filterable types', () => { - ['geo_point', 'geo_shape', 'attachment', 'murmur3', '_source', 'unknown', 'conflict'].forEach( - (type) => { + it('should return false for un-filterable types', () => { + [ + 'geo_point', + 'geo_shape', + 'attachment', + 'murmur3', + '_source', + 'unknown', + 'conflict', + ].forEach((type) => { expect(isFilterable({ ...mockField, type } as DataViewField)).toBe(false); - } - ); + }); + }); }); - }); - it('should return true for scripted fields', () => { - expect(isFilterable({ ...mockField, scripted: true, searchable: false } as DataViewField)).toBe( - true - ); + it('should return true for scripted fields', () => { + expect( + isFilterable({ ...mockField, scripted: true, searchable: false } as DataViewField) + ).toBe(true); + }); + + it('should return true for the _id field', () => { + expect(isFilterable({ ...mockField, name: '_id' } as DataViewField)).toBe(true); + }); }); - it('should return true for the _id field', () => { - expect(isFilterable({ ...mockField, name: '_id' } as DataViewField)).toBe(true); + describe('unwrapEtag', () => { + it('should return the etag without quotes', () => { + expect(unwrapEtag('"foo"')).toBe('foo'); + }); + it('should return the etag without quotes and without gzip', () => { + expect(unwrapEtag('"foo-gzip"')).toBe('foo'); + }); }); }); diff --git a/src/plugins/data_views/common/utils.ts b/src/plugins/data_views/common/utils.ts index e63167ff4d3ff..507f90e764c5e 100644 --- a/src/plugins/data_views/common/utils.ts +++ b/src/plugins/data_views/common/utils.ts @@ -27,3 +27,11 @@ export async function findByName(client: PersistenceAPI, name: string) { return savedObjects ? savedObjects[0] : undefined; } } + +export function unwrapEtag(ifNoneMatch: string) { + let requestHash = (ifNoneMatch as string).slice(1, -1); + if (requestHash.indexOf('-') > -1) { + requestHash = requestHash.split('-')[0]; + } + return requestHash; +} diff --git a/src/plugins/data_views/public/data_views/data_views_api_client.ts b/src/plugins/data_views/public/data_views/data_views_api_client.ts index ff184722a4a3b..4305876720a6a 100644 --- a/src/plugins/data_views/public/data_views/data_views_api_client.ts +++ b/src/plugins/data_views/public/data_views/data_views_api_client.ts @@ -37,6 +37,7 @@ export class DataViewsApiClient implements IDataViewsApiClient { ): Promise | undefined> { const asResponse = true; // circle back to this, will likely need changes to any code that loads fields + // setting to true skips automatic json parsing // const rawResponse = true; const rawResponse = false; const cacheOptions = forceRefresh ? { cache: 'no-cache' as RequestCache } : {}; diff --git a/src/plugins/data_views/server/rest_api_routes/internal/fields.ts b/src/plugins/data_views/server/rest_api_routes/internal/fields.ts index 0e2dd5d0589db..5fbcacc2b2b14 100644 --- a/src/plugins/data_views/server/rest_api_routes/internal/fields.ts +++ b/src/plugins/data_views/server/rest_api_routes/internal/fields.ts @@ -11,6 +11,7 @@ import { estypes } from '@elastic/elasticsearch'; import { schema } from '@kbn/config-schema'; import { IRouter, RequestHandler, StartServicesAccessor } from '@kbn/core/server'; // import { FullValidationConfig } from '@kbn/core-http-server'; +import { unwrapEtag } from '../../../common/utils'; import { IndexPatternsFetcher } from '../../fetcher'; import type { DataViewsServerPluginStart, @@ -144,7 +145,6 @@ const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, I try { const { fields, indices } = await indexPatterns.getFieldsForWildcard({ pattern, - // todo should these be added elsewhere? metaFields: parsedMetaFields, type, rollupIndex, @@ -162,15 +162,9 @@ const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, I const etag = calculateHash(Buffer.from(JSON.stringify(body))); - // todo are these needed? - const headers: Record = { - 'content-type': 'application/json', - // 'If-None-Match': '123456', - etag, - // Expires - }; + const headers: Record = { 'content-type': 'application/json', etag }; - // todo consider running in parallel with the request + // todo examine how long this takes const cacheMaxAge = await uiSettings.get('data_views:cache_max_age'); if (cacheMaxAge) { @@ -180,14 +174,11 @@ const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, I ] = `private, max-age=${cacheMaxAge}, stale-while-revalidate=${stale}`; } - // move to util const ifNoneMatch = request.headers['if-none-match']; - if (ifNoneMatch) { - let requestHash = (ifNoneMatch as string).slice(1, -1); - if (requestHash.indexOf('-') > -1) { - requestHash = requestHash.split('-')[0]; - } + const ifNoneMatchString = Array.isArray(ifNoneMatch) ? ifNoneMatch[0] : ifNoneMatch; + if (ifNoneMatchString) { + const requestHash = unwrapEtag(ifNoneMatchString); if (etag === requestHash) { return response.notModified({ headers }); } @@ -225,14 +216,5 @@ export const registerFields = async ( >, isRollupsEnabled: () => boolean ) => { - // handler - /* This seems to fail due to lack of custom headers on cache requests - router.versioned - .get({ path, access }) - .addVersion( - { version, validate: { request: { query: querySchema }, response: validate.response } }, - handler(isRollupsEnabled) - ); - */ router.get({ path, validate: { query: querySchema } }, handler(isRollupsEnabled)); }; From 4099571b9f0ae693e92c9a415ad8d560dde2c3bb Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Tue, 7 Nov 2023 12:15:27 -0600 Subject: [PATCH 023/168] fix test --- .../apps/discover/feature_controls/discover_security.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/x-pack/test/functional/apps/discover/feature_controls/discover_security.ts b/x-pack/test/functional/apps/discover/feature_controls/discover_security.ts index ed6e1bc27abcb..fa01fc5b8d57f 100644 --- a/x-pack/test/functional/apps/discover/feature_controls/discover_security.ts +++ b/x-pack/test/functional/apps/discover/feature_controls/discover_security.ts @@ -448,8 +448,9 @@ export default function (ctx: FtrProviderContext) { it('allows to access only via a permitted index alias', async () => { await globalNav.badgeExistsOrFail('Read only'); - await PageObjects.settings.refreshDataViewFieldList('logstash-*'); - await PageObjects.common.navigateToApp('discover'); + // this doesn't work since the user doesn't have access + // await PageObjects.settings.refreshDataViewFieldList('logstash-*'); + // await PageObjects.common.navigateToApp('discover'); // can't access logstash index directly await PageObjects.discover.selectIndexPattern('logstash-*'); From 97e7689334b60932dc8dea7bf7ef4c8e9a30a004 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Tue, 7 Nov 2023 14:58:07 -0600 Subject: [PATCH 024/168] restore default cache time length --- src/plugins/data_views/server/ui_settings.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/data_views/server/ui_settings.ts b/src/plugins/data_views/server/ui_settings.ts index a2d0feb05bbe8..7e995112fdc1c 100644 --- a/src/plugins/data_views/server/ui_settings.ts +++ b/src/plugins/data_views/server/ui_settings.ts @@ -14,7 +14,7 @@ export const cacheMaxAge = { name: i18n.translate('dataViews.advancedSettings.cacheMaxAgeTitle', { defaultMessage: 'Field cache max age', }), - value: 0, + value: 300, description: i18n.translate('dataViews.advancedSettings.cacheMaxAgeText', { defaultMessage: "Sets the 'max-age' cache header value for data view fields API requests. A value of 0 will disable caching.", From 7b7189d120e75c866a887c218235fd298c513ef8 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Tue, 7 Nov 2023 21:26:27 -0600 Subject: [PATCH 025/168] functional test fix --- .../apps/discover_ml_uptime/discover/search_source_alert.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts b/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts index 98eb7bd0bbf90..792eb9c042e2d 100644 --- a/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts +++ b/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts @@ -406,6 +406,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { // this test fails with caching in place it('should navigate to alert results via link provided in notification', async () => { + // todo verify this fixes something + await PageObjects.settings.refreshDataViewFieldList(SOURCE_DATA_VIEW); await openAlertResults(RULE_NAME); await checkInitialRuleParamsState(SOURCE_DATA_VIEW); }); From 521e02888d2ec55f3cea51c0540acaa0b2fa648c Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Fri, 10 Nov 2023 00:12:02 -0600 Subject: [PATCH 026/168] hopefully fix some functional tests --- .../cypress/e2e/automated_response_actions/form.cy.ts | 6 +++--- .../test/functional/apps/maps/group4/es_pew_pew_source.js | 6 +++++- x-pack/test/functional/apps/maps/group4/index.js | 2 ++ x-pack/test/functional/apps/maps/group4/mvt_scaling.js | 4 ++++ 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/automated_response_actions/form.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/automated_response_actions/form.cy.ts index 88131f1031f1e..bf60ac20a4fe6 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/automated_response_actions/form.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/automated_response_actions/form.cy.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { FIELDS_PATH } from '@kbn/data-views-plugin/common/constants'; +// import { FIELDS_PATH } from '@kbn/data-views-plugin/common/constants'; import { addEndpointResponseAction, fillUpNewRule, @@ -186,9 +186,9 @@ describe( }); it('All response action controls are disabled', () => { - cy.intercept('GET', `${FIELDS_PATH}*`).as('getFieldsForWildcard'); + // cy.intercept('GET', `${FIELDS_PATH}*`).as('getFieldsForWildcard'); visitRuleActions(ruleId); - cy.wait('@getFieldsForWildcard'); + // cy.wait('@getFieldsForWildcard'); cy.getByTestSubj('edit-rule-actions-tab').click(); cy.getByTestSubj('response-actions-wrapper').within(() => { diff --git a/x-pack/test/functional/apps/maps/group4/es_pew_pew_source.js b/x-pack/test/functional/apps/maps/group4/es_pew_pew_source.js index ea94ee3bc67d8..a61ccc6081ffb 100644 --- a/x-pack/test/functional/apps/maps/group4/es_pew_pew_source.js +++ b/x-pack/test/functional/apps/maps/group4/es_pew_pew_source.js @@ -15,7 +15,11 @@ export default function ({ getPageObjects, getService }) { describe('point to point source', () => { before(async () => { - await security.testUser.setRoles(['global_maps_all', 'geoconnections_data_reader']); + await security.testUser.setRoles([ + 'global_maps_all', + 'geoconnections_data_reader', + 'test_logstash_reader', + ]); await PageObjects.maps.loadSavedMap('pew pew demo'); }); diff --git a/x-pack/test/functional/apps/maps/group4/index.js b/x-pack/test/functional/apps/maps/group4/index.js index 6af1369928d98..9fa5e81bdf242 100644 --- a/x-pack/test/functional/apps/maps/group4/index.js +++ b/x-pack/test/functional/apps/maps/group4/index.js @@ -63,6 +63,7 @@ export default function ({ loadTestFile, getService }) { loadTestFile(require.resolve('./mvt_joins')); loadTestFile(require.resolve('./mapbox_styles')); loadTestFile(require.resolve('./mvt_scaling')); + /* loadTestFile(require.resolve('./mvt_geotile_grid')); loadTestFile(require.resolve('./add_layer_panel')); loadTestFile(require.resolve('./file_upload')); @@ -72,5 +73,6 @@ export default function ({ loadTestFile, getService }) { loadTestFile(require.resolve('./geofile_wizard_auto_open')); loadTestFile(require.resolve('./lens')); loadTestFile(require.resolve('./tile_map')); + */ }); } diff --git a/x-pack/test/functional/apps/maps/group4/mvt_scaling.js b/x-pack/test/functional/apps/maps/group4/mvt_scaling.js index cd779392ba821..a6b2c6256646a 100644 --- a/x-pack/test/functional/apps/maps/group4/mvt_scaling.js +++ b/x-pack/test/functional/apps/maps/group4/mvt_scaling.js @@ -136,6 +136,8 @@ export default function ({ getPageObjects, getService }) { describe('filtering', () => { before(async () => { await PageObjects.maps.loadSavedMap('MVT documents'); + console.log('awaiting'); + await new Promise((resolve) => setTimeout(resolve, 1000 * 60 * 10)); }); async function getTileUrl() { @@ -223,6 +225,8 @@ export default function ({ getPageObjects, getService }) { await PageObjects.maps.openLayerPanel('logstash-*'); await testSubjects.click('mapLayerPanelApplyGlobalTimeCheckbox'); await PageObjects.maps.waitForLayersToLoad(); + console.log('HERE HERE HERE'); + await new Promise((resolve) => setTimeout(resolve, 1000 * 60 * 10)); }); after(async () => { From d93f86b2b43f4bf6000d967bd534b1e6064baf46 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Fri, 10 Nov 2023 06:17:50 +0000 Subject: [PATCH 027/168] [CI] Auto-commit changed files from 'node scripts/lint_ts_projects --fix' --- .../security_solution/public/management/cypress/tsconfig.json | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugins/security_solution/public/management/cypress/tsconfig.json b/x-pack/plugins/security_solution/public/management/cypress/tsconfig.json index 8a38566931281..20cb7534941b3 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/tsconfig.json +++ b/x-pack/plugins/security_solution/public/management/cypress/tsconfig.json @@ -30,7 +30,6 @@ "@kbn/cases-plugin", "@kbn/test", "@kbn/repo-info", - "@kbn/data-views-plugin", "@kbn/tooling-log", ] } From 48455115195404b2e526fa865d7fc3e6e491f209 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Fri, 10 Nov 2023 09:07:42 -0600 Subject: [PATCH 028/168] add vary header which should clear cache on user change --- .../core-http-browser-internal/src/fetch.ts | 1 - .../public/data_views/data_views_api_client.ts | 16 +++++++++++++--- src/plugins/data_views/public/plugin.ts | 7 ++++++- src/plugins/data_views/public/types.ts | 3 +++ x-pack/plugins/security/public/plugin.tsx | 18 ++++++++++++++++-- .../functional/apps/maps/group4/mvt_scaling.js | 4 ---- 6 files changed, 38 insertions(+), 11 deletions(-) diff --git a/packages/core/http/core-http-browser-internal/src/fetch.ts b/packages/core/http/core-http-browser-internal/src/fetch.ts index 60d71ba0dc72e..22b827dc614eb 100644 --- a/packages/core/http/core-http-browser-internal/src/fetch.ts +++ b/packages/core/http/core-http-browser-internal/src/fetch.ts @@ -118,7 +118,6 @@ export class Fetch { private createRequest(options: HttpFetchOptionsWithPath): Request { const context = this.params.executionContext.withGlobalContext(options.context); const { version } = options; - // Merge and destructure options out that are not applicable to the Fetch API. const { query, diff --git a/src/plugins/data_views/public/data_views/data_views_api_client.ts b/src/plugins/data_views/public/data_views/data_views_api_client.ts index 4305876720a6a..d205b1f06bc2d 100644 --- a/src/plugins/data_views/public/data_views/data_views_api_client.ts +++ b/src/plugins/data_views/public/data_views/data_views_api_client.ts @@ -20,16 +20,18 @@ const version = '1'; */ export class DataViewsApiClient implements IDataViewsApiClient { private http: HttpSetup; + private getCurrentUserId: () => Promise; /** * constructor * @param http http dependency */ - constructor(http: HttpSetup) { + constructor(http: HttpSetup, getCurrentUserId: () => Promise) { this.http = http; + this.getCurrentUserId = getCurrentUserId; } - private _request( + private async _request( url: string, query?: {}, body?: string, @@ -41,10 +43,18 @@ export class DataViewsApiClient implements IDataViewsApiClient { // const rawResponse = true; const rawResponse = false; const cacheOptions = forceRefresh ? { cache: 'no-cache' as RequestCache } : {}; + const userId = await this.getCurrentUserId(); const request = body ? this.http.post(url, { query, body, version, asResponse }) - : this.http.fetch(url, { query, version, ...cacheOptions, asResponse, rawResponse }); + : this.http.fetch(url, { + query, + version, + ...cacheOptions, + asResponse, + rawResponse, + headers: { 'user-hash': userId, Vary: 'user-hash' }, + }); return request.catch((resp) => { if (resp.body.statusCode === 404 && resp.body.attributes?.code === 'no_matching_indices') { diff --git a/src/plugins/data_views/public/plugin.ts b/src/plugins/data_views/public/plugin.ts index d00292c3f4fe8..72603ceb8246c 100644 --- a/src/plugins/data_views/public/plugin.ts +++ b/src/plugins/data_views/public/plugin.ts @@ -15,6 +15,7 @@ import { DataViewsPublicPluginStart, DataViewsPublicSetupDependencies, DataViewsPublicStartDependencies, + UserIdGetter, } from './types'; import { DataViewsApiClient } from '.'; @@ -41,6 +42,7 @@ export class DataViewsPublicPlugin { private readonly hasData = new HasData(); private rollupsEnabled: boolean = false; + private userIdGetter: UserIdGetter = async () => undefined; constructor(private readonly initializerContext: PluginInitializerContext) {} @@ -62,6 +64,9 @@ export class DataViewsPublicPlugin return { enableRollups: () => (this.rollupsEnabled = true), + setUserIdGetter: (userIdGetter: UserIdGetter) => { + this.userIdGetter = userIdGetter; + }, }; } @@ -86,7 +91,7 @@ export class DataViewsPublicPlugin hasData: this.hasData.start(core), uiSettings: new UiSettingsPublicToCommon(uiSettings), savedObjectsClient: new ContentMagementWrapper(contentManagement.client), - apiClient: new DataViewsApiClient(http), + apiClient: new DataViewsApiClient(http, this.userIdGetter), fieldFormats, http, onNotification: (toastInputFields, key) => { diff --git a/src/plugins/data_views/public/types.ts b/src/plugins/data_views/public/types.ts index 7d6cbbc6cf533..d01e606b968e3 100644 --- a/src/plugins/data_views/public/types.ts +++ b/src/plugins/data_views/public/types.ts @@ -106,11 +106,14 @@ export interface DataViewsPublicStartDependencies { contentManagement: ContentManagementPublicStart; } +export type UserIdGetter = () => Promise; + /** * Data plugin public Setup contract */ export interface DataViewsPublicPluginSetup { enableRollups: () => void; + setUserIdGetter: (userIdGetter: UserIdGetter) => void; } export interface DataViewsServicePublic extends DataViewsServicePublicMethods { diff --git a/x-pack/plugins/security/public/plugin.tsx b/x-pack/plugins/security/public/plugin.tsx index eb5b2723f9eab..630b98d8ccdf1 100644 --- a/x-pack/plugins/security/public/plugin.tsx +++ b/x-pack/plugins/security/public/plugin.tsx @@ -13,7 +13,10 @@ import type { Plugin, PluginInitializerContext, } from '@kbn/core/public'; -import type { DataViewsPublicPluginStart } from '@kbn/data-views-plugin/public'; +import type { + DataViewsPublicPluginSetup, + DataViewsPublicPluginStart, +} from '@kbn/data-views-plugin/public'; import type { FeaturesPluginStart } from '@kbn/features-plugin/public'; import type { HomePublicPluginSetup } from '@kbn/home-plugin/public'; import { i18n } from '@kbn/i18n'; @@ -45,6 +48,7 @@ export interface PluginSetupDependencies { management?: ManagementSetup; share?: SharePluginSetup; cloud?: CloudSetup; + dataViews?: DataViewsPublicPluginSetup; } export interface PluginStartDependencies { @@ -87,7 +91,7 @@ export class SecurityPlugin public setup( core: CoreSetup, - { cloud, home, licensing, management, share }: PluginSetupDependencies + { cloud, home, licensing, management, share, dataViews }: PluginSetupDependencies ): SecurityPluginSetup { const { license } = this.securityLicenseService.setup({ license$: licensing.license$ }); @@ -159,6 +163,16 @@ export class SecurityPlugin this.anonymousAccessService.setup({ share }); } + const userIdGetter = async () => { + const [, , security] = await core.getStartServices(); + const { profile_uid: profileUid } = await ( + security as SecurityPluginStart + ).authc.getCurrentUser(); + return profileUid; + }; + + dataViews?.setUserIdGetter(userIdGetter); + return { authc: this.authc, license, diff --git a/x-pack/test/functional/apps/maps/group4/mvt_scaling.js b/x-pack/test/functional/apps/maps/group4/mvt_scaling.js index a6b2c6256646a..cd779392ba821 100644 --- a/x-pack/test/functional/apps/maps/group4/mvt_scaling.js +++ b/x-pack/test/functional/apps/maps/group4/mvt_scaling.js @@ -136,8 +136,6 @@ export default function ({ getPageObjects, getService }) { describe('filtering', () => { before(async () => { await PageObjects.maps.loadSavedMap('MVT documents'); - console.log('awaiting'); - await new Promise((resolve) => setTimeout(resolve, 1000 * 60 * 10)); }); async function getTileUrl() { @@ -225,8 +223,6 @@ export default function ({ getPageObjects, getService }) { await PageObjects.maps.openLayerPanel('logstash-*'); await testSubjects.click('mapLayerPanelApplyGlobalTimeCheckbox'); await PageObjects.maps.waitForLayersToLoad(); - console.log('HERE HERE HERE'); - await new Promise((resolve) => setTimeout(resolve, 1000 * 60 * 10)); }); after(async () => { From 7f521306d6d2a2236f1d9c08cea2c84941273fb1 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Fri, 10 Nov 2023 11:18:55 -0600 Subject: [PATCH 029/168] test fixes --- packages/core/http/core-http-browser-internal/src/fetch.ts | 1 + .../public/data_views/data_views_api_client.test.ts | 2 +- src/plugins/data_views/public/mocks.ts | 1 + .../test/functional/apps/maps/group4/es_pew_pew_source.js | 6 +----- x-pack/test/functional/apps/maps/group4/index.js | 2 -- .../apps/observability_log_explorer/columns_selection.ts | 2 +- 6 files changed, 5 insertions(+), 9 deletions(-) diff --git a/packages/core/http/core-http-browser-internal/src/fetch.ts b/packages/core/http/core-http-browser-internal/src/fetch.ts index 22b827dc614eb..60d71ba0dc72e 100644 --- a/packages/core/http/core-http-browser-internal/src/fetch.ts +++ b/packages/core/http/core-http-browser-internal/src/fetch.ts @@ -118,6 +118,7 @@ export class Fetch { private createRequest(options: HttpFetchOptionsWithPath): Request { const context = this.params.executionContext.withGlobalContext(options.context); const { version } = options; + // Merge and destructure options out that are not applicable to the Fetch API. const { query, diff --git a/src/plugins/data_views/public/data_views/data_views_api_client.test.ts b/src/plugins/data_views/public/data_views/data_views_api_client.test.ts index 6320226aea96f..e6c4ed3ddbb4e 100644 --- a/src/plugins/data_views/public/data_views/data_views_api_client.test.ts +++ b/src/plugins/data_views/public/data_views/data_views_api_client.test.ts @@ -16,7 +16,7 @@ describe('IndexPatternsApiClient', () => { beforeEach(() => { fetchSpy = jest.spyOn(http, 'fetch').mockImplementation(() => Promise.resolve({})); - indexPatternsApiClient = new DataViewsApiClient(http); + indexPatternsApiClient = new DataViewsApiClient(http, () => Promise.resolve(undefined)); }); test('uses the right URI to fetch fields for wildcard', async function () { diff --git a/src/plugins/data_views/public/mocks.ts b/src/plugins/data_views/public/mocks.ts index e9d5b487b8b00..0edea939ab8df 100644 --- a/src/plugins/data_views/public/mocks.ts +++ b/src/plugins/data_views/public/mocks.ts @@ -13,6 +13,7 @@ export type Start = jest.Mocked>; const createSetupContract = (): Setup => ({ enableRollups: jest.fn(), + setUserIdGetter: jest.fn(), }); const createStartContract = (): Start => { diff --git a/x-pack/test/functional/apps/maps/group4/es_pew_pew_source.js b/x-pack/test/functional/apps/maps/group4/es_pew_pew_source.js index a61ccc6081ffb..ea94ee3bc67d8 100644 --- a/x-pack/test/functional/apps/maps/group4/es_pew_pew_source.js +++ b/x-pack/test/functional/apps/maps/group4/es_pew_pew_source.js @@ -15,11 +15,7 @@ export default function ({ getPageObjects, getService }) { describe('point to point source', () => { before(async () => { - await security.testUser.setRoles([ - 'global_maps_all', - 'geoconnections_data_reader', - 'test_logstash_reader', - ]); + await security.testUser.setRoles(['global_maps_all', 'geoconnections_data_reader']); await PageObjects.maps.loadSavedMap('pew pew demo'); }); diff --git a/x-pack/test/functional/apps/maps/group4/index.js b/x-pack/test/functional/apps/maps/group4/index.js index 9fa5e81bdf242..6af1369928d98 100644 --- a/x-pack/test/functional/apps/maps/group4/index.js +++ b/x-pack/test/functional/apps/maps/group4/index.js @@ -63,7 +63,6 @@ export default function ({ loadTestFile, getService }) { loadTestFile(require.resolve('./mvt_joins')); loadTestFile(require.resolve('./mapbox_styles')); loadTestFile(require.resolve('./mvt_scaling')); - /* loadTestFile(require.resolve('./mvt_geotile_grid')); loadTestFile(require.resolve('./add_layer_panel')); loadTestFile(require.resolve('./file_upload')); @@ -73,6 +72,5 @@ export default function ({ loadTestFile, getService }) { loadTestFile(require.resolve('./geofile_wizard_auto_open')); loadTestFile(require.resolve('./lens')); loadTestFile(require.resolve('./tile_map')); - */ }); } diff --git a/x-pack/test/functional/apps/observability_log_explorer/columns_selection.ts b/x-pack/test/functional/apps/observability_log_explorer/columns_selection.ts index 83b88d3615547..ef3a8473e320d 100644 --- a/x-pack/test/functional/apps/observability_log_explorer/columns_selection.ts +++ b/x-pack/test/functional/apps/observability_log_explorer/columns_selection.ts @@ -31,7 +31,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { describe('when the log explorer loads', () => { it("should initialize the table columns to logs' default selection", async () => { // did this work? - PageObjects.settings.refreshDataViewFieldList('All logs'); + // PageObjects.settings.refreshDataViewFieldList('All logs'); await PageObjects.observabilityLogExplorer.navigateTo(); await retry.try(async () => { From abcf7d622e37e5a211409ea0721284ad54c5cfbd Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Fri, 10 Nov 2023 21:52:38 -0600 Subject: [PATCH 030/168] I think I implemented the Vary header --- .../data_views/data_views_api_client.ts | 2 +- src/plugins/data_views/server/index.ts | 1 + src/plugins/data_views/server/plugin.ts | 18 ++++++++----- .../server/rest_api_routes/internal/fields.ts | 21 +++++++++++----- src/plugins/data_views/server/routes.ts | 25 +++++++++++++------ src/plugins/data_views/server/types.ts | 6 +++++ x-pack/plugins/security/server/plugin.ts | 18 ++++++++++++- 7 files changed, 69 insertions(+), 22 deletions(-) diff --git a/src/plugins/data_views/public/data_views/data_views_api_client.ts b/src/plugins/data_views/public/data_views/data_views_api_client.ts index d205b1f06bc2d..8b39ab76727f1 100644 --- a/src/plugins/data_views/public/data_views/data_views_api_client.ts +++ b/src/plugins/data_views/public/data_views/data_views_api_client.ts @@ -53,7 +53,7 @@ export class DataViewsApiClient implements IDataViewsApiClient { ...cacheOptions, asResponse, rawResponse, - headers: { 'user-hash': userId, Vary: 'user-hash' }, + headers: { 'user-hash': userId }, }); return request.catch((resp) => { diff --git a/src/plugins/data_views/server/index.ts b/src/plugins/data_views/server/index.ts index a489feabbf935..1bf5cc883ded2 100644 --- a/src/plugins/data_views/server/index.ts +++ b/src/plugins/data_views/server/index.ts @@ -16,6 +16,7 @@ export type { DataViewsServerPluginStart, DataViewsServerPluginSetupDependencies, DataViewsServerPluginStartDependencies, + GetUserId, } from './types'; import { PluginInitializerContext } from '@kbn/core/server'; diff --git a/src/plugins/data_views/server/plugin.ts b/src/plugins/data_views/server/plugin.ts index c08e1533e6140..bdb1ac192d5a8 100644 --- a/src/plugins/data_views/server/plugin.ts +++ b/src/plugins/data_views/server/plugin.ts @@ -21,6 +21,7 @@ import { DataViewsServerPluginStart, DataViewsServerPluginSetupDependencies, DataViewsServerPluginStartDependencies, + GetUserId, } from './types'; import { DataViewsStorage } from './content_management'; import { cacheMaxAge } from './ui_settings'; @@ -36,6 +37,7 @@ export class DataViewsServerPlugin { private readonly logger: Logger; private rollupsEnabled: boolean = false; + private getUserId: GetUserId = async () => undefined; constructor(private readonly initializerContext: PluginInitializerContext) { this.logger = initializerContext.logger.get('dataView'); @@ -50,12 +52,13 @@ export class DataViewsServerPlugin core.uiSettings.register(cacheMaxAge); const dataViewRestCounter = usageCollection?.createUsageCounter('dataViewsRestApi'); - registerRoutes( - core.http, - core.getStartServices, - () => this.rollupsEnabled, - dataViewRestCounter - ); + registerRoutes({ + http: core.http, + getStartServices: core.getStartServices, + isRollupsEnabled: () => this.rollupsEnabled, + dataViewRestCounter, + getUserIdGetter: () => this.getUserId, + }); expressions.registerFunction(getIndexPatternLoad({ getStartServices: core.getStartServices })); registerIndexPatternsUsageCollector(core.getStartServices, usageCollection); @@ -74,6 +77,9 @@ export class DataViewsServerPlugin return { enableRollups: () => (this.rollupsEnabled = true), + setGetUserId: (getUserId: GetUserId) => { + this.getUserId = getUserId; + }, }; } diff --git a/src/plugins/data_views/server/rest_api_routes/internal/fields.ts b/src/plugins/data_views/server/rest_api_routes/internal/fields.ts index 5fbcacc2b2b14..e80ea74387213 100644 --- a/src/plugins/data_views/server/rest_api_routes/internal/fields.ts +++ b/src/plugins/data_views/server/rest_api_routes/internal/fields.ts @@ -9,7 +9,7 @@ import { createHash } from 'crypto'; import { estypes } from '@elastic/elasticsearch'; import { schema } from '@kbn/config-schema'; -import { IRouter, RequestHandler, StartServicesAccessor } from '@kbn/core/server'; +import { IRouter, RequestHandler, StartServicesAccessor, KibanaRequest } from '@kbn/core/server'; // import { FullValidationConfig } from '@kbn/core-http-server'; import { unwrapEtag } from '../../../common/utils'; import { IndexPatternsFetcher } from '../../fetcher'; @@ -118,8 +118,11 @@ function calculateHash(srcBuffer: Buffer) { return hash.digest('hex'); } -const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, IBody> = - (isRollupsEnabled) => async (context, request, response) => { +const handler: ( + isRollupsEnabled: () => boolean, + getUserId: () => (kibanaRequest: KibanaRequest) => Promise +) => RequestHandler<{}, IQuery, IBody> = + (isRollupsEnabled, getUserId) => async (context, request, response) => { const core = await context.core; const uiSettings = core.uiSettings.client; const { asCurrentUser } = core.elasticsearch.client; @@ -162,7 +165,12 @@ const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, I const etag = calculateHash(Buffer.from(JSON.stringify(body))); - const headers: Record = { 'content-type': 'application/json', etag }; + const headers: Record = { + 'content-type': 'application/json', + etag, + vary: 'accept-encoding, user-hash', + 'user-hash': (await getUserId()(request)) || '', + }; // todo examine how long this takes const cacheMaxAge = await uiSettings.get('data_views:cache_max_age'); @@ -214,7 +222,8 @@ export const registerFields = async ( DataViewsServerPluginStartDependencies, DataViewsServerPluginStart >, - isRollupsEnabled: () => boolean + isRollupsEnabled: () => boolean, + getUserId: () => (request: KibanaRequest) => Promise ) => { - router.get({ path, validate: { query: querySchema } }, handler(isRollupsEnabled)); + router.get({ path, validate: { query: querySchema } }, handler(isRollupsEnabled, getUserId)); }; diff --git a/src/plugins/data_views/server/routes.ts b/src/plugins/data_views/server/routes.ts index 51f991449029a..4acc09aafe2ab 100644 --- a/src/plugins/data_views/server/routes.ts +++ b/src/plugins/data_views/server/routes.ts @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -import { HttpServiceSetup, StartServicesAccessor } from '@kbn/core/server'; +import { HttpServiceSetup, StartServicesAccessor, KibanaRequest } from '@kbn/core/server'; import { UsageCounter } from '@kbn/usage-collection-plugin/server'; import { routes } from './rest_api_routes/public'; import type { DataViewsServerPluginStart, DataViewsServerPluginStartDependencies } from './types'; @@ -16,21 +16,30 @@ import { registerFieldForWildcard } from './rest_api_routes/internal/fields_for' import { registerHasDataViewsRoute } from './rest_api_routes/internal/has_data_views'; import { registerFields } from './rest_api_routes/internal/fields'; -export function registerRoutes( - http: HttpServiceSetup, +interface RegisterRoutesArgs { + http: HttpServiceSetup; getStartServices: StartServicesAccessor< DataViewsServerPluginStartDependencies, DataViewsServerPluginStart - >, - isRollupsEnabled: () => boolean, - dataViewRestCounter?: UsageCounter -) { + >; + isRollupsEnabled: () => boolean; + dataViewRestCounter?: UsageCounter; + getUserIdGetter: () => (request: KibanaRequest) => Promise; +} + +export function registerRoutes({ + http, + getStartServices, + dataViewRestCounter, + isRollupsEnabled, + getUserIdGetter, +}: RegisterRoutesArgs) { const router = http.createRouter(); routes.forEach((route) => route(router, getStartServices, dataViewRestCounter)); registerExistingIndicesPath(router); registerFieldForWildcard(router, getStartServices, isRollupsEnabled); - registerFields(router, getStartServices, isRollupsEnabled); + registerFields(router, getStartServices, isRollupsEnabled, getUserIdGetter); registerHasDataViewsRoute(router); } diff --git a/src/plugins/data_views/server/types.ts b/src/plugins/data_views/server/types.ts index 6bf438dc634bd..1349d47234950 100644 --- a/src/plugins/data_views/server/types.ts +++ b/src/plugins/data_views/server/types.ts @@ -16,8 +16,11 @@ import { ExpressionsServerSetup } from '@kbn/expressions-plugin/server'; import { UsageCollectionSetup } from '@kbn/usage-collection-plugin/server'; import { FieldFormatsSetup, FieldFormatsStart } from '@kbn/field-formats-plugin/server'; import type { ContentManagementServerSetup } from '@kbn/content-management-plugin/server'; +import { SecurityPluginStart } from '@kbn/security-plugin/server'; import { DataViewsService } from '../common'; +export type GetUserId = (request: KibanaRequest) => Promise; + /** * Data Views service factory */ @@ -59,6 +62,7 @@ export interface DataViewsServerPluginStart { */ export interface DataViewsServerPluginSetup { enableRollups: () => void; + setGetUserId: (getUserId: GetUserId) => void; } /** @@ -97,4 +101,6 @@ export interface DataViewsServerPluginStartDependencies { * Logger */ logger: Logger; + + security: SecurityPluginStart; } diff --git a/x-pack/plugins/security/server/plugin.ts b/x-pack/plugins/security/server/plugin.ts index 341ab79e97e5b..e87881b1a824e 100644 --- a/x-pack/plugins/security/server/plugin.ts +++ b/x-pack/plugins/security/server/plugin.ts @@ -18,6 +18,7 @@ import type { Plugin, PluginInitializerContext, } from '@kbn/core/server'; +import type { DataViewsServerPluginSetup } from '@kbn/data-views-plugin/server'; import type { PluginSetupContract as FeaturesPluginSetup, PluginStartContract as FeaturesPluginStart, @@ -118,6 +119,7 @@ export interface PluginSetupDependencies { taskManager: TaskManagerSetupContract; usageCollection?: UsageCollectionSetup; spaces?: SpacesPluginSetup; + dataViews: DataViewsServerPluginSetup; } export interface PluginStartDependencies { @@ -242,7 +244,14 @@ export class SecurityPlugin public setup( core: CoreSetup, - { features, licensing, taskManager, usageCollection, spaces }: PluginSetupDependencies + { + features, + licensing, + taskManager, + usageCollection, + spaces, + dataViews, + }: PluginSetupDependencies ) { this.kibanaIndexName = core.savedObjects.getDefaultIndex(); const config$ = this.initializerContext.config.create>().pipe( @@ -259,6 +268,13 @@ export class SecurityPlugin const config = this.getConfig(); const kibanaIndexName = this.getKibanaIndexName(); + async function getUserId(request: KibanaRequest): Promise { + const [, , security] = await core.getStartServices(); + return security.authc.getCurrentUser(request)?.profile_uid; + } + + dataViews.setGetUserId(getUserId); + // A subset of `start` services we need during `setup`. const startServicesPromise = core.getStartServices().then(([coreServices, depsServices]) => ({ elasticsearch: coreServices.elasticsearch, From 5f123aed067c0be4358e70d5c1f8ddf6ac942009 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Sat, 11 Nov 2023 03:58:21 +0000 Subject: [PATCH 031/168] [CI] Auto-commit changed files from 'node scripts/lint_ts_projects --fix' --- src/plugins/data_views/tsconfig.json | 1 + 1 file changed, 1 insertion(+) diff --git a/src/plugins/data_views/tsconfig.json b/src/plugins/data_views/tsconfig.json index e5613323bc222..cf3d7aa75e551 100644 --- a/src/plugins/data_views/tsconfig.json +++ b/src/plugins/data_views/tsconfig.json @@ -33,6 +33,7 @@ "@kbn/object-versioning", "@kbn/core-saved-objects-server", "@kbn/logging", + "@kbn/security-plugin", ], "exclude": [ "target/**/*", From ebc5c678b3acb46ad3d4974a3dc8149398fc4efa Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Fri, 10 Nov 2023 22:08:23 -0600 Subject: [PATCH 032/168] remove unused type --- src/plugins/data_views/server/types.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/plugins/data_views/server/types.ts b/src/plugins/data_views/server/types.ts index 1349d47234950..1fe900dc58113 100644 --- a/src/plugins/data_views/server/types.ts +++ b/src/plugins/data_views/server/types.ts @@ -16,7 +16,6 @@ import { ExpressionsServerSetup } from '@kbn/expressions-plugin/server'; import { UsageCollectionSetup } from '@kbn/usage-collection-plugin/server'; import { FieldFormatsSetup, FieldFormatsStart } from '@kbn/field-formats-plugin/server'; import type { ContentManagementServerSetup } from '@kbn/content-management-plugin/server'; -import { SecurityPluginStart } from '@kbn/security-plugin/server'; import { DataViewsService } from '../common'; export type GetUserId = (request: KibanaRequest) => Promise; @@ -101,6 +100,4 @@ export interface DataViewsServerPluginStartDependencies { * Logger */ logger: Logger; - - security: SecurityPluginStart; } From efdc60aafc58d77e0dc308189e26a307d654de19 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Sat, 11 Nov 2023 04:14:41 +0000 Subject: [PATCH 033/168] [CI] Auto-commit changed files from 'node scripts/lint_ts_projects --fix' --- src/plugins/data_views/tsconfig.json | 1 - 1 file changed, 1 deletion(-) diff --git a/src/plugins/data_views/tsconfig.json b/src/plugins/data_views/tsconfig.json index cf3d7aa75e551..e5613323bc222 100644 --- a/src/plugins/data_views/tsconfig.json +++ b/src/plugins/data_views/tsconfig.json @@ -33,7 +33,6 @@ "@kbn/object-versioning", "@kbn/core-saved-objects-server", "@kbn/logging", - "@kbn/security-plugin", ], "exclude": [ "target/**/*", From 0b90af39cd3a82f5a348de1d28126369873df044 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Fri, 10 Nov 2023 22:23:18 -0600 Subject: [PATCH 034/168] remove unused type --- src/plugins/data_views/tsconfig.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/plugins/data_views/tsconfig.json b/src/plugins/data_views/tsconfig.json index cf3d7aa75e551..66079b0126d6c 100644 --- a/src/plugins/data_views/tsconfig.json +++ b/src/plugins/data_views/tsconfig.json @@ -32,8 +32,7 @@ "@kbn/content-management-utils", "@kbn/object-versioning", "@kbn/core-saved-objects-server", - "@kbn/logging", - "@kbn/security-plugin", + "@kbn/logging" ], "exclude": [ "target/**/*", From cbfc4ded9424f2c0827f12b08a58348cfbd02045 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Fri, 10 Nov 2023 23:18:35 -0600 Subject: [PATCH 035/168] fix jest test --- x-pack/plugins/security/server/plugin.test.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/x-pack/plugins/security/server/plugin.test.ts b/x-pack/plugins/security/server/plugin.test.ts index e838016e94524..0426a1fd674eb 100644 --- a/x-pack/plugins/security/server/plugin.test.ts +++ b/x-pack/plugins/security/server/plugin.test.ts @@ -51,6 +51,9 @@ describe('Security Plugin', () => { licensing: { license$: of({}), featureUsage: { register: jest.fn() } }, features: featuresPluginMock.createSetup(), taskManager: taskManagerMock.createSetup(), + dataViews: { + setGetUserId: jest.fn(), + }, } as unknown as PluginSetupDependencies; mockCoreStart = coreMock.createStart(); From cde6207c3de681cf28aed9fa0340ead878d83ac0 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Sat, 11 Nov 2023 21:22:41 -0600 Subject: [PATCH 036/168] add console.log debugging for functional tests --- src/plugins/data_views/common/data_views/data_views.ts | 9 ++++++++- .../public/data_views/data_views_api_client.ts | 2 ++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/plugins/data_views/common/data_views/data_views.ts b/src/plugins/data_views/common/data_views/data_views.ts index e6ab814ee7bd0..a93c15fd80385 100644 --- a/src/plugins/data_views/common/data_views/data_views.ts +++ b/src/plugins/data_views/common/data_views/data_views.ts @@ -542,7 +542,7 @@ export class DataViewsService { forceRefresh: boolean = false ) => { const metaFields = await this.config.get(META_FIELDS); - return this.apiClient.getFieldsForWildcard({ + const result = await this.apiClient.getFieldsForWildcard({ type: dataView.type, rollupIndex: dataView?.typeMeta?.params?.rollup_index, allowNoIndex: true, @@ -550,6 +550,13 @@ export class DataViewsService { metaFields, forceRefresh, }); + + console.log( + 'getFieldsAndIndicesForDataView result', + dataView.getIndexPattern(), + JSON.stringify(result.fields, 0, 2) + ); + return result; }; private getFieldsAndIndicesForWildcard = async (options: GetFieldsOptions) => { diff --git a/src/plugins/data_views/public/data_views/data_views_api_client.ts b/src/plugins/data_views/public/data_views/data_views_api_client.ts index 8b39ab76727f1..e9e774fa119a1 100644 --- a/src/plugins/data_views/public/data_views/data_views_api_client.ts +++ b/src/plugins/data_views/public/data_views/data_views_api_client.ts @@ -56,6 +56,8 @@ export class DataViewsApiClient implements IDataViewsApiClient { headers: { 'user-hash': userId }, }); + console.log('User id:', userId); + return request.catch((resp) => { if (resp.body.statusCode === 404 && resp.body.attributes?.code === 'no_matching_indices') { throw new DataViewMissingIndices(resp.body.message); From 2c0e5d5b0861c9af62cf6d1514ae9c4984bb9550 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Sat, 11 Nov 2023 21:52:19 -0600 Subject: [PATCH 037/168] add console.log debugging for functional tests --- src/plugins/data_views/common/data_views/data_views.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/data_views/common/data_views/data_views.ts b/src/plugins/data_views/common/data_views/data_views.ts index a93c15fd80385..ec550edc5e4dd 100644 --- a/src/plugins/data_views/common/data_views/data_views.ts +++ b/src/plugins/data_views/common/data_views/data_views.ts @@ -554,7 +554,7 @@ export class DataViewsService { console.log( 'getFieldsAndIndicesForDataView result', dataView.getIndexPattern(), - JSON.stringify(result.fields, 0, 2) + JSON.stringify(result.fields, undefined, 2) ); return result; }; From e2b2ba72a6fc2428122551cde82db51228b76719 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Sun, 12 Nov 2023 00:08:31 -0600 Subject: [PATCH 038/168] add refresh button in data view picker, attempt to fix field formatter functional test --- .../public/dataview_picker/change_dataview.tsx | 17 +++++++++++++++++ .../management/data_views/_field_formatter.ts | 1 + test/functional/page_objects/discover_page.ts | 7 +++++++ test/functional/page_objects/settings_page.ts | 12 +++++++----- 4 files changed, 32 insertions(+), 5 deletions(-) diff --git a/src/plugins/unified_search/public/dataview_picker/change_dataview.tsx b/src/plugins/unified_search/public/dataview_picker/change_dataview.tsx index e1565f1ff6b0d..1a90b47103988 100644 --- a/src/plugins/unified_search/public/dataview_picker/change_dataview.tsx +++ b/src/plugins/unified_search/public/dataview_picker/change_dataview.tsx @@ -217,6 +217,23 @@ export function ChangeDataView({ ) : ( ), + { + const dataView = await dataViews.get(currentDataViewId!); + await dataViews.refreshFields(dataView, undefined, true); + if (onEditDataView) { + onEditDataView(dataView); + } + setPopoverIsOpen(false); + }} + > + {i18n.translate('unifiedSearch.query.queryBar.indexPattern.refreshFieldButton', { + defaultMessage: 'Refresh field list', + })} + , ); } diff --git a/test/functional/apps/management/data_views/_field_formatter.ts b/test/functional/apps/management/data_views/_field_formatter.ts index 881ba6197510a..8473cf4af1736 100644 --- a/test/functional/apps/management/data_views/_field_formatter.ts +++ b/test/functional/apps/management/data_views/_field_formatter.ts @@ -465,6 +465,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.settings.navigateTo(); await PageObjects.settings.clickKibanaIndexPatterns(); await PageObjects.settings.clickIndexPatternByName(indexTitle); + await PageObjects.settings.refreshDataViewFieldList(); }); afterEach(async () => { diff --git a/test/functional/page_objects/discover_page.ts b/test/functional/page_objects/discover_page.ts index 47a79132212cc..c84afdb5d98bd 100644 --- a/test/functional/page_objects/discover_page.ts +++ b/test/functional/page_objects/discover_page.ts @@ -409,6 +409,13 @@ export class DiscoverPageObject extends FtrService { }); } + public async refreshFieldList() { + await this.clickIndexPatternActions(); + await this.testSubjects.click('data-view-refresh-fields'); + // trying to wait for pop over to be closed + // await this.testSubjects.waitForHidden('changeDataViewPopover'); + } + public async clickAddNewField() { await this.retry.try(async () => { await this.testSubjects.click('indexPattern-add-field'); diff --git a/test/functional/page_objects/settings_page.ts b/test/functional/page_objects/settings_page.ts index d81c1e09777d0..177cb50b3c1bb 100644 --- a/test/functional/page_objects/settings_page.ts +++ b/test/functional/page_objects/settings_page.ts @@ -483,11 +483,13 @@ export class SettingsPageObject extends FtrService { await customDataViewIdInput.type(value); } - async refreshDataViewFieldList(dataViewName: string) { - await this.navigateTo(); - await this.clickKibanaIndexPatterns(); - await this.header.waitUntilLoadingHasFinished(); - await this.testSubjects.click(`detail-link-${dataViewName}`); + async refreshDataViewFieldList(dataViewName?: string) { + if (dataViewName) { + await this.navigateTo(); + await this.clickKibanaIndexPatterns(); + await this.header.waitUntilLoadingHasFinished(); + await this.testSubjects.click(`detail-link-${dataViewName}`); + } await this.testSubjects.click('refreshDataViewButton'); } From c929e5f33e1a50236c70600b2e7c5a79d5e109a9 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Sun, 12 Nov 2023 11:21:43 -0600 Subject: [PATCH 039/168] functional test fixes --- .../apps/management/data_views/_index_pattern_filter.ts | 2 ++ .../apps/discover_ml_uptime/discover/search_source_alert.ts | 3 +-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/test/functional/apps/management/data_views/_index_pattern_filter.ts b/test/functional/apps/management/data_views/_index_pattern_filter.ts index 5c0dc5d0284f0..3e462ee6d6673 100644 --- a/test/functional/apps/management/data_views/_index_pattern_filter.ts +++ b/test/functional/apps/management/data_views/_index_pattern_filter.ts @@ -167,6 +167,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.settings.clickIndexPatternLogstash(); + await PageObjects.settings.refreshDataViewFieldList(); + await testSubjects.existOrFail('dataViewMappingConflict'); expect(await PageObjects.settings.getFieldTypes()).to.eql([ diff --git a/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts b/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts index 792eb9c042e2d..9982a8cf1ad3d 100644 --- a/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts +++ b/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts @@ -211,6 +211,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.discover.clickNewSearchButton(); // reset params await PageObjects.discover.selectIndexPattern(OUTPUT_DATA_VIEW); + await PageObjects.discover.refreshFieldList(); let ruleId: string; if (type === 'name') { @@ -406,8 +407,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { // this test fails with caching in place it('should navigate to alert results via link provided in notification', async () => { - // todo verify this fixes something - await PageObjects.settings.refreshDataViewFieldList(SOURCE_DATA_VIEW); await openAlertResults(RULE_NAME); await checkInitialRuleParamsState(SOURCE_DATA_VIEW); }); From 8f59cf5b91f7619c891b8fd2206fe800831bd64e Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Sun, 12 Nov 2023 11:43:13 -0600 Subject: [PATCH 040/168] fix transform functional test --- .../apps/observability_log_explorer/columns_selection.ts | 4 ++-- .../creation_runtime_mappings.ts | 4 +++- .../functional/tests/discover_integration.ts | 2 ++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/x-pack/test/functional/apps/observability_log_explorer/columns_selection.ts b/x-pack/test/functional/apps/observability_log_explorer/columns_selection.ts index ef3a8473e320d..15c1ce1a468eb 100644 --- a/x-pack/test/functional/apps/observability_log_explorer/columns_selection.ts +++ b/x-pack/test/functional/apps/observability_log_explorer/columns_selection.ts @@ -30,11 +30,11 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { describe('when the log explorer loads', () => { it("should initialize the table columns to logs' default selection", async () => { - // did this work? - // PageObjects.settings.refreshDataViewFieldList('All logs'); await PageObjects.observabilityLogExplorer.navigateTo(); await retry.try(async () => { + // did this work? + await PageObjects.discover.refreshFieldList(); expect(await PageObjects.discover.getColumnHeaders()).to.eql(defaultLogColumns); }); }); diff --git a/x-pack/test/functional/apps/transform/creation/runtime_mappings_saved_search/creation_runtime_mappings.ts b/x-pack/test/functional/apps/transform/creation/runtime_mappings_saved_search/creation_runtime_mappings.ts index 3f0adc5783893..f975dec539622 100644 --- a/x-pack/test/functional/apps/transform/creation/runtime_mappings_saved_search/creation_runtime_mappings.ts +++ b/x-pack/test/functional/apps/transform/creation/runtime_mappings_saved_search/creation_runtime_mappings.ts @@ -19,9 +19,10 @@ import { PivotTransformTestData, } from '../../helpers'; -export default function ({ getService }: FtrProviderContext) { +export default function ({ getPageObjects, getService }: FtrProviderContext) { const esArchiver = getService('esArchiver'); const transform = getService('transform'); + const PageObjects = getPageObjects(['discover']); const runtimeMappings = { rt_airline_lower: { @@ -446,6 +447,7 @@ export default function ({ getService }: FtrProviderContext) { await transform.testExecution.logTestStep('redirects to Discover page'); await transform.wizard.redirectToDiscover(); + await PageObjects.discover.refreshFieldList(); if (isLatestTransformTestData(testData)) { const fromTime = 'Feb 7, 2016 @ 00:00:00.000'; diff --git a/x-pack/test/saved_object_tagging/functional/tests/discover_integration.ts b/x-pack/test/saved_object_tagging/functional/tests/discover_integration.ts index 8258b4570ed9b..cac4c36852b56 100644 --- a/x-pack/test/saved_object_tagging/functional/tests/discover_integration.ts +++ b/x-pack/test/saved_object_tagging/functional/tests/discover_integration.ts @@ -104,6 +104,8 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { beforeEach(async () => { await PageObjects.common.navigateToApp('discover'); await PageObjects.discover.selectIndexPattern('logstash-*'); + // did this fix a test? + await PageObjects.discover.refreshFieldList(); await PageObjects.timePicker.setDefaultAbsoluteRange(); await PageObjects.header.waitUntilLoadingHasFinished(); }); From 7e7da4a480cb289c90555b43f6edc04a8f4e8036 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Sun, 12 Nov 2023 17:25:05 -0600 Subject: [PATCH 041/168] fix data view mgmt type filter on refresh --- .../components/edit_index_pattern/edit_index_pattern.tsx | 1 + .../public/components/edit_index_pattern/tabs/tabs.tsx | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/plugins/data_view_management/public/components/edit_index_pattern/edit_index_pattern.tsx b/src/plugins/data_view_management/public/components/edit_index_pattern/edit_index_pattern.tsx index 17835dbdf8aef..de42a9d7bd078 100644 --- a/src/plugins/data_view_management/public/components/edit_index_pattern/edit_index_pattern.tsx +++ b/src/plugins/data_view_management/public/components/edit_index_pattern/edit_index_pattern.tsx @@ -250,6 +250,7 @@ export const EditIndexPattern = withRouter( refreshIndexPatternClick={async () => { setIsRefreshing(true); await dataViews.refreshFields(indexPattern, false, true); + setFields(indexPattern.getNonScriptedFields()); setRefreshCount(refreshCount + 1); // rerender field list setIsRefreshing(false); }} diff --git a/src/plugins/data_view_management/public/components/edit_index_pattern/tabs/tabs.tsx b/src/plugins/data_view_management/public/components/edit_index_pattern/tabs/tabs.tsx index 0128d013af6f5..818127ab9e8ab 100644 --- a/src/plugins/data_view_management/public/components/edit_index_pattern/tabs/tabs.tsx +++ b/src/plugins/data_view_management/public/components/edit_index_pattern/tabs/tabs.tsx @@ -265,7 +265,7 @@ export const Tabs: React.FC = ({ const refreshFilters = useCallback(() => { const tempIndexedFieldTypes: string[] = []; const tempScriptedFieldLanguages: string[] = []; - indexPattern.fields.getAll().forEach((field) => { + fields.forEach((field) => { if (field.scripted) { if (field.lang) { tempScriptedFieldLanguages.push(field.lang); @@ -284,7 +284,7 @@ export const Tabs: React.FC = ({ setIndexedFieldTypes(convertToEuiFilterOptions(tempIndexedFieldTypes)); setScriptedFieldLanguages(convertToEuiFilterOptions(tempScriptedFieldLanguages)); - }, [indexPattern]); + }, [fields]); const closeFieldEditor = useCallback(() => { if (closeEditorHandler.current) { From fd083a1173f7bd90856965093335337a763eaa05 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Sun, 12 Nov 2023 17:44:39 -0600 Subject: [PATCH 042/168] try to fix another test --- .../apps/observability_log_explorer/columns_selection.ts | 4 +++- .../creation/index_pattern/creation_index_pattern.ts | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/x-pack/test/functional/apps/observability_log_explorer/columns_selection.ts b/x-pack/test/functional/apps/observability_log_explorer/columns_selection.ts index 15c1ce1a468eb..f7fd36b3edb02 100644 --- a/x-pack/test/functional/apps/observability_log_explorer/columns_selection.ts +++ b/x-pack/test/functional/apps/observability_log_explorer/columns_selection.ts @@ -31,10 +31,12 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { describe('when the log explorer loads', () => { it("should initialize the table columns to logs' default selection", async () => { await PageObjects.observabilityLogExplorer.navigateTo(); + await new Promise((r) => setTimeout(r, 1000 * 60 * 10)); await retry.try(async () => { // did this work? - await PageObjects.discover.refreshFieldList(); + // no, different ids + // await PageObjects.discover.refreshFieldList(); expect(await PageObjects.discover.getColumnHeaders()).to.eql(defaultLogColumns); }); }); diff --git a/x-pack/test/functional/apps/transform/creation/index_pattern/creation_index_pattern.ts b/x-pack/test/functional/apps/transform/creation/index_pattern/creation_index_pattern.ts index 69406c5830421..2b873b5bfae25 100644 --- a/x-pack/test/functional/apps/transform/creation/index_pattern/creation_index_pattern.ts +++ b/x-pack/test/functional/apps/transform/creation/index_pattern/creation_index_pattern.ts @@ -835,6 +835,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await transform.testExecution.logTestStep('should navigate to discover'); await transform.table.clickTransformRowAction(testData.transformId, 'Discover'); await pageObjects.discover.waitUntilSearchingHasFinished(); + // did this work? + await pageObjects.discover.refreshFieldList(); if (testData.discoverAdjustSuperDatePicker) { await transform.testExecution.logTestStep( From 4e5d94d179e0811fdab5e44d5cfe71948fe9fbc7 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Sun, 12 Nov 2023 19:41:24 -0600 Subject: [PATCH 043/168] script field functional test fixes --- .../public/components/edit_index_pattern/tabs/tabs.tsx | 6 ++++-- .../apps/observability_log_explorer/columns_selection.ts | 1 - 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/plugins/data_view_management/public/components/edit_index_pattern/tabs/tabs.tsx b/src/plugins/data_view_management/public/components/edit_index_pattern/tabs/tabs.tsx index 818127ab9e8ab..0642bd2509e49 100644 --- a/src/plugins/data_view_management/public/components/edit_index_pattern/tabs/tabs.tsx +++ b/src/plugins/data_view_management/public/components/edit_index_pattern/tabs/tabs.tsx @@ -265,7 +265,7 @@ export const Tabs: React.FC = ({ const refreshFilters = useCallback(() => { const tempIndexedFieldTypes: string[] = []; const tempScriptedFieldLanguages: string[] = []; - fields.forEach((field) => { + indexPattern.fields.getAll().forEach((field) => { if (field.scripted) { if (field.lang) { tempScriptedFieldLanguages.push(field.lang); @@ -284,7 +284,9 @@ export const Tabs: React.FC = ({ setIndexedFieldTypes(convertToEuiFilterOptions(tempIndexedFieldTypes)); setScriptedFieldLanguages(convertToEuiFilterOptions(tempScriptedFieldLanguages)); - }, [fields]); + // need to reset based on changes to fields but indexPattern is the same + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [indexPattern, fields]); const closeFieldEditor = useCallback(() => { if (closeEditorHandler.current) { diff --git a/x-pack/test/functional/apps/observability_log_explorer/columns_selection.ts b/x-pack/test/functional/apps/observability_log_explorer/columns_selection.ts index f7fd36b3edb02..b9442c8378126 100644 --- a/x-pack/test/functional/apps/observability_log_explorer/columns_selection.ts +++ b/x-pack/test/functional/apps/observability_log_explorer/columns_selection.ts @@ -31,7 +31,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { describe('when the log explorer loads', () => { it("should initialize the table columns to logs' default selection", async () => { await PageObjects.observabilityLogExplorer.navigateTo(); - await new Promise((r) => setTimeout(r, 1000 * 60 * 10)); await retry.try(async () => { // did this work? From 651a792353eceac6d36a532df34d45ed300bfc4b Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Sun, 12 Nov 2023 20:11:03 -0600 Subject: [PATCH 044/168] I hope I fixed the maps functional test --- x-pack/test/functional/apps/maps/group4/index.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/x-pack/test/functional/apps/maps/group4/index.js b/x-pack/test/functional/apps/maps/group4/index.js index 6af1369928d98..25285f9340962 100644 --- a/x-pack/test/functional/apps/maps/group4/index.js +++ b/x-pack/test/functional/apps/maps/group4/index.js @@ -5,12 +5,13 @@ * 2.0. */ -export default function ({ loadTestFile, getService }) { +export default function ({ loadTestFile, getService, getPageObjects }) { const kibanaServer = getService('kibanaServer'); const esArchiver = getService('esArchiver'); const browser = getService('browser'); const log = getService('log'); const supertest = getService('supertest'); + const PageObjects = getPageObjects(['discover', 'common']); describe('maps app', function () { this.tags(['skipFirefox']); @@ -49,6 +50,9 @@ export default function ({ loadTestFile, getService }) { defaultIndex: 'c698b940-e149-11e8-a35a-370a8516603a', }); await browser.setWindowSize(1600, 1000); + + await PageObjects.common.navigateToApp('discover'); + await PageObjects.discover.refreshFieldList(); // refreshes field list cache from previous tests }); after(async () => { From 86852433567828f6df0f738efa2a855ee0f78544 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Sun, 12 Nov 2023 21:51:10 -0600 Subject: [PATCH 045/168] fix some functional tests --- src/plugins/data_views/common/data_views/data_views.ts | 1 + src/plugins/data_views/server/ui_settings.ts | 2 +- src/plugins/data_views/tsconfig.json | 2 +- test/functional/page_objects/discover_page.ts | 2 -- x-pack/test/functional/apps/maps/group4/index.js | 2 ++ .../apps/observability_log_explorer/columns_selection.ts | 3 +++ 6 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/plugins/data_views/common/data_views/data_views.ts b/src/plugins/data_views/common/data_views/data_views.ts index ec550edc5e4dd..edfb68249e837 100644 --- a/src/plugins/data_views/common/data_views/data_views.ts +++ b/src/plugins/data_views/common/data_views/data_views.ts @@ -551,6 +551,7 @@ export class DataViewsService { forceRefresh, }); + // todo console.log( 'getFieldsAndIndicesForDataView result', dataView.getIndexPattern(), diff --git a/src/plugins/data_views/server/ui_settings.ts b/src/plugins/data_views/server/ui_settings.ts index 7e995112fdc1c..b276d440a06f4 100644 --- a/src/plugins/data_views/server/ui_settings.ts +++ b/src/plugins/data_views/server/ui_settings.ts @@ -17,7 +17,7 @@ export const cacheMaxAge = { value: 300, description: i18n.translate('dataViews.advancedSettings.cacheMaxAgeText', { defaultMessage: - "Sets the 'max-age' cache header value for data view fields API requests. A value of 0 will disable caching.", + "Sets the 'max-age' cache header value for data view fields API requests. A value of 0 will disable caching. A hard reload of Kibana may be necessary for your browser to pick up the new setting.", }), schema: schema.number(), }, diff --git a/src/plugins/data_views/tsconfig.json b/src/plugins/data_views/tsconfig.json index 66079b0126d6c..e5613323bc222 100644 --- a/src/plugins/data_views/tsconfig.json +++ b/src/plugins/data_views/tsconfig.json @@ -32,7 +32,7 @@ "@kbn/content-management-utils", "@kbn/object-versioning", "@kbn/core-saved-objects-server", - "@kbn/logging" + "@kbn/logging", ], "exclude": [ "target/**/*", diff --git a/test/functional/page_objects/discover_page.ts b/test/functional/page_objects/discover_page.ts index c84afdb5d98bd..3604ad349dd39 100644 --- a/test/functional/page_objects/discover_page.ts +++ b/test/functional/page_objects/discover_page.ts @@ -412,8 +412,6 @@ export class DiscoverPageObject extends FtrService { public async refreshFieldList() { await this.clickIndexPatternActions(); await this.testSubjects.click('data-view-refresh-fields'); - // trying to wait for pop over to be closed - // await this.testSubjects.waitForHidden('changeDataViewPopover'); } public async clickAddNewField() { diff --git a/x-pack/test/functional/apps/maps/group4/index.js b/x-pack/test/functional/apps/maps/group4/index.js index 25285f9340962..4186670bb9b73 100644 --- a/x-pack/test/functional/apps/maps/group4/index.js +++ b/x-pack/test/functional/apps/maps/group4/index.js @@ -11,6 +11,7 @@ export default function ({ loadTestFile, getService, getPageObjects }) { const browser = getService('browser'); const log = getService('log'); const supertest = getService('supertest'); + const security = getService('security'); const PageObjects = getPageObjects(['discover', 'common']); describe('maps app', function () { @@ -51,6 +52,7 @@ export default function ({ loadTestFile, getService, getPageObjects }) { }); await browser.setWindowSize(1600, 1000); + await security.testUser.setRoles(['kibana_admin']); // necessary to refresh field list await PageObjects.common.navigateToApp('discover'); await PageObjects.discover.refreshFieldList(); // refreshes field list cache from previous tests }); diff --git a/x-pack/test/functional/apps/observability_log_explorer/columns_selection.ts b/x-pack/test/functional/apps/observability_log_explorer/columns_selection.ts index b9442c8378126..81198bab7ae63 100644 --- a/x-pack/test/functional/apps/observability_log_explorer/columns_selection.ts +++ b/x-pack/test/functional/apps/observability_log_explorer/columns_selection.ts @@ -13,6 +13,7 @@ const defaultLogColumns = ['@timestamp', 'service.name', 'host.name', 'message'] export default function ({ getService, getPageObjects }: FtrProviderContext) { const esArchiver = getService('esArchiver'); const retry = getService('retry'); + const browser = getService('browser'); const PageObjects = getPageObjects(['discover', 'observabilityLogExplorer', 'settings']); describe('Columns selection initialization and update', () => { @@ -31,11 +32,13 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { describe('when the log explorer loads', () => { it("should initialize the table columns to logs' default selection", async () => { await PageObjects.observabilityLogExplorer.navigateTo(); + await browser.refresh(); await retry.try(async () => { // did this work? // no, different ids // await PageObjects.discover.refreshFieldList(); + expect(await PageObjects.discover.getColumnHeaders()).to.eql(defaultLogColumns); }); }); From 7319b1894d7fda0cfe9a8632986985019866bd1d Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Sun, 12 Nov 2023 23:23:13 -0600 Subject: [PATCH 046/168] cleanup and functional test fix --- .../server/rest_api_routes/internal/fields.ts | 96 +------------------ .../rest_api_routes/internal/fields_for.ts | 6 +- .../test/functional/apps/maps/group4/index.js | 6 +- .../columns_selection.ts | 10 +- 4 files changed, 13 insertions(+), 105 deletions(-) diff --git a/src/plugins/data_views/server/rest_api_routes/internal/fields.ts b/src/plugins/data_views/server/rest_api_routes/internal/fields.ts index e80ea74387213..8a8bc13333c18 100644 --- a/src/plugins/data_views/server/rest_api_routes/internal/fields.ts +++ b/src/plugins/data_views/server/rest_api_routes/internal/fields.ts @@ -7,10 +7,7 @@ */ import { createHash } from 'crypto'; -import { estypes } from '@elastic/elasticsearch'; -import { schema } from '@kbn/config-schema'; import { IRouter, RequestHandler, StartServicesAccessor, KibanaRequest } from '@kbn/core/server'; -// import { FullValidationConfig } from '@kbn/core-http-server'; import { unwrapEtag } from '../../../common/utils'; import { IndexPatternsFetcher } from '../../fetcher'; import type { @@ -19,98 +16,7 @@ import type { } from '../../types'; import type { FieldDescriptorRestResponse } from '../route_types'; import { FIELDS_PATH as path } from '../../../common/constants'; - -/** - * Accepts one of the following: - * 1. An array of field names - * 2. A JSON-stringified array of field names - * 3. A single field name (not comma-separated) - * @returns an array of field names - * @param fields - */ -export const parseFields = (fields: string | string[]): string[] => { - if (Array.isArray(fields)) return fields; - try { - return JSON.parse(fields); - } catch (e) { - if (!fields.includes(',')) return [fields]; - throw new Error( - 'metaFields should be an array of field names, a JSON-stringified array of field names, or a single field name' - ); - } -}; - -type IBody = { index_filter?: estypes.QueryDslQueryContainer } | undefined; -interface IQuery { - pattern: string; - meta_fields: string | string[]; - type?: string; - rollup_index?: string; - allow_no_index?: boolean; - include_unmapped?: boolean; - fields?: string[]; -} - -const querySchema = schema.object({ - pattern: schema.string(), - meta_fields: schema.oneOf([schema.string(), schema.arrayOf(schema.string())], { - defaultValue: [], - }), - type: schema.maybe(schema.string()), - rollup_index: schema.maybe(schema.string()), - allow_no_index: schema.maybe(schema.boolean()), - include_unmapped: schema.maybe(schema.boolean()), - fields: schema.maybe(schema.oneOf([schema.string(), schema.arrayOf(schema.string())])), -}); - -/** -const fieldSubTypeSchema = schema.object({ - multi: schema.maybe(schema.object({ parent: schema.string() })), - nested: schema.maybe(schema.object({ path: schema.string() })), -}); - -/* -const FieldDescriptorSchema = schema.object({ - aggregatable: schema.boolean(), - name: schema.string(), - readFromDocValues: schema.boolean(), - searchable: schema.boolean(), - type: schema.string(), - esTypes: schema.maybe(schema.arrayOf(schema.string())), - subType: fieldSubTypeSchema, - metadata_field: schema.maybe(schema.boolean()), - fixedInterval: schema.maybe(schema.arrayOf(schema.string())), - timeZone: schema.maybe(schema.arrayOf(schema.string())), - timeSeriesMetric: schema.maybe( - schema.oneOf([ - schema.literal('histogram'), - schema.literal('summary'), - schema.literal('counter'), - schema.literal('gauge'), - schema.literal('position'), - ]) - ), - timeSeriesDimension: schema.maybe(schema.boolean()), - conflictDescriptions: schema.maybe( - schema.recordOf(schema.string(), schema.arrayOf(schema.string())) - ), -}); - -/* -const validate: FullValidationConfig = { - request: { - query: querySchema, - }, - response: { - 200: { - body: schema.object({ - fields: schema.arrayOf(FieldDescriptorSchema), - indices: schema.arrayOf(schema.string()), - }), - }, - }, -}; -*/ +import { parseFields, IBody, IQuery, querySchema } from './fields_for'; function calculateHash(srcBuffer: Buffer) { const hash = createHash('sha1'); diff --git a/src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts b/src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts index 15d761935c0a7..58016cb6ee899 100644 --- a/src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts +++ b/src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts @@ -41,8 +41,8 @@ export const parseFields = (fields: string | string[]): string[] => { const access = 'internal'; -type IBody = { index_filter?: estypes.QueryDslQueryContainer } | undefined; -interface IQuery { +export type IBody = { index_filter?: estypes.QueryDslQueryContainer } | undefined; +export interface IQuery { pattern: string; meta_fields: string | string[]; type?: string; @@ -52,7 +52,7 @@ interface IQuery { fields?: string[]; } -const querySchema = schema.object({ +export const querySchema = schema.object({ pattern: schema.string(), meta_fields: schema.oneOf([schema.string(), schema.arrayOf(schema.string())], { defaultValue: [], diff --git a/x-pack/test/functional/apps/maps/group4/index.js b/x-pack/test/functional/apps/maps/group4/index.js index 4186670bb9b73..b18800d16f196 100644 --- a/x-pack/test/functional/apps/maps/group4/index.js +++ b/x-pack/test/functional/apps/maps/group4/index.js @@ -52,7 +52,11 @@ export default function ({ loadTestFile, getService, getPageObjects }) { }); await browser.setWindowSize(1600, 1000); - await security.testUser.setRoles(['kibana_admin']); // necessary to refresh field list + await security.testUser.setRoles([ + 'kibana_admin', + 'test_logstash_reader', + 'kibana_sample_admin', + ]); // necessary to refresh field list await PageObjects.common.navigateToApp('discover'); await PageObjects.discover.refreshFieldList(); // refreshes field list cache from previous tests }); diff --git a/x-pack/test/functional/apps/observability_log_explorer/columns_selection.ts b/x-pack/test/functional/apps/observability_log_explorer/columns_selection.ts index 81198bab7ae63..1f2ee69b0308c 100644 --- a/x-pack/test/functional/apps/observability_log_explorer/columns_selection.ts +++ b/x-pack/test/functional/apps/observability_log_explorer/columns_selection.ts @@ -13,7 +13,6 @@ const defaultLogColumns = ['@timestamp', 'service.name', 'host.name', 'message'] export default function ({ getService, getPageObjects }: FtrProviderContext) { const esArchiver = getService('esArchiver'); const retry = getService('retry'); - const browser = getService('browser'); const PageObjects = getPageObjects(['discover', 'observabilityLogExplorer', 'settings']); describe('Columns selection initialization and update', () => { @@ -21,6 +20,10 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await esArchiver.load( 'x-pack/test/functional/es_archives/observability_log_explorer/data_streams' ); + + await PageObjects.settings.navigateTo(); + await PageObjects.settings.createIndexPattern('logs-*-*', '@timestamp'); + await PageObjects.settings.refreshDataViewFieldList(); }); after(async () => { @@ -32,13 +35,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { describe('when the log explorer loads', () => { it("should initialize the table columns to logs' default selection", async () => { await PageObjects.observabilityLogExplorer.navigateTo(); - await browser.refresh(); await retry.try(async () => { - // did this work? - // no, different ids - // await PageObjects.discover.refreshFieldList(); - expect(await PageObjects.discover.getColumnHeaders()).to.eql(defaultLogColumns); }); }); From 45b3dcad524fb3e421f317da25f68ea0f92775ea Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Mon, 13 Nov 2023 09:03:06 -0600 Subject: [PATCH 047/168] don't cache empty responses --- .../data_views/server/rest_api_routes/internal/fields.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/plugins/data_views/server/rest_api_routes/internal/fields.ts b/src/plugins/data_views/server/rest_api_routes/internal/fields.ts index 8a8bc13333c18..cf4e0e0ddf766 100644 --- a/src/plugins/data_views/server/rest_api_routes/internal/fields.ts +++ b/src/plugins/data_views/server/rest_api_routes/internal/fields.ts @@ -81,11 +81,13 @@ const handler: ( // todo examine how long this takes const cacheMaxAge = await uiSettings.get('data_views:cache_max_age'); - if (cacheMaxAge) { + if (cacheMaxAge && fields.length) { const stale = 365 * 24 * 60 * 60 - cacheMaxAge; headers[ 'cache-control' ] = `private, max-age=${cacheMaxAge}, stale-while-revalidate=${stale}`; + } else { + headers['cache-control'] = 'private, no-cache'; } const ifNoneMatch = request.headers['if-none-match']; From 4b4266f3cb1c8f6e002d80c9bd3a991d5da8861b Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Mon, 13 Nov 2023 10:23:02 -0600 Subject: [PATCH 048/168] fix functional tests by removing unneeded code --- x-pack/test/functional/apps/maps/group4/index.js | 8 +++++--- .../apps/observability_log_explorer/columns_selection.ts | 2 ++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/x-pack/test/functional/apps/maps/group4/index.js b/x-pack/test/functional/apps/maps/group4/index.js index b18800d16f196..0a8926158dbfd 100644 --- a/x-pack/test/functional/apps/maps/group4/index.js +++ b/x-pack/test/functional/apps/maps/group4/index.js @@ -5,14 +5,14 @@ * 2.0. */ -export default function ({ loadTestFile, getService, getPageObjects }) { +export default function ({ loadTestFile, getService }) { const kibanaServer = getService('kibanaServer'); const esArchiver = getService('esArchiver'); const browser = getService('browser'); const log = getService('log'); const supertest = getService('supertest'); - const security = getService('security'); - const PageObjects = getPageObjects(['discover', 'common']); + // const security = getService('security'); + // const PageObjects = getPageObjects(['discover', 'common']); describe('maps app', function () { this.tags(['skipFirefox']); @@ -52,6 +52,7 @@ export default function ({ loadTestFile, getService, getPageObjects }) { }); await browser.setWindowSize(1600, 1000); + /* await security.testUser.setRoles([ 'kibana_admin', 'test_logstash_reader', @@ -59,6 +60,7 @@ export default function ({ loadTestFile, getService, getPageObjects }) { ]); // necessary to refresh field list await PageObjects.common.navigateToApp('discover'); await PageObjects.discover.refreshFieldList(); // refreshes field list cache from previous tests + */ }); after(async () => { diff --git a/x-pack/test/functional/apps/observability_log_explorer/columns_selection.ts b/x-pack/test/functional/apps/observability_log_explorer/columns_selection.ts index 1f2ee69b0308c..83384c4d8ed63 100644 --- a/x-pack/test/functional/apps/observability_log_explorer/columns_selection.ts +++ b/x-pack/test/functional/apps/observability_log_explorer/columns_selection.ts @@ -21,9 +21,11 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { 'x-pack/test/functional/es_archives/observability_log_explorer/data_streams' ); + /* await PageObjects.settings.navigateTo(); await PageObjects.settings.createIndexPattern('logs-*-*', '@timestamp'); await PageObjects.settings.refreshDataViewFieldList(); + */ }); after(async () => { From 2f11c36587c231175145832dbdb1d6691062d62b Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Mon, 13 Nov 2023 13:24:42 -0600 Subject: [PATCH 049/168] cleanup --- src/plugins/data_views/common/utils.ts | 2 +- .../server/rest_api_routes/internal/fields.ts | 1 - test/functional/apps/discover/group3/_sidebar.ts | 15 +-------------- .../data_views/_index_pattern_results_sort.ts | 1 - .../e2e/automated_response_actions/form.cy.ts | 3 --- .../feature_controls/discover_security.ts | 5 ----- .../index_pattern/creation_index_pattern.ts | 1 - .../discover/search_source_alert.ts | 1 + .../functional/tests/discover_integration.ts | 1 - 9 files changed, 3 insertions(+), 27 deletions(-) diff --git a/src/plugins/data_views/common/utils.ts b/src/plugins/data_views/common/utils.ts index 507f90e764c5e..b77696e15ba79 100644 --- a/src/plugins/data_views/common/utils.ts +++ b/src/plugins/data_views/common/utils.ts @@ -29,7 +29,7 @@ export async function findByName(client: PersistenceAPI, name: string) { } export function unwrapEtag(ifNoneMatch: string) { - let requestHash = (ifNoneMatch as string).slice(1, -1); + let requestHash = ifNoneMatch.slice(1, -1); if (requestHash.indexOf('-') > -1) { requestHash = requestHash.split('-')[0]; } diff --git a/src/plugins/data_views/server/rest_api_routes/internal/fields.ts b/src/plugins/data_views/server/rest_api_routes/internal/fields.ts index cf4e0e0ddf766..a6ea905829cf4 100644 --- a/src/plugins/data_views/server/rest_api_routes/internal/fields.ts +++ b/src/plugins/data_views/server/rest_api_routes/internal/fields.ts @@ -78,7 +78,6 @@ const handler: ( 'user-hash': (await getUserId()(request)) || '', }; - // todo examine how long this takes const cacheMaxAge = await uiSettings.get('data_views:cache_max_age'); if (cacheMaxAge && fields.length) { diff --git a/test/functional/apps/discover/group3/_sidebar.ts b/test/functional/apps/discover/group3/_sidebar.ts index 218ee9570b520..84a76bd5314e9 100644 --- a/test/functional/apps/discover/group3/_sidebar.ts +++ b/test/functional/apps/discover/group3/_sidebar.ts @@ -495,8 +495,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.header.waitUntilLoadingHasFinished(); await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded(); - // console.log('######################## await timeout'); - // await new Promise((r) => setTimeout(r, 10000 * 60)); expect(await PageObjects.unifiedFieldList.getSidebarAriaDescription()).to.be( '0 available fields. 0 empty fields. 0 meta fields.' ); @@ -531,10 +529,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { ); await browser.refresh(); - // here - await PageObjects.settings.refreshDataViewFieldList('with-timefield'); - await PageObjects.common.navigateToApp('discover'); - await PageObjects.discover.waitUntilSearchingHasFinished(); + await PageObjects.discover.refreshFieldList(); await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded(); expect(await PageObjects.unifiedFieldList.getSidebarAriaDescription()).to.be( @@ -555,12 +550,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.header.waitUntilLoadingHasFinished(); await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded(); - // console.log('######################## await timeout'); - // await new Promise((r) => setTimeout(r, 1000 * 300)); - // todo - comapre to main. This looks correct in this branch expect(await PageObjects.unifiedFieldList.getSidebarAriaDescription()).to.be( '0 available fields. 7 empty fields. 3 meta fields.' - // '0 available fields. 13 empty fields. 0 empty fields. 3 meta fields.' ); await testSubjects.existOrFail( `${PageObjects.unifiedFieldList.getSidebarSectionSelector( @@ -584,8 +575,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await esArchiver.unload( 'test/functional/fixtures/es_archiver/index_pattern_without_timefield' ); - // console.log('######################## await timeout'); - // await new Promise((r) => setTimeout(r, 1000 * 30)); }); it('should work when filters change', async () => { @@ -766,8 +755,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.header.waitUntilLoadingHasFinished(); await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded(); - // console.log('######################## await timeout'); - // await new Promise((r) => setTimeout(r, 1000 * 30)); expect(await PageObjects.unifiedFieldList.getSidebarAriaDescription()).to.be( '0 available fields. 7 empty fields. 3 meta fields.' ); diff --git a/test/functional/apps/management/data_views/_index_pattern_results_sort.ts b/test/functional/apps/management/data_views/_index_pattern_results_sort.ts index aef25d91634e5..8a94f96827a35 100644 --- a/test/functional/apps/management/data_views/_index_pattern_results_sort.ts +++ b/test/functional/apps/management/data_views/_index_pattern_results_sort.ts @@ -68,7 +68,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { describe('field list pagination', function () { const EXPECTED_FIELD_COUNT = 85; it('makelogs data should have expected number of fields', async function () { - // did this work? await testSubjects.click('refreshDataViewButton'); await retry.try(async function () { const TabCount = await PageObjects.settings.getFieldsTabCount(); diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/automated_response_actions/form.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/automated_response_actions/form.cy.ts index bf60ac20a4fe6..27458ef09e15b 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/automated_response_actions/form.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/automated_response_actions/form.cy.ts @@ -5,7 +5,6 @@ * 2.0. */ -// import { FIELDS_PATH } from '@kbn/data-views-plugin/common/constants'; import { addEndpointResponseAction, fillUpNewRule, @@ -186,9 +185,7 @@ describe( }); it('All response action controls are disabled', () => { - // cy.intercept('GET', `${FIELDS_PATH}*`).as('getFieldsForWildcard'); visitRuleActions(ruleId); - // cy.wait('@getFieldsForWildcard'); cy.getByTestSubj('edit-rule-actions-tab').click(); cy.getByTestSubj('response-actions-wrapper').within(() => { diff --git a/x-pack/test/functional/apps/discover/feature_controls/discover_security.ts b/x-pack/test/functional/apps/discover/feature_controls/discover_security.ts index fa01fc5b8d57f..ee810f7ddebfb 100644 --- a/x-pack/test/functional/apps/discover/feature_controls/discover_security.ts +++ b/x-pack/test/functional/apps/discover/feature_controls/discover_security.ts @@ -31,7 +31,6 @@ export default function (ctx: FtrProviderContext) { 'spaceSelector', 'header', 'unifiedFieldList', - 'settings', ]); const testSubjects = getService('testSubjects'); const appsMenu = getService('appsMenu'); @@ -448,10 +447,6 @@ export default function (ctx: FtrProviderContext) { it('allows to access only via a permitted index alias', async () => { await globalNav.badgeExistsOrFail('Read only'); - // this doesn't work since the user doesn't have access - // await PageObjects.settings.refreshDataViewFieldList('logstash-*'); - // await PageObjects.common.navigateToApp('discover'); - // can't access logstash index directly await PageObjects.discover.selectIndexPattern('logstash-*'); await PageObjects.header.waitUntilLoadingHasFinished(); diff --git a/x-pack/test/functional/apps/transform/creation/index_pattern/creation_index_pattern.ts b/x-pack/test/functional/apps/transform/creation/index_pattern/creation_index_pattern.ts index 2b873b5bfae25..7384b5bae2c9a 100644 --- a/x-pack/test/functional/apps/transform/creation/index_pattern/creation_index_pattern.ts +++ b/x-pack/test/functional/apps/transform/creation/index_pattern/creation_index_pattern.ts @@ -835,7 +835,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await transform.testExecution.logTestStep('should navigate to discover'); await transform.table.clickTransformRowAction(testData.transformId, 'Discover'); await pageObjects.discover.waitUntilSearchingHasFinished(); - // did this work? await pageObjects.discover.refreshFieldList(); if (testData.discoverAdjustSuperDatePicker) { diff --git a/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts b/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts index 9982a8cf1ad3d..403522855a7e2 100644 --- a/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts +++ b/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts @@ -308,6 +308,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { describe('Search source Alert', () => { before(async () => { + // todo try removing this await kibanaServer.uiSettings.replace({ 'data_views:fields_max_cache': '0', }); diff --git a/x-pack/test/saved_object_tagging/functional/tests/discover_integration.ts b/x-pack/test/saved_object_tagging/functional/tests/discover_integration.ts index cac4c36852b56..a132c493de83a 100644 --- a/x-pack/test/saved_object_tagging/functional/tests/discover_integration.ts +++ b/x-pack/test/saved_object_tagging/functional/tests/discover_integration.ts @@ -104,7 +104,6 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { beforeEach(async () => { await PageObjects.common.navigateToApp('discover'); await PageObjects.discover.selectIndexPattern('logstash-*'); - // did this fix a test? await PageObjects.discover.refreshFieldList(); await PageObjects.timePicker.setDefaultAbsoluteRange(); await PageObjects.header.waitUntilLoadingHasFinished(); From b0f930e377fd7a145796dfa650e91884379a76f3 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Mon, 13 Nov 2023 15:09:56 -0600 Subject: [PATCH 050/168] remove console.log statements --- src/plugins/data_views/common/data_views/data_views.ts | 10 +--------- .../public/data_views/data_views_api_client.ts | 2 -- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/src/plugins/data_views/common/data_views/data_views.ts b/src/plugins/data_views/common/data_views/data_views.ts index edfb68249e837..cf3f049355c2f 100644 --- a/src/plugins/data_views/common/data_views/data_views.ts +++ b/src/plugins/data_views/common/data_views/data_views.ts @@ -542,7 +542,7 @@ export class DataViewsService { forceRefresh: boolean = false ) => { const metaFields = await this.config.get(META_FIELDS); - const result = await this.apiClient.getFieldsForWildcard({ + return await this.apiClient.getFieldsForWildcard({ type: dataView.type, rollupIndex: dataView?.typeMeta?.params?.rollup_index, allowNoIndex: true, @@ -550,14 +550,6 @@ export class DataViewsService { metaFields, forceRefresh, }); - - // todo - console.log( - 'getFieldsAndIndicesForDataView result', - dataView.getIndexPattern(), - JSON.stringify(result.fields, undefined, 2) - ); - return result; }; private getFieldsAndIndicesForWildcard = async (options: GetFieldsOptions) => { diff --git a/src/plugins/data_views/public/data_views/data_views_api_client.ts b/src/plugins/data_views/public/data_views/data_views_api_client.ts index e9e774fa119a1..8b39ab76727f1 100644 --- a/src/plugins/data_views/public/data_views/data_views_api_client.ts +++ b/src/plugins/data_views/public/data_views/data_views_api_client.ts @@ -56,8 +56,6 @@ export class DataViewsApiClient implements IDataViewsApiClient { headers: { 'user-hash': userId }, }); - console.log('User id:', userId); - return request.catch((resp) => { if (resp.body.statusCode === 404 && resp.body.attributes?.code === 'no_matching_indices') { throw new DataViewMissingIndices(resp.body.message); From 208f64346f99d0e08a44bf9dd19d5b7e21cf05c6 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Mon, 13 Nov 2023 19:39:21 -0600 Subject: [PATCH 051/168] use hash instead of userid --- src/plugins/data_views/common/utils.ts | 7 +++++++ .../public/data_views/data_views_api_client.ts | 5 ++++- src/plugins/data_views/public/index.ts | 1 + .../server/rest_api_routes/internal/fields.ts | 17 +++++++---------- 4 files changed, 19 insertions(+), 11 deletions(-) diff --git a/src/plugins/data_views/common/utils.ts b/src/plugins/data_views/common/utils.ts index b77696e15ba79..cc4d4fd9a1b7f 100644 --- a/src/plugins/data_views/common/utils.ts +++ b/src/plugins/data_views/common/utils.ts @@ -6,6 +6,7 @@ * Side Public License, v 1. */ +import { createHash } from 'crypto'; import type { PersistenceAPI } from './types'; /** @@ -35,3 +36,9 @@ export function unwrapEtag(ifNoneMatch: string) { } return requestHash; } + +export function calculateHash(srcBuffer: Buffer) { + const hash = createHash('sha1'); + hash.update(srcBuffer); + return hash.digest('hex'); +} diff --git a/src/plugins/data_views/public/data_views/data_views_api_client.ts b/src/plugins/data_views/public/data_views/data_views_api_client.ts index 8b39ab76727f1..2bcb79c24e323 100644 --- a/src/plugins/data_views/public/data_views/data_views_api_client.ts +++ b/src/plugins/data_views/public/data_views/data_views_api_client.ts @@ -7,6 +7,7 @@ */ import { HttpSetup, HttpResponse } from '@kbn/core/public'; +import { calculateHash } from '../../common/utils'; import { DataViewMissingIndices } from '../../common/lib'; import { GetFieldsOptions, IDataViewsApiClient } from '../../common'; import { FieldsForWildcardResponse } from '../../common/types'; @@ -45,6 +46,8 @@ export class DataViewsApiClient implements IDataViewsApiClient { const cacheOptions = forceRefresh ? { cache: 'no-cache' as RequestCache } : {}; const userId = await this.getCurrentUserId(); + const userHash = userId ? calculateHash(Buffer.from(userId)) : ''; + const request = body ? this.http.post(url, { query, body, version, asResponse }) : this.http.fetch(url, { @@ -53,7 +56,7 @@ export class DataViewsApiClient implements IDataViewsApiClient { ...cacheOptions, asResponse, rawResponse, - headers: { 'user-hash': userId }, + headers: { 'user-hash': userHash }, }); return request.catch((resp) => { diff --git a/src/plugins/data_views/public/index.ts b/src/plugins/data_views/public/index.ts index f690552b5a147..96380665548dd 100644 --- a/src/plugins/data_views/public/index.ts +++ b/src/plugins/data_views/public/index.ts @@ -67,6 +67,7 @@ export type { DataViewsContract, HasDataViewsResponse, IndicesViaSearchResponse, + UserIdGetter, } from './types'; // Export plugin after all other imports diff --git a/src/plugins/data_views/server/rest_api_routes/internal/fields.ts b/src/plugins/data_views/server/rest_api_routes/internal/fields.ts index a6ea905829cf4..6ed385d1f7758 100644 --- a/src/plugins/data_views/server/rest_api_routes/internal/fields.ts +++ b/src/plugins/data_views/server/rest_api_routes/internal/fields.ts @@ -6,9 +6,8 @@ * Side Public License, v 1. */ -import { createHash } from 'crypto'; import { IRouter, RequestHandler, StartServicesAccessor, KibanaRequest } from '@kbn/core/server'; -import { unwrapEtag } from '../../../common/utils'; +import { unwrapEtag, calculateHash } from '../../../common/utils'; import { IndexPatternsFetcher } from '../../fetcher'; import type { DataViewsServerPluginStart, @@ -18,17 +17,11 @@ import type { FieldDescriptorRestResponse } from '../route_types'; import { FIELDS_PATH as path } from '../../../common/constants'; import { parseFields, IBody, IQuery, querySchema } from './fields_for'; -function calculateHash(srcBuffer: Buffer) { - const hash = createHash('sha1'); - hash.update(srcBuffer); - return hash.digest('hex'); -} - const handler: ( isRollupsEnabled: () => boolean, getUserId: () => (kibanaRequest: KibanaRequest) => Promise ) => RequestHandler<{}, IQuery, IBody> = - (isRollupsEnabled, getUserId) => async (context, request, response) => { + (isRollupsEnabled, getUserIdGetter) => async (context, request, response) => { const core = await context.core; const uiSettings = core.uiSettings.client; const { asCurrentUser } = core.elasticsearch.client; @@ -71,11 +64,15 @@ const handler: ( const etag = calculateHash(Buffer.from(JSON.stringify(body))); + const getUserId = getUserIdGetter(); + const userId = await getUserId(request); + const userHash = userId ? calculateHash(Buffer.from(userId)) : ''; + const headers: Record = { 'content-type': 'application/json', etag, vary: 'accept-encoding, user-hash', - 'user-hash': (await getUserId()(request)) || '', + 'user-hash': userHash, }; const cacheMaxAge = await uiSettings.get('data_views:cache_max_age'); From e101dbaf0f478ab8268e1fbe08c93b9e789faa3e Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Mon, 13 Nov 2023 21:08:05 -0600 Subject: [PATCH 052/168] fix browser bundle size --- src/plugins/data_views/common/utils.ts | 7 ------- .../public/data_views/data_views_api_client.ts | 11 +++++++++-- .../server/rest_api_routes/internal/fields.ts | 9 ++++++++- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/src/plugins/data_views/common/utils.ts b/src/plugins/data_views/common/utils.ts index cc4d4fd9a1b7f..b77696e15ba79 100644 --- a/src/plugins/data_views/common/utils.ts +++ b/src/plugins/data_views/common/utils.ts @@ -6,7 +6,6 @@ * Side Public License, v 1. */ -import { createHash } from 'crypto'; import type { PersistenceAPI } from './types'; /** @@ -36,9 +35,3 @@ export function unwrapEtag(ifNoneMatch: string) { } return requestHash; } - -export function calculateHash(srcBuffer: Buffer) { - const hash = createHash('sha1'); - hash.update(srcBuffer); - return hash.digest('hex'); -} diff --git a/src/plugins/data_views/public/data_views/data_views_api_client.ts b/src/plugins/data_views/public/data_views/data_views_api_client.ts index 2bcb79c24e323..e0052ff459c4f 100644 --- a/src/plugins/data_views/public/data_views/data_views_api_client.ts +++ b/src/plugins/data_views/public/data_views/data_views_api_client.ts @@ -7,7 +7,6 @@ */ import { HttpSetup, HttpResponse } from '@kbn/core/public'; -import { calculateHash } from '../../common/utils'; import { DataViewMissingIndices } from '../../common/lib'; import { GetFieldsOptions, IDataViewsApiClient } from '../../common'; import { FieldsForWildcardResponse } from '../../common/types'; @@ -16,6 +15,14 @@ import { FIELDS_FOR_WILDCARD_PATH, FIELDS_PATH } from '../../common/constants'; const API_BASE_URL: string = `/api/index_patterns/`; const version = '1'; +async function sha1(str: string) { + const enc = new TextEncoder(); + const hash = await crypto.subtle.digest('SHA-1', enc.encode(str)); + return Array.from(new Uint8Array(hash)) + .map((v) => v.toString(16).padStart(2, '0')) + .join(''); +} + /** * Data Views API Client - client implementation */ @@ -46,7 +53,7 @@ export class DataViewsApiClient implements IDataViewsApiClient { const cacheOptions = forceRefresh ? { cache: 'no-cache' as RequestCache } : {}; const userId = await this.getCurrentUserId(); - const userHash = userId ? calculateHash(Buffer.from(userId)) : ''; + const userHash = userId ? await sha1(userId) : ''; const request = body ? this.http.post(url, { query, body, version, asResponse }) diff --git a/src/plugins/data_views/server/rest_api_routes/internal/fields.ts b/src/plugins/data_views/server/rest_api_routes/internal/fields.ts index 6ed385d1f7758..2838c47668b5b 100644 --- a/src/plugins/data_views/server/rest_api_routes/internal/fields.ts +++ b/src/plugins/data_views/server/rest_api_routes/internal/fields.ts @@ -6,8 +6,9 @@ * Side Public License, v 1. */ +import { createHash } from 'crypto'; import { IRouter, RequestHandler, StartServicesAccessor, KibanaRequest } from '@kbn/core/server'; -import { unwrapEtag, calculateHash } from '../../../common/utils'; +import { unwrapEtag } from '../../../common/utils'; import { IndexPatternsFetcher } from '../../fetcher'; import type { DataViewsServerPluginStart, @@ -17,6 +18,12 @@ import type { FieldDescriptorRestResponse } from '../route_types'; import { FIELDS_PATH as path } from '../../../common/constants'; import { parseFields, IBody, IQuery, querySchema } from './fields_for'; +export function calculateHash(srcBuffer: Buffer) { + const hash = createHash('sha1'); + hash.update(srcBuffer); + return hash.digest('hex'); +} + const handler: ( isRollupsEnabled: () => boolean, getUserId: () => (kibanaRequest: KibanaRequest) => Promise From eb2e989d1bb54cf37defcef14b91c18748d5d991 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Mon, 13 Nov 2023 22:05:22 -0600 Subject: [PATCH 053/168] add integration tests --- .../apis/data_views/fields_route/cache.ts | 65 +++++ .../apis/data_views/fields_route/conflicts.ts | 81 ++++++ .../apis/data_views/fields_route/index.ts | 18 ++ .../apis/data_views/fields_route/params.ts | 148 +++++++++++ .../apis/data_views/fields_route/response.ts | 231 ++++++++++++++++++ test/api_integration/apis/data_views/index.ts | 1 + 6 files changed, 544 insertions(+) create mode 100644 test/api_integration/apis/data_views/fields_route/cache.ts create mode 100644 test/api_integration/apis/data_views/fields_route/conflicts.ts create mode 100644 test/api_integration/apis/data_views/fields_route/index.ts create mode 100644 test/api_integration/apis/data_views/fields_route/params.ts create mode 100644 test/api_integration/apis/data_views/fields_route/response.ts diff --git a/test/api_integration/apis/data_views/fields_route/cache.ts b/test/api_integration/apis/data_views/fields_route/cache.ts new file mode 100644 index 0000000000000..4e852949405cf --- /dev/null +++ b/test/api_integration/apis/data_views/fields_route/cache.ts @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { ELASTIC_HTTP_VERSION_HEADER } from '@kbn/core-http-common'; +import { INITIAL_REST_VERSION_INTERNAL } from '@kbn/data-views-plugin/server/constants'; +import { FIELDS_PATH } from '@kbn/data-views-plugin/common/constants'; +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../../ftr_provider_context'; + +export default function ({ getService }: FtrProviderContext) { + const esArchiver = getService('esArchiver'); + const supertest = getService('supertest'); + + describe('cache headers', () => { + before(() => + esArchiver.load('test/api_integration/fixtures/es_archiver/index_patterns/basic_index') + ); + after(() => + esArchiver.unload('test/api_integration/fixtures/es_archiver/index_patterns/basic_index') + ); + + it('are present', async () => { + const response = await supertest + .get(FIELDS_PATH) + .set(ELASTIC_HTTP_VERSION_HEADER, INITIAL_REST_VERSION_INTERNAL) + .query({ + pattern: '*', + include_unmapped: true, + }); + + const cacheControlHeader = response.get('cache-control'); + + expect(cacheControlHeader).to.contain('max-age'); + expect(cacheControlHeader).to.contain('stale-while-revalidate'); + expect(response.get('vary')).to.equal('accept-encoding, user-hash'); + expect(response.get('etag')).to.not.be.empty(); + expect(response.get('user-hash')).to.equal(''); + }); + + it('returns 304 on matching etag', async () => { + const response = await supertest + .get(FIELDS_PATH) + .set(ELASTIC_HTTP_VERSION_HEADER, INITIAL_REST_VERSION_INTERNAL) + .query({ + pattern: '*', + include_unmapped: true, + }); + + await supertest + .get(FIELDS_PATH) + .set(ELASTIC_HTTP_VERSION_HEADER, INITIAL_REST_VERSION_INTERNAL) + .set('If-None-Match', response.get('etag')) + .query({ + pattern: '*', + include_unmapped: true, + }) + .expect(304); + }); + }); +} diff --git a/test/api_integration/apis/data_views/fields_route/conflicts.ts b/test/api_integration/apis/data_views/fields_route/conflicts.ts new file mode 100644 index 0000000000000..9ad861272c4cd --- /dev/null +++ b/test/api_integration/apis/data_views/fields_route/conflicts.ts @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { ELASTIC_HTTP_VERSION_HEADER } from '@kbn/core-http-common'; +import { INITIAL_REST_VERSION_INTERNAL } from '@kbn/data-views-plugin/server/constants'; +import { FIELDS_PATH } from '@kbn/data-views-plugin/common/constants'; +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../../ftr_provider_context'; + +export default function ({ getService }: FtrProviderContext) { + const supertest = getService('supertest'); + const esArchiver = getService('esArchiver'); + + describe('conflicts', () => { + before(() => + esArchiver.load('test/api_integration/fixtures/es_archiver/index_patterns/conflicts') + ); + after(() => + esArchiver.unload('test/api_integration/fixtures/es_archiver/index_patterns/conflicts') + ); + + it('flags fields with mismatched types as conflicting', () => + supertest + .get(FIELDS_PATH) + .set(ELASTIC_HTTP_VERSION_HEADER, INITIAL_REST_VERSION_INTERNAL) + .query({ pattern: 'logs-*' }) + .expect(200) + .then((resp) => { + expect(resp.body).to.eql({ + fields: [ + { + name: '@timestamp', + type: 'date', + esTypes: ['date'], + aggregatable: true, + searchable: true, + readFromDocValues: true, + metadata_field: false, + }, + { + name: 'number_conflict', + type: 'number', + esTypes: ['float', 'integer'], + aggregatable: true, + searchable: true, + readFromDocValues: true, + metadata_field: false, + }, + { + name: 'string_conflict', + type: 'string', + esTypes: ['keyword', 'text'], + aggregatable: true, + searchable: true, + readFromDocValues: true, + metadata_field: false, + }, + { + name: 'success', + type: 'conflict', + esTypes: ['keyword', 'boolean'], + aggregatable: true, + searchable: true, + readFromDocValues: false, + conflictDescriptions: { + boolean: ['logs-2017.01.02'], + keyword: ['logs-2017.01.01'], + }, + metadata_field: false, + }, + ], + indices: ['logs-2017.01.01', 'logs-2017.01.02'], + }); + })); + }); +} diff --git a/test/api_integration/apis/data_views/fields_route/index.ts b/test/api_integration/apis/data_views/fields_route/index.ts new file mode 100644 index 0000000000000..4b03c2b58afbe --- /dev/null +++ b/test/api_integration/apis/data_views/fields_route/index.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { FtrProviderContext } from '../../../ftr_provider_context'; + +export default function ({ loadTestFile }: FtrProviderContext) { + describe('data_views/fields route', () => { + loadTestFile(require.resolve('./params')); + loadTestFile(require.resolve('./conflicts')); + loadTestFile(require.resolve('./response')); + loadTestFile(require.resolve('./cache')); + }); +} diff --git a/test/api_integration/apis/data_views/fields_route/params.ts b/test/api_integration/apis/data_views/fields_route/params.ts new file mode 100644 index 0000000000000..29d6c06e3a60b --- /dev/null +++ b/test/api_integration/apis/data_views/fields_route/params.ts @@ -0,0 +1,148 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { ELASTIC_HTTP_VERSION_HEADER } from '@kbn/core-http-common'; +import { INITIAL_REST_VERSION_INTERNAL } from '@kbn/data-views-plugin/server/constants'; +import { FIELDS_PATH } from '@kbn/data-views-plugin/common/constants'; +import { FtrProviderContext } from '../../../ftr_provider_context'; + +export default function ({ getService }: FtrProviderContext) { + const esArchiver = getService('esArchiver'); + const supertest = getService('supertest'); + const randomness = getService('randomness'); + + describe('params', () => { + before(() => + esArchiver.load('test/api_integration/fixtures/es_archiver/index_patterns/basic_index') + ); + after(() => + esArchiver.unload('test/api_integration/fixtures/es_archiver/index_patterns/basic_index') + ); + + it('requires a pattern query param', () => + supertest + .get(FIELDS_PATH) + .set(ELASTIC_HTTP_VERSION_HEADER, INITIAL_REST_VERSION_INTERNAL) + .query({}) + .expect(400)); + + it('accepts include_unmapped param', () => + supertest + .get(FIELDS_PATH) + .set(ELASTIC_HTTP_VERSION_HEADER, INITIAL_REST_VERSION_INTERNAL) + .query({ + pattern: '*', + include_unmapped: true, + }) + .expect(200)); + + it('rejects unexpected query params', () => + supertest + .get(FIELDS_PATH) + .set(ELASTIC_HTTP_VERSION_HEADER, INITIAL_REST_VERSION_INTERNAL) + .query({ + pattern: randomness.word(), + [randomness.word()]: randomness.word(), + }) + .expect(400)); + + describe('fields', () => { + it('accepts a JSON formatted fields query param', () => + supertest + .get(FIELDS_PATH) + .set(ELASTIC_HTTP_VERSION_HEADER, INITIAL_REST_VERSION_INTERNAL) + .query({ + pattern: '*', + fields: JSON.stringify(['baz']), + }) + .expect(200)); + + it('accepts meta_fields query param in string array', () => + supertest + .get(FIELDS_PATH) + .set(ELASTIC_HTTP_VERSION_HEADER, INITIAL_REST_VERSION_INTERNAL) + .query({ + pattern: '*', + fields: ['baz', 'foo'], + }) + .expect(200)); + + it('accepts single array fields query param', () => + supertest + .get(FIELDS_PATH) + .set(ELASTIC_HTTP_VERSION_HEADER, INITIAL_REST_VERSION_INTERNAL) + .query({ + pattern: '*', + fields: ['baz'], + }) + .expect(200)); + + it('accepts single fields query param', () => + supertest + .get(FIELDS_PATH) + .set(ELASTIC_HTTP_VERSION_HEADER, INITIAL_REST_VERSION_INTERNAL) + .query({ + pattern: '*', + fields: 'baz', + }) + .expect(200)); + + it('rejects a comma-separated list of fields', () => + supertest + .get(FIELDS_PATH) + .set(ELASTIC_HTTP_VERSION_HEADER, INITIAL_REST_VERSION_INTERNAL) + .query({ + pattern: '*', + fields: 'foo,bar', + }) + .expect(400)); + }); + + describe('meta_fields', () => { + it('accepts a JSON formatted meta_fields query param', () => + supertest + .get(FIELDS_PATH) + .set(ELASTIC_HTTP_VERSION_HEADER, INITIAL_REST_VERSION_INTERNAL) + .query({ + pattern: '*', + meta_fields: JSON.stringify(['meta']), + }) + .expect(200)); + + it('accepts meta_fields query param in string array', () => + supertest + .get(FIELDS_PATH) + .set(ELASTIC_HTTP_VERSION_HEADER, INITIAL_REST_VERSION_INTERNAL) + .query({ + pattern: '*', + meta_fields: ['_id', 'meta'], + }) + .expect(200)); + + it('accepts single meta_fields query param', () => + supertest + .get(FIELDS_PATH) + .set(ELASTIC_HTTP_VERSION_HEADER, INITIAL_REST_VERSION_INTERNAL) + .query({ + pattern: '*', + meta_fields: ['_id'], + }) + .expect(200)); + + it('rejects a comma-separated list of meta_fields', () => + supertest + .get(FIELDS_PATH) + .set(ELASTIC_HTTP_VERSION_HEADER, INITIAL_REST_VERSION_INTERNAL) + .query({ + pattern: '*', + meta_fields: 'foo,bar', + }) + .expect(400)); + }); + }); +} diff --git a/test/api_integration/apis/data_views/fields_route/response.ts b/test/api_integration/apis/data_views/fields_route/response.ts new file mode 100644 index 0000000000000..47c8814b0c4b0 --- /dev/null +++ b/test/api_integration/apis/data_views/fields_route/response.ts @@ -0,0 +1,231 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { ELASTIC_HTTP_VERSION_HEADER } from '@kbn/core-http-common'; +import { INITIAL_REST_VERSION_INTERNAL } from '@kbn/data-views-plugin/server/constants'; +import { FIELDS_PATH } from '@kbn/data-views-plugin/common/constants'; +import expect from '@kbn/expect'; +import { sortBy } from 'lodash'; +import { FtrProviderContext } from '../../../ftr_provider_context'; + +export default function ({ getService }: FtrProviderContext) { + const esArchiver = getService('esArchiver'); + const supertest = getService('supertest'); + + const ensureFieldsAreSorted = (resp: { body: { fields: { name: string } } }) => { + expect(resp.body.fields).to.eql(sortBy(resp.body.fields, 'name')); + }; + + const testFields = [ + { + type: 'boolean', + esTypes: ['boolean'], + searchable: true, + aggregatable: true, + name: 'bar', + readFromDocValues: true, + metadata_field: false, + }, + { + type: 'string', + esTypes: ['text'], + searchable: true, + aggregatable: false, + name: 'baz', + readFromDocValues: false, + metadata_field: false, + }, + { + type: 'string', + esTypes: ['keyword'], + searchable: true, + aggregatable: true, + name: 'baz.keyword', + readFromDocValues: true, + subType: { multi: { parent: 'baz' } }, + metadata_field: false, + }, + { + type: 'number', + esTypes: ['long'], + searchable: true, + aggregatable: true, + name: 'foo', + readFromDocValues: true, + metadata_field: false, + }, + { + aggregatable: true, + esTypes: ['keyword'], + name: 'nestedField.child', + readFromDocValues: true, + searchable: true, + subType: { + nested: { + path: 'nestedField', + }, + }, + type: 'string', + metadata_field: false, + }, + ]; + + describe('fields route response', () => { + before(() => + esArchiver.load('test/api_integration/fixtures/es_archiver/index_patterns/basic_index') + ); + after(() => + esArchiver.unload('test/api_integration/fixtures/es_archiver/index_patterns/basic_index') + ); + + it('returns a flattened version of the fields in es', async () => { + await supertest + .get(FIELDS_PATH) + .set(ELASTIC_HTTP_VERSION_HEADER, INITIAL_REST_VERSION_INTERNAL) + .query({ pattern: 'basic_index' }) + .expect(200, { + fields: testFields, + indices: ['basic_index'], + }) + .then(ensureFieldsAreSorted); + }); + + it('returns a single field as requested', async () => { + await supertest + .get(FIELDS_PATH) + .set(ELASTIC_HTTP_VERSION_HEADER, INITIAL_REST_VERSION_INTERNAL) + .query({ pattern: 'basic_index', fields: JSON.stringify(['bar']) }) + .expect(200, { + fields: [testFields[0]], + indices: ['basic_index'], + }); + }); + + it('always returns a field for all passed meta fields', async () => { + await supertest + .get(FIELDS_PATH) + .set(ELASTIC_HTTP_VERSION_HEADER, INITIAL_REST_VERSION_INTERNAL) + .query({ + pattern: 'basic_index', + meta_fields: JSON.stringify(['_id', '_source', 'crazy_meta_field']), + }) + .expect(200, { + fields: [ + { + aggregatable: false, + name: '_id', + esTypes: ['_id'], + readFromDocValues: false, + searchable: true, + type: 'string', + metadata_field: true, + }, + { + aggregatable: false, + name: '_source', + esTypes: ['_source'], + readFromDocValues: false, + searchable: false, + type: '_source', + metadata_field: true, + }, + { + type: 'boolean', + esTypes: ['boolean'], + searchable: true, + aggregatable: true, + name: 'bar', + readFromDocValues: true, + metadata_field: false, + }, + { + aggregatable: false, + name: 'baz', + esTypes: ['text'], + readFromDocValues: false, + searchable: true, + type: 'string', + metadata_field: false, + }, + { + type: 'string', + esTypes: ['keyword'], + searchable: true, + aggregatable: true, + name: 'baz.keyword', + readFromDocValues: true, + subType: { multi: { parent: 'baz' } }, + metadata_field: false, + }, + { + aggregatable: false, + name: 'crazy_meta_field', + readFromDocValues: false, + searchable: false, + type: 'string', + metadata_field: true, + }, + { + type: 'number', + esTypes: ['long'], + searchable: true, + aggregatable: true, + name: 'foo', + readFromDocValues: true, + metadata_field: false, + }, + { + aggregatable: true, + esTypes: ['keyword'], + name: 'nestedField.child', + readFromDocValues: true, + searchable: true, + subType: { + nested: { + path: 'nestedField', + }, + }, + type: 'string', + metadata_field: false, + }, + ], + indices: ['basic_index'], + }) + .then(ensureFieldsAreSorted); + }); + + it('returns fields when one pattern exists and the other does not', async () => { + await supertest + .get(FIELDS_PATH) + .set(ELASTIC_HTTP_VERSION_HEADER, INITIAL_REST_VERSION_INTERNAL) + .query({ pattern: 'bad_index,basic_index' }) + .expect(200, { + fields: testFields, + indices: ['basic_index'], + }); + }); + + it('returns 404 when neither exists', async () => { + await supertest + .get(FIELDS_PATH) + .set(ELASTIC_HTTP_VERSION_HEADER, INITIAL_REST_VERSION_INTERNAL) + .query({ pattern: 'bad_index,bad_index_2' }) + .expect(404); + }); + + it('returns 404 when no patterns exist', async () => { + await supertest + .get(FIELDS_PATH) + .set(ELASTIC_HTTP_VERSION_HEADER, INITIAL_REST_VERSION_INTERNAL) + .query({ + pattern: 'bad_index', + }) + .expect(404); + }); + }); +} diff --git a/test/api_integration/apis/data_views/index.ts b/test/api_integration/apis/data_views/index.ts index a6589d5680b3d..654421937f44b 100644 --- a/test/api_integration/apis/data_views/index.ts +++ b/test/api_integration/apis/data_views/index.ts @@ -23,5 +23,6 @@ export default function ({ loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./has_user_index_pattern')); loadTestFile(require.resolve('./swap_references')); loadTestFile(require.resolve('./resolve_index')); + loadTestFile(require.resolve('./fields_route')); }); } From 121c2e86ab6ef124a4c8d4249c3f4d4593e3db24 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Mon, 13 Nov 2023 22:14:06 -0600 Subject: [PATCH 054/168] update refresh button text --- .../unified_search/public/dataview_picker/change_dataview.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/unified_search/public/dataview_picker/change_dataview.tsx b/src/plugins/unified_search/public/dataview_picker/change_dataview.tsx index 1a90b47103988..f1e19be1c72ce 100644 --- a/src/plugins/unified_search/public/dataview_picker/change_dataview.tsx +++ b/src/plugins/unified_search/public/dataview_picker/change_dataview.tsx @@ -231,7 +231,7 @@ export function ChangeDataView({ }} > {i18n.translate('unifiedSearch.query.queryBar.indexPattern.refreshFieldButton', { - defaultMessage: 'Refresh field list', + defaultMessage: 'Refresh field list for this data view', })} , From aa3503bd5e30cc4f7c9c03074678e71649369753 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Mon, 13 Nov 2023 22:56:46 -0600 Subject: [PATCH 055/168] increase bundle limit --- packages/kbn-optimizer/limits.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/kbn-optimizer/limits.yml b/packages/kbn-optimizer/limits.yml index 68521a9b35d12..fb5b7c0aaad23 100644 --- a/packages/kbn-optimizer/limits.yml +++ b/packages/kbn-optimizer/limits.yml @@ -32,7 +32,7 @@ pageLoadAssetSize: dataViewEditor: 28082 dataViewFieldEditor: 27000 dataViewManagement: 5100 - dataViews: 48800 + dataViews: 50000 dataVisualizer: 27530 devTools: 38637 discover: 99999 From 773411839a9ea0fba22b2c6a0a3ea85e561cada4 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Tue, 14 Nov 2023 08:15:32 -0600 Subject: [PATCH 056/168] skip flakey test --- test/functional/apps/management/data_views/_scripted_fields.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/functional/apps/management/data_views/_scripted_fields.ts b/test/functional/apps/management/data_views/_scripted_fields.ts index 2c70161a3bc43..a85947730f3fd 100644 --- a/test/functional/apps/management/data_views/_scripted_fields.ts +++ b/test/functional/apps/management/data_views/_scripted_fields.ts @@ -152,7 +152,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.common.setTime({ from, to }); }); - it('should see scripted field value in Discover', async function () { + it.skip('should see scripted field value in Discover', async function () { await PageObjects.common.navigateToApp('discover'); await retry.try(async function () { From 247bcaaefc060ac70c3b85b73ac77b53c49c8e55 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Tue, 14 Nov 2023 09:28:48 -0600 Subject: [PATCH 057/168] skip flakey test --- .../functional/apps/management/data_views/_scripted_fields.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/functional/apps/management/data_views/_scripted_fields.ts b/test/functional/apps/management/data_views/_scripted_fields.ts index a85947730f3fd..d284d06547d90 100644 --- a/test/functional/apps/management/data_views/_scripted_fields.ts +++ b/test/functional/apps/management/data_views/_scripted_fields.ts @@ -145,14 +145,14 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); }); - describe('discover scripted field', async () => { + describe.skip('discover scripted field', async () => { before(async () => { const from = 'Sep 17, 2015 @ 06:31:44.000'; const to = 'Sep 18, 2015 @ 18:31:44.000'; await PageObjects.common.setTime({ from, to }); }); - it.skip('should see scripted field value in Discover', async function () { + it('should see scripted field value in Discover', async function () { await PageObjects.common.navigateToApp('discover'); await retry.try(async function () { From 66836bd059c444fcb284b79ad7d3d8f2b3218064 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Tue, 14 Nov 2023 10:10:20 -0600 Subject: [PATCH 058/168] skip flaky test --- test/functional/apps/management/data_views/_scripted_fields.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/functional/apps/management/data_views/_scripted_fields.ts b/test/functional/apps/management/data_views/_scripted_fields.ts index d284d06547d90..a9cf454331456 100644 --- a/test/functional/apps/management/data_views/_scripted_fields.ts +++ b/test/functional/apps/management/data_views/_scripted_fields.ts @@ -226,7 +226,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); }); - describe('creating and using Painless string scripted fields', function describeIndexTests() { + describe.skip('creating and using Painless string scripted fields', function describeIndexTests() { const scriptedPainlessFieldName2 = 'painString'; before(async () => { From 387b2036414a3dbc77a9f09e6c5c49732d35e6f2 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Tue, 14 Nov 2023 16:10:46 -0600 Subject: [PATCH 059/168] update snapshot --- .../services/log_views/log_views_client.test.ts | 4 ++-- x-pack/test/functional/apps/maps/group4/index.js | 12 ------------ .../observability_log_explorer/columns_selection.ts | 8 +------- 3 files changed, 3 insertions(+), 21 deletions(-) diff --git a/x-pack/plugins/logs_shared/server/services/log_views/log_views_client.test.ts b/x-pack/plugins/logs_shared/server/services/log_views/log_views_client.test.ts index 4d863667235a8..e2741bea55931 100644 --- a/x-pack/plugins/logs_shared/server/services/log_views/log_views_client.test.ts +++ b/x-pack/plugins/logs_shared/server/services/log_views/log_views_client.test.ts @@ -276,8 +276,8 @@ describe('LogViewsClient class', () => { }, "fields": FldList [], "flattenHit": [Function], - "getEtag": [Function], "getAllowHidden": [Function], + "getEtag": [Function], "getFieldAttrs": [Function], "getIndexPattern": [Function], "getName": [Function], @@ -302,8 +302,8 @@ describe('LogViewsClient class', () => { }, }, "scriptedFields": Array [], - "setEtag": [Function], "setAllowHidden": [Function], + "setEtag": [Function], "setFieldFormat": [Function], "setIndexPattern": [Function], "shortDotsEnable": false, diff --git a/x-pack/test/functional/apps/maps/group4/index.js b/x-pack/test/functional/apps/maps/group4/index.js index 0a8926158dbfd..6af1369928d98 100644 --- a/x-pack/test/functional/apps/maps/group4/index.js +++ b/x-pack/test/functional/apps/maps/group4/index.js @@ -11,8 +11,6 @@ export default function ({ loadTestFile, getService }) { const browser = getService('browser'); const log = getService('log'); const supertest = getService('supertest'); - // const security = getService('security'); - // const PageObjects = getPageObjects(['discover', 'common']); describe('maps app', function () { this.tags(['skipFirefox']); @@ -51,16 +49,6 @@ export default function ({ loadTestFile, getService }) { defaultIndex: 'c698b940-e149-11e8-a35a-370a8516603a', }); await browser.setWindowSize(1600, 1000); - - /* - await security.testUser.setRoles([ - 'kibana_admin', - 'test_logstash_reader', - 'kibana_sample_admin', - ]); // necessary to refresh field list - await PageObjects.common.navigateToApp('discover'); - await PageObjects.discover.refreshFieldList(); // refreshes field list cache from previous tests - */ }); after(async () => { diff --git a/x-pack/test/functional/apps/observability_log_explorer/columns_selection.ts b/x-pack/test/functional/apps/observability_log_explorer/columns_selection.ts index 83384c4d8ed63..7738d3b97b2bc 100644 --- a/x-pack/test/functional/apps/observability_log_explorer/columns_selection.ts +++ b/x-pack/test/functional/apps/observability_log_explorer/columns_selection.ts @@ -13,19 +13,13 @@ const defaultLogColumns = ['@timestamp', 'service.name', 'host.name', 'message'] export default function ({ getService, getPageObjects }: FtrProviderContext) { const esArchiver = getService('esArchiver'); const retry = getService('retry'); - const PageObjects = getPageObjects(['discover', 'observabilityLogExplorer', 'settings']); + const PageObjects = getPageObjects(['discover', 'observabilityLogExplorer']); describe('Columns selection initialization and update', () => { before(async () => { await esArchiver.load( 'x-pack/test/functional/es_archives/observability_log_explorer/data_streams' ); - - /* - await PageObjects.settings.navigateTo(); - await PageObjects.settings.createIndexPattern('logs-*-*', '@timestamp'); - await PageObjects.settings.refreshDataViewFieldList(); - */ }); after(async () => { From 99b03af22ee29a28712f58356925e76074aa8be6 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Tue, 14 Nov 2023 18:07:12 -0600 Subject: [PATCH 060/168] skip scripted field test, its being flaky --- test/functional/apps/management/data_views/_scripted_fields.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/functional/apps/management/data_views/_scripted_fields.ts b/test/functional/apps/management/data_views/_scripted_fields.ts index a9cf454331456..f7324e9b83934 100644 --- a/test/functional/apps/management/data_views/_scripted_fields.ts +++ b/test/functional/apps/management/data_views/_scripted_fields.ts @@ -330,7 +330,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); }); - describe('creating and using Painless boolean scripted fields', function describeIndexTests() { + describe.skip('creating and using Painless boolean scripted fields', function describeIndexTests() { const scriptedPainlessFieldName2 = 'painBool'; before(async () => { From 4246c60a9364e0958dd09ebc647c19d1155fec1d Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Tue, 14 Nov 2023 19:00:25 -0600 Subject: [PATCH 061/168] skip scripted field test, its being flaky --- test/functional/apps/management/data_views/_scripted_fields.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/functional/apps/management/data_views/_scripted_fields.ts b/test/functional/apps/management/data_views/_scripted_fields.ts index f7324e9b83934..1116a482a77f1 100644 --- a/test/functional/apps/management/data_views/_scripted_fields.ts +++ b/test/functional/apps/management/data_views/_scripted_fields.ts @@ -44,7 +44,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { 'unifiedFieldList', ]); - describe('scripted fields', function () { + describe.skip('scripted fields', function () { this.tags(['skipFirefox']); before(async function () { From 42bd2f41f21bf49031bef64b4c44531486c60660 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Tue, 14 Nov 2023 21:14:09 -0600 Subject: [PATCH 062/168] refactor security / data view plugin dependency --- src/plugins/data_views/kibana.jsonc | 1 + src/plugins/data_views/server/plugin.ts | 25 ++++++++++++++++++++---- src/plugins/data_views/server/types.ts | 1 - x-pack/plugins/security/server/plugin.ts | 16 +-------------- 4 files changed, 23 insertions(+), 20 deletions(-) diff --git a/src/plugins/data_views/kibana.jsonc b/src/plugins/data_views/kibana.jsonc index 7595f8e68b3c4..7789383b48ba4 100644 --- a/src/plugins/data_views/kibana.jsonc +++ b/src/plugins/data_views/kibana.jsonc @@ -18,6 +18,7 @@ "requiredBundles": [ "kibanaUtils" ], + "runtimePluginDependencies" : ["security"], "extraPublicDirs": [ "common" ] diff --git a/src/plugins/data_views/server/plugin.ts b/src/plugins/data_views/server/plugin.ts index bdb1ac192d5a8..91746655e7521 100644 --- a/src/plugins/data_views/server/plugin.ts +++ b/src/plugins/data_views/server/plugin.ts @@ -6,7 +6,15 @@ * Side Public License, v 1. */ -import { CoreSetup, CoreStart, Logger, Plugin, PluginInitializerContext } from '@kbn/core/server'; +import { + CoreSetup, + CoreStart, + Logger, + Plugin, + PluginInitializerContext, + KibanaRequest, +} from '@kbn/core/server'; +import { SecurityPluginStart } from '@kbn/security-plugin/server'; import { dataViewsServiceFactory } from './data_views_service_factory'; import { registerRoutes } from './routes'; import { dataViewSavedObjectType } from './saved_objects'; @@ -51,6 +59,18 @@ export class DataViewsServerPlugin core.capabilities.registerProvider(capabilitiesProvider); core.uiSettings.register(cacheMaxAge); const dataViewRestCounter = usageCollection?.createUsageCounter('dataViewsRestApi'); + core.plugins.onStart<{ security: SecurityPluginStart }>('security').then(({ security }) => { + if (security.found) { + const getUserId = async function getUserId( + request: KibanaRequest + ): Promise { + return security.contract.authc.getCurrentUser(request)?.profile_uid; + }; + this.getUserId = getUserId; + } else { + throw new Error('Security plugin is not available, but is required for Data Views plugin'); + } + }); registerRoutes({ http: core.http, @@ -77,9 +97,6 @@ export class DataViewsServerPlugin return { enableRollups: () => (this.rollupsEnabled = true), - setGetUserId: (getUserId: GetUserId) => { - this.getUserId = getUserId; - }, }; } diff --git a/src/plugins/data_views/server/types.ts b/src/plugins/data_views/server/types.ts index 1fe900dc58113..2222e7d77a681 100644 --- a/src/plugins/data_views/server/types.ts +++ b/src/plugins/data_views/server/types.ts @@ -61,7 +61,6 @@ export interface DataViewsServerPluginStart { */ export interface DataViewsServerPluginSetup { enableRollups: () => void; - setGetUserId: (getUserId: GetUserId) => void; } /** diff --git a/x-pack/plugins/security/server/plugin.ts b/x-pack/plugins/security/server/plugin.ts index e87881b1a824e..2e3f5199d584c 100644 --- a/x-pack/plugins/security/server/plugin.ts +++ b/x-pack/plugins/security/server/plugin.ts @@ -244,14 +244,7 @@ export class SecurityPlugin public setup( core: CoreSetup, - { - features, - licensing, - taskManager, - usageCollection, - spaces, - dataViews, - }: PluginSetupDependencies + { features, licensing, taskManager, usageCollection, spaces }: PluginSetupDependencies ) { this.kibanaIndexName = core.savedObjects.getDefaultIndex(); const config$ = this.initializerContext.config.create>().pipe( @@ -268,13 +261,6 @@ export class SecurityPlugin const config = this.getConfig(); const kibanaIndexName = this.getKibanaIndexName(); - async function getUserId(request: KibanaRequest): Promise { - const [, , security] = await core.getStartServices(); - return security.authc.getCurrentUser(request)?.profile_uid; - } - - dataViews.setGetUserId(getUserId); - // A subset of `start` services we need during `setup`. const startServicesPromise = core.getStartServices().then(([coreServices, depsServices]) => ({ elasticsearch: coreServices.elasticsearch, From 6693cf681f3b8d966a34e330df1cea5d0a1ac9ad Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 15 Nov 2023 03:22:12 +0000 Subject: [PATCH 063/168] [CI] Auto-commit changed files from 'node scripts/lint_ts_projects --fix' --- src/plugins/data_views/tsconfig.json | 1 + 1 file changed, 1 insertion(+) diff --git a/src/plugins/data_views/tsconfig.json b/src/plugins/data_views/tsconfig.json index e5613323bc222..cf3d7aa75e551 100644 --- a/src/plugins/data_views/tsconfig.json +++ b/src/plugins/data_views/tsconfig.json @@ -33,6 +33,7 @@ "@kbn/object-versioning", "@kbn/core-saved-objects-server", "@kbn/logging", + "@kbn/security-plugin", ], "exclude": [ "target/**/*", From 7b384d4841dabfc42e1b796711c3a35b8f574eab Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Tue, 14 Nov 2023 21:29:40 -0600 Subject: [PATCH 064/168] Revert "refactor security / data view plugin dependency" This reverts commit 42bd2f41f21bf49031bef64b4c44531486c60660. --- src/plugins/data_views/kibana.jsonc | 1 - src/plugins/data_views/server/plugin.ts | 25 ++++-------------------- src/plugins/data_views/server/types.ts | 1 + x-pack/plugins/security/server/plugin.ts | 16 ++++++++++++++- 4 files changed, 20 insertions(+), 23 deletions(-) diff --git a/src/plugins/data_views/kibana.jsonc b/src/plugins/data_views/kibana.jsonc index 7789383b48ba4..7595f8e68b3c4 100644 --- a/src/plugins/data_views/kibana.jsonc +++ b/src/plugins/data_views/kibana.jsonc @@ -18,7 +18,6 @@ "requiredBundles": [ "kibanaUtils" ], - "runtimePluginDependencies" : ["security"], "extraPublicDirs": [ "common" ] diff --git a/src/plugins/data_views/server/plugin.ts b/src/plugins/data_views/server/plugin.ts index 91746655e7521..bdb1ac192d5a8 100644 --- a/src/plugins/data_views/server/plugin.ts +++ b/src/plugins/data_views/server/plugin.ts @@ -6,15 +6,7 @@ * Side Public License, v 1. */ -import { - CoreSetup, - CoreStart, - Logger, - Plugin, - PluginInitializerContext, - KibanaRequest, -} from '@kbn/core/server'; -import { SecurityPluginStart } from '@kbn/security-plugin/server'; +import { CoreSetup, CoreStart, Logger, Plugin, PluginInitializerContext } from '@kbn/core/server'; import { dataViewsServiceFactory } from './data_views_service_factory'; import { registerRoutes } from './routes'; import { dataViewSavedObjectType } from './saved_objects'; @@ -59,18 +51,6 @@ export class DataViewsServerPlugin core.capabilities.registerProvider(capabilitiesProvider); core.uiSettings.register(cacheMaxAge); const dataViewRestCounter = usageCollection?.createUsageCounter('dataViewsRestApi'); - core.plugins.onStart<{ security: SecurityPluginStart }>('security').then(({ security }) => { - if (security.found) { - const getUserId = async function getUserId( - request: KibanaRequest - ): Promise { - return security.contract.authc.getCurrentUser(request)?.profile_uid; - }; - this.getUserId = getUserId; - } else { - throw new Error('Security plugin is not available, but is required for Data Views plugin'); - } - }); registerRoutes({ http: core.http, @@ -97,6 +77,9 @@ export class DataViewsServerPlugin return { enableRollups: () => (this.rollupsEnabled = true), + setGetUserId: (getUserId: GetUserId) => { + this.getUserId = getUserId; + }, }; } diff --git a/src/plugins/data_views/server/types.ts b/src/plugins/data_views/server/types.ts index 2222e7d77a681..1fe900dc58113 100644 --- a/src/plugins/data_views/server/types.ts +++ b/src/plugins/data_views/server/types.ts @@ -61,6 +61,7 @@ export interface DataViewsServerPluginStart { */ export interface DataViewsServerPluginSetup { enableRollups: () => void; + setGetUserId: (getUserId: GetUserId) => void; } /** diff --git a/x-pack/plugins/security/server/plugin.ts b/x-pack/plugins/security/server/plugin.ts index 2e3f5199d584c..e87881b1a824e 100644 --- a/x-pack/plugins/security/server/plugin.ts +++ b/x-pack/plugins/security/server/plugin.ts @@ -244,7 +244,14 @@ export class SecurityPlugin public setup( core: CoreSetup, - { features, licensing, taskManager, usageCollection, spaces }: PluginSetupDependencies + { + features, + licensing, + taskManager, + usageCollection, + spaces, + dataViews, + }: PluginSetupDependencies ) { this.kibanaIndexName = core.savedObjects.getDefaultIndex(); const config$ = this.initializerContext.config.create>().pipe( @@ -261,6 +268,13 @@ export class SecurityPlugin const config = this.getConfig(); const kibanaIndexName = this.getKibanaIndexName(); + async function getUserId(request: KibanaRequest): Promise { + const [, , security] = await core.getStartServices(); + return security.authc.getCurrentUser(request)?.profile_uid; + } + + dataViews.setGetUserId(getUserId); + // A subset of `start` services we need during `setup`. const startServicesPromise = core.getStartServices().then(([coreServices, depsServices]) => ({ elasticsearch: coreServices.elasticsearch, From b009721abd6f625335f6adeb187db9c471cd9803 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 15 Nov 2023 03:41:09 +0000 Subject: [PATCH 065/168] [CI] Auto-commit changed files from 'node scripts/lint_ts_projects --fix' --- src/plugins/data_views/tsconfig.json | 1 - 1 file changed, 1 deletion(-) diff --git a/src/plugins/data_views/tsconfig.json b/src/plugins/data_views/tsconfig.json index cf3d7aa75e551..e5613323bc222 100644 --- a/src/plugins/data_views/tsconfig.json +++ b/src/plugins/data_views/tsconfig.json @@ -33,7 +33,6 @@ "@kbn/object-versioning", "@kbn/core-saved-objects-server", "@kbn/logging", - "@kbn/security-plugin", ], "exclude": [ "target/**/*", From 5c582327feecda80d5cf8b92c9ae03c7b03e35a5 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Tue, 14 Nov 2023 22:35:57 -0600 Subject: [PATCH 066/168] remove circ dependency --- src/plugins/data_views/tsconfig.json | 1 - 1 file changed, 1 deletion(-) diff --git a/src/plugins/data_views/tsconfig.json b/src/plugins/data_views/tsconfig.json index cf3d7aa75e551..e5613323bc222 100644 --- a/src/plugins/data_views/tsconfig.json +++ b/src/plugins/data_views/tsconfig.json @@ -33,7 +33,6 @@ "@kbn/object-versioning", "@kbn/core-saved-objects-server", "@kbn/logging", - "@kbn/security-plugin", ], "exclude": [ "target/**/*", From 6ea9098ae910f896fa881b8c520efa409bda6051 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Tue, 14 Nov 2023 23:38:37 -0600 Subject: [PATCH 067/168] disable flaky test --- .../management/data_views/_scripted_fields_classic_table.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/functional/apps/management/data_views/_scripted_fields_classic_table.ts b/test/functional/apps/management/data_views/_scripted_fields_classic_table.ts index 0ca095811c0b4..7cbda9f04353f 100644 --- a/test/functional/apps/management/data_views/_scripted_fields_classic_table.ts +++ b/test/functional/apps/management/data_views/_scripted_fields_classic_table.ts @@ -109,7 +109,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); }); - describe('creating and using Painless numeric scripted fields', function describeIndexTests() { + describe.skip('creating and using Painless numeric scripted fields', function describeIndexTests() { const scriptedPainlessFieldName = 'ram_Pain1'; it('should create scripted field', async function () { From 864d9ef49e11fdaa58f8d2412f1edffb08860a43 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Wed, 15 Nov 2023 08:04:02 -0600 Subject: [PATCH 068/168] remove discover refresh button and turn off cache --- src/plugins/data_views/server/ui_settings.ts | 2 +- .../public/dataview_picker/change_dataview.tsx | 17 ----------------- test/functional/page_objects/discover_page.ts | 5 +++-- 3 files changed, 4 insertions(+), 20 deletions(-) diff --git a/src/plugins/data_views/server/ui_settings.ts b/src/plugins/data_views/server/ui_settings.ts index b276d440a06f4..99175f0137dc9 100644 --- a/src/plugins/data_views/server/ui_settings.ts +++ b/src/plugins/data_views/server/ui_settings.ts @@ -14,7 +14,7 @@ export const cacheMaxAge = { name: i18n.translate('dataViews.advancedSettings.cacheMaxAgeTitle', { defaultMessage: 'Field cache max age', }), - value: 300, + value: 0, description: i18n.translate('dataViews.advancedSettings.cacheMaxAgeText', { defaultMessage: "Sets the 'max-age' cache header value for data view fields API requests. A value of 0 will disable caching. A hard reload of Kibana may be necessary for your browser to pick up the new setting.", diff --git a/src/plugins/unified_search/public/dataview_picker/change_dataview.tsx b/src/plugins/unified_search/public/dataview_picker/change_dataview.tsx index f1e19be1c72ce..e1565f1ff6b0d 100644 --- a/src/plugins/unified_search/public/dataview_picker/change_dataview.tsx +++ b/src/plugins/unified_search/public/dataview_picker/change_dataview.tsx @@ -217,23 +217,6 @@ export function ChangeDataView({ ) : ( ), - { - const dataView = await dataViews.get(currentDataViewId!); - await dataViews.refreshFields(dataView, undefined, true); - if (onEditDataView) { - onEditDataView(dataView); - } - setPopoverIsOpen(false); - }} - > - {i18n.translate('unifiedSearch.query.queryBar.indexPattern.refreshFieldButton', { - defaultMessage: 'Refresh field list for this data view', - })} - , ); } diff --git a/test/functional/page_objects/discover_page.ts b/test/functional/page_objects/discover_page.ts index 0d9bdd9b5c338..903b8875405a1 100644 --- a/test/functional/page_objects/discover_page.ts +++ b/test/functional/page_objects/discover_page.ts @@ -433,9 +433,10 @@ export class DiscoverPageObject extends FtrService { }); } + // disabled while `data_views:cache_max_age` ui setting is set to zero public async refreshFieldList() { - await this.clickIndexPatternActions(); - await this.testSubjects.click('data-view-refresh-fields'); + // await this.clickIndexPatternActions(); + // await this.testSubjects.click('data-view-refresh-fields'); } public async clickAddNewField() { From 2d7ab2f5297b4ba6eecb0055cd570ff77fd31e29 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Wed, 15 Nov 2023 09:11:08 -0600 Subject: [PATCH 069/168] fix api integration test --- test/api_integration/apis/data_views/fields_route/cache.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/api_integration/apis/data_views/fields_route/cache.ts b/test/api_integration/apis/data_views/fields_route/cache.ts index 4e852949405cf..727a64fdfa061 100644 --- a/test/api_integration/apis/data_views/fields_route/cache.ts +++ b/test/api_integration/apis/data_views/fields_route/cache.ts @@ -24,7 +24,8 @@ export default function ({ getService }: FtrProviderContext) { esArchiver.unload('test/api_integration/fixtures/es_archiver/index_patterns/basic_index') ); - it('are present', async () => { + // disabled since caching is disabled. Try to load uiSettings so this test can be enabled + it.skip('are present', async () => { const response = await supertest .get(FIELDS_PATH) .set(ELASTIC_HTTP_VERSION_HEADER, INITIAL_REST_VERSION_INTERNAL) From dbee9016f70168682fa63873d1060dae93ba5673 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Wed, 15 Nov 2023 16:47:28 -0600 Subject: [PATCH 070/168] cleanup --- .../data_views/public/data_views/data_views_api_client.ts | 7 ------- src/plugins/data_views/server/ui_settings.ts | 2 +- test/functional/apps/discover/group3/_sidebar.ts | 2 -- .../discover_ml_uptime/discover/search_source_alert.ts | 8 -------- 4 files changed, 1 insertion(+), 18 deletions(-) diff --git a/src/plugins/data_views/public/data_views/data_views_api_client.ts b/src/plugins/data_views/public/data_views/data_views_api_client.ts index 43d670136a586..2612ddcf98bb0 100644 --- a/src/plugins/data_views/public/data_views/data_views_api_client.ts +++ b/src/plugins/data_views/public/data_views/data_views_api_client.ts @@ -46,10 +46,6 @@ export class DataViewsApiClient implements IDataViewsApiClient { forceRefresh?: boolean ): Promise | undefined> { const asResponse = true; - // circle back to this, will likely need changes to any code that loads fields - // setting to true skips automatic json parsing - // const rawResponse = true; - const rawResponse = false; const cacheOptions = forceRefresh ? { cache: 'no-cache' as RequestCache } : {}; const userId = await this.getCurrentUserId(); @@ -62,7 +58,6 @@ export class DataViewsApiClient implements IDataViewsApiClient { version, ...cacheOptions, asResponse, - rawResponse, headers: { 'user-hash': userHash }, }); @@ -129,8 +124,6 @@ export class DataViewsApiClient implements IDataViewsApiClient { this._getUrl(['has_user_index_pattern']) ); - // const body = await response?.response?.json(); - // return body?.result ?? false; return response?.body?.result ?? false; } } diff --git a/src/plugins/data_views/server/ui_settings.ts b/src/plugins/data_views/server/ui_settings.ts index 99175f0137dc9..b4abf665709b5 100644 --- a/src/plugins/data_views/server/ui_settings.ts +++ b/src/plugins/data_views/server/ui_settings.ts @@ -12,7 +12,7 @@ import { schema } from '@kbn/config-schema'; export const cacheMaxAge = { 'data_views:cache_max_age': { name: i18n.translate('dataViews.advancedSettings.cacheMaxAgeTitle', { - defaultMessage: 'Field cache max age', + defaultMessage: 'Field cache max age (in seconds)', }), value: 0, description: i18n.translate('dataViews.advancedSettings.cacheMaxAgeText', { diff --git a/test/functional/apps/discover/group3/_sidebar.ts b/test/functional/apps/discover/group3/_sidebar.ts index 84a76bd5314e9..259ac5066e5d6 100644 --- a/test/functional/apps/discover/group3/_sidebar.ts +++ b/test/functional/apps/discover/group3/_sidebar.ts @@ -286,7 +286,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); }); - // describe('renders field groups', function () { it('should show field list groups excluding subfields', async function () { await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded(); @@ -477,7 +476,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { ); }); - // it('should work correctly for a data view for a missing index', async function () { // but we are skipping importing the index itself await kibanaServer.importExport.load( diff --git a/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts b/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts index 403522855a7e2..929e582d36191 100644 --- a/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts +++ b/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts @@ -308,10 +308,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { describe('Search source Alert', () => { before(async () => { - // todo try removing this - await kibanaServer.uiSettings.replace({ - 'data_views:fields_max_cache': '0', - }); await security.testUser.setRoles(['discover_alert']); log.debug('create source indices'); @@ -335,9 +331,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await deleteConnector(connectorId); await security.testUser.restoreDefaults(); await kibanaServer.savedObjects.cleanStandardList(); - await kibanaServer.uiSettings.replace({ - 'data_views:fields_max_cache': '300', - }); }); it('should create an alert when there is no data view', async () => { @@ -406,7 +399,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await checkInitialRuleParamsState(SOURCE_DATA_VIEW, true); }); - // this test fails with caching in place it('should navigate to alert results via link provided in notification', async () => { await openAlertResults(RULE_NAME); await checkInitialRuleParamsState(SOURCE_DATA_VIEW); From 80249e55e0d12f6af4abb6d9573c72638a1d57a4 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Wed, 15 Nov 2023 20:45:03 -0600 Subject: [PATCH 071/168] remove skips --- .../apps/management/data_views/_scripted_fields.ts | 8 ++++---- .../data_views/_scripted_fields_classic_table.ts | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/test/functional/apps/management/data_views/_scripted_fields.ts b/test/functional/apps/management/data_views/_scripted_fields.ts index 1116a482a77f1..2c70161a3bc43 100644 --- a/test/functional/apps/management/data_views/_scripted_fields.ts +++ b/test/functional/apps/management/data_views/_scripted_fields.ts @@ -44,7 +44,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { 'unifiedFieldList', ]); - describe.skip('scripted fields', function () { + describe('scripted fields', function () { this.tags(['skipFirefox']); before(async function () { @@ -145,7 +145,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); }); - describe.skip('discover scripted field', async () => { + describe('discover scripted field', async () => { before(async () => { const from = 'Sep 17, 2015 @ 06:31:44.000'; const to = 'Sep 18, 2015 @ 18:31:44.000'; @@ -226,7 +226,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); }); - describe.skip('creating and using Painless string scripted fields', function describeIndexTests() { + describe('creating and using Painless string scripted fields', function describeIndexTests() { const scriptedPainlessFieldName2 = 'painString'; before(async () => { @@ -330,7 +330,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); }); - describe.skip('creating and using Painless boolean scripted fields', function describeIndexTests() { + describe('creating and using Painless boolean scripted fields', function describeIndexTests() { const scriptedPainlessFieldName2 = 'painBool'; before(async () => { diff --git a/test/functional/apps/management/data_views/_scripted_fields_classic_table.ts b/test/functional/apps/management/data_views/_scripted_fields_classic_table.ts index 7cbda9f04353f..0ca095811c0b4 100644 --- a/test/functional/apps/management/data_views/_scripted_fields_classic_table.ts +++ b/test/functional/apps/management/data_views/_scripted_fields_classic_table.ts @@ -109,7 +109,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); }); - describe.skip('creating and using Painless numeric scripted fields', function describeIndexTests() { + describe('creating and using Painless numeric scripted fields', function describeIndexTests() { const scriptedPainlessFieldName = 'ram_Pain1'; it('should create scripted field', async function () { From cd4049b76e69aae8700a8824bfc2b6b20581182e Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Fri, 17 Nov 2023 00:01:54 +0000 Subject: [PATCH 072/168] [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' --- .../public/data_views/data_views_api_client.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/plugins/data_views/public/data_views/data_views_api_client.test.ts b/src/plugins/data_views/public/data_views/data_views_api_client.test.ts index fcf7694ea5793..6aada9616737b 100644 --- a/src/plugins/data_views/public/data_views/data_views_api_client.test.ts +++ b/src/plugins/data_views/public/data_views/data_views_api_client.test.ts @@ -17,7 +17,9 @@ describe('IndexPatternsApiClient', () => { beforeEach(() => { fetchSpy = jest.spyOn(http, 'fetch').mockImplementation(() => Promise.resolve({})); - indexPatternsApiClient = new DataViewsApiClient(http as HttpSetup, () => Promise.resolve(undefined)); + indexPatternsApiClient = new DataViewsApiClient(http as HttpSetup, () => + Promise.resolve(undefined) + ); }); test('uses the right URI to fetch fields for wildcard', async function () { From e03a3ec0ca2b136014a720fa2aa9b2a789ebe71e Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Tue, 21 Nov 2023 10:29:03 -0600 Subject: [PATCH 073/168] reenable cache but only for classic environment --- src/plugins/data_views/common/types.ts | 1 + src/plugins/data_views/server/constants.ts | 2 ++ src/plugins/data_views/server/index.ts | 6 ++++++ src/plugins/data_views/server/plugin.ts | 9 ++++++++- .../data_views/server/rest_api_routes/internal/fields.ts | 6 +++++- src/plugins/data_views/server/ui_settings.ts | 3 ++- 6 files changed, 24 insertions(+), 3 deletions(-) diff --git a/src/plugins/data_views/common/types.ts b/src/plugins/data_views/common/types.ts index 43e1ec991776f..7fb7ddedbccd4 100644 --- a/src/plugins/data_views/common/types.ts +++ b/src/plugins/data_views/common/types.ts @@ -544,4 +544,5 @@ export interface HasDataService { export interface ClientConfigType { scriptedFieldsEnabled?: boolean; + fieldListCachingEnabled?: boolean; } diff --git a/src/plugins/data_views/server/constants.ts b/src/plugins/data_views/server/constants.ts index 040a321978e17..f204fffe8fc6e 100644 --- a/src/plugins/data_views/server/constants.ts +++ b/src/plugins/data_views/server/constants.ts @@ -93,3 +93,5 @@ export const INITIAL_REST_VERSION = '2023-10-31'; */ export const INITIAL_REST_VERSION_INTERNAL = '1'; + +export const DEFAULT_FIELD_CACHE_FRESHNESS = 5; diff --git a/src/plugins/data_views/server/index.ts b/src/plugins/data_views/server/index.ts index 0e4034105d8bf..c1070e1cce541 100644 --- a/src/plugins/data_views/server/index.ts +++ b/src/plugins/data_views/server/index.ts @@ -47,6 +47,12 @@ const configSchema = schema.object({ schema.boolean({ defaultValue: false }), schema.never() ), + fieldListCachingEnabled: schema.conditional( + schema.contextRef('serverless'), + true, + schema.boolean({ defaultValue: false }), + schema.never() + ), }); type ConfigType = TypeOf; diff --git a/src/plugins/data_views/server/plugin.ts b/src/plugins/data_views/server/plugin.ts index bdb1ac192d5a8..c037545429247 100644 --- a/src/plugins/data_views/server/plugin.ts +++ b/src/plugins/data_views/server/plugin.ts @@ -49,7 +49,14 @@ export class DataViewsServerPlugin ) { core.savedObjects.registerType(dataViewSavedObjectType); core.capabilities.registerProvider(capabilitiesProvider); - core.uiSettings.register(cacheMaxAge); + + const config = this.initializerContext.config.get(); + + // add conditional + if (config.fieldListCachingEnabled) { + core.uiSettings.register(cacheMaxAge); + } + const dataViewRestCounter = usageCollection?.createUsageCounter('dataViewsRestApi'); registerRoutes({ diff --git a/src/plugins/data_views/server/rest_api_routes/internal/fields.ts b/src/plugins/data_views/server/rest_api_routes/internal/fields.ts index 2838c47668b5b..fc56c10e92bfd 100644 --- a/src/plugins/data_views/server/rest_api_routes/internal/fields.ts +++ b/src/plugins/data_views/server/rest_api_routes/internal/fields.ts @@ -17,6 +17,7 @@ import type { import type { FieldDescriptorRestResponse } from '../route_types'; import { FIELDS_PATH as path } from '../../../common/constants'; import { parseFields, IBody, IQuery, querySchema } from './fields_for'; +import { DEFAULT_FIELD_CACHE_FRESHNESS } from '../../constants'; export function calculateHash(srcBuffer: Buffer) { const hash = createHash('sha1'); @@ -82,7 +83,10 @@ const handler: ( 'user-hash': userHash, }; - const cacheMaxAge = await uiSettings.get('data_views:cache_max_age'); + // field cache is configurable in classic environment but not on serverless + const cacheMaxAge = + (await uiSettings.get('data_views:cache_max_age')) || + DEFAULT_FIELD_CACHE_FRESHNESS; if (cacheMaxAge && fields.length) { const stale = 365 * 24 * 60 * 60 - cacheMaxAge; diff --git a/src/plugins/data_views/server/ui_settings.ts b/src/plugins/data_views/server/ui_settings.ts index b4abf665709b5..7e4dabc441bf9 100644 --- a/src/plugins/data_views/server/ui_settings.ts +++ b/src/plugins/data_views/server/ui_settings.ts @@ -8,13 +8,14 @@ import { i18n } from '@kbn/i18n'; import { schema } from '@kbn/config-schema'; +import { DEFAULT_FIELD_CACHE_FRESHNESS } from './constants'; export const cacheMaxAge = { 'data_views:cache_max_age': { name: i18n.translate('dataViews.advancedSettings.cacheMaxAgeTitle', { defaultMessage: 'Field cache max age (in seconds)', }), - value: 0, + value: DEFAULT_FIELD_CACHE_FRESHNESS, description: i18n.translate('dataViews.advancedSettings.cacheMaxAgeText', { defaultMessage: "Sets the 'max-age' cache header value for data view fields API requests. A value of 0 will disable caching. A hard reload of Kibana may be necessary for your browser to pick up the new setting.", From 5c69712ed10ad66a374f41abaea47e1d6e018afc Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Wed, 22 Nov 2023 00:19:49 -0600 Subject: [PATCH 074/168] type cleanup --- src/plugins/data_views/common/data_views/data_view.ts | 2 +- src/plugins/data_views/common/data_views/data_views.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/plugins/data_views/common/data_views/data_view.ts b/src/plugins/data_views/common/data_views/data_view.ts index 7e33e3f16b97e..55c709f31e596 100644 --- a/src/plugins/data_views/common/data_views/data_view.ts +++ b/src/plugins/data_views/common/data_views/data_view.ts @@ -81,7 +81,7 @@ export class DataView extends AbstractDataView implements DataViewBase { getEtag = () => this.etag; - setEtag = (etag: string) => (this.etag = etag); + setEtag = (etag: string | undefined) => (this.etag = etag); /** * Returns scripted fields diff --git a/src/plugins/data_views/common/data_views/data_views.ts b/src/plugins/data_views/common/data_views/data_views.ts index 27d77b39a8e52..fd2b2b5e8d1ff 100644 --- a/src/plugins/data_views/common/data_views/data_views.ts +++ b/src/plugins/data_views/common/data_views/data_views.ts @@ -577,7 +577,7 @@ export class DataViewsService { if (indexPattern.getEtag() && etag === indexPattern.getEtag()) { return; } else { - indexPattern.setEtag(etag!); + indexPattern.setEtag(etag); } fields.forEach((field) => (field.isMapped = true)); @@ -862,7 +862,7 @@ export class DataViewsService { : {}; const indexPattern = await this.createFromSpec(spec, true, displayErrors); - indexPattern.setEtag(etag!); + indexPattern.setEtag(etag); indexPattern.matchedIndices = indices; indexPattern.resetOriginalSavedObjectBody(); return indexPattern; From 377a5a65b1a601f01f6be620710625cf2af305a8 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Wed, 22 Nov 2023 14:04:16 -0600 Subject: [PATCH 075/168] remove user hash from server response --- .../data_views/server/rest_api_routes/internal/fields.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/plugins/data_views/server/rest_api_routes/internal/fields.ts b/src/plugins/data_views/server/rest_api_routes/internal/fields.ts index fc56c10e92bfd..715ace2975584 100644 --- a/src/plugins/data_views/server/rest_api_routes/internal/fields.ts +++ b/src/plugins/data_views/server/rest_api_routes/internal/fields.ts @@ -72,15 +72,15 @@ const handler: ( const etag = calculateHash(Buffer.from(JSON.stringify(body))); - const getUserId = getUserIdGetter(); - const userId = await getUserId(request); - const userHash = userId ? calculateHash(Buffer.from(userId)) : ''; + // const getUserId = getUserIdGetter(); + // const userId = await getUserId(request); + // const userHash = userId ? calculateHash(Buffer.from(userId)) : ''; const headers: Record = { 'content-type': 'application/json', etag, vary: 'accept-encoding, user-hash', - 'user-hash': userHash, + // 'user-hash': userHash, }; // field cache is configurable in classic environment but not on serverless From 1a9a450af4ddd037517092ef5629683dce569b11 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Wed, 22 Nov 2023 15:59:33 -0600 Subject: [PATCH 076/168] remove server side user hash header --- src/plugins/data_views/server/plugin.ts | 6 ------ .../server/rest_api_routes/internal/fields.ts | 19 +++++-------------- src/plugins/data_views/server/routes.ts | 6 ++---- src/plugins/data_views/server/types.ts | 3 --- x-pack/plugins/security/server/plugin.ts | 18 +----------------- 5 files changed, 8 insertions(+), 44 deletions(-) diff --git a/src/plugins/data_views/server/plugin.ts b/src/plugins/data_views/server/plugin.ts index c037545429247..828736feaaa93 100644 --- a/src/plugins/data_views/server/plugin.ts +++ b/src/plugins/data_views/server/plugin.ts @@ -21,7 +21,6 @@ import { DataViewsServerPluginStart, DataViewsServerPluginSetupDependencies, DataViewsServerPluginStartDependencies, - GetUserId, } from './types'; import { DataViewsStorage } from './content_management'; import { cacheMaxAge } from './ui_settings'; @@ -37,7 +36,6 @@ export class DataViewsServerPlugin { private readonly logger: Logger; private rollupsEnabled: boolean = false; - private getUserId: GetUserId = async () => undefined; constructor(private readonly initializerContext: PluginInitializerContext) { this.logger = initializerContext.logger.get('dataView'); @@ -64,7 +62,6 @@ export class DataViewsServerPlugin getStartServices: core.getStartServices, isRollupsEnabled: () => this.rollupsEnabled, dataViewRestCounter, - getUserIdGetter: () => this.getUserId, }); expressions.registerFunction(getIndexPatternLoad({ getStartServices: core.getStartServices })); @@ -84,9 +81,6 @@ export class DataViewsServerPlugin return { enableRollups: () => (this.rollupsEnabled = true), - setGetUserId: (getUserId: GetUserId) => { - this.getUserId = getUserId; - }, }; } diff --git a/src/plugins/data_views/server/rest_api_routes/internal/fields.ts b/src/plugins/data_views/server/rest_api_routes/internal/fields.ts index 715ace2975584..c711ae405a647 100644 --- a/src/plugins/data_views/server/rest_api_routes/internal/fields.ts +++ b/src/plugins/data_views/server/rest_api_routes/internal/fields.ts @@ -7,7 +7,7 @@ */ import { createHash } from 'crypto'; -import { IRouter, RequestHandler, StartServicesAccessor, KibanaRequest } from '@kbn/core/server'; +import { IRouter, RequestHandler, StartServicesAccessor } from '@kbn/core/server'; import { unwrapEtag } from '../../../common/utils'; import { IndexPatternsFetcher } from '../../fetcher'; import type { @@ -25,11 +25,8 @@ export function calculateHash(srcBuffer: Buffer) { return hash.digest('hex'); } -const handler: ( - isRollupsEnabled: () => boolean, - getUserId: () => (kibanaRequest: KibanaRequest) => Promise -) => RequestHandler<{}, IQuery, IBody> = - (isRollupsEnabled, getUserIdGetter) => async (context, request, response) => { +const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, IBody> = + (isRollupsEnabled) => async (context, request, response) => { const core = await context.core; const uiSettings = core.uiSettings.client; const { asCurrentUser } = core.elasticsearch.client; @@ -72,15 +69,10 @@ const handler: ( const etag = calculateHash(Buffer.from(JSON.stringify(body))); - // const getUserId = getUserIdGetter(); - // const userId = await getUserId(request); - // const userHash = userId ? calculateHash(Buffer.from(userId)) : ''; - const headers: Record = { 'content-type': 'application/json', etag, vary: 'accept-encoding, user-hash', - // 'user-hash': userHash, }; // field cache is configurable in classic environment but not on serverless @@ -137,8 +129,7 @@ export const registerFields = async ( DataViewsServerPluginStartDependencies, DataViewsServerPluginStart >, - isRollupsEnabled: () => boolean, - getUserId: () => (request: KibanaRequest) => Promise + isRollupsEnabled: () => boolean ) => { - router.get({ path, validate: { query: querySchema } }, handler(isRollupsEnabled, getUserId)); + router.get({ path, validate: { query: querySchema } }, handler(isRollupsEnabled)); }; diff --git a/src/plugins/data_views/server/routes.ts b/src/plugins/data_views/server/routes.ts index 4acc09aafe2ab..d6ee36927ff80 100644 --- a/src/plugins/data_views/server/routes.ts +++ b/src/plugins/data_views/server/routes.ts @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -import { HttpServiceSetup, StartServicesAccessor, KibanaRequest } from '@kbn/core/server'; +import { HttpServiceSetup, StartServicesAccessor } from '@kbn/core/server'; import { UsageCounter } from '@kbn/usage-collection-plugin/server'; import { routes } from './rest_api_routes/public'; import type { DataViewsServerPluginStart, DataViewsServerPluginStartDependencies } from './types'; @@ -24,7 +24,6 @@ interface RegisterRoutesArgs { >; isRollupsEnabled: () => boolean; dataViewRestCounter?: UsageCounter; - getUserIdGetter: () => (request: KibanaRequest) => Promise; } export function registerRoutes({ @@ -32,7 +31,6 @@ export function registerRoutes({ getStartServices, dataViewRestCounter, isRollupsEnabled, - getUserIdGetter, }: RegisterRoutesArgs) { const router = http.createRouter(); @@ -40,6 +38,6 @@ export function registerRoutes({ registerExistingIndicesPath(router); registerFieldForWildcard(router, getStartServices, isRollupsEnabled); - registerFields(router, getStartServices, isRollupsEnabled, getUserIdGetter); + registerFields(router, getStartServices, isRollupsEnabled); registerHasDataViewsRoute(router); } diff --git a/src/plugins/data_views/server/types.ts b/src/plugins/data_views/server/types.ts index 1fe900dc58113..6bf438dc634bd 100644 --- a/src/plugins/data_views/server/types.ts +++ b/src/plugins/data_views/server/types.ts @@ -18,8 +18,6 @@ import { FieldFormatsSetup, FieldFormatsStart } from '@kbn/field-formats-plugin/ import type { ContentManagementServerSetup } from '@kbn/content-management-plugin/server'; import { DataViewsService } from '../common'; -export type GetUserId = (request: KibanaRequest) => Promise; - /** * Data Views service factory */ @@ -61,7 +59,6 @@ export interface DataViewsServerPluginStart { */ export interface DataViewsServerPluginSetup { enableRollups: () => void; - setGetUserId: (getUserId: GetUserId) => void; } /** diff --git a/x-pack/plugins/security/server/plugin.ts b/x-pack/plugins/security/server/plugin.ts index e87881b1a824e..341ab79e97e5b 100644 --- a/x-pack/plugins/security/server/plugin.ts +++ b/x-pack/plugins/security/server/plugin.ts @@ -18,7 +18,6 @@ import type { Plugin, PluginInitializerContext, } from '@kbn/core/server'; -import type { DataViewsServerPluginSetup } from '@kbn/data-views-plugin/server'; import type { PluginSetupContract as FeaturesPluginSetup, PluginStartContract as FeaturesPluginStart, @@ -119,7 +118,6 @@ export interface PluginSetupDependencies { taskManager: TaskManagerSetupContract; usageCollection?: UsageCollectionSetup; spaces?: SpacesPluginSetup; - dataViews: DataViewsServerPluginSetup; } export interface PluginStartDependencies { @@ -244,14 +242,7 @@ export class SecurityPlugin public setup( core: CoreSetup, - { - features, - licensing, - taskManager, - usageCollection, - spaces, - dataViews, - }: PluginSetupDependencies + { features, licensing, taskManager, usageCollection, spaces }: PluginSetupDependencies ) { this.kibanaIndexName = core.savedObjects.getDefaultIndex(); const config$ = this.initializerContext.config.create>().pipe( @@ -268,13 +259,6 @@ export class SecurityPlugin const config = this.getConfig(); const kibanaIndexName = this.getKibanaIndexName(); - async function getUserId(request: KibanaRequest): Promise { - const [, , security] = await core.getStartServices(); - return security.authc.getCurrentUser(request)?.profile_uid; - } - - dataViews.setGetUserId(getUserId); - // A subset of `start` services we need during `setup`. const startServicesPromise = core.getStartServices().then(([coreServices, depsServices]) => ({ elasticsearch: coreServices.elasticsearch, From 5971b41217fb798392f55e4ced91250663952e74 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Wed, 22 Nov 2023 16:10:52 -0600 Subject: [PATCH 077/168] typefix --- src/plugins/data_views/server/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/plugins/data_views/server/index.ts b/src/plugins/data_views/server/index.ts index c1070e1cce541..591870bbad029 100644 --- a/src/plugins/data_views/server/index.ts +++ b/src/plugins/data_views/server/index.ts @@ -16,7 +16,6 @@ export type { DataViewsServerPluginStart, DataViewsServerPluginSetupDependencies, DataViewsServerPluginStartDependencies, - GetUserId, } from './types'; import { PluginInitializerContext } from '@kbn/core/server'; From fd7adb23c53ec7a7adbf2ea3366d3e5f0450cfba Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Wed, 22 Nov 2023 17:43:51 -0600 Subject: [PATCH 078/168] remove unneeded functional test changes --- test/functional/apps/discover/group3/_sidebar.ts | 2 -- test/functional/page_objects/discover_page.ts | 6 ------ x-pack/plugins/security/server/plugin.test.ts | 3 --- .../creation/index_pattern/creation_index_pattern.ts | 1 - .../creation_runtime_mappings.ts | 4 +--- .../apps/discover_ml_uptime/discover/search_source_alert.ts | 1 - .../functional/tests/discover_integration.ts | 1 - 7 files changed, 1 insertion(+), 17 deletions(-) diff --git a/test/functional/apps/discover/group3/_sidebar.ts b/test/functional/apps/discover/group3/_sidebar.ts index 259ac5066e5d6..01e357b7f01e6 100644 --- a/test/functional/apps/discover/group3/_sidebar.ts +++ b/test/functional/apps/discover/group3/_sidebar.ts @@ -17,7 +17,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { 'discover', 'timePicker', 'header', - 'settings', 'unifiedSearch', 'unifiedFieldList', ]); @@ -527,7 +526,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { ); await browser.refresh(); - await PageObjects.discover.refreshFieldList(); await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded(); expect(await PageObjects.unifiedFieldList.getSidebarAriaDescription()).to.be( diff --git a/test/functional/page_objects/discover_page.ts b/test/functional/page_objects/discover_page.ts index 903b8875405a1..6005bbd2de7ad 100644 --- a/test/functional/page_objects/discover_page.ts +++ b/test/functional/page_objects/discover_page.ts @@ -433,12 +433,6 @@ export class DiscoverPageObject extends FtrService { }); } - // disabled while `data_views:cache_max_age` ui setting is set to zero - public async refreshFieldList() { - // await this.clickIndexPatternActions(); - // await this.testSubjects.click('data-view-refresh-fields'); - } - public async clickAddNewField() { await this.retry.try(async () => { await this.testSubjects.click('indexPattern-add-field'); diff --git a/x-pack/plugins/security/server/plugin.test.ts b/x-pack/plugins/security/server/plugin.test.ts index 0426a1fd674eb..e838016e94524 100644 --- a/x-pack/plugins/security/server/plugin.test.ts +++ b/x-pack/plugins/security/server/plugin.test.ts @@ -51,9 +51,6 @@ describe('Security Plugin', () => { licensing: { license$: of({}), featureUsage: { register: jest.fn() } }, features: featuresPluginMock.createSetup(), taskManager: taskManagerMock.createSetup(), - dataViews: { - setGetUserId: jest.fn(), - }, } as unknown as PluginSetupDependencies; mockCoreStart = coreMock.createStart(); diff --git a/x-pack/test/functional/apps/transform/creation/index_pattern/creation_index_pattern.ts b/x-pack/test/functional/apps/transform/creation/index_pattern/creation_index_pattern.ts index 7384b5bae2c9a..69406c5830421 100644 --- a/x-pack/test/functional/apps/transform/creation/index_pattern/creation_index_pattern.ts +++ b/x-pack/test/functional/apps/transform/creation/index_pattern/creation_index_pattern.ts @@ -835,7 +835,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await transform.testExecution.logTestStep('should navigate to discover'); await transform.table.clickTransformRowAction(testData.transformId, 'Discover'); await pageObjects.discover.waitUntilSearchingHasFinished(); - await pageObjects.discover.refreshFieldList(); if (testData.discoverAdjustSuperDatePicker) { await transform.testExecution.logTestStep( diff --git a/x-pack/test/functional/apps/transform/creation/runtime_mappings_saved_search/creation_runtime_mappings.ts b/x-pack/test/functional/apps/transform/creation/runtime_mappings_saved_search/creation_runtime_mappings.ts index f975dec539622..3f0adc5783893 100644 --- a/x-pack/test/functional/apps/transform/creation/runtime_mappings_saved_search/creation_runtime_mappings.ts +++ b/x-pack/test/functional/apps/transform/creation/runtime_mappings_saved_search/creation_runtime_mappings.ts @@ -19,10 +19,9 @@ import { PivotTransformTestData, } from '../../helpers'; -export default function ({ getPageObjects, getService }: FtrProviderContext) { +export default function ({ getService }: FtrProviderContext) { const esArchiver = getService('esArchiver'); const transform = getService('transform'); - const PageObjects = getPageObjects(['discover']); const runtimeMappings = { rt_airline_lower: { @@ -447,7 +446,6 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { await transform.testExecution.logTestStep('redirects to Discover page'); await transform.wizard.redirectToDiscover(); - await PageObjects.discover.refreshFieldList(); if (isLatestTransformTestData(testData)) { const fromTime = 'Feb 7, 2016 @ 00:00:00.000'; diff --git a/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts b/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts index 929e582d36191..6b668f75d56d3 100644 --- a/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts +++ b/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts @@ -211,7 +211,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.discover.clickNewSearchButton(); // reset params await PageObjects.discover.selectIndexPattern(OUTPUT_DATA_VIEW); - await PageObjects.discover.refreshFieldList(); let ruleId: string; if (type === 'name') { diff --git a/x-pack/test/saved_object_tagging/functional/tests/discover_integration.ts b/x-pack/test/saved_object_tagging/functional/tests/discover_integration.ts index a132c493de83a..8258b4570ed9b 100644 --- a/x-pack/test/saved_object_tagging/functional/tests/discover_integration.ts +++ b/x-pack/test/saved_object_tagging/functional/tests/discover_integration.ts @@ -104,7 +104,6 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { beforeEach(async () => { await PageObjects.common.navigateToApp('discover'); await PageObjects.discover.selectIndexPattern('logstash-*'); - await PageObjects.discover.refreshFieldList(); await PageObjects.timePicker.setDefaultAbsoluteRange(); await PageObjects.header.waitUntilLoadingHasFinished(); }); From 55b8191b510ad8a1e174e606ec0f6794d3d8fe89 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Sun, 26 Nov 2023 20:47:08 -0600 Subject: [PATCH 079/168] implement versioned router on fields endpoint --- .../data_views/data_views_api_client.ts | 2 ++ .../apis/data_views/fields_route/cache.ts | 30 ++++++++----------- .../apis/data_views/fields_route/conflicts.ts | 4 +-- .../apis/data_views/fields_route/params.ts | 28 ++++++++--------- .../apis/data_views/fields_route/response.ts | 20 ++++++------- 5 files changed, 39 insertions(+), 45 deletions(-) diff --git a/src/plugins/data_views/public/data_views/data_views_api_client.ts b/src/plugins/data_views/public/data_views/data_views_api_client.ts index 2612ddcf98bb0..c65db9e5c736f 100644 --- a/src/plugins/data_views/public/data_views/data_views_api_client.ts +++ b/src/plugins/data_views/public/data_views/data_views_api_client.ts @@ -92,6 +92,7 @@ export class DataViewsApiClient implements IDataViewsApiClient { allowHidden, } = options; const path = indexFilter ? FIELDS_FOR_WILDCARD_PATH : FIELDS_PATH; + const versionQueryParam = indexFilter ? {} : { apiVersion: version }; return this._request( path, @@ -104,6 +105,7 @@ export class DataViewsApiClient implements IDataViewsApiClient { include_unmapped: includeUnmapped, fields, allow_hidden: allowHidden, + ...versionQueryParam, }, indexFilter ? JSON.stringify({ index_filter: indexFilter }) : undefined, forceRefresh diff --git a/test/api_integration/apis/data_views/fields_route/cache.ts b/test/api_integration/apis/data_views/fields_route/cache.ts index 727a64fdfa061..f5b33de590c3b 100644 --- a/test/api_integration/apis/data_views/fields_route/cache.ts +++ b/test/api_integration/apis/data_views/fields_route/cache.ts @@ -6,7 +6,6 @@ * Side Public License, v 1. */ -import { ELASTIC_HTTP_VERSION_HEADER } from '@kbn/core-http-common'; import { INITIAL_REST_VERSION_INTERNAL } from '@kbn/data-views-plugin/server/constants'; import { FIELDS_PATH } from '@kbn/data-views-plugin/common/constants'; import expect from '@kbn/expect'; @@ -24,15 +23,12 @@ export default function ({ getService }: FtrProviderContext) { esArchiver.unload('test/api_integration/fixtures/es_archiver/index_patterns/basic_index') ); - // disabled since caching is disabled. Try to load uiSettings so this test can be enabled - it.skip('are present', async () => { - const response = await supertest - .get(FIELDS_PATH) - .set(ELASTIC_HTTP_VERSION_HEADER, INITIAL_REST_VERSION_INTERNAL) - .query({ - pattern: '*', - include_unmapped: true, - }); + it('are present', async () => { + const response = await supertest.get(FIELDS_PATH).query({ + pattern: '*', + include_unmapped: true, + apiVersion: INITIAL_REST_VERSION_INTERNAL, + }); const cacheControlHeader = response.get('cache-control'); @@ -44,21 +40,19 @@ export default function ({ getService }: FtrProviderContext) { }); it('returns 304 on matching etag', async () => { - const response = await supertest - .get(FIELDS_PATH) - .set(ELASTIC_HTTP_VERSION_HEADER, INITIAL_REST_VERSION_INTERNAL) - .query({ - pattern: '*', - include_unmapped: true, - }); + const response = await supertest.get(FIELDS_PATH).query({ + pattern: '*', + include_unmapped: true, + apiVersion: INITIAL_REST_VERSION_INTERNAL, + }); await supertest .get(FIELDS_PATH) - .set(ELASTIC_HTTP_VERSION_HEADER, INITIAL_REST_VERSION_INTERNAL) .set('If-None-Match', response.get('etag')) .query({ pattern: '*', include_unmapped: true, + apiVersion: INITIAL_REST_VERSION_INTERNAL, }) .expect(304); }); diff --git a/test/api_integration/apis/data_views/fields_route/conflicts.ts b/test/api_integration/apis/data_views/fields_route/conflicts.ts index 9ad861272c4cd..026ca58b6dba4 100644 --- a/test/api_integration/apis/data_views/fields_route/conflicts.ts +++ b/test/api_integration/apis/data_views/fields_route/conflicts.ts @@ -6,7 +6,6 @@ * Side Public License, v 1. */ -import { ELASTIC_HTTP_VERSION_HEADER } from '@kbn/core-http-common'; import { INITIAL_REST_VERSION_INTERNAL } from '@kbn/data-views-plugin/server/constants'; import { FIELDS_PATH } from '@kbn/data-views-plugin/common/constants'; import expect from '@kbn/expect'; @@ -27,8 +26,7 @@ export default function ({ getService }: FtrProviderContext) { it('flags fields with mismatched types as conflicting', () => supertest .get(FIELDS_PATH) - .set(ELASTIC_HTTP_VERSION_HEADER, INITIAL_REST_VERSION_INTERNAL) - .query({ pattern: 'logs-*' }) + .query({ pattern: 'logs-*', apiVersion: INITIAL_REST_VERSION_INTERNAL }) .expect(200) .then((resp) => { expect(resp.body).to.eql({ diff --git a/test/api_integration/apis/data_views/fields_route/params.ts b/test/api_integration/apis/data_views/fields_route/params.ts index 29d6c06e3a60b..7d111936426bb 100644 --- a/test/api_integration/apis/data_views/fields_route/params.ts +++ b/test/api_integration/apis/data_views/fields_route/params.ts @@ -6,7 +6,6 @@ * Side Public License, v 1. */ -import { ELASTIC_HTTP_VERSION_HEADER } from '@kbn/core-http-common'; import { INITIAL_REST_VERSION_INTERNAL } from '@kbn/data-views-plugin/server/constants'; import { FIELDS_PATH } from '@kbn/data-views-plugin/common/constants'; import { FtrProviderContext } from '../../../ftr_provider_context'; @@ -27,27 +26,28 @@ export default function ({ getService }: FtrProviderContext) { it('requires a pattern query param', () => supertest .get(FIELDS_PATH) - .set(ELASTIC_HTTP_VERSION_HEADER, INITIAL_REST_VERSION_INTERNAL) - .query({}) + .query({ + apiVersion: INITIAL_REST_VERSION_INTERNAL, + }) .expect(400)); it('accepts include_unmapped param', () => supertest .get(FIELDS_PATH) - .set(ELASTIC_HTTP_VERSION_HEADER, INITIAL_REST_VERSION_INTERNAL) .query({ pattern: '*', include_unmapped: true, + apiVersion: INITIAL_REST_VERSION_INTERNAL, }) .expect(200)); it('rejects unexpected query params', () => supertest .get(FIELDS_PATH) - .set(ELASTIC_HTTP_VERSION_HEADER, INITIAL_REST_VERSION_INTERNAL) .query({ pattern: randomness.word(), [randomness.word()]: randomness.word(), + apiVersion: INITIAL_REST_VERSION_INTERNAL, }) .expect(400)); @@ -55,50 +55,50 @@ export default function ({ getService }: FtrProviderContext) { it('accepts a JSON formatted fields query param', () => supertest .get(FIELDS_PATH) - .set(ELASTIC_HTTP_VERSION_HEADER, INITIAL_REST_VERSION_INTERNAL) .query({ pattern: '*', fields: JSON.stringify(['baz']), + apiVersion: INITIAL_REST_VERSION_INTERNAL, }) .expect(200)); it('accepts meta_fields query param in string array', () => supertest .get(FIELDS_PATH) - .set(ELASTIC_HTTP_VERSION_HEADER, INITIAL_REST_VERSION_INTERNAL) .query({ pattern: '*', fields: ['baz', 'foo'], + apiVersion: INITIAL_REST_VERSION_INTERNAL, }) .expect(200)); it('accepts single array fields query param', () => supertest .get(FIELDS_PATH) - .set(ELASTIC_HTTP_VERSION_HEADER, INITIAL_REST_VERSION_INTERNAL) .query({ pattern: '*', fields: ['baz'], + apiVersion: INITIAL_REST_VERSION_INTERNAL, }) .expect(200)); it('accepts single fields query param', () => supertest .get(FIELDS_PATH) - .set(ELASTIC_HTTP_VERSION_HEADER, INITIAL_REST_VERSION_INTERNAL) .query({ pattern: '*', fields: 'baz', + apiVersion: INITIAL_REST_VERSION_INTERNAL, }) .expect(200)); it('rejects a comma-separated list of fields', () => supertest .get(FIELDS_PATH) - .set(ELASTIC_HTTP_VERSION_HEADER, INITIAL_REST_VERSION_INTERNAL) .query({ pattern: '*', fields: 'foo,bar', + apiVersion: INITIAL_REST_VERSION_INTERNAL, }) .expect(400)); }); @@ -107,40 +107,40 @@ export default function ({ getService }: FtrProviderContext) { it('accepts a JSON formatted meta_fields query param', () => supertest .get(FIELDS_PATH) - .set(ELASTIC_HTTP_VERSION_HEADER, INITIAL_REST_VERSION_INTERNAL) .query({ pattern: '*', meta_fields: JSON.stringify(['meta']), + apiVersion: INITIAL_REST_VERSION_INTERNAL, }) .expect(200)); it('accepts meta_fields query param in string array', () => supertest .get(FIELDS_PATH) - .set(ELASTIC_HTTP_VERSION_HEADER, INITIAL_REST_VERSION_INTERNAL) .query({ pattern: '*', meta_fields: ['_id', 'meta'], + apiVersion: INITIAL_REST_VERSION_INTERNAL, }) .expect(200)); it('accepts single meta_fields query param', () => supertest .get(FIELDS_PATH) - .set(ELASTIC_HTTP_VERSION_HEADER, INITIAL_REST_VERSION_INTERNAL) .query({ pattern: '*', meta_fields: ['_id'], + apiVersion: INITIAL_REST_VERSION_INTERNAL, }) .expect(200)); it('rejects a comma-separated list of meta_fields', () => supertest .get(FIELDS_PATH) - .set(ELASTIC_HTTP_VERSION_HEADER, INITIAL_REST_VERSION_INTERNAL) .query({ pattern: '*', meta_fields: 'foo,bar', + apiVersion: INITIAL_REST_VERSION_INTERNAL, }) .expect(400)); }); diff --git a/test/api_integration/apis/data_views/fields_route/response.ts b/test/api_integration/apis/data_views/fields_route/response.ts index 47c8814b0c4b0..2c6aeb8d75ab7 100644 --- a/test/api_integration/apis/data_views/fields_route/response.ts +++ b/test/api_integration/apis/data_views/fields_route/response.ts @@ -6,7 +6,6 @@ * Side Public License, v 1. */ -import { ELASTIC_HTTP_VERSION_HEADER } from '@kbn/core-http-common'; import { INITIAL_REST_VERSION_INTERNAL } from '@kbn/data-views-plugin/server/constants'; import { FIELDS_PATH } from '@kbn/data-views-plugin/common/constants'; import expect from '@kbn/expect'; @@ -86,11 +85,11 @@ export default function ({ getService }: FtrProviderContext) { it('returns a flattened version of the fields in es', async () => { await supertest .get(FIELDS_PATH) - .set(ELASTIC_HTTP_VERSION_HEADER, INITIAL_REST_VERSION_INTERNAL) .query({ pattern: 'basic_index' }) .expect(200, { fields: testFields, indices: ['basic_index'], + apiVersion: INITIAL_REST_VERSION_INTERNAL, }) .then(ensureFieldsAreSorted); }); @@ -98,8 +97,11 @@ export default function ({ getService }: FtrProviderContext) { it('returns a single field as requested', async () => { await supertest .get(FIELDS_PATH) - .set(ELASTIC_HTTP_VERSION_HEADER, INITIAL_REST_VERSION_INTERNAL) - .query({ pattern: 'basic_index', fields: JSON.stringify(['bar']) }) + .query({ + pattern: 'basic_index', + fields: JSON.stringify(['bar']), + apiVersion: INITIAL_REST_VERSION_INTERNAL, + }) .expect(200, { fields: [testFields[0]], indices: ['basic_index'], @@ -109,10 +111,10 @@ export default function ({ getService }: FtrProviderContext) { it('always returns a field for all passed meta fields', async () => { await supertest .get(FIELDS_PATH) - .set(ELASTIC_HTTP_VERSION_HEADER, INITIAL_REST_VERSION_INTERNAL) .query({ pattern: 'basic_index', meta_fields: JSON.stringify(['_id', '_source', 'crazy_meta_field']), + apiVersion: INITIAL_REST_VERSION_INTERNAL, }) .expect(200, { fields: [ @@ -202,8 +204,7 @@ export default function ({ getService }: FtrProviderContext) { it('returns fields when one pattern exists and the other does not', async () => { await supertest .get(FIELDS_PATH) - .set(ELASTIC_HTTP_VERSION_HEADER, INITIAL_REST_VERSION_INTERNAL) - .query({ pattern: 'bad_index,basic_index' }) + .query({ pattern: 'bad_index,basic_index', apiVersion: INITIAL_REST_VERSION_INTERNAL }) .expect(200, { fields: testFields, indices: ['basic_index'], @@ -213,17 +214,16 @@ export default function ({ getService }: FtrProviderContext) { it('returns 404 when neither exists', async () => { await supertest .get(FIELDS_PATH) - .set(ELASTIC_HTTP_VERSION_HEADER, INITIAL_REST_VERSION_INTERNAL) - .query({ pattern: 'bad_index,bad_index_2' }) + .query({ pattern: 'bad_index,bad_index_2', apiVersion: INITIAL_REST_VERSION_INTERNAL }) .expect(404); }); it('returns 404 when no patterns exist', async () => { await supertest .get(FIELDS_PATH) - .set(ELASTIC_HTTP_VERSION_HEADER, INITIAL_REST_VERSION_INTERNAL) .query({ pattern: 'bad_index', + apiVersion: INITIAL_REST_VERSION_INTERNAL, }) .expect(404); }); From 89c4ff29e57e3c6d2fa54932e327b26bd040b912 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Sun, 26 Nov 2023 22:02:23 -0600 Subject: [PATCH 080/168] fix fields route --- .../data_views/server/rest_api_routes/internal/fields.ts | 9 +++++++-- .../server/rest_api_routes/internal/fields_for.ts | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/plugins/data_views/server/rest_api_routes/internal/fields.ts b/src/plugins/data_views/server/rest_api_routes/internal/fields.ts index c711ae405a647..b55b742535479 100644 --- a/src/plugins/data_views/server/rest_api_routes/internal/fields.ts +++ b/src/plugins/data_views/server/rest_api_routes/internal/fields.ts @@ -16,7 +16,7 @@ import type { } from '../../types'; import type { FieldDescriptorRestResponse } from '../route_types'; import { FIELDS_PATH as path } from '../../../common/constants'; -import { parseFields, IBody, IQuery, querySchema } from './fields_for'; +import { parseFields, IBody, IQuery, querySchema, validate } from './fields_for'; import { DEFAULT_FIELD_CACHE_FRESHNESS } from '../../constants'; export function calculateHash(srcBuffer: Buffer) { @@ -131,5 +131,10 @@ export const registerFields = async ( >, isRollupsEnabled: () => boolean ) => { - router.get({ path, validate: { query: querySchema } }, handler(isRollupsEnabled)); + router.versioned + .get({ path, access: 'internal', enableQueryVersion: true }) + .addVersion( + { version: '1', validate: { request: { query: querySchema }, response: validate.response } }, + handler(isRollupsEnabled) + ); }; diff --git a/src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts b/src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts index 7931dcce51e37..3c89cdef17955 100644 --- a/src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts +++ b/src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts @@ -97,7 +97,7 @@ const FieldDescriptorSchema = schema.object({ ), }); -const validate: FullValidationConfig = { +export const validate: FullValidationConfig = { request: { query: querySchema, // not available to get request From 6efa5f1766e11c281640ecd18a5d801ab583a4e2 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Sun, 26 Nov 2023 23:15:03 -0600 Subject: [PATCH 081/168] fix test --- test/api_integration/apis/data_views/fields_route/response.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/api_integration/apis/data_views/fields_route/response.ts b/test/api_integration/apis/data_views/fields_route/response.ts index 2c6aeb8d75ab7..880206f6fb498 100644 --- a/test/api_integration/apis/data_views/fields_route/response.ts +++ b/test/api_integration/apis/data_views/fields_route/response.ts @@ -85,11 +85,10 @@ export default function ({ getService }: FtrProviderContext) { it('returns a flattened version of the fields in es', async () => { await supertest .get(FIELDS_PATH) - .query({ pattern: 'basic_index' }) + .query({ pattern: 'basic_index', apiVersion: INITIAL_REST_VERSION_INTERNAL }) .expect(200, { fields: testFields, indices: ['basic_index'], - apiVersion: INITIAL_REST_VERSION_INTERNAL, }) .then(ensureFieldsAreSorted); }); From c5c993ee7af0590829000cc2c23be04ebde7e867 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Mon, 27 Nov 2023 00:03:09 -0600 Subject: [PATCH 082/168] add performance journey --- x-pack/test/scalability/apis/api.fields.json | 50 ++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 x-pack/test/scalability/apis/api.fields.json diff --git a/x-pack/test/scalability/apis/api.fields.json b/x-pack/test/scalability/apis/api.fields.json new file mode 100644 index 0000000000000..7e60dcdf6c4d6 --- /dev/null +++ b/x-pack/test/scalability/apis/api.fields.json @@ -0,0 +1,50 @@ +{ + "journeyName": "GET /internal/data_views/fields", + "scalabilitySetup": { + "responseTimeThreshold": { + "threshold1": 5000, + "threshold2": 10000, + "threshold3": 20000 + }, + "warmup": [ + { + "action": "constantUsersPerSec", + "userCount": 1, + "duration": "30s" + } + ], + "test": [ + { + "action": "rampUsersPerSec", + "minUsersCount": 1, + "maxUsersCount": 7, + "duration": "140s" + } + ], + "maxDuration": "5m" + }, + "testData": { + "esArchives": ["test/functional/fixtures/es_archiver/many_fields"] + }, + "streams": [ + { + "requests": [ + { + "http": { + "method": "GET", + "path": "/internal/data_views/fields", + "query": "?pattern=indices*&apiVersion=1", + "headers": { + "Cookie": "", + "Kbn-Version": "", + "Accept-Encoding": "gzip, deflate, br", + "Content-Type": "application/json", + "x-elastic-internal-origin": "kibana" + }, + "statusCode": 200 + } + } + ] + } + ] +} From 6d7212a29140145ee533d365381e2ccf3438423f Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Mon, 27 Nov 2023 00:20:59 -0600 Subject: [PATCH 083/168] remove one serialization --- .../data_views/server/rest_api_routes/internal/fields.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/plugins/data_views/server/rest_api_routes/internal/fields.ts b/src/plugins/data_views/server/rest_api_routes/internal/fields.ts index b55b742535479..454035f3b812f 100644 --- a/src/plugins/data_views/server/rest_api_routes/internal/fields.ts +++ b/src/plugins/data_views/server/rest_api_routes/internal/fields.ts @@ -67,7 +67,9 @@ const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, I indices, }; - const etag = calculateHash(Buffer.from(JSON.stringify(body))); + const bodyAsString = JSON.stringify(body); + + const etag = calculateHash(Buffer.from(bodyAsString)); const headers: Record = { 'content-type': 'application/json', @@ -100,7 +102,7 @@ const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, I } return response.ok({ - body, + body: bodyAsString, headers, }); } catch (error) { From ed73b7b7bb11416c322e41cbc5e04d905b5cadf8 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Mon, 27 Nov 2023 07:56:05 -0600 Subject: [PATCH 084/168] fix api integration test --- test/api_integration/apis/data_views/fields_route/cache.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/test/api_integration/apis/data_views/fields_route/cache.ts b/test/api_integration/apis/data_views/fields_route/cache.ts index f5b33de590c3b..978c048ee8b8e 100644 --- a/test/api_integration/apis/data_views/fields_route/cache.ts +++ b/test/api_integration/apis/data_views/fields_route/cache.ts @@ -36,7 +36,6 @@ export default function ({ getService }: FtrProviderContext) { expect(cacheControlHeader).to.contain('stale-while-revalidate'); expect(response.get('vary')).to.equal('accept-encoding, user-hash'); expect(response.get('etag')).to.not.be.empty(); - expect(response.get('user-hash')).to.equal(''); }); it('returns 304 on matching etag', async () => { From cdeec6877e718696ce82bf6f389fb43b36ae1301 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Wed, 29 Nov 2023 23:20:52 -0600 Subject: [PATCH 085/168] add performance journeys --- src/plugins/data_views/server/plugin.ts | 1 - .../apis/api.fields.32_fields.json | 50 +++++++++++++++++++ ...ields.json => api.fields.6800_fields.json} | 2 +- 3 files changed, 51 insertions(+), 2 deletions(-) create mode 100644 x-pack/test/scalability/apis/api.fields.32_fields.json rename x-pack/test/scalability/apis/{api.fields.json => api.fields.6800_fields.json} (94%) diff --git a/src/plugins/data_views/server/plugin.ts b/src/plugins/data_views/server/plugin.ts index 828736feaaa93..2da7d128b837a 100644 --- a/src/plugins/data_views/server/plugin.ts +++ b/src/plugins/data_views/server/plugin.ts @@ -50,7 +50,6 @@ export class DataViewsServerPlugin const config = this.initializerContext.config.get(); - // add conditional if (config.fieldListCachingEnabled) { core.uiSettings.register(cacheMaxAge); } diff --git a/x-pack/test/scalability/apis/api.fields.32_fields.json b/x-pack/test/scalability/apis/api.fields.32_fields.json new file mode 100644 index 0000000000000..03837b238e44e --- /dev/null +++ b/x-pack/test/scalability/apis/api.fields.32_fields.json @@ -0,0 +1,50 @@ +{ + "journeyName": "GET /internal/data_views/fields - 32 fields", + "scalabilitySetup": { + "responseTimeThreshold": { + "threshold1": 5000, + "threshold2": 10000, + "threshold3": 20000 + }, + "warmup": [ + { + "action": "constantUsersPerSec", + "userCount": 10, + "duration": "30s" + } + ], + "test": [ + { + "action": "rampUsersPerSec", + "minUsersCount": 10, + "maxUsersCount": 375, + "duration": "140s" + } + ], + "maxDuration": "5m" + }, + "testData": { + "esArchives": ["test/functional/fixtures/es_archiver/kibana_sample_data_flights"] + }, + "streams": [ + { + "requests": [ + { + "http": { + "method": "GET", + "path": "/internal/data_views/fields", + "query": "?pattern=kibana*&apiVersion=1", + "headers": { + "Cookie": "", + "Kbn-Version": "", + "Accept-Encoding": "gzip, deflate, br", + "Content-Type": "application/json", + "x-elastic-internal-origin": "kibana" + }, + "statusCode": 200 + } + } + ] + } + ] +} diff --git a/x-pack/test/scalability/apis/api.fields.json b/x-pack/test/scalability/apis/api.fields.6800_fields.json similarity index 94% rename from x-pack/test/scalability/apis/api.fields.json rename to x-pack/test/scalability/apis/api.fields.6800_fields.json index 7e60dcdf6c4d6..362eaa211acae 100644 --- a/x-pack/test/scalability/apis/api.fields.json +++ b/x-pack/test/scalability/apis/api.fields.6800_fields.json @@ -1,5 +1,5 @@ { - "journeyName": "GET /internal/data_views/fields", + "journeyName": "GET /internal/data_views/fields - 6800 fields", "scalabilitySetup": { "responseTimeThreshold": { "threshold1": 5000, From 26d3d6de42c5b0fcdcc3df7c7643b6fa6f13cb69 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Thu, 30 Nov 2023 22:38:54 +0100 Subject: [PATCH 086/168] Add new fields to list with correct types --- .../src/hooks/use_existing_fields.ts | 22 ++++++++++++- .../src/hooks/use_grouped_fields.ts | 32 +++++++++++++------ .../field_existing/field_existing_utils.ts | 11 +++++-- .../field_existing/load_field_existing.ts | 3 +- 4 files changed, 54 insertions(+), 14 deletions(-) diff --git a/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts b/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts index d099a6f2b85e2..588e0f60384b3 100644 --- a/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts +++ b/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts @@ -15,6 +15,7 @@ import { type DataPublicPluginStart } from '@kbn/data-plugin/public'; import type { DataView, DataViewsContract } from '@kbn/data-views-plugin/common'; import { getEsQueryConfig } from '@kbn/data-service/src/es_query'; import { reportPerformanceMetricEvent } from '@kbn/ebt-tools'; +import { DataViewField } from '@kbn/data-views-plugin/common'; import { loadFieldExisting } from '../services/field_existing'; import { ExistenceFetchStatus } from '../types'; @@ -24,6 +25,8 @@ const generateId = htmlIdGenerator(); export interface ExistingFieldsInfo { fetchStatus: ExistenceFetchStatus; existingFieldsByFieldNameMap: Record; + newFieldsByFieldNameMap: Record; + newFields?: DataViewField[]; numberOfFetches: number; hasDataViewRestrictions?: boolean; } @@ -54,12 +57,14 @@ export interface ExistingFieldsReader { hasFieldData: (dataViewId: string, fieldName: string) => boolean; getFieldsExistenceStatus: (dataViewId: string) => ExistenceFetchStatus; isFieldsExistenceInfoUnavailable: (dataViewId: string) => boolean; + getNewFields: (dataViewId: string) => DataViewField[]; } const initialData: ExistingFieldsByDataViewMap = {}; const unknownInfo: ExistingFieldsInfo = { fetchStatus: ExistenceFetchStatus.unknown, existingFieldsByFieldNameMap: {}, + newFieldsByFieldNameMap: {}, numberOfFetches: 0, }; @@ -157,6 +162,7 @@ export const useExistingFieldsFetcher = ( } info.existingFieldsByFieldNameMap = booleanMap(existingFieldNames); + info.newFields = result.newFields; info.fetchStatus = ExistenceFetchStatus.succeeded; } catch (error) { info.fetchStatus = ExistenceFetchStatus.failed; @@ -286,6 +292,19 @@ export const useExistingFieldsReader: () => ExistingFieldsReader = () => { [existingFieldsByDataViewMap] ); + const getNewFields = useCallback( + (dataViewId: string) => { + const info = existingFieldsByDataViewMap[dataViewId]; + + if (info?.fetchStatus === ExistenceFetchStatus.succeeded) { + return info?.newFields ?? []; + } + + return []; + }, + [existingFieldsByDataViewMap] + ); + const getFieldsExistenceInfo = useCallback( (dataViewId: string) => { return dataViewId ? existingFieldsByDataViewMap[dataViewId] : unknownInfo; @@ -321,8 +340,9 @@ export const useExistingFieldsReader: () => ExistingFieldsReader = () => { hasFieldData, getFieldsExistenceStatus, isFieldsExistenceInfoUnavailable, + getNewFields, }), - [hasFieldData, getFieldsExistenceStatus, isFieldsExistenceInfoUnavailable] + [hasFieldData, getFieldsExistenceStatus, isFieldsExistenceInfoUnavailable, getNewFields] ); }; diff --git a/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts b/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts index f51f9b0fffb2d..7822e71244d10 100644 --- a/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts +++ b/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts @@ -120,13 +120,23 @@ export function useGroupedFields({ }; const selectedFields = sortedSelectedFields || []; - const sortedFields = [...(allFields || [])].sort(sortFields); + const newFields = dataViewId ? fieldsExistenceReader.getNewFields(dataViewId) : []; + console.log({ newFields }); + const fieldsToSort = + allFields && newFields.length + ? allFields.map((field) => { + return (dataView?.getFieldByName(field.name) as unknown as T) ?? field; + }) + : allFields; + const sortedFields = [...(fieldsToSort || [])].sort(sortFields); + const groupedFields = { ...getDefaultFieldGroups(), ...groupBy(sortedFields, (field) => { if (!sortedSelectedFields && onSelectedFieldFilter && onSelectedFieldFilter(field)) { selectedFields.push(field); } + if (onSupportedFieldFilter && !onSupportedFieldFilter(field)) { return 'skippedFields'; } @@ -311,18 +321,19 @@ export function useGroupedFields({ return fieldGroupDefinitions; }, [ + sortedSelectedFields, allFields, - onSupportedFieldFilter, - onSelectedFieldFilter, - onOverrideFieldGroupDetails, - dataView, - dataViewId, - hasFieldDataHandler, - fieldsExistenceInfoUnavailable, + popularFieldsLimit, isAffectedByGlobalFilter, isAffectedByTimeFilter, - popularFieldsLimit, - sortedSelectedFields, + dataViewId, + fieldsExistenceInfoUnavailable, + onOverrideFieldGroupDetails, + hasFieldDataHandler, + fieldsExistenceReader, + onSelectedFieldFilter, + onSupportedFieldFilter, + dataView, ]); const fieldGroups: FieldListGroups = useMemo(() => { @@ -403,5 +414,6 @@ function getDefaultFieldGroups() { metaFields: [], unmappedFields: [], skippedFields: [], + newFields: [], }; } diff --git a/packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts b/packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts index 2aa6137ffb1fb..0a487368748cf 100644 --- a/packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts +++ b/packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts @@ -7,7 +7,7 @@ */ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import { RuntimeField } from '@kbn/data-views-plugin/common'; +import { DataViewField, RuntimeField } from '@kbn/data-views-plugin/common'; import type { DataViewsContract, DataView, FieldSpec } from '@kbn/data-views-plugin/common'; import type { IKibanaSearchRequest } from '@kbn/data-plugin/common'; @@ -49,15 +49,22 @@ export async function fetchFieldExistence({ metaFields: string[]; dataViewsService: DataViewsContract; }) { - const allFields = buildFieldList(dataView, metaFields); const existingFieldList = await dataViewsService.getFieldsForIndexPattern(dataView, { // filled in by data views service pattern: '', indexFilter: toQuery(timeFieldName, fromDate, toDate, dslQuery), }); + const newFields = existingFieldList + .filter((field) => dataView.getFieldByName(field.name) === undefined) + .map((field) => dataView.getFieldByName(field.name) ?? new DataViewField(field)); + if (newFields.length) { + await dataViewsService.refreshFields(dataView, false); + } + const allFields = buildFieldList(dataView, metaFields); return { indexPatternTitle: dataView.title, existingFieldNames: existingFields(existingFieldList, allFields), + newFields, }; } diff --git a/packages/kbn-unified-field-list/src/services/field_existing/load_field_existing.ts b/packages/kbn-unified-field-list/src/services/field_existing/load_field_existing.ts index 796062bc60559..889c92008554b 100644 --- a/packages/kbn-unified-field-list/src/services/field_existing/load_field_existing.ts +++ b/packages/kbn-unified-field-list/src/services/field_existing/load_field_existing.ts @@ -9,7 +9,7 @@ import { IUiSettingsClient } from '@kbn/core/public'; import { type DataPublicPluginStart } from '@kbn/data-plugin/public'; import { UI_SETTINGS } from '@kbn/data-service/src/constants'; -import type { DataView, DataViewsContract } from '@kbn/data-views-plugin/common'; +import type { DataView, DataViewField, DataViewsContract } from '@kbn/data-views-plugin/common'; import { lastValueFrom } from 'rxjs'; import { fetchFieldExistence } from './field_existing_utils'; @@ -27,6 +27,7 @@ interface FetchFieldExistenceParams { export type LoadFieldExistingHandler = (params: FetchFieldExistenceParams) => Promise<{ existingFieldNames: string[]; indexPatternTitle: string; + newFields?: DataViewField[]; }>; export const loadFieldExisting: LoadFieldExistingHandler = async ({ From 7b999db6641a1919a09d3ee4aec052be9d9200c6 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Fri, 1 Dec 2023 09:36:14 +0100 Subject: [PATCH 087/168] Improve code --- .../src/hooks/use_grouped_fields.test.tsx | 3 +++ .../kbn-unified-field-list/src/hooks/use_grouped_fields.ts | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.test.tsx b/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.test.tsx index 053e7d912d375..78b34329e0bf0 100644 --- a/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.test.tsx +++ b/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.test.tsx @@ -96,6 +96,7 @@ describe('UnifiedFieldList useGroupedFields()', () => { ? ExistenceFetchStatus.succeeded : ExistenceFetchStatus.unknown, isFieldsExistenceInfoUnavailable: (dataViewId) => dataViewId !== props.dataViewId, + getNewFields: () => [], }) ); @@ -156,6 +157,7 @@ describe('UnifiedFieldList useGroupedFields()', () => { ? ExistenceFetchStatus.succeeded : ExistenceFetchStatus.unknown, isFieldsExistenceInfoUnavailable: (dataViewId) => dataViewId !== props.dataViewId, + getNewFields: () => [], }) ); @@ -438,6 +440,7 @@ describe('UnifiedFieldList useGroupedFields()', () => { ? ExistenceFetchStatus.succeeded : ExistenceFetchStatus.unknown, isFieldsExistenceInfoUnavailable: (dataViewId) => dataViewId !== knownDataViewId, + getNewFields: () => [], }) ); diff --git a/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts b/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts index 7822e71244d10..731eb63dd7934 100644 --- a/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts +++ b/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts @@ -121,7 +121,6 @@ export function useGroupedFields({ const selectedFields = sortedSelectedFields || []; const newFields = dataViewId ? fieldsExistenceReader.getNewFields(dataViewId) : []; - console.log({ newFields }); const fieldsToSort = allFields && newFields.length ? allFields.map((field) => { From 3a6c239cb35b64871f1bea7080bec7c8183ce154 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Fri, 1 Dec 2023 09:46:44 +0100 Subject: [PATCH 088/168] Remove redundant newFieldsByFieldNameMap --- .../kbn-unified-field-list/src/hooks/use_existing_fields.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts b/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts index 588e0f60384b3..f756b4cc19126 100644 --- a/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts +++ b/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts @@ -25,7 +25,6 @@ const generateId = htmlIdGenerator(); export interface ExistingFieldsInfo { fetchStatus: ExistenceFetchStatus; existingFieldsByFieldNameMap: Record; - newFieldsByFieldNameMap: Record; newFields?: DataViewField[]; numberOfFetches: number; hasDataViewRestrictions?: boolean; @@ -64,7 +63,6 @@ const initialData: ExistingFieldsByDataViewMap = {}; const unknownInfo: ExistingFieldsInfo = { fetchStatus: ExistenceFetchStatus.unknown, existingFieldsByFieldNameMap: {}, - newFieldsByFieldNameMap: {}, numberOfFetches: 0, }; From 3fa136908494ac59c8b0e1817550044647e819e0 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Fri, 1 Dec 2023 22:53:55 -0600 Subject: [PATCH 089/168] update setting text --- src/plugins/data_views/server/ui_settings.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/data_views/server/ui_settings.ts b/src/plugins/data_views/server/ui_settings.ts index 7e4dabc441bf9..3ba489a4b6b28 100644 --- a/src/plugins/data_views/server/ui_settings.ts +++ b/src/plugins/data_views/server/ui_settings.ts @@ -18,7 +18,7 @@ export const cacheMaxAge = { value: DEFAULT_FIELD_CACHE_FRESHNESS, description: i18n.translate('dataViews.advancedSettings.cacheMaxAgeText', { defaultMessage: - "Sets the 'max-age' cache header value for data view fields API requests. A value of 0 will disable caching. A hard reload of Kibana may be necessary for your browser to pick up the new setting.", + 'Sets how long data view fields API requests are cached in seconds. A value of 0 turns off caching. Modifying this value may not take immediate effect, users need to clear browser cache or wait until the current cache expires. To see immediate changes, try a hard reload of Kibana.', }), schema: schema.number(), }, From f40eb1d6fbc46dbf8816e8c8a6f84cf12ef8113c Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Sat, 2 Dec 2023 00:28:55 -0600 Subject: [PATCH 090/168] use new method for security dependency --- docs/concepts/data-views.asciidoc | 12 ++++++++++++ src/plugins/data_views/kibana.jsonc | 1 + src/plugins/data_views/public/mocks.ts | 1 - src/plugins/data_views/public/plugin.ts | 16 +++++++++++++--- src/plugins/data_views/public/types.ts | 1 - test/functional/page_objects/settings_page.ts | 1 + x-pack/plugins/security/public/plugin.tsx | 10 ---------- 7 files changed, 27 insertions(+), 15 deletions(-) diff --git a/docs/concepts/data-views.asciidoc b/docs/concepts/data-views.asciidoc index 992741be518a7..56b1df33803b3 100644 --- a/docs/concepts/data-views.asciidoc +++ b/docs/concepts/data-views.asciidoc @@ -172,3 +172,15 @@ WARNING: Deleting a {data-source} breaks all visualizations, saved searches, and . Find the {data-source} that you want to delete, and then click image:management/index-patterns/images/delete.png[Delete icon] in the *Actions* column. + +[float] +[[data-view-field-cache]] +=== {data-source} field cache + +The browser caches {data-source} field lists for increased performance. This is particularly impactful +for {data-sources} with a high field count that span a large number of indices and clusters. The field +list will be updated every couple of minutes in typical {kib} usage. Alternatively, the {data-source} +management detail page provides a refresh button that gets an updated field list. A force reload of {kib} +will do the same. + +The field list may be impacted by changes in indices and user permissions. \ No newline at end of file diff --git a/src/plugins/data_views/kibana.jsonc b/src/plugins/data_views/kibana.jsonc index 7595f8e68b3c4..7789383b48ba4 100644 --- a/src/plugins/data_views/kibana.jsonc +++ b/src/plugins/data_views/kibana.jsonc @@ -18,6 +18,7 @@ "requiredBundles": [ "kibanaUtils" ], + "runtimePluginDependencies" : ["security"], "extraPublicDirs": [ "common" ] diff --git a/src/plugins/data_views/public/mocks.ts b/src/plugins/data_views/public/mocks.ts index 0edea939ab8df..e9d5b487b8b00 100644 --- a/src/plugins/data_views/public/mocks.ts +++ b/src/plugins/data_views/public/mocks.ts @@ -13,7 +13,6 @@ export type Start = jest.Mocked>; const createSetupContract = (): Setup => ({ enableRollups: jest.fn(), - setUserIdGetter: jest.fn(), }); const createStartContract = (): Start => { diff --git a/src/plugins/data_views/public/plugin.ts b/src/plugins/data_views/public/plugin.ts index 72603ceb8246c..6d8d2d9fde217 100644 --- a/src/plugins/data_views/public/plugin.ts +++ b/src/plugins/data_views/public/plugin.ts @@ -8,6 +8,7 @@ import { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from '@kbn/core/public'; import { i18n } from '@kbn/i18n'; +import type { SecurityPluginStart } from '@kbn/security-plugin-types-public'; import { getIndexPatternLoad } from './expressions'; import type { ClientConfigType } from '../common/types'; import { @@ -62,11 +63,20 @@ export class DataViewsPublicPlugin }), }); + core.plugins.onStart<{ security: SecurityPluginStart }>('security').then(({ security }) => { + if (security.found) { + const getUserId = async function getUserId(): Promise { + const currentUser = await security.contract.authc.getCurrentUser(); + return currentUser?.profile_uid; + }; + this.userIdGetter = getUserId; + } else { + throw new Error('Security plugin is not available, but is required for Data Views plugin'); + } + }); + return { enableRollups: () => (this.rollupsEnabled = true), - setUserIdGetter: (userIdGetter: UserIdGetter) => { - this.userIdGetter = userIdGetter; - }, }; } diff --git a/src/plugins/data_views/public/types.ts b/src/plugins/data_views/public/types.ts index d01e606b968e3..98db0382f641f 100644 --- a/src/plugins/data_views/public/types.ts +++ b/src/plugins/data_views/public/types.ts @@ -113,7 +113,6 @@ export type UserIdGetter = () => Promise; */ export interface DataViewsPublicPluginSetup { enableRollups: () => void; - setUserIdGetter: (userIdGetter: UserIdGetter) => void; } export interface DataViewsServicePublic extends DataViewsServicePublicMethods { diff --git a/test/functional/page_objects/settings_page.ts b/test/functional/page_objects/settings_page.ts index 6b3840faad3f4..d0994451cabdb 100644 --- a/test/functional/page_objects/settings_page.ts +++ b/test/functional/page_objects/settings_page.ts @@ -491,6 +491,7 @@ export class SettingsPageObject extends FtrService { await this.testSubjects.click(`detail-link-${dataViewName}`); } await this.testSubjects.click('refreshDataViewButton'); + // todo - await on refresh button being available again } async allowHiddenClick() { diff --git a/x-pack/plugins/security/public/plugin.tsx b/x-pack/plugins/security/public/plugin.tsx index 6208d487e408a..be0abf3f828b8 100644 --- a/x-pack/plugins/security/public/plugin.tsx +++ b/x-pack/plugins/security/public/plugin.tsx @@ -166,16 +166,6 @@ export class SecurityPlugin this.anonymousAccessService.setup({ share }); } - const userIdGetter = async () => { - const [, , security] = await core.getStartServices(); - const { profile_uid: profileUid } = await ( - security as SecurityPluginStart - ).authc.getCurrentUser(); - return profileUid; - }; - - dataViews?.setUserIdGetter(userIdGetter); - return { authc: this.authc, license, From cf7c8933d74f43fa95c24c84a5760e8ac0631108 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Sat, 2 Dec 2023 06:35:33 +0000 Subject: [PATCH 091/168] [CI] Auto-commit changed files from 'node scripts/lint_ts_projects --fix' --- src/plugins/data_views/tsconfig.json | 1 + 1 file changed, 1 insertion(+) diff --git a/src/plugins/data_views/tsconfig.json b/src/plugins/data_views/tsconfig.json index e5613323bc222..1d414cb98ee95 100644 --- a/src/plugins/data_views/tsconfig.json +++ b/src/plugins/data_views/tsconfig.json @@ -33,6 +33,7 @@ "@kbn/object-versioning", "@kbn/core-saved-objects-server", "@kbn/logging", + "@kbn/security-plugin-types-public", ], "exclude": [ "target/**/*", From 9b719cc000c291fadf5d7be99ac51d77800da2d7 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Sat, 2 Dec 2023 09:58:16 -0600 Subject: [PATCH 092/168] Revert "use new method for security dependency" This reverts commit f40eb1d6fbc46dbf8816e8c8a6f84cf12ef8113c. --- docs/concepts/data-views.asciidoc | 12 ------------ src/plugins/data_views/kibana.jsonc | 1 - src/plugins/data_views/public/mocks.ts | 1 + src/plugins/data_views/public/plugin.ts | 16 +++------------- src/plugins/data_views/public/types.ts | 1 + test/functional/page_objects/settings_page.ts | 1 - x-pack/plugins/security/public/plugin.tsx | 10 ++++++++++ 7 files changed, 15 insertions(+), 27 deletions(-) diff --git a/docs/concepts/data-views.asciidoc b/docs/concepts/data-views.asciidoc index 56b1df33803b3..992741be518a7 100644 --- a/docs/concepts/data-views.asciidoc +++ b/docs/concepts/data-views.asciidoc @@ -172,15 +172,3 @@ WARNING: Deleting a {data-source} breaks all visualizations, saved searches, and . Find the {data-source} that you want to delete, and then click image:management/index-patterns/images/delete.png[Delete icon] in the *Actions* column. - -[float] -[[data-view-field-cache]] -=== {data-source} field cache - -The browser caches {data-source} field lists for increased performance. This is particularly impactful -for {data-sources} with a high field count that span a large number of indices and clusters. The field -list will be updated every couple of minutes in typical {kib} usage. Alternatively, the {data-source} -management detail page provides a refresh button that gets an updated field list. A force reload of {kib} -will do the same. - -The field list may be impacted by changes in indices and user permissions. \ No newline at end of file diff --git a/src/plugins/data_views/kibana.jsonc b/src/plugins/data_views/kibana.jsonc index 7789383b48ba4..7595f8e68b3c4 100644 --- a/src/plugins/data_views/kibana.jsonc +++ b/src/plugins/data_views/kibana.jsonc @@ -18,7 +18,6 @@ "requiredBundles": [ "kibanaUtils" ], - "runtimePluginDependencies" : ["security"], "extraPublicDirs": [ "common" ] diff --git a/src/plugins/data_views/public/mocks.ts b/src/plugins/data_views/public/mocks.ts index e9d5b487b8b00..0edea939ab8df 100644 --- a/src/plugins/data_views/public/mocks.ts +++ b/src/plugins/data_views/public/mocks.ts @@ -13,6 +13,7 @@ export type Start = jest.Mocked>; const createSetupContract = (): Setup => ({ enableRollups: jest.fn(), + setUserIdGetter: jest.fn(), }); const createStartContract = (): Start => { diff --git a/src/plugins/data_views/public/plugin.ts b/src/plugins/data_views/public/plugin.ts index 6d8d2d9fde217..72603ceb8246c 100644 --- a/src/plugins/data_views/public/plugin.ts +++ b/src/plugins/data_views/public/plugin.ts @@ -8,7 +8,6 @@ import { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from '@kbn/core/public'; import { i18n } from '@kbn/i18n'; -import type { SecurityPluginStart } from '@kbn/security-plugin-types-public'; import { getIndexPatternLoad } from './expressions'; import type { ClientConfigType } from '../common/types'; import { @@ -63,20 +62,11 @@ export class DataViewsPublicPlugin }), }); - core.plugins.onStart<{ security: SecurityPluginStart }>('security').then(({ security }) => { - if (security.found) { - const getUserId = async function getUserId(): Promise { - const currentUser = await security.contract.authc.getCurrentUser(); - return currentUser?.profile_uid; - }; - this.userIdGetter = getUserId; - } else { - throw new Error('Security plugin is not available, but is required for Data Views plugin'); - } - }); - return { enableRollups: () => (this.rollupsEnabled = true), + setUserIdGetter: (userIdGetter: UserIdGetter) => { + this.userIdGetter = userIdGetter; + }, }; } diff --git a/src/plugins/data_views/public/types.ts b/src/plugins/data_views/public/types.ts index 98db0382f641f..d01e606b968e3 100644 --- a/src/plugins/data_views/public/types.ts +++ b/src/plugins/data_views/public/types.ts @@ -113,6 +113,7 @@ export type UserIdGetter = () => Promise; */ export interface DataViewsPublicPluginSetup { enableRollups: () => void; + setUserIdGetter: (userIdGetter: UserIdGetter) => void; } export interface DataViewsServicePublic extends DataViewsServicePublicMethods { diff --git a/test/functional/page_objects/settings_page.ts b/test/functional/page_objects/settings_page.ts index d0994451cabdb..6b3840faad3f4 100644 --- a/test/functional/page_objects/settings_page.ts +++ b/test/functional/page_objects/settings_page.ts @@ -491,7 +491,6 @@ export class SettingsPageObject extends FtrService { await this.testSubjects.click(`detail-link-${dataViewName}`); } await this.testSubjects.click('refreshDataViewButton'); - // todo - await on refresh button being available again } async allowHiddenClick() { diff --git a/x-pack/plugins/security/public/plugin.tsx b/x-pack/plugins/security/public/plugin.tsx index be0abf3f828b8..6208d487e408a 100644 --- a/x-pack/plugins/security/public/plugin.tsx +++ b/x-pack/plugins/security/public/plugin.tsx @@ -166,6 +166,16 @@ export class SecurityPlugin this.anonymousAccessService.setup({ share }); } + const userIdGetter = async () => { + const [, , security] = await core.getStartServices(); + const { profile_uid: profileUid } = await ( + security as SecurityPluginStart + ).authc.getCurrentUser(); + return profileUid; + }; + + dataViews?.setUserIdGetter(userIdGetter); + return { authc: this.authc, license, From ccd243a663e85df5d264032766161b5d7d7f574c Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Sat, 2 Dec 2023 16:04:39 +0000 Subject: [PATCH 093/168] [CI] Auto-commit changed files from 'node scripts/lint_ts_projects --fix' --- src/plugins/data_views/tsconfig.json | 1 - 1 file changed, 1 deletion(-) diff --git a/src/plugins/data_views/tsconfig.json b/src/plugins/data_views/tsconfig.json index 1d414cb98ee95..e5613323bc222 100644 --- a/src/plugins/data_views/tsconfig.json +++ b/src/plugins/data_views/tsconfig.json @@ -33,7 +33,6 @@ "@kbn/object-versioning", "@kbn/core-saved-objects-server", "@kbn/logging", - "@kbn/security-plugin-types-public", ], "exclude": [ "target/**/*", From 9ec277309b7e60d0a9d98cba8ddcbed208b5f89b Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Sat, 2 Dec 2023 12:55:33 -0600 Subject: [PATCH 094/168] Revert "Revert "use new method for security dependency"" This reverts commit 9b719cc000c291fadf5d7be99ac51d77800da2d7. --- docs/concepts/data-views.asciidoc | 12 ++++++++++++ src/plugins/data_views/kibana.jsonc | 1 + src/plugins/data_views/public/mocks.ts | 1 - src/plugins/data_views/public/plugin.ts | 16 +++++++++++++--- src/plugins/data_views/public/types.ts | 1 - test/functional/page_objects/settings_page.ts | 1 + x-pack/plugins/security/public/plugin.tsx | 10 ---------- 7 files changed, 27 insertions(+), 15 deletions(-) diff --git a/docs/concepts/data-views.asciidoc b/docs/concepts/data-views.asciidoc index 992741be518a7..56b1df33803b3 100644 --- a/docs/concepts/data-views.asciidoc +++ b/docs/concepts/data-views.asciidoc @@ -172,3 +172,15 @@ WARNING: Deleting a {data-source} breaks all visualizations, saved searches, and . Find the {data-source} that you want to delete, and then click image:management/index-patterns/images/delete.png[Delete icon] in the *Actions* column. + +[float] +[[data-view-field-cache]] +=== {data-source} field cache + +The browser caches {data-source} field lists for increased performance. This is particularly impactful +for {data-sources} with a high field count that span a large number of indices and clusters. The field +list will be updated every couple of minutes in typical {kib} usage. Alternatively, the {data-source} +management detail page provides a refresh button that gets an updated field list. A force reload of {kib} +will do the same. + +The field list may be impacted by changes in indices and user permissions. \ No newline at end of file diff --git a/src/plugins/data_views/kibana.jsonc b/src/plugins/data_views/kibana.jsonc index 7595f8e68b3c4..7789383b48ba4 100644 --- a/src/plugins/data_views/kibana.jsonc +++ b/src/plugins/data_views/kibana.jsonc @@ -18,6 +18,7 @@ "requiredBundles": [ "kibanaUtils" ], + "runtimePluginDependencies" : ["security"], "extraPublicDirs": [ "common" ] diff --git a/src/plugins/data_views/public/mocks.ts b/src/plugins/data_views/public/mocks.ts index 0edea939ab8df..e9d5b487b8b00 100644 --- a/src/plugins/data_views/public/mocks.ts +++ b/src/plugins/data_views/public/mocks.ts @@ -13,7 +13,6 @@ export type Start = jest.Mocked>; const createSetupContract = (): Setup => ({ enableRollups: jest.fn(), - setUserIdGetter: jest.fn(), }); const createStartContract = (): Start => { diff --git a/src/plugins/data_views/public/plugin.ts b/src/plugins/data_views/public/plugin.ts index 72603ceb8246c..6d8d2d9fde217 100644 --- a/src/plugins/data_views/public/plugin.ts +++ b/src/plugins/data_views/public/plugin.ts @@ -8,6 +8,7 @@ import { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from '@kbn/core/public'; import { i18n } from '@kbn/i18n'; +import type { SecurityPluginStart } from '@kbn/security-plugin-types-public'; import { getIndexPatternLoad } from './expressions'; import type { ClientConfigType } from '../common/types'; import { @@ -62,11 +63,20 @@ export class DataViewsPublicPlugin }), }); + core.plugins.onStart<{ security: SecurityPluginStart }>('security').then(({ security }) => { + if (security.found) { + const getUserId = async function getUserId(): Promise { + const currentUser = await security.contract.authc.getCurrentUser(); + return currentUser?.profile_uid; + }; + this.userIdGetter = getUserId; + } else { + throw new Error('Security plugin is not available, but is required for Data Views plugin'); + } + }); + return { enableRollups: () => (this.rollupsEnabled = true), - setUserIdGetter: (userIdGetter: UserIdGetter) => { - this.userIdGetter = userIdGetter; - }, }; } diff --git a/src/plugins/data_views/public/types.ts b/src/plugins/data_views/public/types.ts index d01e606b968e3..98db0382f641f 100644 --- a/src/plugins/data_views/public/types.ts +++ b/src/plugins/data_views/public/types.ts @@ -113,7 +113,6 @@ export type UserIdGetter = () => Promise; */ export interface DataViewsPublicPluginSetup { enableRollups: () => void; - setUserIdGetter: (userIdGetter: UserIdGetter) => void; } export interface DataViewsServicePublic extends DataViewsServicePublicMethods { diff --git a/test/functional/page_objects/settings_page.ts b/test/functional/page_objects/settings_page.ts index 6b3840faad3f4..d0994451cabdb 100644 --- a/test/functional/page_objects/settings_page.ts +++ b/test/functional/page_objects/settings_page.ts @@ -491,6 +491,7 @@ export class SettingsPageObject extends FtrService { await this.testSubjects.click(`detail-link-${dataViewName}`); } await this.testSubjects.click('refreshDataViewButton'); + // todo - await on refresh button being available again } async allowHiddenClick() { diff --git a/x-pack/plugins/security/public/plugin.tsx b/x-pack/plugins/security/public/plugin.tsx index 6208d487e408a..be0abf3f828b8 100644 --- a/x-pack/plugins/security/public/plugin.tsx +++ b/x-pack/plugins/security/public/plugin.tsx @@ -166,16 +166,6 @@ export class SecurityPlugin this.anonymousAccessService.setup({ share }); } - const userIdGetter = async () => { - const [, , security] = await core.getStartServices(); - const { profile_uid: profileUid } = await ( - security as SecurityPluginStart - ).authc.getCurrentUser(); - return profileUid; - }; - - dataViews?.setUserIdGetter(userIdGetter); - return { authc: this.authc, license, From c51b7d2d7c622d2acb8683d556b708d8ab55d74f Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Sat, 2 Dec 2023 19:03:49 +0000 Subject: [PATCH 095/168] [CI] Auto-commit changed files from 'node scripts/lint_ts_projects --fix' --- src/plugins/data_views/tsconfig.json | 1 + 1 file changed, 1 insertion(+) diff --git a/src/plugins/data_views/tsconfig.json b/src/plugins/data_views/tsconfig.json index e5613323bc222..1d414cb98ee95 100644 --- a/src/plugins/data_views/tsconfig.json +++ b/src/plugins/data_views/tsconfig.json @@ -33,6 +33,7 @@ "@kbn/object-versioning", "@kbn/core-saved-objects-server", "@kbn/logging", + "@kbn/security-plugin-types-public", ], "exclude": [ "target/**/*", From 6d5df9d983b37c6557ec6a0f2ff8b97c33b72928 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Sat, 2 Dec 2023 14:44:15 -0600 Subject: [PATCH 096/168] add loading text for button, better button push test logic --- .../edit_index_pattern/index_header/index_header.tsx | 9 +++++++-- test/functional/page_objects/settings_page.ts | 12 +++++++++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/plugins/data_view_management/public/components/edit_index_pattern/index_header/index_header.tsx b/src/plugins/data_view_management/public/components/edit_index_pattern/index_header/index_header.tsx index cf1ce9faedb55..44d8888b98afc 100644 --- a/src/plugins/data_view_management/public/components/edit_index_pattern/index_header/index_header.tsx +++ b/src/plugins/data_view_management/public/components/edit_index_pattern/index_header/index_header.tsx @@ -58,6 +58,10 @@ const refreshLabel = i18n.translate('indexPatternManagement.editDataView.refresh defaultMessage: 'Refresh', }); +const isLoadingLabel = i18n.translate('indexPatternManagement.editDataView.refreshLabel', { + defaultMessage: 'Loading...', +}); + export const IndexHeader: React.FC = ({ defaultIndex, indexPattern, @@ -79,9 +83,10 @@ export const IndexHeader: React.FC = ({ iconType="refresh" aria-label={refreshAriaLabel} data-test-subj="refreshDataViewButton" - disabled={isRefreshing} + isLoading={isRefreshing} + isDisabled={isRefreshing} > - {refreshLabel} + {isRefreshing ? isLoadingLabel : refreshLabel} , canSave && ( diff --git a/test/functional/page_objects/settings_page.ts b/test/functional/page_objects/settings_page.ts index d0994451cabdb..fde7e2343f10f 100644 --- a/test/functional/page_objects/settings_page.ts +++ b/test/functional/page_objects/settings_page.ts @@ -491,7 +491,17 @@ export class SettingsPageObject extends FtrService { await this.testSubjects.click(`detail-link-${dataViewName}`); } await this.testSubjects.click('refreshDataViewButton'); - // todo - await on refresh button being available again + await this.retry.try(async () => { + const btn = await this.testSubjects.find('refreshDataViewButton'); + const disabled = await btn.getAttribute('disabled'); + expect(disabled).to.be('true'); + }); + + await this.retry.try(async () => { + const btn = await this.testSubjects.find('refreshDataViewButton'); + const disabled = await btn.getAttribute('disabled'); + expect(disabled).to.be(''); + }); } async allowHiddenClick() { From 3f7fb79cccb24ccab670dd3c14921eca8c2d0814 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Sat, 2 Dec 2023 17:27:02 -0600 Subject: [PATCH 097/168] fix i18n name --- .../components/edit_index_pattern/index_header/index_header.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/data_view_management/public/components/edit_index_pattern/index_header/index_header.tsx b/src/plugins/data_view_management/public/components/edit_index_pattern/index_header/index_header.tsx index 44d8888b98afc..cd03e4b0aa54f 100644 --- a/src/plugins/data_view_management/public/components/edit_index_pattern/index_header/index_header.tsx +++ b/src/plugins/data_view_management/public/components/edit_index_pattern/index_header/index_header.tsx @@ -58,7 +58,7 @@ const refreshLabel = i18n.translate('indexPatternManagement.editDataView.refresh defaultMessage: 'Refresh', }); -const isLoadingLabel = i18n.translate('indexPatternManagement.editDataView.refreshLabel', { +const isLoadingLabel = i18n.translate('indexPatternManagement.editDataView.refreshLoading', { defaultMessage: 'Loading...', }); From 3c1ab442b26beea2a67cf9141bdba5e8c229123c Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Sat, 2 Dec 2023 18:39:44 -0600 Subject: [PATCH 098/168] wait on field list refresh --- test/functional/page_objects/settings_page.ts | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/test/functional/page_objects/settings_page.ts b/test/functional/page_objects/settings_page.ts index fde7e2343f10f..d1c49c89923b7 100644 --- a/test/functional/page_objects/settings_page.ts +++ b/test/functional/page_objects/settings_page.ts @@ -491,16 +491,15 @@ export class SettingsPageObject extends FtrService { await this.testSubjects.click(`detail-link-${dataViewName}`); } await this.testSubjects.click('refreshDataViewButton'); - await this.retry.try(async () => { - const btn = await this.testSubjects.find('refreshDataViewButton'); - const disabled = await btn.getAttribute('disabled'); - expect(disabled).to.be('true'); - }); + // wait for refresh to start + await new Promise((r) => setTimeout(r, 500)); + + // wait for refresh to finish await this.retry.try(async () => { const btn = await this.testSubjects.find('refreshDataViewButton'); const disabled = await btn.getAttribute('disabled'); - expect(disabled).to.be(''); + expect(disabled).to.be(null); }); } From fdcd16cf44eb4c5f5ffe892960fb00e415b1b1c2 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Sat, 2 Dec 2023 18:47:08 -0600 Subject: [PATCH 099/168] add refresh button push to serverless index filter test --- .../common/management/data_views/_index_pattern_filter.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/test_serverless/functional/test_suites/common/management/data_views/_index_pattern_filter.ts b/x-pack/test_serverless/functional/test_suites/common/management/data_views/_index_pattern_filter.ts index 9da55f122114c..2dba717645ff2 100644 --- a/x-pack/test_serverless/functional/test_suites/common/management/data_views/_index_pattern_filter.ts +++ b/x-pack/test_serverless/functional/test_suites/common/management/data_views/_index_pattern_filter.ts @@ -171,6 +171,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.settings.clickIndexPatternLogstash(); + await PageObjects.settings.refreshDataViewFieldList(); + await testSubjects.existOrFail('dataViewMappingConflict'); expect(await PageObjects.settings.getFieldTypes()).to.eql([ From 53f1f19b7e7dd689e725763a9c10a623e21255b8 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Sat, 2 Dec 2023 21:24:46 -0600 Subject: [PATCH 100/168] fix ml alert test --- .../apps/discover_ml_uptime/discover/search_source_alert.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts b/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts index 6b668f75d56d3..d9ed0604ff55c 100644 --- a/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts +++ b/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts @@ -210,6 +210,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.header.waitUntilLoadingHasFinished(); await PageObjects.discover.clickNewSearchButton(); // reset params + await PageObjects.settings.refreshDataViewFieldList(OUTPUT_DATA_VIEW); + await PageObjects.common.navigateToApp('discover'); + await PageObjects.discover.selectIndexPattern(OUTPUT_DATA_VIEW); let ruleId: string; From ebd04c7088827980a7c27b714def6476ce635b3b Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Sat, 2 Dec 2023 22:29:16 -0600 Subject: [PATCH 101/168] fix discover security functional test --- .../apps/discover/feature_controls/discover_security.ts | 3 +++ .../apps/discover_ml_uptime/discover/search_source_alert.ts | 1 + 2 files changed, 4 insertions(+) diff --git a/x-pack/test/functional/apps/discover/feature_controls/discover_security.ts b/x-pack/test/functional/apps/discover/feature_controls/discover_security.ts index ee810f7ddebfb..ef0efbcc6fb31 100644 --- a/x-pack/test/functional/apps/discover/feature_controls/discover_security.ts +++ b/x-pack/test/functional/apps/discover/feature_controls/discover_security.ts @@ -31,6 +31,7 @@ export default function (ctx: FtrProviderContext) { 'spaceSelector', 'header', 'unifiedFieldList', + 'settings', ]); const testSubjects = getService('testSubjects'); const appsMenu = getService('appsMenu'); @@ -448,6 +449,8 @@ export default function (ctx: FtrProviderContext) { await globalNav.badgeExistsOrFail('Read only'); // can't access logstash index directly + await PageObjects.settings.refreshDataViewFieldList('logstash-*'); + await PageObjects.common.navigateToApp('discover'); await PageObjects.discover.selectIndexPattern('logstash-*'); await PageObjects.header.waitUntilLoadingHasFinished(); await testSubjects.existOrFail('discoverNoResultsCheckIndices'); diff --git a/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts b/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts index d9ed0604ff55c..cdf25a28ad099 100644 --- a/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts +++ b/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts @@ -210,6 +210,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.header.waitUntilLoadingHasFinished(); await PageObjects.discover.clickNewSearchButton(); // reset params + // todo this didn't work await PageObjects.settings.refreshDataViewFieldList(OUTPUT_DATA_VIEW); await PageObjects.common.navigateToApp('discover'); From 46c9ee753b5f1cc78ed0d3e6be9537c24af3078f Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Sat, 2 Dec 2023 23:46:59 -0600 Subject: [PATCH 102/168] functional test fix attempts --- .../apps/discover/feature_controls/discover_security.ts | 6 ++++-- .../apps/discover_ml_uptime/discover/search_source_alert.ts | 2 ++ .../test_suites/search/cases/attachment_framework.ts | 3 +++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/x-pack/test/functional/apps/discover/feature_controls/discover_security.ts b/x-pack/test/functional/apps/discover/feature_controls/discover_security.ts index ef0efbcc6fb31..d9e0834f9e2cd 100644 --- a/x-pack/test/functional/apps/discover/feature_controls/discover_security.ts +++ b/x-pack/test/functional/apps/discover/feature_controls/discover_security.ts @@ -449,8 +449,10 @@ export default function (ctx: FtrProviderContext) { await globalNav.badgeExistsOrFail('Read only'); // can't access logstash index directly - await PageObjects.settings.refreshDataViewFieldList('logstash-*'); - await PageObjects.common.navigateToApp('discover'); + // swapping index patterns so we get an updated field list + await new Promise((resolve) => setTimeout(resolve, 5000)); + await PageObjects.discover.selectIndexPattern('alias-logstash-discover'); + // await PageObjects.discover.selectIndexPattern('logstash-*'); await PageObjects.header.waitUntilLoadingHasFinished(); await testSubjects.existOrFail('discoverNoResultsCheckIndices'); diff --git a/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts b/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts index cdf25a28ad099..5cdd190458f96 100644 --- a/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts +++ b/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts @@ -403,6 +403,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); it('should navigate to alert results via link provided in notification', async () => { + // todo did this work? + await PageObjects.settings.refreshDataViewFieldList(OUTPUT_DATA_VIEW); await openAlertResults(RULE_NAME); await checkInitialRuleParamsState(SOURCE_DATA_VIEW); }); diff --git a/x-pack/test_serverless/functional/test_suites/search/cases/attachment_framework.ts b/x-pack/test_serverless/functional/test_suites/search/cases/attachment_framework.ts index 3cf087eb1ae9b..c7252bebffd80 100644 --- a/x-pack/test_serverless/functional/test_suites/search/cases/attachment_framework.ts +++ b/x-pack/test_serverless/functional/test_suites/search/cases/attachment_framework.ts @@ -16,6 +16,7 @@ export default ({ getPageObject, getService }: FtrProviderContext) => { const svlSearchNavigation = getService('svlSearchNavigation'); const svlCommonNavigation = getPageObject('svlCommonNavigation'); const svlCommonPage = getPageObject('svlCommonPage'); + const settings = getPageObject('settings'); describe('persistable attachment', () => { before(async () => { @@ -33,6 +34,8 @@ export default ({ getPageObject, getService }: FtrProviderContext) => { 'x-pack/test/functional/fixtures/kbn_archiver/lens/lens_basic.json' ); + await settings.refreshDataViewFieldList('logstash-*'); + await svlSearchNavigation.navigateToLandingPage(); await svlCommonNavigation.sidenav.clickLink({ deepLinkId: 'dashboards' }); From 60b0675f73e34ad45db5e8203c736ab54a549c91 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Sun, 3 Dec 2023 00:38:51 -0600 Subject: [PATCH 103/168] another attempt at fixing the discover security test --- .../apps/discover/feature_controls/discover_security.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/x-pack/test/functional/apps/discover/feature_controls/discover_security.ts b/x-pack/test/functional/apps/discover/feature_controls/discover_security.ts index d9e0834f9e2cd..93a62c370d561 100644 --- a/x-pack/test/functional/apps/discover/feature_controls/discover_security.ts +++ b/x-pack/test/functional/apps/discover/feature_controls/discover_security.ts @@ -452,6 +452,10 @@ export default function (ctx: FtrProviderContext) { // swapping index patterns so we get an updated field list await new Promise((resolve) => setTimeout(resolve, 5000)); await PageObjects.discover.selectIndexPattern('alias-logstash-discover'); + await new Promise((resolve) => setTimeout(resolve, 1000)); + await PageObjects.discover.selectIndexPattern('logstash-*'); + await new Promise((resolve) => setTimeout(resolve, 1000)); + await PageObjects.discover.selectIndexPattern('alias-logstash-discover'); // await PageObjects.discover.selectIndexPattern('logstash-*'); await PageObjects.header.waitUntilLoadingHasFinished(); From b563acdd9c1c1d88ca55d9db045ed904fb6a4cd1 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Sun, 3 Dec 2023 10:03:16 -0600 Subject: [PATCH 104/168] add some debugging to hopefully help with remaining errors --- src/plugins/data_views/common/data_views/data_views.ts | 5 +++++ .../data_views/public/data_views/data_views_api_client.ts | 1 + 2 files changed, 6 insertions(+) diff --git a/src/plugins/data_views/common/data_views/data_views.ts b/src/plugins/data_views/common/data_views/data_views.ts index fd2b2b5e8d1ff..2aa28d2e3a927 100644 --- a/src/plugins/data_views/common/data_views/data_views.ts +++ b/src/plugins/data_views/common/data_views/data_views.ts @@ -923,6 +923,7 @@ export class DataViewsService { displayErrors: boolean = true, refreshFields = false ): Promise => { + console.log('DEBUG: get data view', id); const dataViewFromCache = this.dataViewCache.get(id)?.then(async (dataView) => { if (dataView && refreshFields) { await this.refreshFields(dataView, displayErrors); @@ -939,6 +940,10 @@ export class DataViewsService { this.dataViewCache.clear(id); }); + indexPatternPromise.then((indexPattern) => { + console.log('DEBUG: get data view resolved', id, indexPattern.getIndexPattern()); + }); + return indexPatternPromise; }; diff --git a/src/plugins/data_views/public/data_views/data_views_api_client.ts b/src/plugins/data_views/public/data_views/data_views_api_client.ts index c65db9e5c736f..5e90cf220fc2a 100644 --- a/src/plugins/data_views/public/data_views/data_views_api_client.ts +++ b/src/plugins/data_views/public/data_views/data_views_api_client.ts @@ -110,6 +110,7 @@ export class DataViewsApiClient implements IDataViewsApiClient { indexFilter ? JSON.stringify({ index_filter: indexFilter }) : undefined, forceRefresh ).then((response) => { + console.log('DEBUG: api_client', pattern, JSON.stringify(response?.body, null, 2)); return { indices: response?.body?.indices || [], fields: response?.body?.fields || [], From b1453754dc06a1c55011d38f67936acf69ef7e0d Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Sun, 3 Dec 2023 16:54:13 -0600 Subject: [PATCH 105/168] a little more debugging --- .../apps/discover/feature_controls/discover_security.ts | 7 +++++-- .../discover_ml_uptime/discover/search_source_alert.ts | 9 ++++----- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/x-pack/test/functional/apps/discover/feature_controls/discover_security.ts b/x-pack/test/functional/apps/discover/feature_controls/discover_security.ts index 93a62c370d561..d8c26060716bf 100644 --- a/x-pack/test/functional/apps/discover/feature_controls/discover_security.ts +++ b/x-pack/test/functional/apps/discover/feature_controls/discover_security.ts @@ -11,6 +11,7 @@ import { getSavedQuerySecurityUtils } from '../../saved_query_management/utils/s export default function (ctx: FtrProviderContext) { const { getPageObjects, getService } = ctx; + const log = getService('log'); const savedQuerySecurityUtils = getSavedQuerySecurityUtils(ctx); const esArchiver = getService('esArchiver'); const esSupertest = getService('esSupertest'); @@ -450,12 +451,14 @@ export default function (ctx: FtrProviderContext) { // can't access logstash index directly // swapping index patterns so we get an updated field list + log.debug('DEBUG a bunch of data view switching'); await new Promise((resolve) => setTimeout(resolve, 5000)); await PageObjects.discover.selectIndexPattern('alias-logstash-discover'); - await new Promise((resolve) => setTimeout(resolve, 1000)); + await new Promise((resolve) => setTimeout(resolve, 5000)); await PageObjects.discover.selectIndexPattern('logstash-*'); - await new Promise((resolve) => setTimeout(resolve, 1000)); + await new Promise((resolve) => setTimeout(resolve, 5000)); await PageObjects.discover.selectIndexPattern('alias-logstash-discover'); + log.debug('DEBUG a bunch of data view switching done'); // await PageObjects.discover.selectIndexPattern('logstash-*'); await PageObjects.header.waitUntilLoadingHasFinished(); diff --git a/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts b/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts index 5cdd190458f96..777bec2788ae0 100644 --- a/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts +++ b/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts @@ -209,11 +209,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.common.navigateToApp('discover'); await PageObjects.header.waitUntilLoadingHasFinished(); await PageObjects.discover.clickNewSearchButton(); // reset params - - // todo this didn't work - await PageObjects.settings.refreshDataViewFieldList(OUTPUT_DATA_VIEW); - await PageObjects.common.navigateToApp('discover'); - await PageObjects.discover.selectIndexPattern(OUTPUT_DATA_VIEW); let ruleId: string; @@ -404,7 +399,11 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { it('should navigate to alert results via link provided in notification', async () => { // todo did this work? + log.debug('DATA VIEW REFRESH'); + await PageObjects.settings.refreshDataViewFieldList(OUTPUT_DATA_VIEW); + await new Promise((resolve) => setTimeout(resolve, 5000)); await PageObjects.settings.refreshDataViewFieldList(OUTPUT_DATA_VIEW); + log.debug('DATA VIEW REFRESH COMPLETE'); await openAlertResults(RULE_NAME); await checkInitialRuleParamsState(SOURCE_DATA_VIEW); }); From eb0604da3294423505887b488400b493969db156 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Sun, 3 Dec 2023 18:09:44 -0600 Subject: [PATCH 106/168] a little more debugging --- .../apps/discover_ml_uptime/discover/search_source_alert.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts b/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts index 777bec2788ae0..2431ce7f7a6b7 100644 --- a/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts +++ b/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts @@ -399,11 +399,11 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { it('should navigate to alert results via link provided in notification', async () => { // todo did this work? - log.debug('DATA VIEW REFRESH'); + log.write('DATA VIEW REFRESH'); await PageObjects.settings.refreshDataViewFieldList(OUTPUT_DATA_VIEW); await new Promise((resolve) => setTimeout(resolve, 5000)); await PageObjects.settings.refreshDataViewFieldList(OUTPUT_DATA_VIEW); - log.debug('DATA VIEW REFRESH COMPLETE'); + log.write('DATA VIEW REFRESH COMPLETE'); await openAlertResults(RULE_NAME); await checkInitialRuleParamsState(SOURCE_DATA_VIEW); }); From 6216b37e2f8dbde31b0422935043c462fc9332be Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Sun, 3 Dec 2023 23:52:15 -0600 Subject: [PATCH 107/168] attempt to fix lens functional test with additional data view refresh --- .../apps/discover_ml_uptime/discover/search_source_alert.ts | 2 ++ .../functional/test_suites/search/cases/attachment_framework.ts | 1 + 2 files changed, 3 insertions(+) diff --git a/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts b/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts index 2431ce7f7a6b7..8b48c0e04b5b1 100644 --- a/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts +++ b/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts @@ -210,6 +210,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.header.waitUntilLoadingHasFinished(); await PageObjects.discover.clickNewSearchButton(); // reset params await PageObjects.discover.selectIndexPattern(OUTPUT_DATA_VIEW); + // this fixed it before + // await PageObjects.discover.refreshFieldList(); let ruleId: string; if (type === 'name') { diff --git a/x-pack/test_serverless/functional/test_suites/search/cases/attachment_framework.ts b/x-pack/test_serverless/functional/test_suites/search/cases/attachment_framework.ts index c7252bebffd80..e786f5c7e7ec8 100644 --- a/x-pack/test_serverless/functional/test_suites/search/cases/attachment_framework.ts +++ b/x-pack/test_serverless/functional/test_suites/search/cases/attachment_framework.ts @@ -34,6 +34,7 @@ export default ({ getPageObject, getService }: FtrProviderContext) => { 'x-pack/test/functional/fixtures/kbn_archiver/lens/lens_basic.json' ); + await settings.refreshDataViewFieldList('log*'); await settings.refreshDataViewFieldList('logstash-*'); await svlSearchNavigation.navigateToLandingPage(); From 307a16d9fb4059aad0c3aaedd4ff88c9b689754b Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Mon, 4 Dec 2023 17:06:38 +0100 Subject: [PATCH 108/168] Add multi field support --- .../field_list_sidebar.tsx | 46 +++++++++++-------- .../src/hooks/use_grouped_fields.ts | 10 ++-- 2 files changed, 33 insertions(+), 23 deletions(-) diff --git a/packages/kbn-unified-field-list/src/containers/unified_field_list_sidebar/field_list_sidebar.tsx b/packages/kbn-unified-field-list/src/containers/unified_field_list_sidebar/field_list_sidebar.tsx index 4bc54069336b0..e1e72ca0a9aa9 100644 --- a/packages/kbn-unified-field-list/src/containers/unified_field_list_sidebar/field_list_sidebar.tsx +++ b/packages/kbn-unified-field-list/src/containers/unified_field_list_sidebar/field_list_sidebar.tsx @@ -191,25 +191,6 @@ export const UnifiedFieldListSidebarComponent: React.FC { - if ( - searchMode !== 'documents' || - !useNewFieldsApi || - stateService.creationOptions.disableMultiFieldsGroupingByParent - ) { - setMultiFieldsMap(undefined); // we don't have to calculate multifields in this case - } else { - setMultiFieldsMap(calculateMultiFields(allFields, selectedFieldsState.selectedFieldsMap)); - } - }, [ - stateService.creationOptions.disableMultiFieldsGroupingByParent, - selectedFieldsState.selectedFieldsMap, - allFields, - useNewFieldsApi, - setMultiFieldsMap, - searchMode, - ]); - const popularFieldsLimit = useMemo( () => core.uiSettings.get(FIELDS_LIMIT_SETTING), [core.uiSettings] @@ -226,7 +207,11 @@ export const UnifiedFieldListSidebarComponent: React.FC({ + const { + fieldListFiltersProps, + fieldListGroupedProps, + allFields: allFieldsModified, + } = useGroupedFields({ dataViewId: (searchMode === 'documents' && dataView?.id) || null, // passing `null` for text-based queries allFields, popularFieldsLimit: @@ -245,6 +230,27 @@ export const UnifiedFieldListSidebarComponent: React.FC { + if ( + searchMode !== 'documents' || + !useNewFieldsApi || + stateService.creationOptions.disableMultiFieldsGroupingByParent + ) { + setMultiFieldsMap(undefined); // we don't have to calculate multifields in this case + } else { + setMultiFieldsMap( + calculateMultiFields(allFieldsModified, selectedFieldsState.selectedFieldsMap) + ); + } + }, [ + stateService.creationOptions.disableMultiFieldsGroupingByParent, + selectedFieldsState.selectedFieldsMap, + allFieldsModified, + useNewFieldsApi, + setMultiFieldsMap, + searchMode, + ]); + const renderFieldItem: FieldListGroupedProps['renderFieldItem'] = useCallback( ({ field, groupName, groupIndex, itemIndex, fieldSearchHighlight }) => (
  • diff --git a/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts b/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts index 731eb63dd7934..87b9ab5187139 100644 --- a/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts +++ b/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts @@ -7,7 +7,7 @@ */ import { groupBy } from 'lodash'; -import { useEffect, useMemo, useState } from 'react'; +import { useEffect, useMemo, useState, useRef } from 'react'; import { i18n } from '@kbn/i18n'; import { type CoreStart } from '@kbn/core-lifecycle-browser'; import { type DataView, type DataViewField } from '@kbn/data-views-plugin/common'; @@ -52,6 +52,7 @@ export interface GroupedFieldsResult { fieldsExistInIndex: boolean; screenReaderDescriptionId?: string; }; + allFields: T[] | null; // `null` is for loading indicator } export function useGroupedFields({ @@ -73,6 +74,7 @@ export function useGroupedFields({ getCustomFieldType, onSupportedFieldFilter, }); + const fieldsToSort = useRef(allFields); const onFilterFieldList = fieldListFilters.onFilterField; const [dataView, setDataView] = useState(null); const isAffectedByTimeFilter = Boolean(dataView?.timeFieldName); @@ -121,13 +123,14 @@ export function useGroupedFields({ const selectedFields = sortedSelectedFields || []; const newFields = dataViewId ? fieldsExistenceReader.getNewFields(dataViewId) : []; - const fieldsToSort = + fieldsToSort.current = allFields && newFields.length ? allFields.map((field) => { return (dataView?.getFieldByName(field.name) as unknown as T) ?? field; }) : allFields; - const sortedFields = [...(fieldsToSort || [])].sort(sortFields); + + const sortedFields = [...(fieldsToSort.current || [])].sort(sortFields); const groupedFields = { ...getDefaultFieldGroups(), @@ -391,6 +394,7 @@ export function useGroupedFields({ return { fieldListGroupedProps, fieldListFiltersProps: fieldListFilters.fieldListFiltersProps, + allFields: fieldsToSort.current, }; } From af83362507c45c2765bec1915b8dd8211c971496 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Tue, 5 Dec 2023 13:16:27 -0600 Subject: [PATCH 109/168] fix discover alert functional test --- src/plugins/data_views/common/data_views/data_views.ts | 5 ----- test/functional/page_objects/settings_page.ts | 3 +-- .../discover_ml_uptime/discover/search_source_alert.ts | 9 +-------- .../discover_ml_uptime/discover/search_source_alert.ts | 8 ++++++-- 4 files changed, 8 insertions(+), 17 deletions(-) diff --git a/src/plugins/data_views/common/data_views/data_views.ts b/src/plugins/data_views/common/data_views/data_views.ts index 2aa28d2e3a927..fd2b2b5e8d1ff 100644 --- a/src/plugins/data_views/common/data_views/data_views.ts +++ b/src/plugins/data_views/common/data_views/data_views.ts @@ -923,7 +923,6 @@ export class DataViewsService { displayErrors: boolean = true, refreshFields = false ): Promise => { - console.log('DEBUG: get data view', id); const dataViewFromCache = this.dataViewCache.get(id)?.then(async (dataView) => { if (dataView && refreshFields) { await this.refreshFields(dataView, displayErrors); @@ -940,10 +939,6 @@ export class DataViewsService { this.dataViewCache.clear(id); }); - indexPatternPromise.then((indexPattern) => { - console.log('DEBUG: get data view resolved', id, indexPattern.getIndexPattern()); - }); - return indexPatternPromise; }; diff --git a/test/functional/page_objects/settings_page.ts b/test/functional/page_objects/settings_page.ts index d1c49c89923b7..92b6a4eebf3a8 100644 --- a/test/functional/page_objects/settings_page.ts +++ b/test/functional/page_objects/settings_page.ts @@ -485,8 +485,7 @@ export class SettingsPageObject extends FtrService { async refreshDataViewFieldList(dataViewName?: string) { if (dataViewName) { - await this.navigateTo(); - await this.clickKibanaIndexPatterns(); + await this.common.navigateToApp('management/kibana/dataViews'); await this.header.waitUntilLoadingHasFinished(); await this.testSubjects.click(`detail-link-${dataViewName}`); } diff --git a/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts b/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts index 8b48c0e04b5b1..6b668f75d56d3 100644 --- a/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts +++ b/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts @@ -209,9 +209,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.common.navigateToApp('discover'); await PageObjects.header.waitUntilLoadingHasFinished(); await PageObjects.discover.clickNewSearchButton(); // reset params + await PageObjects.discover.selectIndexPattern(OUTPUT_DATA_VIEW); - // this fixed it before - // await PageObjects.discover.refreshFieldList(); let ruleId: string; if (type === 'name') { @@ -400,12 +399,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); it('should navigate to alert results via link provided in notification', async () => { - // todo did this work? - log.write('DATA VIEW REFRESH'); - await PageObjects.settings.refreshDataViewFieldList(OUTPUT_DATA_VIEW); - await new Promise((resolve) => setTimeout(resolve, 5000)); - await PageObjects.settings.refreshDataViewFieldList(OUTPUT_DATA_VIEW); - log.write('DATA VIEW REFRESH COMPLETE'); await openAlertResults(RULE_NAME); await checkInitialRuleParamsState(SOURCE_DATA_VIEW); }); diff --git a/x-pack/test_serverless/functional/test_suites/common/discover_ml_uptime/discover/search_source_alert.ts b/x-pack/test_serverless/functional/test_suites/common/discover_ml_uptime/discover/search_source_alert.ts index f683e02237996..3c431190c8abb 100644 --- a/x-pack/test_serverless/functional/test_suites/common/discover_ml_uptime/discover/search_source_alert.ts +++ b/x-pack/test_serverless/functional/test_suites/common/discover_ml_uptime/discover/search_source_alert.ts @@ -238,6 +238,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }; const openAlertResults = async (value: string, type: 'id' | 'name' = 'name') => { + await PageObjects.settings.refreshDataViewFieldList(OUTPUT_DATA_VIEW); await PageObjects.common.navigateToApp('discover'); await PageObjects.header.waitUntilLoadingHasFinished(); await PageObjects.discover.clickNewSearchButton(); // reset params @@ -378,9 +379,12 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { // should not have data view selected by default const dataViewSelector = await testSubjects.find('selectDataViewExpression'); - // TODO: Serverless Security has an existing data view by default + // TODO: Serverless Security and Search have an existing data view by default const dataViewSelectorText = await dataViewSelector.getVisibleText(); - if (!dataViewSelectorText.includes('.alerts-security')) { + if ( + !dataViewSelectorText.includes('.alerts-security') && + !dataViewSelectorText.includes('default:all-data') + ) { expect(await dataViewSelector.getVisibleText()).to.eql('DATA VIEW\nSelect a data view'); } From 1d0dcd5e788a73b39a4918058ac548e2118e00e5 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Tue, 5 Dec 2023 21:00:52 -0600 Subject: [PATCH 110/168] get fresh field list vith data view swapping. --- .../apps/discover/feature_controls/discover_security.ts | 8 ++++---- .../discover_ml_uptime/discover/search_source_alert.ts | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/x-pack/test/functional/apps/discover/feature_controls/discover_security.ts b/x-pack/test/functional/apps/discover/feature_controls/discover_security.ts index d8c26060716bf..be105c4467696 100644 --- a/x-pack/test/functional/apps/discover/feature_controls/discover_security.ts +++ b/x-pack/test/functional/apps/discover/feature_controls/discover_security.ts @@ -11,7 +11,6 @@ import { getSavedQuerySecurityUtils } from '../../saved_query_management/utils/s export default function (ctx: FtrProviderContext) { const { getPageObjects, getService } = ctx; - const log = getService('log'); const savedQuerySecurityUtils = getSavedQuerySecurityUtils(ctx); const esArchiver = getService('esArchiver'); const esSupertest = getService('esSupertest'); @@ -451,15 +450,16 @@ export default function (ctx: FtrProviderContext) { // can't access logstash index directly // swapping index patterns so we get an updated field list - log.debug('DEBUG a bunch of data view switching'); + await new Promise((resolve) => setTimeout(resolve, 5000)); + await PageObjects.discover.selectIndexPattern('logstash-*'); await new Promise((resolve) => setTimeout(resolve, 5000)); await PageObjects.discover.selectIndexPattern('alias-logstash-discover'); await new Promise((resolve) => setTimeout(resolve, 5000)); await PageObjects.discover.selectIndexPattern('logstash-*'); await new Promise((resolve) => setTimeout(resolve, 5000)); await PageObjects.discover.selectIndexPattern('alias-logstash-discover'); - log.debug('DEBUG a bunch of data view switching done'); - // + + await new Promise((resolve) => setTimeout(resolve, 5000)); await PageObjects.discover.selectIndexPattern('logstash-*'); await PageObjects.header.waitUntilLoadingHasFinished(); await testSubjects.existOrFail('discoverNoResultsCheckIndices'); diff --git a/x-pack/test_serverless/functional/test_suites/common/discover_ml_uptime/discover/search_source_alert.ts b/x-pack/test_serverless/functional/test_suites/common/discover_ml_uptime/discover/search_source_alert.ts index 3c431190c8abb..ef9a62018fbe7 100644 --- a/x-pack/test_serverless/functional/test_suites/common/discover_ml_uptime/discover/search_source_alert.ts +++ b/x-pack/test_serverless/functional/test_suites/common/discover_ml_uptime/discover/search_source_alert.ts @@ -238,8 +238,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }; const openAlertResults = async (value: string, type: 'id' | 'name' = 'name') => { - await PageObjects.settings.refreshDataViewFieldList(OUTPUT_DATA_VIEW); - await PageObjects.common.navigateToApp('discover'); await PageObjects.header.waitUntilLoadingHasFinished(); await PageObjects.discover.clickNewSearchButton(); // reset params @@ -448,6 +446,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); it('should navigate to alert results via link provided in notification', async () => { + await PageObjects.settings.refreshDataViewFieldList(OUTPUT_DATA_VIEW); + await PageObjects.common.navigateToApp('discover'); await openAlertResults(RULE_NAME); await checkInitialRuleParamsState(SOURCE_DATA_VIEW); }); From 797f6ce508c71c832292e868f6aa1015427af415 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Tue, 5 Dec 2023 22:31:39 -0600 Subject: [PATCH 111/168] fix discover alert functional test --- .../common/discover_ml_uptime/discover/search_source_alert.ts | 2 +- .../test_suites/search/cases/attachment_framework.ts | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/x-pack/test_serverless/functional/test_suites/common/discover_ml_uptime/discover/search_source_alert.ts b/x-pack/test_serverless/functional/test_suites/common/discover_ml_uptime/discover/search_source_alert.ts index ef9a62018fbe7..d2e8f863d5fe9 100644 --- a/x-pack/test_serverless/functional/test_suites/common/discover_ml_uptime/discover/search_source_alert.ts +++ b/x-pack/test_serverless/functional/test_suites/common/discover_ml_uptime/discover/search_source_alert.ts @@ -238,6 +238,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }; const openAlertResults = async (value: string, type: 'id' | 'name' = 'name') => { + await PageObjects.common.navigateToApp('discover'); await PageObjects.header.waitUntilLoadingHasFinished(); await PageObjects.discover.clickNewSearchButton(); // reset params @@ -447,7 +448,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { it('should navigate to alert results via link provided in notification', async () => { await PageObjects.settings.refreshDataViewFieldList(OUTPUT_DATA_VIEW); - await PageObjects.common.navigateToApp('discover'); await openAlertResults(RULE_NAME); await checkInitialRuleParamsState(SOURCE_DATA_VIEW); }); diff --git a/x-pack/test_serverless/functional/test_suites/search/cases/attachment_framework.ts b/x-pack/test_serverless/functional/test_suites/search/cases/attachment_framework.ts index e786f5c7e7ec8..26e0812400283 100644 --- a/x-pack/test_serverless/functional/test_suites/search/cases/attachment_framework.ts +++ b/x-pack/test_serverless/functional/test_suites/search/cases/attachment_framework.ts @@ -16,7 +16,7 @@ export default ({ getPageObject, getService }: FtrProviderContext) => { const svlSearchNavigation = getService('svlSearchNavigation'); const svlCommonNavigation = getPageObject('svlCommonNavigation'); const svlCommonPage = getPageObject('svlCommonPage'); - const settings = getPageObject('settings'); + // const settings = getPageObject('settings'); describe('persistable attachment', () => { before(async () => { @@ -34,8 +34,10 @@ export default ({ getPageObject, getService }: FtrProviderContext) => { 'x-pack/test/functional/fixtures/kbn_archiver/lens/lens_basic.json' ); + /* await settings.refreshDataViewFieldList('log*'); await settings.refreshDataViewFieldList('logstash-*'); + */ await svlSearchNavigation.navigateToLandingPage(); From 913bedbc3fdc7545861f3553f8d85c2670b70891 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Tue, 5 Dec 2023 23:34:37 -0600 Subject: [PATCH 112/168] fix discover alert functional test --- .../apps/discover_ml_uptime/discover/search_source_alert.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts b/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts index 6b668f75d56d3..bffa564fba774 100644 --- a/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts +++ b/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts @@ -399,6 +399,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); it('should navigate to alert results via link provided in notification', async () => { + await PageObjects.settings.refreshDataViewFieldList(OUTPUT_DATA_VIEW); await openAlertResults(RULE_NAME); await checkInitialRuleParamsState(SOURCE_DATA_VIEW); }); From 3f58ecdc347c9ab7da118b3e3bb8972126d3dea6 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Wed, 6 Dec 2023 15:49:58 -0600 Subject: [PATCH 113/168] fix functional test --- .../data_views/public/data_views/data_views_api_client.ts | 1 - .../test_suites/search/cases/attachment_framework.ts | 7 ++----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/plugins/data_views/public/data_views/data_views_api_client.ts b/src/plugins/data_views/public/data_views/data_views_api_client.ts index 5e90cf220fc2a..c65db9e5c736f 100644 --- a/src/plugins/data_views/public/data_views/data_views_api_client.ts +++ b/src/plugins/data_views/public/data_views/data_views_api_client.ts @@ -110,7 +110,6 @@ export class DataViewsApiClient implements IDataViewsApiClient { indexFilter ? JSON.stringify({ index_filter: indexFilter }) : undefined, forceRefresh ).then((response) => { - console.log('DEBUG: api_client', pattern, JSON.stringify(response?.body, null, 2)); return { indices: response?.body?.indices || [], fields: response?.body?.fields || [], diff --git a/x-pack/test_serverless/functional/test_suites/search/cases/attachment_framework.ts b/x-pack/test_serverless/functional/test_suites/search/cases/attachment_framework.ts index 26e0812400283..e74262f60ac3e 100644 --- a/x-pack/test_serverless/functional/test_suites/search/cases/attachment_framework.ts +++ b/x-pack/test_serverless/functional/test_suites/search/cases/attachment_framework.ts @@ -16,7 +16,7 @@ export default ({ getPageObject, getService }: FtrProviderContext) => { const svlSearchNavigation = getService('svlSearchNavigation'); const svlCommonNavigation = getPageObject('svlCommonNavigation'); const svlCommonPage = getPageObject('svlCommonPage'); - // const settings = getPageObject('settings'); + const settings = getPageObject('settings'); describe('persistable attachment', () => { before(async () => { @@ -34,10 +34,7 @@ export default ({ getPageObject, getService }: FtrProviderContext) => { 'x-pack/test/functional/fixtures/kbn_archiver/lens/lens_basic.json' ); - /* - await settings.refreshDataViewFieldList('log*'); - await settings.refreshDataViewFieldList('logstash-*'); - */ + await settings.refreshDataViewFieldList('default:all-data'); await svlSearchNavigation.navigateToLandingPage(); From 08a3ea8ea313198004d19dd4e83ea1da73a51d9c Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Wed, 6 Dec 2023 17:59:31 -0600 Subject: [PATCH 114/168] cleanup --- src/plugins/data_views/common/data_views/data_views.ts | 2 +- .../apps/discover/feature_controls/discover_security.ts | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/plugins/data_views/common/data_views/data_views.ts b/src/plugins/data_views/common/data_views/data_views.ts index fd2b2b5e8d1ff..f694bcccb9009 100644 --- a/src/plugins/data_views/common/data_views/data_views.ts +++ b/src/plugins/data_views/common/data_views/data_views.ts @@ -544,7 +544,7 @@ export class DataViewsService { forceRefresh: boolean = false ) => { const metaFields = await this.config.get(META_FIELDS); - return await this.apiClient.getFieldsForWildcard({ + return this.apiClient.getFieldsForWildcard({ type: dataView.type, rollupIndex: dataView?.typeMeta?.params?.rollup_index, allowNoIndex: true, diff --git a/x-pack/test/functional/apps/discover/feature_controls/discover_security.ts b/x-pack/test/functional/apps/discover/feature_controls/discover_security.ts index be105c4467696..0db9064f9cc4e 100644 --- a/x-pack/test/functional/apps/discover/feature_controls/discover_security.ts +++ b/x-pack/test/functional/apps/discover/feature_controls/discover_security.ts @@ -450,6 +450,8 @@ export default function (ctx: FtrProviderContext) { // can't access logstash index directly // swapping index patterns so we get an updated field list + // this is necessary since we don't have access to data view management + // nor can we force reload the browser in a test await new Promise((resolve) => setTimeout(resolve, 5000)); await PageObjects.discover.selectIndexPattern('logstash-*'); await new Promise((resolve) => setTimeout(resolve, 5000)); From b73f727e7dad44f31597721aea0382ef3a615adb Mon Sep 17 00:00:00 2001 From: Matthew Kime Date: Thu, 7 Dec 2023 15:33:55 -0600 Subject: [PATCH 115/168] Update docs/concepts/data-views.asciidoc Co-authored-by: amyjtechwriter <61687663+amyjtechwriter@users.noreply.github.com> --- docs/concepts/data-views.asciidoc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/concepts/data-views.asciidoc b/docs/concepts/data-views.asciidoc index 56b1df33803b3..2eba42aed3051 100644 --- a/docs/concepts/data-views.asciidoc +++ b/docs/concepts/data-views.asciidoc @@ -179,8 +179,7 @@ click image:management/index-patterns/images/delete.png[Delete icon] in the *Act The browser caches {data-source} field lists for increased performance. This is particularly impactful for {data-sources} with a high field count that span a large number of indices and clusters. The field -list will be updated every couple of minutes in typical {kib} usage. Alternatively, the {data-source} -management detail page provides a refresh button that gets an updated field list. A force reload of {kib} -will do the same. +list is updated every couple of minutes in typical {kib} usage. Alternatively, use the refresh button on the {data-source} +management detail page to get an updated field list. A force reload of {kib} has the same effect. The field list may be impacted by changes in indices and user permissions. \ No newline at end of file From 9c4e41679d47479f191f24219a5c76ca501f051d Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Thu, 7 Dec 2023 19:57:56 -0600 Subject: [PATCH 116/168] better rendering of refresh button --- .../index_header/index_header.tsx | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/plugins/data_view_management/public/components/edit_index_pattern/index_header/index_header.tsx b/src/plugins/data_view_management/public/components/edit_index_pattern/index_header/index_header.tsx index cd03e4b0aa54f..8a3792c6fb333 100644 --- a/src/plugins/data_view_management/public/components/edit_index_pattern/index_header/index_header.tsx +++ b/src/plugins/data_view_management/public/components/edit_index_pattern/index_header/index_header.tsx @@ -77,28 +77,29 @@ export const IndexHeader: React.FC = ({ {indexPattern.getName()}} rightSideItems={[ - {refreshTooltip}

    }> - - {isRefreshing ? isLoadingLabel : refreshLabel} - -     , canSave && ( {editTooltip} ), + {refreshTooltip}

    }> + + {isRefreshing ? isLoadingLabel : refreshLabel} + +     , defaultIndex !== indexPattern.id && setDefault && canSave && indexPattern.isPersisted() && ( Date: Sat, 9 Dec 2023 14:30:04 +0100 Subject: [PATCH 117/168] Add support for Lens --- .../src/hooks/use_grouped_fields.ts | 36 ++++++++++++------- .../datasources/form_based/datapanel.tsx | 32 ++++++++++------- .../datasources/text_based/datapanel.tsx | 2 ++ 3 files changed, 46 insertions(+), 24 deletions(-) diff --git a/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts b/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts index 87b9ab5187139..8da8646da8e5c 100644 --- a/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts +++ b/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts @@ -41,6 +41,7 @@ export interface GroupedFieldsParams { onOverrideFieldGroupDetails?: OverrideFieldGroupDetails; onSupportedFieldFilter?: (field: T) => boolean; onSelectedFieldFilter?: (field: T) => boolean; + isCompatibleField?: (fields: DataViewField) => boolean; } export interface GroupedFieldsResult { @@ -53,6 +54,7 @@ export interface GroupedFieldsResult { screenReaderDescriptionId?: string; }; allFields: T[] | null; // `null` is for loading indicator + hasNewFields: boolean; } export function useGroupedFields({ @@ -66,6 +68,7 @@ export function useGroupedFields({ onOverrideFieldGroupDetails, onSupportedFieldFilter, onSelectedFieldFilter, + isCompatibleField, }: GroupedFieldsParams): GroupedFieldsResult { const fieldsExistenceReader = useExistingFieldsReader(); const fieldListFilters = useFieldFilters({ @@ -74,7 +77,8 @@ export function useGroupedFields({ getCustomFieldType, onSupportedFieldFilter, }); - const fieldsToSort = useRef(allFields); + const allFieldsInclNew = useRef(allFields); + const hasNewFields = useRef(false); const onFilterFieldList = fieldListFilters.onFilterField; const [dataView, setDataView] = useState(null); const isAffectedByTimeFilter = Boolean(dataView?.timeFieldName); @@ -122,16 +126,23 @@ export function useGroupedFields({ }; const selectedFields = sortedSelectedFields || []; - const newFields = dataViewId ? fieldsExistenceReader.getNewFields(dataViewId) : []; - fieldsToSort.current = - allFields && newFields.length - ? allFields.map((field) => { - return (dataView?.getFieldByName(field.name) as unknown as T) ?? field; - }) - : allFields; - - const sortedFields = [...(fieldsToSort.current || [])].sort(sortFields); + const newFields = dataViewId + ? fieldsExistenceReader + .getNewFields(dataViewId) + .filter((field) => (isCompatibleField ? isCompatibleField(field) : true)) + : []; + // remove fields from allFields that are available in newFields, because they can be provided in unmapped state + const allFieldsWithoutNewFields = !newFields.length + ? allFields + : allFields?.filter((field) => !newFields.find((newField) => newField.name === field.name)); + // append new fields to the end of the list allFieldsWithoutNewFields + const allFieldsWithNewFields = allFieldsWithoutNewFields + ? [...allFieldsWithoutNewFields, ...newFields] + : newFields; + const sortedFields = [...((allFieldsWithNewFields as unknown as T[]) || [])].sort(sortFields); + allFieldsInclNew.current = sortedFields; + hasNewFields.current = Boolean(newFields.length); const groupedFields = { ...getDefaultFieldGroups(), ...groupBy(sortedFields, (field) => { @@ -336,6 +347,7 @@ export function useGroupedFields({ onSelectedFieldFilter, onSupportedFieldFilter, dataView, + isCompatibleField, ]); const fieldGroups: FieldListGroups = useMemo(() => { @@ -394,7 +406,8 @@ export function useGroupedFields({ return { fieldListGroupedProps, fieldListFiltersProps: fieldListFilters.fieldListFiltersProps, - allFields: fieldsToSort.current, + allFields: allFieldsInclNew.current, + hasNewFields: hasNewFields.current, }; } @@ -417,6 +430,5 @@ function getDefaultFieldGroups() { metaFields: [], unmappedFields: [], skippedFields: [], - newFields: [], }; } diff --git a/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx b/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx index 53a26693b7a7a..58c4e1bddaa27 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx +++ b/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx @@ -28,6 +28,7 @@ import { useGroupedFields, } from '@kbn/unified-field-list'; import { ChartsPluginSetup } from '@kbn/charts-plugin/public'; +import { isFieldLensCompatible } from '@kbn/visualization-ui-components'; import type { DatasourceDataPanelProps, FramePublicAPI, @@ -249,18 +250,20 @@ export const InnerFormBasedDataPanel = function InnerFormBasedDataPanel({ } }, []); - const { fieldListFiltersProps, fieldListGroupedProps } = useGroupedFields({ - dataViewId: currentIndexPatternId, - allFields, - services: { - dataViews, - core, - }, - isAffectedByGlobalFilter: Boolean(filters.length), - onSupportedFieldFilter, - onSelectedFieldFilter, - onOverrideFieldGroupDetails, - }); + const { fieldListFiltersProps, fieldListGroupedProps, hasNewFields } = + useGroupedFields({ + dataViewId: currentIndexPatternId, + allFields, + services: { + dataViews, + core, + }, + isAffectedByGlobalFilter: Boolean(filters.length), + onSupportedFieldFilter, + onSelectedFieldFilter, + onOverrideFieldGroupDetails, + isCompatibleField: isFieldLensCompatible, + }); const closeFieldEditor = useRef<() => void | undefined>(); @@ -296,6 +299,11 @@ export const InnerFormBasedDataPanel = function InnerFormBasedDataPanel({ frame.dataViews.indexPatterns, data.search.session, ]); + useEffect(() => { + if (hasNewFields) { + refreshFieldList(); + } + }, [hasNewFields, refreshFieldList]); const editField = useMemo( () => diff --git a/x-pack/plugins/lens/public/datasources/text_based/datapanel.tsx b/x-pack/plugins/lens/public/datasources/text_based/datapanel.tsx index 113125484cddf..ac62636ad50c1 100644 --- a/x-pack/plugins/lens/public/datasources/text_based/datapanel.tsx +++ b/x-pack/plugins/lens/public/datasources/text_based/datapanel.tsx @@ -24,6 +24,7 @@ import { GetCustomFieldType, useGroupedFields, } from '@kbn/unified-field-list'; +import { isFieldLensCompatible } from '@kbn/visualization-ui-components'; import type { DatasourceDataPanelProps } from '../../types'; import type { TextBasedPrivateState } from './types'; import { getStateFromAggregateQuery } from './utils'; @@ -105,6 +106,7 @@ export function TextBasedDataPanel({ getCustomFieldType, onSelectedFieldFilter, onOverrideFieldGroupDetails, + isCompatibleField: isFieldLensCompatible, }); const renderFieldItem: FieldListGroupedProps['renderFieldItem'] = useCallback( From dc9fa0e12261039dc72942708d3ec422ebe02d90 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Sun, 10 Dec 2023 22:08:21 -0600 Subject: [PATCH 118/168] cleanup --- .../components/edit_index_pattern/edit_index_pattern.tsx | 1 - x-pack/plugins/security/public/plugin.tsx | 8 ++------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/src/plugins/data_view_management/public/components/edit_index_pattern/edit_index_pattern.tsx b/src/plugins/data_view_management/public/components/edit_index_pattern/edit_index_pattern.tsx index 839d04665ae09..20855e4dbbc0c 100644 --- a/src/plugins/data_view_management/public/components/edit_index_pattern/edit_index_pattern.tsx +++ b/src/plugins/data_view_management/public/components/edit_index_pattern/edit_index_pattern.tsx @@ -252,7 +252,6 @@ export const EditIndexPattern = withRouter( refreshIndexPatternClick={async () => { setIsRefreshing(true); await dataViews.refreshFields(indexPattern, false, true); - setFields(indexPattern.getNonScriptedFields()); setRefreshCount(refreshCount + 1); // rerender field list setIsRefreshing(false); }} diff --git a/x-pack/plugins/security/public/plugin.tsx b/x-pack/plugins/security/public/plugin.tsx index be0abf3f828b8..1927ebb21d8e1 100644 --- a/x-pack/plugins/security/public/plugin.tsx +++ b/x-pack/plugins/security/public/plugin.tsx @@ -13,10 +13,7 @@ import type { Plugin, PluginInitializerContext, } from '@kbn/core/public'; -import type { - DataViewsPublicPluginSetup, - DataViewsPublicPluginStart, -} from '@kbn/data-views-plugin/public'; +import type { DataViewsPublicPluginStart } from '@kbn/data-views-plugin/public'; import type { FeaturesPluginStart } from '@kbn/features-plugin/public'; import type { HomePublicPluginSetup } from '@kbn/home-plugin/public'; import { i18n } from '@kbn/i18n'; @@ -51,7 +48,6 @@ export interface PluginSetupDependencies { management?: ManagementSetup; share?: SharePluginSetup; cloud?: CloudSetup; - dataViews?: DataViewsPublicPluginSetup; } export interface PluginStartDependencies { @@ -94,7 +90,7 @@ export class SecurityPlugin public setup( core: CoreSetup, - { cloud, home, licensing, management, share, dataViews }: PluginSetupDependencies + { cloud, home, licensing, management, share }: PluginSetupDependencies ): SecurityPluginSetup { const { license } = this.securityLicenseService.setup({ license$: licensing.license$ }); From 985ed118d81d1394ae6a182cf74e1b8bf0e29f09 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Mon, 11 Dec 2023 08:09:46 -0600 Subject: [PATCH 119/168] better caching, add tests --- .../common/data_views/data_views.test.ts | 13 +++++++ .../common/data_views/data_views.ts | 6 ++-- .../server/rest_api_routes/internal/fields.ts | 10 ++++-- .../apis/data_views/fields_route/cache.ts | 34 +++++++++++++++++++ 4 files changed, 58 insertions(+), 5 deletions(-) diff --git a/src/plugins/data_views/common/data_views/data_views.test.ts b/src/plugins/data_views/common/data_views/data_views.test.ts index e95e5c6736d17..4d0fa196e9ccc 100644 --- a/src/plugins/data_views/common/data_views/data_views.test.ts +++ b/src/plugins/data_views/common/data_views/data_views.test.ts @@ -158,10 +158,23 @@ describe('IndexPatterns', () => { test('force field refresh', async () => { const id = '1'; + // make sure initial load and subsequent reloads use same params + const args = { + allowHidden: undefined, + allowNoIndex: true, + indexFilter: undefined, + metaFields: false, + pattern: 'something', + rollupIndex: undefined, + type: undefined, + }; + await indexPatterns.get(id); expect(apiClient.getFieldsForWildcard).toBeCalledTimes(1); + expect(apiClient.getFieldsForWildcard).toBeCalledWith(args); await indexPatterns.get(id, undefined, true); expect(apiClient.getFieldsForWildcard).toBeCalledTimes(2); + expect(apiClient.getFieldsForWildcard).toBeCalledWith(args); }); test('getFieldsForWildcard called with allowNoIndex set to true as default ', async () => { diff --git a/src/plugins/data_views/common/data_views/data_views.ts b/src/plugins/data_views/common/data_views/data_views.ts index f694bcccb9009..dfdb8025b182c 100644 --- a/src/plugins/data_views/common/data_views/data_views.ts +++ b/src/plugins/data_views/common/data_views/data_views.ts @@ -536,7 +536,9 @@ export class DataViewsService { ...options, pattern: indexPattern.title as string, allowHidden: - (indexPattern as DataViewSpec).allowHidden || (indexPattern as DataView)?.getAllowHidden(), + (indexPattern as DataViewSpec).allowHidden || + (indexPattern as DataView)?.getAllowHidden() || + undefined, }); private getFieldsAndIndicesForDataView = async ( @@ -551,7 +553,7 @@ export class DataViewsService { pattern: dataView.getIndexPattern(), metaFields, forceRefresh, - allowHidden: dataView.getAllowHidden(), + allowHidden: dataView.getAllowHidden() || undefined, }); }; diff --git a/src/plugins/data_views/server/rest_api_routes/internal/fields.ts b/src/plugins/data_views/server/rest_api_routes/internal/fields.ts index 454035f3b812f..cee8efc8b6d1d 100644 --- a/src/plugins/data_views/server/rest_api_routes/internal/fields.ts +++ b/src/plugins/data_views/server/rest_api_routes/internal/fields.ts @@ -78,9 +78,13 @@ const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, I }; // field cache is configurable in classic environment but not on serverless - const cacheMaxAge = - (await uiSettings.get('data_views:cache_max_age')) || - DEFAULT_FIELD_CACHE_FRESHNESS; + let cacheMaxAge = DEFAULT_FIELD_CACHE_FRESHNESS; + const cacheMaxAgeSetting = await uiSettings.get( + 'data_views:cache_max_age' + ); + if (cacheMaxAgeSetting !== undefined) { + cacheMaxAge = cacheMaxAgeSetting; + } if (cacheMaxAge && fields.length) { const stale = 365 * 24 * 60 * 60 - cacheMaxAge; diff --git a/test/api_integration/apis/data_views/fields_route/cache.ts b/test/api_integration/apis/data_views/fields_route/cache.ts index 978c048ee8b8e..c97be4b7411f3 100644 --- a/test/api_integration/apis/data_views/fields_route/cache.ts +++ b/test/api_integration/apis/data_views/fields_route/cache.ts @@ -14,6 +14,7 @@ import { FtrProviderContext } from '../../../ftr_provider_context'; export default function ({ getService }: FtrProviderContext) { const esArchiver = getService('esArchiver'); const supertest = getService('supertest'); + const kibanaServer = getService('kibanaServer'); describe('cache headers', () => { before(() => @@ -32,12 +33,34 @@ export default function ({ getService }: FtrProviderContext) { const cacheControlHeader = response.get('cache-control'); + expect(cacheControlHeader).to.contain('private'); expect(cacheControlHeader).to.contain('max-age'); expect(cacheControlHeader).to.contain('stale-while-revalidate'); expect(response.get('vary')).to.equal('accept-encoding, user-hash'); expect(response.get('etag')).to.not.be.empty(); }); + it('no-cache when data_views:cache_max_age set to zero', async () => { + await kibanaServer.uiSettings.update({ 'data_views:cache_max_age': 0 }); + + const response = await supertest.get(FIELDS_PATH).query({ + pattern: 'b*', + include_unmapped: true, + apiVersion: INITIAL_REST_VERSION_INTERNAL, + }); + + const cacheControlHeader = response.get('cache-control'); + + expect(cacheControlHeader).to.contain('private'); + expect(cacheControlHeader).to.contain('no-cache'); + expect(cacheControlHeader).to.not.contain('max-age'); + expect(cacheControlHeader).to.not.contain('stale-while-revalidate'); + expect(response.get('vary')).to.equal('accept-encoding, user-hash'); + expect(response.get('etag')).to.not.be.empty(); + + kibanaServer.uiSettings.replace({ 'data_views:cache_max_age': 5 }); + }); + it('returns 304 on matching etag', async () => { const response = await supertest.get(FIELDS_PATH).query({ pattern: '*', @@ -55,5 +78,16 @@ export default function ({ getService }: FtrProviderContext) { }) .expect(304); }); + + it('handles empty field lists', async () => { + const response = await supertest.get(FIELDS_PATH).query({ + pattern: 'xyz', + include_unmapped: true, + apiVersion: INITIAL_REST_VERSION_INTERNAL, + allow_no_index: true, + }); + + expect(response.body.fields).to.be.empty(); + }); }); } From 15cc582977bb76e683d223e8a9167add0264471d Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Wed, 13 Dec 2023 19:11:49 +0100 Subject: [PATCH 120/168] Add functional test --- .../discover/group4/_field_list_new_fields.ts | 86 +++++++++++++++++++ test/functional/apps/discover/group4/index.ts | 1 + 2 files changed, 87 insertions(+) create mode 100644 test/functional/apps/discover/group4/_field_list_new_fields.ts diff --git a/test/functional/apps/discover/group4/_field_list_new_fields.ts b/test/functional/apps/discover/group4/_field_list_new_fields.ts new file mode 100644 index 0000000000000..3c24bcf613ae4 --- /dev/null +++ b/test/functional/apps/discover/group4/_field_list_new_fields.ts @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../ftr_provider_context'; + +export default function ({ getService, getPageObjects }: FtrProviderContext) { + const esArchiver = getService('esArchiver'); + const kibanaServer = getService('kibanaServer'); + const security = getService('security'); + const es = getService('es'); + const retry = getService('retry'); + const queryBar = getService('queryBar'); + const PageObjects = getPageObjects(['common', 'discover', 'timePicker', 'unifiedFieldList']); + + describe('Field list new fields in background handling', function () { + before(async () => { + await security.testUser.setRoles(['kibana_admin', 'test_logstash_reader']); + await kibanaServer.importExport.load('test/functional/fixtures/kbn_archiver/discover.json'); + await esArchiver.loadIfNeeded('test/functional/fixtures/es_archiver/logstash_functional'); + + await PageObjects.common.navigateToApp('discover'); + await PageObjects.timePicker.setCommonlyUsedTime('This_week'); + }); + + after(async () => { + await kibanaServer.savedObjects.cleanStandardList(); + await esArchiver.unload('x-pack/test/functional/es_archives/logstash_functional'); + await es.transport.request({ + path: '/my-index-000001', + method: 'DELETE', + }); + }); + + it('Check that new ingested fields are added to the available fields section', async function () { + const initialPattern = 'my-index-'; + await es.transport.request({ + path: '/my-index-000001/_doc', + method: 'POST', + body: { + '@timestamp': new Date().toISOString(), + a: 'GET /search HTTP/1.1 200 1070000', + }, + }); + + await PageObjects.discover.createAdHocDataView(initialPattern, true); + + await retry.waitFor('current data view to get updated', async () => { + return (await PageObjects.discover.getCurrentlySelectedDataView()) === `${initialPattern}*`; + }); + await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded(); + + expect(await PageObjects.discover.getHitCountInt()).to.be(1); + expect(await PageObjects.unifiedFieldList.getSidebarSectionFieldNames('available')).to.eql([ + '@timestamp', + 'a', + ]); + + await es.transport.request({ + path: '/my-index-000001/_doc', + method: 'POST', + body: { + '@timestamp': new Date().toISOString(), + b: 'GET /search HTTP/1.1 200 1070000', + }, + }); + + await retry.waitFor('the new record was found', async () => { + await queryBar.submitQuery(); + await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded(); + return (await PageObjects.discover.getHitCountInt()) === 2; + }); + + expect(await PageObjects.unifiedFieldList.getSidebarSectionFieldNames('available')).to.eql([ + '@timestamp', + 'a', + 'b', + ]); + }); + }); +} diff --git a/test/functional/apps/discover/group4/index.ts b/test/functional/apps/discover/group4/index.ts index 1aab3db2bfc43..656a116551db8 100644 --- a/test/functional/apps/discover/group4/index.ts +++ b/test/functional/apps/discover/group4/index.ts @@ -33,5 +33,6 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./_context_encoded_url_params')); loadTestFile(require.resolve('./_hide_announcements')); loadTestFile(require.resolve('./_data_view_edit')); + loadTestFile(require.resolve('./_field_list_new_fields')); }); } From 065a50c9da9b3e1ae2e3c6e2aa4abf38d6e97acf Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Thu, 14 Dec 2023 06:06:19 +0100 Subject: [PATCH 121/168] Add documentation --- .../src/services/field_existing/field_existing_utils.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts b/packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts index 0a487368748cf..e4dab136c5a52 100644 --- a/packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts +++ b/packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts @@ -54,15 +54,20 @@ export async function fetchFieldExistence({ pattern: '', indexFilter: toQuery(timeFieldName, fromDate, toDate, dslQuery), }); + + // take care of fields of existingFieldList, that are not yet available + // in the given data view. Those fields we consider as new fields, + // that were ingested after the data view was loaded const newFields = existingFieldList .filter((field) => dataView.getFieldByName(field.name) === undefined) .map((field) => dataView.getFieldByName(field.name) ?? new DataViewField(field)); + // refresh the data view in case there are new fields if (newFields.length) { await dataViewsService.refreshFields(dataView, false); } const allFields = buildFieldList(dataView, metaFields); return { - indexPatternTitle: dataView.title, + indexPatternTitle: dataView.getIndexPattern(), existingFieldNames: existingFields(existingFieldList, allFields), newFields, }; From e9fd00eecc06b6cc287c039d302e307fec9c7831 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Thu, 14 Dec 2023 10:12:32 +0100 Subject: [PATCH 122/168] Improve code --- .../src/hooks/use_grouped_fields.ts | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts b/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts index 8da8646da8e5c..c079b6b9503e9 100644 --- a/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts +++ b/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts @@ -77,7 +77,7 @@ export function useGroupedFields({ getCustomFieldType, onSupportedFieldFilter, }); - const allFieldsInclNew = useRef(allFields); + const allFieldsToReturn = useRef(allFields); const hasNewFields = useRef(false); const onFilterFieldList = fieldListFilters.onFilterField; const [dataView, setDataView] = useState(null); @@ -126,22 +126,22 @@ export function useGroupedFields({ }; const selectedFields = sortedSelectedFields || []; + + // Taking care of new fields that were ingested after the selected data view was loaded + // Those replace existing fields if updated, or are added to the list const newFields = dataViewId - ? fieldsExistenceReader - .getNewFields(dataViewId) - .filter((field) => (isCompatibleField ? isCompatibleField(field) : true)) + ? fieldsExistenceReader.getNewFields(dataViewId).filter(isCompatibleField || (() => true)) + : []; + // Filtering out fields that e.g. Discover provides by analyzing the resultset, that are not part of the loaded DataView + // These can be replaced by the new fields, which are mapped correctly, and therefore can be used in the right way + const allFieldsExlNew = allFields + ? allFields.filter((field) => !newFields.some((newField) => newField.name === field.name)) : []; - // remove fields from allFields that are available in newFields, because they can be provided in unmapped state - const allFieldsWithoutNewFields = !newFields.length - ? allFields - : allFields?.filter((field) => !newFields.find((newField) => newField.name === field.name)); - // append new fields to the end of the list allFieldsWithoutNewFields - const allFieldsWithNewFields = allFieldsWithoutNewFields - ? [...allFieldsWithoutNewFields, ...newFields] - : newFields; - const sortedFields = [...((allFieldsWithNewFields as unknown as T[]) || [])].sort(sortFields); - allFieldsInclNew.current = sortedFields; + const allFieldsInclNew = [...allFieldsExlNew, ...newFields] as unknown as T[]; + + const sortedFields = allFieldsInclNew.sort(sortFields); + allFieldsToReturn.current = sortedFields; hasNewFields.current = Boolean(newFields.length); const groupedFields = { ...getDefaultFieldGroups(), @@ -406,7 +406,7 @@ export function useGroupedFields({ return { fieldListGroupedProps, fieldListFiltersProps: fieldListFilters.fieldListFiltersProps, - allFields: allFieldsInclNew.current, + allFields: allFieldsToReturn.current, hasNewFields: hasNewFields.current, }; } From 454592c884f3194c9149296feb6740e652c14417 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Fri, 15 Dec 2023 09:45:19 +0100 Subject: [PATCH 123/168] Add Lens test --- .../apps/lens/group2/fields_list.ts | 50 ++++++++++++++++++- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/x-pack/test/functional/apps/lens/group2/fields_list.ts b/x-pack/test/functional/apps/lens/group2/fields_list.ts index 4e1f771d0b042..79baafe6100a6 100644 --- a/x-pack/test/functional/apps/lens/group2/fields_list.ts +++ b/x-pack/test/functional/apps/lens/group2/fields_list.ts @@ -9,13 +9,15 @@ import expect from '@kbn/expect'; import { FtrProviderContext } from '../../../ftr_provider_context'; export default function ({ getService, getPageObjects }: FtrProviderContext) { - const PageObjects = getPageObjects(['visualize', 'lens', 'common', 'header']); + const PageObjects = getPageObjects(['visualize', 'lens', 'common', 'header', 'timePicker']); const find = getService('find'); const log = getService('log'); const testSubjects = getService('testSubjects'); const filterBar = getService('filterBar'); const fieldEditor = getService('fieldEditor'); const retry = getService('retry'); + const es = getService('es'); + const queryBar = getService('queryBar'); describe('lens fields list tests', () => { for (const datasourceType of ['form-based', 'ad-hoc', 'ad-hoc-no-timefield']) { @@ -48,7 +50,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.header.waitUntilLoadingHasFinished(); }); }); - it('should show all fields as available', async () => { expect( await (await testSubjects.find('lnsIndexPatternAvailableFields-count')).getVisibleText() @@ -231,5 +232,50 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { } }); } + + describe(`update field list test`, () => { + before(async () => { + await es.transport.request({ + path: '/field-update-test/_doc', + method: 'POST', + body: { + '@timestamp': new Date().toISOString(), + oldField: 10, + }, + }); + await PageObjects.visualize.navigateToNewVisualization(); + await PageObjects.visualize.clickVisType('lens'); + await PageObjects.timePicker.setCommonlyUsedTime('This_week'); + + await PageObjects.lens.createAdHocDataView('field-update-test', true); + await retry.try(async () => { + const selectedPattern = await PageObjects.lens.getDataPanelIndexPattern(); + expect(selectedPattern).to.eql('field-update-test*'); + }); + }); + after(async () => { + await es.transport.request({ + path: '/field-update-test', + method: 'DELETE', + }); + }); + + it('should show new fields Available fields', async () => { + await es.transport.request({ + path: '/field-update-test/_doc', + method: 'POST', + body: { + '@timestamp': new Date().toISOString(), + oldField: 10, + newField: 20, + }, + }); + await PageObjects.lens.waitForField('oldField'); + await queryBar.setQuery('oldField: 10'); + await queryBar.submitQuery(); + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.lens.waitForField('newField'); + }); + }); }); } From 75cc2355c61ac0dde3066972f11fd7b827adb2cc Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Fri, 15 Dec 2023 13:08:28 +0100 Subject: [PATCH 124/168] Improve Lens code preventing an endless loop of update --- .../plugins/lens/public/datasources/form_based/datapanel.tsx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx b/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx index 58c4e1bddaa27..1d7b3603482c8 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx +++ b/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx @@ -299,11 +299,14 @@ export const InnerFormBasedDataPanel = function InnerFormBasedDataPanel({ frame.dataViews.indexPatterns, data.search.session, ]); + useEffect(() => { if (hasNewFields) { refreshFieldList(); } - }, [hasNewFields, refreshFieldList]); + // Preventing a race condition, making sure refreshFieldList is just executed once when hasNewFields is true + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [hasNewFields]); const editField = useMemo( () => From 3a746002e7b11ce77277a2d75eb3ae3555aa7a0c Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Tue, 19 Dec 2023 07:58:30 +0100 Subject: [PATCH 125/168] Update packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts Co-authored-by: Julia Rechkunova --- .../kbn-unified-field-list/src/hooks/use_existing_fields.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts b/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts index f756b4cc19126..dba018e59a86b 100644 --- a/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts +++ b/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts @@ -15,7 +15,7 @@ import { type DataPublicPluginStart } from '@kbn/data-plugin/public'; import type { DataView, DataViewsContract } from '@kbn/data-views-plugin/common'; import { getEsQueryConfig } from '@kbn/data-service/src/es_query'; import { reportPerformanceMetricEvent } from '@kbn/ebt-tools'; -import { DataViewField } from '@kbn/data-views-plugin/common'; +import type { DataViewField } from '@kbn/data-views-plugin/common'; import { loadFieldExisting } from '../services/field_existing'; import { ExistenceFetchStatus } from '../types'; From 15e5168f8ea9ad54d0f404a3ef225ea0bbb53b1e Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Tue, 19 Dec 2023 07:58:57 +0100 Subject: [PATCH 126/168] Update packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts Co-authored-by: Julia Rechkunova --- packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts b/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts index c079b6b9503e9..0d7f92983c230 100644 --- a/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts +++ b/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts @@ -41,7 +41,7 @@ export interface GroupedFieldsParams { onOverrideFieldGroupDetails?: OverrideFieldGroupDetails; onSupportedFieldFilter?: (field: T) => boolean; onSelectedFieldFilter?: (field: T) => boolean; - isCompatibleField?: (fields: DataViewField) => boolean; + isCompatibleField?: (field: DataViewField) => boolean; } export interface GroupedFieldsResult { From 139b65165a75a7eae384aa02091ce2e183da94d9 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Tue, 19 Dec 2023 07:59:17 +0100 Subject: [PATCH 127/168] Update packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts Co-authored-by: Julia Rechkunova --- .../src/services/field_existing/field_existing_utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts b/packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts index e4dab136c5a52..912ef361dd9a8 100644 --- a/packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts +++ b/packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts @@ -7,7 +7,7 @@ */ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import { DataViewField, RuntimeField } from '@kbn/data-views-plugin/common'; +import { type DataViewField, RuntimeField } from '@kbn/data-views-plugin/common'; import type { DataViewsContract, DataView, FieldSpec } from '@kbn/data-views-plugin/common'; import type { IKibanaSearchRequest } from '@kbn/data-plugin/common'; From ab514e4c4fcc44459bfb7539dad4a3b0fb55650e Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Tue, 19 Dec 2023 08:00:34 +0100 Subject: [PATCH 128/168] Update packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts Co-authored-by: Julia Rechkunova --- .../src/services/field_existing/field_existing_utils.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts b/packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts index 912ef361dd9a8..75ac7b159f113 100644 --- a/packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts +++ b/packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts @@ -59,8 +59,8 @@ export async function fetchFieldExistence({ // in the given data view. Those fields we consider as new fields, // that were ingested after the data view was loaded const newFields = existingFieldList - .filter((field) => dataView.getFieldByName(field.name) === undefined) - .map((field) => dataView.getFieldByName(field.name) ?? new DataViewField(field)); + .filter((field) => !dataView.getFieldByName(field.name)) + .map((field) => new DataViewField(field)); // refresh the data view in case there are new fields if (newFields.length) { await dataViewsService.refreshFields(dataView, false); From b424159790279852eec8d655172b2ee6060c5ef0 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Wed, 20 Dec 2023 09:54:32 +0100 Subject: [PATCH 129/168] Improve code --- .../field_list_sidebar.tsx | 46 ++++----- .../src/hooks/use_existing_fields.ts | 14 ++- .../src/hooks/use_grouped_fields.ts | 36 +++---- .../field_existing/field_existing_utils.ts | 30 +++--- .../field_existing/load_field_existing.ts | 4 +- .../lens/public/data_views_service/loader.ts | 94 ++++++++++--------- .../datasources/form_based/datapanel.tsx | 10 +- .../datasources/text_based/datapanel.tsx | 2 - 8 files changed, 129 insertions(+), 107 deletions(-) diff --git a/packages/kbn-unified-field-list/src/containers/unified_field_list_sidebar/field_list_sidebar.tsx b/packages/kbn-unified-field-list/src/containers/unified_field_list_sidebar/field_list_sidebar.tsx index e1e72ca0a9aa9..256d6a8f148fc 100644 --- a/packages/kbn-unified-field-list/src/containers/unified_field_list_sidebar/field_list_sidebar.tsx +++ b/packages/kbn-unified-field-list/src/containers/unified_field_list_sidebar/field_list_sidebar.tsx @@ -22,7 +22,7 @@ import { useEuiTheme, } from '@elastic/eui'; import { ToolbarButton } from '@kbn/shared-ux-button-toolbar'; -import { type DataViewField } from '@kbn/data-views-plugin/public'; +import { DataViewField } from '@kbn/data-views-plugin/common'; import { getDataViewFieldSubtypeMulti } from '@kbn/es-query/src/utils'; import { FIELDS_LIMIT_SETTING, SEARCH_FIELDS_FROM_SOURCE } from '@kbn/discover-utils'; import { FieldList } from '../../components/field_list'; @@ -207,28 +207,28 @@ export const UnifiedFieldListSidebarComponent: React.FC({ - dataViewId: (searchMode === 'documents' && dataView?.id) || null, // passing `null` for text-based queries - allFields, - popularFieldsLimit: - searchMode !== 'documents' || stateService.creationOptions.disablePopularFields - ? 0 - : popularFieldsLimit, - isAffectedByGlobalFilter, - services: { - dataViews, - core, - }, - sortedSelectedFields: onSelectedFieldFilter ? undefined : selectedFieldsState.selectedFields, - onSelectedFieldFilter, - onSupportedFieldFilter: - stateService.creationOptions.onSupportedFieldFilter ?? onSupportedFieldFilter, - onOverrideFieldGroupDetails: stateService.creationOptions.onOverrideFieldGroupDetails, - }); + const { fieldListFiltersProps, fieldListGroupedProps, allFieldsModified } = + useGroupedFields({ + dataViewId: (searchMode === 'documents' && dataView?.id) || null, // passing `null` for text-based queries + allFields, + popularFieldsLimit: + searchMode !== 'documents' || stateService.creationOptions.disablePopularFields + ? 0 + : popularFieldsLimit, + isAffectedByGlobalFilter, + services: { + dataViews, + core, + }, + sortedSelectedFields: onSelectedFieldFilter ? undefined : selectedFieldsState.selectedFields, + onSelectedFieldFilter, + onSupportedFieldFilter: + stateService.creationOptions.onSupportedFieldFilter ?? onSupportedFieldFilter, + onOverrideFieldGroupDetails: stateService.creationOptions.onOverrideFieldGroupDetails, + getNewFieldsBySpec: (fieldSpecArr) => { + return fieldSpecArr.map((fieldSpec) => new DataViewField(fieldSpec)); + }, + }); useEffect(() => { if ( diff --git a/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts b/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts index dba018e59a86b..7826c815b9af3 100644 --- a/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts +++ b/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts @@ -12,10 +12,9 @@ import { BehaviorSubject } from 'rxjs'; import type { CoreStart } from '@kbn/core/public'; import type { AggregateQuery, EsQueryConfig, Filter, Query } from '@kbn/es-query'; import { type DataPublicPluginStart } from '@kbn/data-plugin/public'; -import type { DataView, DataViewsContract } from '@kbn/data-views-plugin/common'; +import type { DataView, DataViewsContract, FieldSpec } from '@kbn/data-views-plugin/common'; import { getEsQueryConfig } from '@kbn/data-service/src/es_query'; import { reportPerformanceMetricEvent } from '@kbn/ebt-tools'; -import type { DataViewField } from '@kbn/data-views-plugin/common'; import { loadFieldExisting } from '../services/field_existing'; import { ExistenceFetchStatus } from '../types'; @@ -25,7 +24,7 @@ const generateId = htmlIdGenerator(); export interface ExistingFieldsInfo { fetchStatus: ExistenceFetchStatus; existingFieldsByFieldNameMap: Record; - newFields?: DataViewField[]; + newFields?: FieldSpec[]; numberOfFetches: number; hasDataViewRestrictions?: boolean; } @@ -56,7 +55,7 @@ export interface ExistingFieldsReader { hasFieldData: (dataViewId: string, fieldName: string) => boolean; getFieldsExistenceStatus: (dataViewId: string) => ExistenceFetchStatus; isFieldsExistenceInfoUnavailable: (dataViewId: string) => boolean; - getNewFields: (dataViewId: string) => DataViewField[]; + getNewFields: (dataViewId: string) => FieldSpec[]; } const initialData: ExistingFieldsByDataViewMap = {}; @@ -255,7 +254,12 @@ export const useExistingFieldsFetcher = ( ); }; -export const useExistingFieldsReader: () => ExistingFieldsReader = () => { +export const useExistingFieldsReader: () => { + getFieldsExistenceStatus: (dataViewId: string) => ExistenceFetchStatus; + isFieldsExistenceInfoUnavailable: (dataViewId: string) => boolean; + getNewFields: (dataViewId: string) => FieldSpec[]; + hasFieldData: (dataViewId: string, fieldName: string) => boolean; +} = () => { const mountedRef = useRef(true); const [existingFieldsByDataViewMap, setExistingFieldsByDataViewMap] = useState(globalMap$.getValue()); diff --git a/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts b/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts index 0d7f92983c230..bfc32be695120 100644 --- a/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts +++ b/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts @@ -10,7 +10,7 @@ import { groupBy } from 'lodash'; import { useEffect, useMemo, useState, useRef } from 'react'; import { i18n } from '@kbn/i18n'; import { type CoreStart } from '@kbn/core-lifecycle-browser'; -import { type DataView, type DataViewField } from '@kbn/data-views-plugin/common'; +import type { DataView, DataViewField, FieldSpec } from '@kbn/data-views-plugin/common'; import { type DataViewsContract } from '@kbn/data-views-plugin/public'; import { type FieldListGroups, @@ -41,7 +41,7 @@ export interface GroupedFieldsParams { onOverrideFieldGroupDetails?: OverrideFieldGroupDetails; onSupportedFieldFilter?: (field: T) => boolean; onSelectedFieldFilter?: (field: T) => boolean; - isCompatibleField?: (field: DataViewField) => boolean; + getNewFieldsBySpec?: (field: FieldSpec[], dataView: DataView | null) => T[]; } export interface GroupedFieldsResult { @@ -53,7 +53,7 @@ export interface GroupedFieldsResult { fieldsExistInIndex: boolean; screenReaderDescriptionId?: string; }; - allFields: T[] | null; // `null` is for loading indicator + allFieldsModified: T[] | null; // `null` is for loading indicator hasNewFields: boolean; } @@ -68,7 +68,7 @@ export function useGroupedFields({ onOverrideFieldGroupDetails, onSupportedFieldFilter, onSelectedFieldFilter, - isCompatibleField, + getNewFieldsBySpec, }: GroupedFieldsParams): GroupedFieldsResult { const fieldsExistenceReader = useExistingFieldsReader(); const fieldListFilters = useFieldFilters({ @@ -129,20 +129,24 @@ export function useGroupedFields({ // Taking care of new fields that were ingested after the selected data view was loaded // Those replace existing fields if updated, or are added to the list - const newFields = dataViewId - ? fieldsExistenceReader.getNewFields(dataViewId).filter(isCompatibleField || (() => true)) - : []; + const newFields = + dataViewId && getNewFieldsBySpec + ? getNewFieldsBySpec(fieldsExistenceReader.getNewFields(dataViewId), dataView) + : []; // Filtering out fields that e.g. Discover provides by analyzing the resultset, that are not part of the loaded DataView // These can be replaced by the new fields, which are mapped correctly, and therefore can be used in the right way - const allFieldsExlNew = allFields - ? allFields.filter((field) => !newFields.some((newField) => newField.name === field.name)) - : []; + const allFieldsExlNew = + allFields && newFields.length + ? allFields.filter((field) => !newFields.some((newField) => newField.name === field.name)) + : allFields; - const allFieldsInclNew = [...allFieldsExlNew, ...newFields] as unknown as T[]; - - const sortedFields = allFieldsInclNew.sort(sortFields); - allFieldsToReturn.current = sortedFields; + const allFieldsInclNew = newFields.length + ? [...(allFieldsExlNew || []), ...newFields] + : allFields || []; + allFieldsToReturn.current = newFields.length ? allFieldsInclNew : allFields; hasNewFields.current = Boolean(newFields.length); + const sortedFields = [...allFieldsInclNew].sort(sortFields); + const groupedFields = { ...getDefaultFieldGroups(), ...groupBy(sortedFields, (field) => { @@ -347,7 +351,7 @@ export function useGroupedFields({ onSelectedFieldFilter, onSupportedFieldFilter, dataView, - isCompatibleField, + getNewFieldsBySpec, ]); const fieldGroups: FieldListGroups = useMemo(() => { @@ -406,7 +410,7 @@ export function useGroupedFields({ return { fieldListGroupedProps, fieldListFiltersProps: fieldListFilters.fieldListFiltersProps, - allFields: allFieldsToReturn.current, + allFieldsModified: allFieldsToReturn.current, hasNewFields: hasNewFields.current, }; } diff --git a/packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts b/packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts index 75ac7b159f113..41a1e53a33849 100644 --- a/packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts +++ b/packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts @@ -7,7 +7,7 @@ */ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import { type DataViewField, RuntimeField } from '@kbn/data-views-plugin/common'; +import type { DataViewField, RuntimeField } from '@kbn/data-views-plugin/common'; import type { DataViewsContract, DataView, FieldSpec } from '@kbn/data-views-plugin/common'; import type { IKibanaSearchRequest } from '@kbn/data-plugin/common'; @@ -58,9 +58,7 @@ export async function fetchFieldExistence({ // take care of fields of existingFieldList, that are not yet available // in the given data view. Those fields we consider as new fields, // that were ingested after the data view was loaded - const newFields = existingFieldList - .filter((field) => !dataView.getFieldByName(field.name)) - .map((field) => new DataViewField(field)); + const newFields = existingFieldList.filter((field) => !dataView.getFieldByName(field.name)); // refresh the data view in case there are new fields if (newFields.length) { await dataViewsService.refreshFields(dataView, false); @@ -78,19 +76,23 @@ export async function fetchFieldExistence({ */ export function buildFieldList(indexPattern: DataView, metaFields: string[]): Field[] { return indexPattern.fields.map((field) => { - return { - name: field.name, - isScript: !!field.scripted, - lang: field.lang, - script: field.script, - // id is a special case - it doesn't show up in the meta field list, - // but as it's not part of source, it has to be handled separately. - isMeta: metaFields?.includes(field.name) || field.name === '_id', - runtimeField: !field.isMapped ? field.runtimeField : undefined, - }; + return buildField(field, metaFields); }); } +export function buildField(field: DataViewField, metaFields: string[]): Field { + return { + name: field.name, + isScript: !!field.scripted, + lang: field.lang, + script: field.script, + // id is a special case - it doesn't show up in the meta field list, + // but as it's not part of source, it has to be handled separately. + isMeta: metaFields?.includes(field.name) || field.name === '_id', + runtimeField: !field.isMapped ? field.runtimeField : undefined, + }; +} + function toQuery( timeFieldName: string | undefined, fromDate: string | undefined, diff --git a/packages/kbn-unified-field-list/src/services/field_existing/load_field_existing.ts b/packages/kbn-unified-field-list/src/services/field_existing/load_field_existing.ts index 889c92008554b..590bc46eff573 100644 --- a/packages/kbn-unified-field-list/src/services/field_existing/load_field_existing.ts +++ b/packages/kbn-unified-field-list/src/services/field_existing/load_field_existing.ts @@ -9,7 +9,7 @@ import { IUiSettingsClient } from '@kbn/core/public'; import { type DataPublicPluginStart } from '@kbn/data-plugin/public'; import { UI_SETTINGS } from '@kbn/data-service/src/constants'; -import type { DataView, DataViewField, DataViewsContract } from '@kbn/data-views-plugin/common'; +import type { DataView, DataViewsContract, FieldSpec } from '@kbn/data-views-plugin/common'; import { lastValueFrom } from 'rxjs'; import { fetchFieldExistence } from './field_existing_utils'; @@ -27,7 +27,7 @@ interface FetchFieldExistenceParams { export type LoadFieldExistingHandler = (params: FetchFieldExistenceParams) => Promise<{ existingFieldNames: string[]; indexPatternTitle: string; - newFields?: DataViewField[]; + newFields?: FieldSpec[]; }>; export const loadFieldExisting: LoadFieldExistingHandler = async ({ diff --git a/x-pack/plugins/lens/public/data_views_service/loader.ts b/x-pack/plugins/lens/public/data_views_service/loader.ts index 784c97d832e34..00bd291ef9c24 100644 --- a/x-pack/plugins/lens/public/data_views_service/loader.ts +++ b/x-pack/plugins/lens/public/data_views_service/loader.ts @@ -6,7 +6,12 @@ */ import { isFieldLensCompatible } from '@kbn/visualization-ui-components'; -import type { DataViewsContract, DataView, DataViewSpec } from '@kbn/data-views-plugin/public'; +import { + DataViewsContract, + DataView, + DataViewSpec, + DataViewField, +} from '@kbn/data-views-plugin/public'; import { keyBy } from 'lodash'; import { IndexPattern, IndexPatternField, IndexPatternMap, IndexPatternRef } from '../types'; import { documentField } from '../datasources/form_based/document_field'; @@ -29,49 +34,9 @@ export function convertDataViewIntoLensIndexPattern( dataView: DataView, restrictionRemapper: (name: string) => string = onRestrictionMapping ): IndexPattern { - const metaKeys = new Set(dataView.metaFields); const newFields = dataView.fields .filter(isFieldLensCompatible) - .map((field): IndexPatternField => { - // Convert the getters on the index pattern service into plain JSON - const base = { - name: field.name, - displayName: field.displayName, - type: field.type, - aggregatable: field.aggregatable, - filterable: field.filterable, - searchable: field.searchable, - meta: metaKeys.has(field.name), - esTypes: field.esTypes, - scripted: field.scripted, - isMapped: field.isMapped, - customLabel: field.customLabel, - runtimeField: field.runtimeField, - runtime: Boolean(field.runtimeField), - timeSeriesDimension: field.timeSeriesDimension, - timeSeriesMetric: field.timeSeriesMetric, - timeSeriesRollup: field.isRolledUpField, - partiallyApplicableFunctions: field.isRolledUpField - ? { - percentile: true, - percentile_rank: true, - median: true, - last_value: true, - unique_count: true, - standard_deviation: true, - } - : undefined, - }; - - // Simplifies tests by hiding optional properties instead of undefined - return base.scripted - ? { - ...base, - lang: field.lang, - script: field.script, - } - : base; - }) + .map((field) => buildIndexPatternField(field, dataView)) .concat(documentField); const { typeMeta, title, name, timeFieldName, fieldFormatMap } = dataView; @@ -113,6 +78,51 @@ export function convertDataViewIntoLensIndexPattern( }; } +export function buildIndexPatternField( + field: DataViewField, + dataView: DataView | null +): IndexPatternField { + const meta = dataView?.metaFields.includes(field.name); + // Convert the getters on the index pattern service into plain JSON + const base = { + name: field.name, + displayName: field.displayName, + type: field.type, + aggregatable: field.aggregatable, + filterable: field.filterable, + searchable: field.searchable, + meta, + esTypes: field.esTypes, + scripted: field.scripted, + isMapped: field.isMapped, + customLabel: field.customLabel, + runtimeField: field.runtimeField, + runtime: Boolean(field.runtimeField), + timeSeriesDimension: field.timeSeriesDimension, + timeSeriesMetric: field.timeSeriesMetric, + timeSeriesRollup: field.isRolledUpField, + partiallyApplicableFunctions: field.isRolledUpField + ? { + percentile: true, + percentile_rank: true, + median: true, + last_value: true, + unique_count: true, + standard_deviation: true, + } + : undefined, + }; + + // Simplifies tests by hiding optional properties instead of undefined + return base.scripted + ? { + ...base, + lang: field.lang, + script: field.script, + } + : base; +} + export async function loadIndexPatternRefs( dataViews: MinimalDataViewsContract ): Promise { diff --git a/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx b/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx index 1d7b3603482c8..5108cfbf8019a 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx +++ b/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx @@ -13,7 +13,7 @@ import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n-react'; import type { CoreStart } from '@kbn/core/public'; import type { DataPublicPluginStart } from '@kbn/data-plugin/public'; -import { type DataView } from '@kbn/data-plugin/common'; +import { type DataView, DataViewField } from '@kbn/data-plugin/common'; import type { FieldFormatsStart } from '@kbn/field-formats-plugin/public'; import { IndexPatternFieldEditorStart } from '@kbn/data-view-field-editor-plugin/public'; import { VISUALIZE_GEO_FIELD_TRIGGER } from '@kbn/ui-actions-plugin/public'; @@ -28,7 +28,7 @@ import { useGroupedFields, } from '@kbn/unified-field-list'; import { ChartsPluginSetup } from '@kbn/charts-plugin/public'; -import { isFieldLensCompatible } from '@kbn/visualization-ui-components'; +import { buildIndexPatternField } from '../../data_views_service/loader'; import type { DatasourceDataPanelProps, FramePublicAPI, @@ -262,7 +262,11 @@ export const InnerFormBasedDataPanel = function InnerFormBasedDataPanel({ onSupportedFieldFilter, onSelectedFieldFilter, onOverrideFieldGroupDetails, - isCompatibleField: isFieldLensCompatible, + getNewFieldsBySpec: (spec, dataView) => { + return spec.map((fieldSpec) => + buildIndexPatternField(new DataViewField(fieldSpec), dataView) + ); + }, }); const closeFieldEditor = useRef<() => void | undefined>(); diff --git a/x-pack/plugins/lens/public/datasources/text_based/datapanel.tsx b/x-pack/plugins/lens/public/datasources/text_based/datapanel.tsx index ac62636ad50c1..113125484cddf 100644 --- a/x-pack/plugins/lens/public/datasources/text_based/datapanel.tsx +++ b/x-pack/plugins/lens/public/datasources/text_based/datapanel.tsx @@ -24,7 +24,6 @@ import { GetCustomFieldType, useGroupedFields, } from '@kbn/unified-field-list'; -import { isFieldLensCompatible } from '@kbn/visualization-ui-components'; import type { DatasourceDataPanelProps } from '../../types'; import type { TextBasedPrivateState } from './types'; import { getStateFromAggregateQuery } from './utils'; @@ -106,7 +105,6 @@ export function TextBasedDataPanel({ getCustomFieldType, onSelectedFieldFilter, onOverrideFieldGroupDetails, - isCompatibleField: isFieldLensCompatible, }); const renderFieldItem: FieldListGroupedProps['renderFieldItem'] = useCallback( From 30dfec6cc0a971a298beb17c9eaa1587e3b8e0f8 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Wed, 20 Dec 2023 17:09:56 +0100 Subject: [PATCH 130/168] Add isFieldLensCompatible for new fields in Lens --- .../public/datasources/form_based/datapanel.tsx | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx b/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx index 5108cfbf8019a..6ee8483658e68 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx +++ b/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx @@ -13,7 +13,7 @@ import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n-react'; import type { CoreStart } from '@kbn/core/public'; import type { DataPublicPluginStart } from '@kbn/data-plugin/public'; -import { type DataView, DataViewField } from '@kbn/data-plugin/common'; +import { type DataView, DataViewField, FieldSpec } from '@kbn/data-plugin/common'; import type { FieldFormatsStart } from '@kbn/field-formats-plugin/public'; import { IndexPatternFieldEditorStart } from '@kbn/data-view-field-editor-plugin/public'; import { VISUALIZE_GEO_FIELD_TRIGGER } from '@kbn/ui-actions-plugin/public'; @@ -28,6 +28,7 @@ import { useGroupedFields, } from '@kbn/unified-field-list'; import { ChartsPluginSetup } from '@kbn/charts-plugin/public'; +import { isFieldLensCompatible } from '@kbn/visualization-ui-components'; import { buildIndexPatternField } from '../../data_views_service/loader'; import type { DatasourceDataPanelProps, @@ -263,9 +264,13 @@ export const InnerFormBasedDataPanel = function InnerFormBasedDataPanel({ onSelectedFieldFilter, onOverrideFieldGroupDetails, getNewFieldsBySpec: (spec, dataView) => { - return spec.map((fieldSpec) => - buildIndexPatternField(new DataViewField(fieldSpec), dataView) - ); + return spec.reduce((result: IndexPatternField[], fieldSpec: FieldSpec) => { + const field = new DataViewField(fieldSpec); + if (isFieldLensCompatible(field)) { + result.push(buildIndexPatternField(field, dataView)); + } + return result; + }, []); }, }); From f747c2e945cb383f8ed04e54a82ff1abc95d20a5 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Thu, 21 Dec 2023 07:10:11 +0100 Subject: [PATCH 131/168] Improve multi field code --- .../field_list_item.tsx | 18 +++++--- .../field_list_sidebar.tsx | 41 +++++++++---------- 2 files changed, 32 insertions(+), 27 deletions(-) diff --git a/packages/kbn-unified-field-list/src/containers/unified_field_list_item/field_list_item.tsx b/packages/kbn-unified-field-list/src/containers/unified_field_list_item/field_list_item.tsx index 745d463b28386..306d348855202 100644 --- a/packages/kbn-unified-field-list/src/containers/unified_field_list_item/field_list_item.tsx +++ b/packages/kbn-unified-field-list/src/containers/unified_field_list_item/field_list_item.tsx @@ -67,9 +67,14 @@ function getCommonFieldItemButtonProps({ }; } +export interface FieldListMultiField { + field: DataViewField; + isSelected: boolean; +} + interface MultiFieldsProps { stateService: UnifiedFieldListSidebarContainerStateService; - multiFields: NonNullable; + multiFields: NonNullable; toggleDisplay: (field: DataViewField) => void; alwaysShowActionButton: boolean; size: FieldItemButtonProps['size']; @@ -163,9 +168,9 @@ export interface UnifiedFieldListItemProps { */ trackUiMetric?: (metricType: UiCounterMetricType, eventName: string | string[]) => void; /** - * Multi fields for the current field + * Get multi fields for the given field */ - multiFields?: Array<{ field: DataViewField; isSelected: boolean }>; + getMultiFields?: (field: DataViewField) => FieldListMultiField[] | undefined; /** * Callback to edit a field from data view * @param fieldName name of the field to edit @@ -212,7 +217,7 @@ function UnifiedFieldListItemComponent({ isEmpty, isSelected, trackUiMetric, - multiFields, + getMultiFields, onEditField, onDeleteField, workspaceSelectedFieldNames, @@ -253,8 +258,6 @@ function UnifiedFieldListItemComponent({ [onAddFieldToWorkspace, onRemoveFieldFromWorkspace, closePopover] ); - const rawMultiFields = useMemo(() => multiFields?.map((f) => f.field), [multiFields]); - const customPopoverHeaderProps: Partial = useMemo(() => { const dataTestSubjPrefix = stateService.creationOptions.dataTestSubj?.fieldListItemPopoverHeaderDataTestSubjPrefix; @@ -275,6 +278,9 @@ function UnifiedFieldListItemComponent({ }, [field.name, stateService.creationOptions]); const renderPopover = () => { + const multiFields = getMultiFields?.(field); + const rawMultiFields = multiFields?.map((f) => f.field); + return ( <> ( INITIAL_SELECTED_FIELDS_RESULT ); - const [multiFieldsMap, setMultiFieldsMap] = useState< - Map> | undefined - >(undefined); useEffect(() => { const result = getSelectedFields({ @@ -230,27 +228,28 @@ export const UnifiedFieldListSidebarComponent: React.FC { + const useMultiFields = useMemo(() => { if ( searchMode !== 'documents' || !useNewFieldsApi || stateService.creationOptions.disableMultiFieldsGroupingByParent ) { - setMultiFieldsMap(undefined); // we don't have to calculate multifields in this case + return false; } else { - setMultiFieldsMap( - calculateMultiFields(allFieldsModified, selectedFieldsState.selectedFieldsMap) - ); + return true; } }, [ stateService.creationOptions.disableMultiFieldsGroupingByParent, - selectedFieldsState.selectedFieldsMap, - allFieldsModified, useNewFieldsApi, - setMultiFieldsMap, searchMode, ]); + const getMultiFieldsByField = useCallback( + (field: DataViewField) => + getMultiFieldsByParent(field, allFieldsModified, selectedFieldsState.selectedFieldsMap), + [allFieldsModified, selectedFieldsState.selectedFieldsMap] + ); + const renderFieldItem: FieldListGroupedProps['renderFieldItem'] = useCallback( ({ field, groupName, groupIndex, itemIndex, fieldSearchHighlight }) => (
  • @@ -267,7 +266,7 @@ export const UnifiedFieldListSidebarComponent: React.FC>(); + const result: FieldListMultiField[] = []; allFields.forEach((field) => { const subTypeMulti = getDataViewFieldSubtypeMulti(field); const parent = subTypeMulti?.multi.parent; - if (!parent) { + if (!parent || parent !== parentField.name) { return; } const multiField = { field, isSelected: Boolean(selectedFieldsMap?.[field.name]), }; - const value = map.get(parent) ?? []; - value.push(multiField); - map.set(parent, value); + result.push(multiField); }); - return map; + return result.length ? result : undefined; } From 9ed0c495247bbf63d742effa90260b31570c08ef Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Thu, 21 Dec 2023 08:50:45 +0100 Subject: [PATCH 132/168] Improve code --- .../field_list_sidebar.tsx | 24 +++++-------------- 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/packages/kbn-unified-field-list/src/containers/unified_field_list_sidebar/field_list_sidebar.tsx b/packages/kbn-unified-field-list/src/containers/unified_field_list_sidebar/field_list_sidebar.tsx index 3ed0e7dcbf68b..a98b8567aad78 100644 --- a/packages/kbn-unified-field-list/src/containers/unified_field_list_sidebar/field_list_sidebar.tsx +++ b/packages/kbn-unified-field-list/src/containers/unified_field_list_sidebar/field_list_sidebar.tsx @@ -223,26 +223,14 @@ export const UnifiedFieldListSidebarComponent: React.FC { - return fieldSpecArr.map((fieldSpec) => new DataViewField(fieldSpec)); - }, + getNewFieldsBySpec: (fieldSpecArr) => + fieldSpecArr.map((fieldSpec) => new DataViewField(fieldSpec)), }); - const useMultiFields = useMemo(() => { - if ( - searchMode !== 'documents' || - !useNewFieldsApi || - stateService.creationOptions.disableMultiFieldsGroupingByParent - ) { - return false; - } else { - return true; - } - }, [ - stateService.creationOptions.disableMultiFieldsGroupingByParent, - useNewFieldsApi, - searchMode, - ]); + const useMultiFields = + searchMode === 'documents' && + useNewFieldsApi && + !stateService.creationOptions.disableMultiFieldsGroupingByParent; const getMultiFieldsByField = useCallback( (field: DataViewField) => From 5420c7708910b945b85434bd5549f0a559301016 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Thu, 21 Dec 2023 08:51:05 +0100 Subject: [PATCH 133/168] Remove redundant field list refreshing --- .../public/dataview_picker/change_dataview.tsx | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/plugins/unified_search/public/dataview_picker/change_dataview.tsx b/src/plugins/unified_search/public/dataview_picker/change_dataview.tsx index 9076bcb37c7df..2d857de608834 100644 --- a/src/plugins/unified_search/public/dataview_picker/change_dataview.tsx +++ b/src/plugins/unified_search/public/dataview_picker/change_dataview.tsx @@ -307,12 +307,6 @@ export function ChangeDataView({ isTextBasedLangSelected={isTextBasedLangSelected} setPopoverIsOpen={setPopoverIsOpen} onChangeDataView={async (newId) => { - try { - // refreshing the field list - await dataViews.get(newId, false, true); - } catch (e) { - // - } setSelectedDataViewId(newId); setPopoverIsOpen(false); if (isTextBasedLangSelected && !isTextLangTransitionModalDismissed) { From f340fae4cae5bfe9533fd3c8bcf082248c58b8b3 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Fri, 22 Dec 2023 12:09:49 +0100 Subject: [PATCH 134/168] Undo multi field changes --- .../field_list_item.tsx | 18 +++---- .../field_list_sidebar.tsx | 54 +++++++++++-------- 2 files changed, 39 insertions(+), 33 deletions(-) diff --git a/packages/kbn-unified-field-list/src/containers/unified_field_list_item/field_list_item.tsx b/packages/kbn-unified-field-list/src/containers/unified_field_list_item/field_list_item.tsx index 306d348855202..745d463b28386 100644 --- a/packages/kbn-unified-field-list/src/containers/unified_field_list_item/field_list_item.tsx +++ b/packages/kbn-unified-field-list/src/containers/unified_field_list_item/field_list_item.tsx @@ -67,14 +67,9 @@ function getCommonFieldItemButtonProps({ }; } -export interface FieldListMultiField { - field: DataViewField; - isSelected: boolean; -} - interface MultiFieldsProps { stateService: UnifiedFieldListSidebarContainerStateService; - multiFields: NonNullable; + multiFields: NonNullable; toggleDisplay: (field: DataViewField) => void; alwaysShowActionButton: boolean; size: FieldItemButtonProps['size']; @@ -168,9 +163,9 @@ export interface UnifiedFieldListItemProps { */ trackUiMetric?: (metricType: UiCounterMetricType, eventName: string | string[]) => void; /** - * Get multi fields for the given field + * Multi fields for the current field */ - getMultiFields?: (field: DataViewField) => FieldListMultiField[] | undefined; + multiFields?: Array<{ field: DataViewField; isSelected: boolean }>; /** * Callback to edit a field from data view * @param fieldName name of the field to edit @@ -217,7 +212,7 @@ function UnifiedFieldListItemComponent({ isEmpty, isSelected, trackUiMetric, - getMultiFields, + multiFields, onEditField, onDeleteField, workspaceSelectedFieldNames, @@ -258,6 +253,8 @@ function UnifiedFieldListItemComponent({ [onAddFieldToWorkspace, onRemoveFieldFromWorkspace, closePopover] ); + const rawMultiFields = useMemo(() => multiFields?.map((f) => f.field), [multiFields]); + const customPopoverHeaderProps: Partial = useMemo(() => { const dataTestSubjPrefix = stateService.creationOptions.dataTestSubj?.fieldListItemPopoverHeaderDataTestSubjPrefix; @@ -278,9 +275,6 @@ function UnifiedFieldListItemComponent({ }, [field.name, stateService.creationOptions]); const renderPopover = () => { - const multiFields = getMultiFields?.(field); - const rawMultiFields = multiFields?.map((f) => f.field); - return ( <> ( INITIAL_SELECTED_FIELDS_RESULT ); + const [multiFieldsMap, setMultiFieldsMap] = useState< + Map> | undefined + >(undefined); useEffect(() => { const result = getSelectedFields({ @@ -227,16 +229,26 @@ export const UnifiedFieldListSidebarComponent: React.FC new DataViewField(fieldSpec)), }); - const useMultiFields = - searchMode === 'documents' && - useNewFieldsApi && - !stateService.creationOptions.disableMultiFieldsGroupingByParent; - - const getMultiFieldsByField = useCallback( - (field: DataViewField) => - getMultiFieldsByParent(field, allFieldsModified, selectedFieldsState.selectedFieldsMap), - [allFieldsModified, selectedFieldsState.selectedFieldsMap] - ); + useEffect(() => { + if ( + searchMode !== 'documents' || + !useNewFieldsApi || + stateService.creationOptions.disableMultiFieldsGroupingByParent + ) { + setMultiFieldsMap(undefined); // we don't have to calculate multifields in this case + } else { + setMultiFieldsMap( + calculateMultiFields(allFieldsModified, selectedFieldsState.selectedFieldsMap) + ); + } + }, [ + stateService.creationOptions.disableMultiFieldsGroupingByParent, + selectedFieldsState.selectedFieldsMap, + allFieldsModified, + useNewFieldsApi, + setMultiFieldsMap, + searchMode, + ]); const renderFieldItem: FieldListGroupedProps['renderFieldItem'] = useCallback( ({ field, groupName, groupIndex, itemIndex, fieldSearchHighlight }) => ( @@ -254,7 +266,7 @@ export const UnifiedFieldListSidebarComponent: React.FC>(); allFields.forEach((field) => { const subTypeMulti = getDataViewFieldSubtypeMulti(field); const parent = subTypeMulti?.multi.parent; - if (!parent || parent !== parentField.name) { + if (!parent) { return; } const multiField = { field, isSelected: Boolean(selectedFieldsMap?.[field.name]), }; - result.push(multiField); + const value = map.get(parent) ?? []; + value.push(multiField); + map.set(parent, value); }); - return result.length ? result : undefined; + return map; } From 26c4d72c10ff4f4f3a70209a679625eba20d2bec Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Fri, 22 Dec 2023 15:20:42 +0100 Subject: [PATCH 135/168] Address feedback --- .../field_list_sidebar.tsx | 9 ++- .../src/hooks/use_existing_fields.ts | 9 +-- .../src/hooks/use_grouped_fields.ts | 61 ++++++++----------- .../src/hooks/use_new_fields.ts | 51 ++++++++++++++++ 4 files changed, 83 insertions(+), 47 deletions(-) create mode 100644 packages/kbn-unified-field-list/src/hooks/use_new_fields.ts diff --git a/packages/kbn-unified-field-list/src/containers/unified_field_list_sidebar/field_list_sidebar.tsx b/packages/kbn-unified-field-list/src/containers/unified_field_list_sidebar/field_list_sidebar.tsx index 7b4e69b3ea739..17ebe5b19286f 100644 --- a/packages/kbn-unified-field-list/src/containers/unified_field_list_sidebar/field_list_sidebar.tsx +++ b/packages/kbn-unified-field-list/src/containers/unified_field_list_sidebar/field_list_sidebar.tsx @@ -22,7 +22,7 @@ import { useEuiTheme, } from '@elastic/eui'; import { ToolbarButton } from '@kbn/shared-ux-button-toolbar'; -import { DataViewField } from '@kbn/data-views-plugin/common'; +import { DataViewField, FieldSpec } from '@kbn/data-views-plugin/common'; import { getDataViewFieldSubtypeMulti } from '@kbn/es-query/src/utils'; import { FIELDS_LIMIT_SETTING, SEARCH_FIELDS_FROM_SOURCE } from '@kbn/discover-utils'; import { FieldList } from '../../components/field_list'; @@ -225,8 +225,7 @@ export const UnifiedFieldListSidebarComponent: React.FC - fieldSpecArr.map((fieldSpec) => new DataViewField(fieldSpec)), + getNewFieldsBySpec, }); useEffect(() => { @@ -461,3 +460,7 @@ function calculateMultiFields( }); return map; } + +function getNewFieldsBySpec(fieldSpecArr: FieldSpec[]): DataViewField[] { + return fieldSpecArr.map((fieldSpec) => new DataViewField(fieldSpec)); +} diff --git a/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts b/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts index 7826c815b9af3..a733643008b11 100644 --- a/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts +++ b/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts @@ -120,7 +120,7 @@ export const useExistingFieldsFetcher = ( setActiveRequests((value) => value + 1); - const hasRestrictions = Boolean(dataView.getAggregationRestrictions?.()); + const hasRestrictions = Boolean(dataView?.getAggregationRestrictions?.()); const info: ExistingFieldsInfo = { ...unknownInfo, numberOfFetches, @@ -254,12 +254,7 @@ export const useExistingFieldsFetcher = ( ); }; -export const useExistingFieldsReader: () => { - getFieldsExistenceStatus: (dataViewId: string) => ExistenceFetchStatus; - isFieldsExistenceInfoUnavailable: (dataViewId: string) => boolean; - getNewFields: (dataViewId: string) => FieldSpec[]; - hasFieldData: (dataViewId: string, fieldName: string) => boolean; -} = () => { +export const useExistingFieldsReader: () => ExistingFieldsReader = () => { const mountedRef = useRef(true); const [existingFieldsByDataViewMap, setExistingFieldsByDataViewMap] = useState(globalMap$.getValue()); diff --git a/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts b/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts index bfc32be695120..381803ac74345 100644 --- a/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts +++ b/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts @@ -7,11 +7,12 @@ */ import { groupBy } from 'lodash'; -import { useEffect, useMemo, useState, useRef } from 'react'; +import { useEffect, useMemo, useState } from 'react'; import { i18n } from '@kbn/i18n'; import { type CoreStart } from '@kbn/core-lifecycle-browser'; import type { DataView, DataViewField, FieldSpec } from '@kbn/data-views-plugin/common'; import { type DataViewsContract } from '@kbn/data-views-plugin/public'; +import { useNewFields } from './use_new_fields'; import { type FieldListGroups, type FieldsGroup, @@ -41,7 +42,7 @@ export interface GroupedFieldsParams { onOverrideFieldGroupDetails?: OverrideFieldGroupDetails; onSupportedFieldFilter?: (field: T) => boolean; onSelectedFieldFilter?: (field: T) => boolean; - getNewFieldsBySpec?: (field: FieldSpec[], dataView: DataView | null) => T[]; + getNewFieldsBySpec?: (fields: FieldSpec[], dataView: DataView | null) => T[]; } export interface GroupedFieldsResult { @@ -77,8 +78,7 @@ export function useGroupedFields({ getCustomFieldType, onSupportedFieldFilter, }); - const allFieldsToReturn = useRef(allFields); - const hasNewFields = useRef(false); + const onFilterFieldList = fieldListFilters.onFilterField; const [dataView, setDataView] = useState(null); const isAffectedByTimeFilter = Boolean(dataView?.timeFieldName); @@ -107,6 +107,13 @@ export function useGroupedFields({ // if field existence information changed, reload the data view too }, [dataViewId, services.dataViews, setDataView, hasFieldDataHandler]); + const { allFieldsModified, hasNewFields } = useNewFields({ + dataView, + allFields, + getNewFieldsBySpec, + fieldsExistenceReader, + }); + // important when switching from a known dataViewId to no data view (like in text-based queries) useEffect(() => { if (dataView && !dataViewId) { @@ -127,25 +134,7 @@ export function useGroupedFields({ const selectedFields = sortedSelectedFields || []; - // Taking care of new fields that were ingested after the selected data view was loaded - // Those replace existing fields if updated, or are added to the list - const newFields = - dataViewId && getNewFieldsBySpec - ? getNewFieldsBySpec(fieldsExistenceReader.getNewFields(dataViewId), dataView) - : []; - // Filtering out fields that e.g. Discover provides by analyzing the resultset, that are not part of the loaded DataView - // These can be replaced by the new fields, which are mapped correctly, and therefore can be used in the right way - const allFieldsExlNew = - allFields && newFields.length - ? allFields.filter((field) => !newFields.some((newField) => newField.name === field.name)) - : allFields; - - const allFieldsInclNew = newFields.length - ? [...(allFieldsExlNew || []), ...newFields] - : allFields || []; - allFieldsToReturn.current = newFields.length ? allFieldsInclNew : allFields; - hasNewFields.current = Boolean(newFields.length); - const sortedFields = [...allFieldsInclNew].sort(sortFields); + const sortedFields = [...(allFieldsModified || [])].sort(sortFields); const groupedFields = { ...getDefaultFieldGroups(), @@ -338,20 +327,18 @@ export function useGroupedFields({ return fieldGroupDefinitions; }, [ - sortedSelectedFields, - allFields, - popularFieldsLimit, - isAffectedByGlobalFilter, - isAffectedByTimeFilter, - dataViewId, - fieldsExistenceInfoUnavailable, - onOverrideFieldGroupDetails, - hasFieldDataHandler, - fieldsExistenceReader, - onSelectedFieldFilter, + allFieldsModified, onSupportedFieldFilter, + onSelectedFieldFilter, + onOverrideFieldGroupDetails, dataView, - getNewFieldsBySpec, + dataViewId, + hasFieldDataHandler, + fieldsExistenceInfoUnavailable, + isAffectedByGlobalFilter, + isAffectedByTimeFilter, + popularFieldsLimit, + sortedSelectedFields, ]); const fieldGroups: FieldListGroups = useMemo(() => { @@ -410,8 +397,8 @@ export function useGroupedFields({ return { fieldListGroupedProps, fieldListFiltersProps: fieldListFilters.fieldListFiltersProps, - allFieldsModified: allFieldsToReturn.current, - hasNewFields: hasNewFields.current, + allFieldsModified, + hasNewFields, }; } diff --git a/packages/kbn-unified-field-list/src/hooks/use_new_fields.ts b/packages/kbn-unified-field-list/src/hooks/use_new_fields.ts new file mode 100644 index 0000000000000..5f16ad63a1101 --- /dev/null +++ b/packages/kbn-unified-field-list/src/hooks/use_new_fields.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { useMemo } from 'react'; +import type { DataView, DataViewField } from '@kbn/data-views-plugin/common'; +import { ExistingFieldsReader, type FieldListItem, GroupedFieldsParams } from '../..'; + +/** + * This hook is used to get the new fields of previous fields for wildcards request, and merges those + * with the existing fields. + * @param dataView + * @param allFields + * @param getNewFieldsBySpec + * @param fieldsExistenceReader + */ +export function useNewFields({ + dataView, + allFields, + getNewFieldsBySpec, + fieldsExistenceReader, +}: { + dataView?: DataView | null; + allFields: GroupedFieldsParams['allFields']; + getNewFieldsBySpec: GroupedFieldsParams['getNewFieldsBySpec']; + fieldsExistenceReader: ExistingFieldsReader; +}) { + const newFields = + allFields && dataView?.id && getNewFieldsBySpec + ? getNewFieldsBySpec(fieldsExistenceReader.getNewFields(dataView?.id), dataView) + : null; + const hasNewFields = Boolean(allFields && newFields && newFields.length > 0); + + const allFieldsModified = useMemo(() => { + if (!allFields || !newFields?.length) return allFields; + // Filtering out fields that e.g. Discover provides with fields that were provided by the previous fieldsForWildcards request + // These can be replaced by the new fields, which are mapped correctly, and therefore can be used in the right way + const allFieldsExlNew = + allFields && newFields + ? allFields.filter((field) => !newFields.some((newField) => newField.name === field.name)) + : allFields; + + return newFields ? [...allFieldsExlNew, ...newFields] : allFields; + }, [newFields, allFields]); + + return { allFieldsModified, hasNewFields }; +} From 921a59825b51f1d68e7c99438a78c660d4a3ec4b Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Fri, 22 Dec 2023 15:48:44 +0100 Subject: [PATCH 136/168] Address review feedback --- packages/kbn-unified-field-list/src/hooks/use_new_fields.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/kbn-unified-field-list/src/hooks/use_new_fields.ts b/packages/kbn-unified-field-list/src/hooks/use_new_fields.ts index 5f16ad63a1101..a03b319858738 100644 --- a/packages/kbn-unified-field-list/src/hooks/use_new_fields.ts +++ b/packages/kbn-unified-field-list/src/hooks/use_new_fields.ts @@ -36,7 +36,7 @@ export function useNewFields({ const hasNewFields = Boolean(allFields && newFields && newFields.length > 0); const allFieldsModified = useMemo(() => { - if (!allFields || !newFields?.length) return allFields; + if (!allFields || !newFields?.length || !dataView) return allFields; // Filtering out fields that e.g. Discover provides with fields that were provided by the previous fieldsForWildcards request // These can be replaced by the new fields, which are mapped correctly, and therefore can be used in the right way const allFieldsExlNew = From ec55f89b965903975b541f9d2bb7f7fadd446490 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Fri, 22 Dec 2023 15:49:50 +0100 Subject: [PATCH 137/168] Fix error --- packages/kbn-unified-field-list/src/hooks/use_new_fields.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/kbn-unified-field-list/src/hooks/use_new_fields.ts b/packages/kbn-unified-field-list/src/hooks/use_new_fields.ts index a03b319858738..b866aea306d2b 100644 --- a/packages/kbn-unified-field-list/src/hooks/use_new_fields.ts +++ b/packages/kbn-unified-field-list/src/hooks/use_new_fields.ts @@ -13,10 +13,6 @@ import { ExistingFieldsReader, type FieldListItem, GroupedFieldsParams } from '. /** * This hook is used to get the new fields of previous fields for wildcards request, and merges those * with the existing fields. - * @param dataView - * @param allFields - * @param getNewFieldsBySpec - * @param fieldsExistenceReader */ export function useNewFields({ dataView, @@ -45,7 +41,7 @@ export function useNewFields({ : allFields; return newFields ? [...allFieldsExlNew, ...newFields] : allFields; - }, [newFields, allFields]); + }, [newFields, allFields, dataView]); return { allFieldsModified, hasNewFields }; } From c76c87e375c9e639e86b9fa4cf802f55d7ddc497 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Fri, 22 Dec 2023 15:57:07 +0100 Subject: [PATCH 138/168] Add useMemo for new fields --- packages/kbn-unified-field-list/src/hooks/use_new_fields.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/kbn-unified-field-list/src/hooks/use_new_fields.ts b/packages/kbn-unified-field-list/src/hooks/use_new_fields.ts index b866aea306d2b..ee18f31363fab 100644 --- a/packages/kbn-unified-field-list/src/hooks/use_new_fields.ts +++ b/packages/kbn-unified-field-list/src/hooks/use_new_fields.ts @@ -25,10 +25,12 @@ export function useNewFields({ getNewFieldsBySpec: GroupedFieldsParams['getNewFieldsBySpec']; fieldsExistenceReader: ExistingFieldsReader; }) { - const newFields = - allFields && dataView?.id && getNewFieldsBySpec + const newFields = useMemo(() => { + return allFields && dataView?.id && getNewFieldsBySpec ? getNewFieldsBySpec(fieldsExistenceReader.getNewFields(dataView?.id), dataView) : null; + }, [allFields, dataView, fieldsExistenceReader, getNewFieldsBySpec]); + const hasNewFields = Boolean(allFields && newFields && newFields.length > 0); const allFieldsModified = useMemo(() => { From 165c3cde88970ed8f0794e4c1e8ffdc3bd4504f6 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Fri, 22 Dec 2023 16:04:53 +0100 Subject: [PATCH 139/168] Address review feedback --- .../datasources/form_based/datapanel.tsx | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx b/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx index 6ee8483658e68..2674293da399a 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx +++ b/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx @@ -263,15 +263,7 @@ export const InnerFormBasedDataPanel = function InnerFormBasedDataPanel({ onSupportedFieldFilter, onSelectedFieldFilter, onOverrideFieldGroupDetails, - getNewFieldsBySpec: (spec, dataView) => { - return spec.reduce((result: IndexPatternField[], fieldSpec: FieldSpec) => { - const field = new DataViewField(fieldSpec); - if (isFieldLensCompatible(field)) { - result.push(buildIndexPatternField(field, dataView)); - } - return result; - }, []); - }, + getNewFieldsBySpec, }); const closeFieldEditor = useRef<() => void | undefined>(); @@ -428,4 +420,14 @@ export const InnerFormBasedDataPanel = function InnerFormBasedDataPanel({ ); }; +function getNewFieldsBySpec(spec: FieldSpec[], dataView: DataView | null) { + return spec.reduce((result: IndexPatternField[], fieldSpec: FieldSpec) => { + const field = new DataViewField(fieldSpec); + if (isFieldLensCompatible(field)) { + result.push(buildIndexPatternField(field, dataView)); + } + return result; + }, []); +} + export const MemoizedDataPanel = memo(InnerFormBasedDataPanel); From 07f50583e26932b8ba718346a91131d076b633f7 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 2 Jan 2024 06:06:03 +0000 Subject: [PATCH 140/168] [CI] Auto-commit changed files from 'node scripts/notice' --- NOTICE.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NOTICE.txt b/NOTICE.txt index 45af6e5231783..d02031c4b5a2b 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -1,5 +1,5 @@ Kibana source code with Kibana X-Pack source code -Copyright 2012-2023 Elasticsearch B.V. +Copyright 2012-2024 Elasticsearch B.V. --- Pretty handling of logarithmic axes. From 9170300a6a04c02cccadceb1dab38c27ed29a7ab Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 2 Jan 2024 10:17:40 +0000 Subject: [PATCH 141/168] [CI] Auto-commit changed files from 'node scripts/notice' --- NOTICE.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NOTICE.txt b/NOTICE.txt index 45af6e5231783..d02031c4b5a2b 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -1,5 +1,5 @@ Kibana source code with Kibana X-Pack source code -Copyright 2012-2023 Elasticsearch B.V. +Copyright 2012-2024 Elasticsearch B.V. --- Pretty handling of logarithmic axes. From 02a4537c466e228687adaf78d2281740f37faf04 Mon Sep 17 00:00:00 2001 From: Julia Rechkunova Date: Wed, 3 Jan 2024 14:33:18 +0100 Subject: [PATCH 142/168] [Discover] Small fixes for types and memo deps. Add tests. --- .../field_list_sidebar.tsx | 2 +- .../src/hooks/use_existing_fields.test.tsx | 48 +++++++++++ .../src/hooks/use_existing_fields.ts | 5 +- .../src/hooks/use_grouped_fields.test.tsx | 78 ++++++++++++++++++ .../src/hooks/use_grouped_fields.ts | 6 +- .../src/hooks/use_new_fields.test.tsx | 80 +++++++++++++++++++ .../src/hooks/use_new_fields.ts | 47 +++++++---- 7 files changed, 243 insertions(+), 23 deletions(-) create mode 100644 packages/kbn-unified-field-list/src/hooks/use_new_fields.test.tsx diff --git a/packages/kbn-unified-field-list/src/containers/unified_field_list_sidebar/field_list_sidebar.tsx b/packages/kbn-unified-field-list/src/containers/unified_field_list_sidebar/field_list_sidebar.tsx index 17ebe5b19286f..f8438b0917577 100644 --- a/packages/kbn-unified-field-list/src/containers/unified_field_list_sidebar/field_list_sidebar.tsx +++ b/packages/kbn-unified-field-list/src/containers/unified_field_list_sidebar/field_list_sidebar.tsx @@ -22,7 +22,7 @@ import { useEuiTheme, } from '@elastic/eui'; import { ToolbarButton } from '@kbn/shared-ux-button-toolbar'; -import { DataViewField, FieldSpec } from '@kbn/data-views-plugin/common'; +import { DataViewField, type FieldSpec } from '@kbn/data-views-plugin/common'; import { getDataViewFieldSubtypeMulti } from '@kbn/es-query/src/utils'; import { FIELDS_LIMIT_SETTING, SEARCH_FIELDS_FROM_SOURCE } from '@kbn/discover-utils'; import { FieldList } from '../../components/field_list'; diff --git a/packages/kbn-unified-field-list/src/hooks/use_existing_fields.test.tsx b/packages/kbn-unified-field-list/src/hooks/use_existing_fields.test.tsx index 7cf22950a4a7d..0af0e4abc8e0d 100644 --- a/packages/kbn-unified-field-list/src/hooks/use_existing_fields.test.tsx +++ b/packages/kbn-unified-field-list/src/hooks/use_existing_fields.test.tsx @@ -125,6 +125,7 @@ describe('UnifiedFieldList useExistingFields', () => { expect(hookReader.result.current.getFieldsExistenceStatus(dataViewId)).toBe( ExistenceFetchStatus.succeeded ); + expect(hookReader.result.current.getNewFields(dataViewId)).toStrictEqual([]); // does not have existence info => works less restrictive const anotherDataViewId = 'test-id'; @@ -140,6 +141,7 @@ describe('UnifiedFieldList useExistingFields', () => { expect(hookReader.result.current.getFieldsExistenceStatus(anotherDataViewId)).toBe( ExistenceFetchStatus.unknown ); + expect(hookReader.result.current.getNewFields(dataViewId)).toStrictEqual([]); }); it('should work correctly with multiple readers', async () => { @@ -217,6 +219,7 @@ describe('UnifiedFieldList useExistingFields', () => { expect(currentResult.isFieldsExistenceInfoUnavailable(dataViewId)).toBe(true); expect(currentResult.hasFieldData(dataViewId, dataView.fields[0].name)).toBe(true); expect(currentResult.getFieldsExistenceStatus(dataViewId)).toBe(ExistenceFetchStatus.failed); + expect(currentResult.getNewFields(dataViewId)).toStrictEqual([]); }); it('should work correctly for multiple data views', async () => { @@ -533,4 +536,49 @@ describe('UnifiedFieldList useExistingFields', () => { expect(params.onNoData).toHaveBeenCalledTimes(1); // still 1 time }); + + it('should include newFields', async () => { + const newFields = [{ name: 'test', type: 'keyword', searchable: true, aggregatable: true }]; + + (ExistingFieldsServiceApi.loadFieldExisting as jest.Mock).mockImplementation( + async ({ dataView: currentDataView }) => { + return { + existingFieldNames: [currentDataView.fields[0].name], + newFields, + }; + } + ); + + const params: ExistingFieldsFetcherParams = { + dataViews: [dataView], + services: mockedServices, + fromDate: '2019-01-01', + toDate: '2020-01-01', + query: { query: '', language: 'lucene' }, + filters: [], + }; + const hookFetcher = renderHook(useExistingFieldsFetcher, { + initialProps: params, + }); + + const hookReader = renderHook(useExistingFieldsReader); + await hookFetcher.waitForNextUpdate(); + + expect(ExistingFieldsServiceApi.loadFieldExisting).toHaveBeenCalledWith( + expect.objectContaining({ + fromDate: '2019-01-01', + toDate: '2020-01-01', + dslQuery, + dataView, + timeFieldName: dataView.timeFieldName, + }) + ); + + expect(hookReader.result.current.getFieldsExistenceStatus(dataView.id!)).toBe( + ExistenceFetchStatus.succeeded + ); + + expect(hookReader.result.current.getNewFields(dataView.id!)).toBe(newFields); + expect(hookReader.result.current.getNewFields('another-id')).toStrictEqual([]); + }); }); diff --git a/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts b/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts index a733643008b11..e71d89d6b3626 100644 --- a/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts +++ b/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts @@ -20,6 +20,7 @@ import { ExistenceFetchStatus } from '../types'; const getBuildEsQueryAsync = async () => (await import('@kbn/es-query')).buildEsQuery; const generateId = htmlIdGenerator(); +const DEFAULT_EMPTY_NEW_FIELDS: FieldSpec[] = []; export interface ExistingFieldsInfo { fetchStatus: ExistenceFetchStatus; @@ -294,10 +295,10 @@ export const useExistingFieldsReader: () => ExistingFieldsReader = () => { const info = existingFieldsByDataViewMap[dataViewId]; if (info?.fetchStatus === ExistenceFetchStatus.succeeded) { - return info?.newFields ?? []; + return info?.newFields ?? DEFAULT_EMPTY_NEW_FIELDS; } - return []; + return DEFAULT_EMPTY_NEW_FIELDS; }, [existingFieldsByDataViewMap] ); diff --git a/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.test.tsx b/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.test.tsx index 78b34329e0bf0..4a937f86bd5c0 100644 --- a/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.test.tsx +++ b/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.test.tsx @@ -187,6 +187,8 @@ describe('UnifiedFieldList useGroupedFields()', () => { expect(fieldListGroupedProps.fieldsExistenceStatus).toBe(ExistenceFetchStatus.succeeded); expect(fieldListGroupedProps.fieldsExistInIndex).toBe(true); + expect(result.current.allFieldsModified).toBe(allFields); + expect(result.current.hasNewFields).toBe(false); rerender({ ...props, @@ -200,6 +202,82 @@ describe('UnifiedFieldList useGroupedFields()', () => { expect(result.current.fieldListGroupedProps.scrollToTopResetCounter).not.toBe( scrollToTopResetCounter1 ); + expect(result.current.allFieldsModified).toBe(allFields); + expect(result.current.hasNewFields).toBe(false); + + (ExistenceApi.useExistingFieldsReader as jest.Mock).mockRestore(); + }); + + it('should work correctly with new fields', async () => { + const props: GroupedFieldsParams = { + dataViewId: dataView.id!, + allFields, + services: mockedServices, + getNewFieldsBySpec: (spec) => spec.map((field) => new DataViewField(field)), + }; + + const newField = { name: 'test', type: 'keyword', searchable: true, aggregatable: true }; + + jest.spyOn(ExistenceApi, 'useExistingFieldsReader').mockImplementation( + (): ExistingFieldsReader => ({ + hasFieldData: (dataViewId) => { + return dataViewId === props.dataViewId; + }, + getFieldsExistenceStatus: (dataViewId) => + dataViewId === props.dataViewId + ? ExistenceFetchStatus.succeeded + : ExistenceFetchStatus.unknown, + isFieldsExistenceInfoUnavailable: (dataViewId) => dataViewId !== props.dataViewId, + getNewFields: () => [newField], + }) + ); + + const { result, waitForNextUpdate, rerender } = renderHook(useGroupedFields, { + initialProps: props, + }); + + await waitForNextUpdate(); + + let fieldListGroupedProps = result.current.fieldListGroupedProps; + const fieldGroups = fieldListGroupedProps.fieldGroups; + const scrollToTopResetCounter1 = fieldListGroupedProps.scrollToTopResetCounter; + + expect( + Object.keys(fieldGroups!).map( + (key) => `${key}-${fieldGroups![key as FieldsGroupNames]?.fields.length}` + ) + ).toStrictEqual([ + 'SpecialFields-0', + 'SelectedFields-0', + 'PopularFields-0', + 'AvailableFields-25', + 'UnmappedFields-1', + 'EmptyFields-0', + 'MetaFields-3', + ]); + + expect(fieldListGroupedProps.fieldsExistenceStatus).toBe(ExistenceFetchStatus.succeeded); + expect(fieldListGroupedProps.fieldsExistInIndex).toBe(true); + expect(result.current.allFieldsModified).toStrictEqual([ + ...allFields, + new DataViewField(newField), + ]); + expect(result.current.hasNewFields).toBe(true); + + rerender({ + ...props, + dataViewId: null, // for text-based queries + allFields, + }); + + fieldListGroupedProps = result.current.fieldListGroupedProps; + expect(fieldListGroupedProps.fieldsExistenceStatus).toBe(ExistenceFetchStatus.succeeded); + expect(fieldListGroupedProps.fieldsExistInIndex).toBe(true); + expect(result.current.fieldListGroupedProps.scrollToTopResetCounter).not.toBe( + scrollToTopResetCounter1 + ); + expect(result.current.allFieldsModified).toBe(allFields); + expect(result.current.hasNewFields).toBe(false); (ExistenceApi.useExistingFieldsReader as jest.Mock).mockRestore(); }); diff --git a/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts b/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts index a11d897c37199..7853c7e67800b 100644 --- a/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts +++ b/packages/kbn-unified-field-list/src/hooks/use_grouped_fields.ts @@ -10,9 +10,9 @@ import { groupBy } from 'lodash'; import { useEffect, useMemo, useState } from 'react'; import { i18n } from '@kbn/i18n'; import { type CoreStart } from '@kbn/core-lifecycle-browser'; -import type { DataView, DataViewField, FieldSpec } from '@kbn/data-views-plugin/common'; +import type { DataView, DataViewField } from '@kbn/data-views-plugin/common'; import { type DataViewsContract } from '@kbn/data-views-plugin/public'; -import { useNewFields } from './use_new_fields'; +import { type UseNewFieldsParams, useNewFields } from './use_new_fields'; import { type FieldListGroups, type FieldsGroup, @@ -42,7 +42,7 @@ export interface GroupedFieldsParams { onOverrideFieldGroupDetails?: OverrideFieldGroupDetails; onSupportedFieldFilter?: (field: T) => boolean; onSelectedFieldFilter?: (field: T) => boolean; - getNewFieldsBySpec?: (fields: FieldSpec[], dataView: DataView | null) => T[]; + getNewFieldsBySpec?: UseNewFieldsParams['getNewFieldsBySpec']; } export interface GroupedFieldsResult { diff --git a/packages/kbn-unified-field-list/src/hooks/use_new_fields.test.tsx b/packages/kbn-unified-field-list/src/hooks/use_new_fields.test.tsx new file mode 100644 index 0000000000000..2be5d50764bc9 --- /dev/null +++ b/packages/kbn-unified-field-list/src/hooks/use_new_fields.test.tsx @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { renderHook } from '@testing-library/react-hooks'; +import { stubLogstashDataView as dataView } from '@kbn/data-views-plugin/common/data_view.stub'; +import { DataViewField } from '@kbn/data-views-plugin/common'; +import { useNewFields, type UseNewFieldsParams } from './use_new_fields'; +import { type ExistingFieldsReader } from './use_existing_fields'; +import { ExistenceFetchStatus } from '../types'; + +const fieldsExistenceReader: ExistingFieldsReader = { + hasFieldData: (dataViewId) => { + return dataViewId === dataView.id; + }, + getFieldsExistenceStatus: (dataViewId) => + dataViewId === dataView.id ? ExistenceFetchStatus.succeeded : ExistenceFetchStatus.unknown, + isFieldsExistenceInfoUnavailable: (dataViewId) => dataViewId !== dataView.id, + getNewFields: () => [], +}; + +describe('UnifiedFieldList useNewFields()', () => { + const allFields = dataView.fields; + + it('should work correctly in loading state', async () => { + const props: UseNewFieldsParams = { + dataView, + allFields: null, + fieldsExistenceReader, + }; + const { result } = renderHook(useNewFields, { + initialProps: props, + }); + + expect(result.current.allFieldsModified).toBe(null); + expect(result.current.hasNewFields).toBe(false); + }); + + it('should work correctly with empty new fields', async () => { + const props: UseNewFieldsParams = { + dataView, + allFields, + fieldsExistenceReader, + }; + const { result } = renderHook(useNewFields, { + initialProps: props, + }); + + expect(result.current.allFieldsModified).toBe(allFields); + expect(result.current.hasNewFields).toBe(false); + }); + + it('should work correctly with new fields', async () => { + const newField = { name: 'test', type: 'keyword', searchable: true, aggregatable: true }; + const newField2 = { ...newField, name: 'test2' }; + const props: UseNewFieldsParams = { + dataView, + allFields, + fieldsExistenceReader: { + ...fieldsExistenceReader, + getNewFields: () => [newField, newField2], + }, + getNewFieldsBySpec: (spec) => spec.map((field) => new DataViewField(field)), + }; + const { result } = renderHook(useNewFields, { + initialProps: props, + }); + + expect(result.current.allFieldsModified).toStrictEqual([ + ...allFields, + new DataViewField(newField), + new DataViewField(newField2), + ]); + expect(result.current.hasNewFields).toBe(true); + }); +}); diff --git a/packages/kbn-unified-field-list/src/hooks/use_new_fields.ts b/packages/kbn-unified-field-list/src/hooks/use_new_fields.ts index ee18f31363fab..51e143cc524c3 100644 --- a/packages/kbn-unified-field-list/src/hooks/use_new_fields.ts +++ b/packages/kbn-unified-field-list/src/hooks/use_new_fields.ts @@ -7,8 +7,22 @@ */ import { useMemo } from 'react'; +import type { FieldSpec } from '@kbn/data-views-plugin/common'; import type { DataView, DataViewField } from '@kbn/data-views-plugin/common'; -import { ExistingFieldsReader, type FieldListItem, GroupedFieldsParams } from '../..'; +import type { FieldListItem } from '../types'; +import type { ExistingFieldsReader } from './use_existing_fields'; + +export interface UseNewFieldsParams { + dataView?: DataView | null; + allFields: T[] | null; // `null` is for loading indicator + getNewFieldsBySpec?: (fields: FieldSpec[], dataView: DataView | null) => T[]; + fieldsExistenceReader: ExistingFieldsReader; +} + +export interface UseNewFieldsResult { + allFieldsModified: T[] | null; + hasNewFields: boolean; +} /** * This hook is used to get the new fields of previous fields for wildcards request, and merges those @@ -19,31 +33,30 @@ export function useNewFields({ allFields, getNewFieldsBySpec, fieldsExistenceReader, -}: { - dataView?: DataView | null; - allFields: GroupedFieldsParams['allFields']; - getNewFieldsBySpec: GroupedFieldsParams['getNewFieldsBySpec']; - fieldsExistenceReader: ExistingFieldsReader; -}) { +}: UseNewFieldsParams): UseNewFieldsResult { + const dataViewId = dataView?.id; + const newFields = useMemo(() => { - return allFields && dataView?.id && getNewFieldsBySpec - ? getNewFieldsBySpec(fieldsExistenceReader.getNewFields(dataView?.id), dataView) - : null; + const newLoadedFields = + allFields && dataView?.id && getNewFieldsBySpec + ? getNewFieldsBySpec(fieldsExistenceReader.getNewFields(dataView?.id), dataView) + : null; + + return newLoadedFields?.length ? newLoadedFields : null; }, [allFields, dataView, fieldsExistenceReader, getNewFieldsBySpec]); const hasNewFields = Boolean(allFields && newFields && newFields.length > 0); const allFieldsModified = useMemo(() => { - if (!allFields || !newFields?.length || !dataView) return allFields; + if (!allFields || !newFields?.length || !dataViewId) return allFields; // Filtering out fields that e.g. Discover provides with fields that were provided by the previous fieldsForWildcards request // These can be replaced by the new fields, which are mapped correctly, and therefore can be used in the right way - const allFieldsExlNew = - allFields && newFields - ? allFields.filter((field) => !newFields.some((newField) => newField.name === field.name)) - : allFields; + const allFieldsExlNew = allFields.filter( + (field) => !newFields.some((newField) => newField.name === field.name) + ); - return newFields ? [...allFieldsExlNew, ...newFields] : allFields; - }, [newFields, allFields, dataView]); + return [...allFieldsExlNew, ...newFields]; + }, [newFields, allFields, dataViewId]); return { allFieldsModified, hasNewFields }; } From 04e40bed882e18b359afa4af70589a172ec110d8 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Thu, 4 Jan 2024 23:21:00 -0600 Subject: [PATCH 143/168] cleanup --- .../edit_index_pattern/edit_index_pattern.tsx | 14 +++---- .../index_header/index_header.tsx | 33 +-------------- .../edit_index_pattern/tabs/tabs.tsx | 40 +++++++++++++++++++ src/plugins/data_views/common/utils.ts | 2 +- .../data_views/data_views_api_client.ts | 2 +- src/plugins/data_views/public/plugin.ts | 2 +- src/plugins/data_views/server/constants.ts | 3 ++ 7 files changed, 55 insertions(+), 41 deletions(-) diff --git a/src/plugins/data_view_management/public/components/edit_index_pattern/edit_index_pattern.tsx b/src/plugins/data_view_management/public/components/edit_index_pattern/edit_index_pattern.tsx index 20855e4dbbc0c..f850ef4ec3671 100644 --- a/src/plugins/data_view_management/public/components/edit_index_pattern/edit_index_pattern.tsx +++ b/src/plugins/data_view_management/public/components/edit_index_pattern/edit_index_pattern.tsx @@ -249,15 +249,8 @@ export const EditIndexPattern = withRouter( deleteIndexPatternClick={() => removeHandler([indexPattern as RemoveDataViewProps],
    {warning}
    ) } - refreshIndexPatternClick={async () => { - setIsRefreshing(true); - await dataViews.refreshFields(indexPattern, false, true); - setRefreshCount(refreshCount + 1); // rerender field list - setIsRefreshing(false); - }} defaultIndex={defaultIndex} canSave={userEditPermission} - isRefreshing={isRefreshing} > @@ -342,6 +335,13 @@ export const EditIndexPattern = withRouter( setFields(indexPattern.getNonScriptedFields()); setCompositeRuntimeFields(getCompositeRuntimeFields(indexPattern)); }} + refreshIndexPatternClick={async () => { + setIsRefreshing(true); + await dataViews.refreshFields(indexPattern, false, true); + setRefreshCount(refreshCount + 1); // rerender field list + setIsRefreshing(false); + }} + isRefreshing={isRefreshing} /> {displayIndexPatternEditor} diff --git a/src/plugins/data_view_management/public/components/edit_index_pattern/index_header/index_header.tsx b/src/plugins/data_view_management/public/components/edit_index_pattern/index_header/index_header.tsx index 8a3792c6fb333..1d73ef615e26c 100644 --- a/src/plugins/data_view_management/public/components/edit_index_pattern/index_header/index_header.tsx +++ b/src/plugins/data_view_management/public/components/edit_index_pattern/index_header/index_header.tsx @@ -8,7 +8,7 @@ import React from 'react'; import { i18n } from '@kbn/i18n'; -import { EuiButton, EuiButtonEmpty, EuiPageHeader, EuiToolTip } from '@elastic/eui'; +import { EuiButton, EuiButtonEmpty, EuiPageHeader } from '@elastic/eui'; import { DataView } from '@kbn/data-views-plugin/public'; interface IndexHeaderProps { @@ -46,32 +46,14 @@ const removeTooltip = i18n.translate('indexPatternManagement.editDataView.remove defaultMessage: 'Delete', }); -const refreshAriaLabel = i18n.translate('indexPatternManagement.editDataView.refreshAria', { - defaultMessage: 'Refresh', -}); - -const refreshTooltip = i18n.translate('indexPatternManagement.editDataView.refreshTooltip', { - defaultMessage: 'Refresh local copy of data view field list', -}); - -const refreshLabel = i18n.translate('indexPatternManagement.editDataView.refreshLabel', { - defaultMessage: 'Refresh', -}); - -const isLoadingLabel = i18n.translate('indexPatternManagement.editDataView.refreshLoading', { - defaultMessage: 'Loading...', -}); - export const IndexHeader: React.FC = ({ defaultIndex, indexPattern, setDefault, editIndexPatternClick, deleteIndexPatternClick, - refreshIndexPatternClick, children, canSave, - isRefreshing, }) => { return ( = ({ {editTooltip} ), - {refreshTooltip}

    }> - - {isRefreshing ? isLoadingLabel : refreshLabel} - -     , + , defaultIndex !== indexPattern.id && setDefault && canSave && indexPattern.isPersisted() && ( { relationships: SavedObjectRelation[]; allowedTypes: SavedObjectManagementTypeInfo[]; compositeRuntimeFields: Record; + refreshIndexPatternClick: () => void; + isRefreshing?: boolean; } interface FilterItems { @@ -139,6 +143,14 @@ const addFieldButtonLabel = i18n.translate( } ); +const refreshAriaLabel = i18n.translate('indexPatternManagement.editDataView.refreshAria', { + defaultMessage: 'Refresh', +}); + +const refreshTooltip = i18n.translate('indexPatternManagement.editDataView.refreshTooltip', { + defaultMessage: 'Refresh local copy of data view field list', +}); + const SCHEMA_ITEMS: FilterItems[] = [ { value: 'runtime', @@ -159,6 +171,8 @@ export const Tabs: React.FC = ({ relationships, allowedTypes, compositeRuntimeFields, + refreshIndexPatternClick, + isRefreshing, }) => { const { uiSettings, @@ -323,6 +337,8 @@ export const Tabs: React.FC = ({ [uiSettings] ); + const refreshRef = useRef(null); + const userEditPermission = dataViews.getCanSaveSync(); const getFilterSection = useCallback( (type: string) => { @@ -428,6 +444,28 @@ export const Tabs: React.FC = ({ + + + {refreshTooltip}

    }> + { + refreshIndexPatternClick(); + // clear tooltip focus + if (refreshRef.current) { + refreshRef.current.blur(); + } + }} + iconType="refresh" + aria-label={refreshAriaLabel} + data-test-subj="refreshDataViewButton" + isLoading={isRefreshing} + isDisabled={isRefreshing} + display="base" + size="m" + /> +     +     {userEditPermission && ( openFieldEditor()} data-test-subj="addField"> @@ -505,6 +543,8 @@ export const Tabs: React.FC = ({ updateFieldFilter, updateFieldTypeFilter, updateSchemaFieldTypeFilter, + isRefreshing, + refreshIndexPatternClick, ] ); diff --git a/src/plugins/data_views/common/utils.ts b/src/plugins/data_views/common/utils.ts index b77696e15ba79..61de24dfcf0c3 100644 --- a/src/plugins/data_views/common/utils.ts +++ b/src/plugins/data_views/common/utils.ts @@ -29,7 +29,7 @@ export async function findByName(client: PersistenceAPI, name: string) { } export function unwrapEtag(ifNoneMatch: string) { - let requestHash = ifNoneMatch.slice(1, -1); + let requestHash = ifNoneMatch.replace(/^"(.+)"$/, '$1'); if (requestHash.indexOf('-') > -1) { requestHash = requestHash.split('-')[0]; } diff --git a/src/plugins/data_views/public/data_views/data_views_api_client.ts b/src/plugins/data_views/public/data_views/data_views_api_client.ts index c65db9e5c736f..29791d3567761 100644 --- a/src/plugins/data_views/public/data_views/data_views_api_client.ts +++ b/src/plugins/data_views/public/data_views/data_views_api_client.ts @@ -46,7 +46,7 @@ export class DataViewsApiClient implements IDataViewsApiClient { forceRefresh?: boolean ): Promise | undefined> { const asResponse = true; - const cacheOptions = forceRefresh ? { cache: 'no-cache' as RequestCache } : {}; + const cacheOptions: { cache?: RequestCache } = forceRefresh ? { cache: 'no-cache' } : {}; const userId = await this.getCurrentUserId(); const userHash = userId ? await sha1(userId) : ''; diff --git a/src/plugins/data_views/public/plugin.ts b/src/plugins/data_views/public/plugin.ts index 6d8d2d9fde217..8ff545d8585e1 100644 --- a/src/plugins/data_views/public/plugin.ts +++ b/src/plugins/data_views/public/plugin.ts @@ -101,7 +101,7 @@ export class DataViewsPublicPlugin hasData: this.hasData.start(core), uiSettings: new UiSettingsPublicToCommon(uiSettings), savedObjectsClient: new ContentMagementWrapper(contentManagement.client), - apiClient: new DataViewsApiClient(http, this.userIdGetter), + apiClient: new DataViewsApiClient(http, () => this.userIdGetter()), fieldFormats, http, onNotification: (toastInputFields, key) => { diff --git a/src/plugins/data_views/server/constants.ts b/src/plugins/data_views/server/constants.ts index f204fffe8fc6e..67235ae3af16d 100644 --- a/src/plugins/data_views/server/constants.ts +++ b/src/plugins/data_views/server/constants.ts @@ -94,4 +94,7 @@ export const INITIAL_REST_VERSION = '2023-10-31'; export const INITIAL_REST_VERSION_INTERNAL = '1'; +/** + * Default field caps cache max-age in seconds + */ export const DEFAULT_FIELD_CACHE_FRESHNESS = 5; From a01dd651384e0224da862ffc536e85bcd49ea3d2 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Fri, 5 Jan 2024 02:15:38 -0600 Subject: [PATCH 144/168] fix display of advanced setting --- src/plugins/data_views/server/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/data_views/server/index.ts b/src/plugins/data_views/server/index.ts index 591870bbad029..d8382046183a3 100644 --- a/src/plugins/data_views/server/index.ts +++ b/src/plugins/data_views/server/index.ts @@ -50,7 +50,7 @@ const configSchema = schema.object({ schema.contextRef('serverless'), true, schema.boolean({ defaultValue: false }), - schema.never() + schema.boolean({ defaultValue: true }) ), }); From f67dc854da4738c28651853340fc36cb37960bf3 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Fri, 5 Jan 2024 21:57:48 -0600 Subject: [PATCH 145/168] add functional test to check for presence and absense of advanced setting --- .../apps/management/data_views/_cache.ts | 22 +++++++++++++++++++ test/functional/apps/management/index.ts | 1 + .../common/management/data_views/_cache.ts | 21 ++++++++++++++++++ .../common/management/data_views/index.ts | 1 + 4 files changed, 45 insertions(+) create mode 100644 test/functional/apps/management/data_views/_cache.ts create mode 100644 x-pack/test_serverless/functional/test_suites/common/management/data_views/_cache.ts diff --git a/test/functional/apps/management/data_views/_cache.ts b/test/functional/apps/management/data_views/_cache.ts new file mode 100644 index 0000000000000..f384d253de79d --- /dev/null +++ b/test/functional/apps/management/data_views/_cache.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../../ftr_provider_context'; + +export default function ({ getService, getPageObjects }: FtrProviderContext) { + const PageObjects = getPageObjects(['settings', 'common', 'header']); + const find = getService('find'); + + describe('"Create Index Pattern" wizard', async function () { + await PageObjects.settings.navigateTo(); + await PageObjects.settings.clickKibanaSettings(); + const cacheSetting = await find.byCssSelector('#data_views:cache_max_age-group'); + expect(cacheSetting).to.not.be(undefined); + }); +} diff --git a/test/functional/apps/management/index.ts b/test/functional/apps/management/index.ts index 85de9b5de0926..7b4a0dd5af000 100644 --- a/test/functional/apps/management/index.ts +++ b/test/functional/apps/management/index.ts @@ -21,6 +21,7 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { await esArchiver.unload('test/functional/fixtures/es_archiver/makelogs'); }); + loadTestFile(require.resolve('./data_views/_cache')); loadTestFile(require.resolve('./data_views/_create_index_pattern_wizard')); loadTestFile(require.resolve('./data_views/_data_view_create_delete')); loadTestFile(require.resolve('./data_views/_index_pattern_results_sort')); diff --git a/x-pack/test_serverless/functional/test_suites/common/management/data_views/_cache.ts b/x-pack/test_serverless/functional/test_suites/common/management/data_views/_cache.ts new file mode 100644 index 0000000000000..3696bf9965a7b --- /dev/null +++ b/x-pack/test_serverless/functional/test_suites/common/management/data_views/_cache.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../../../ftr_provider_context'; + +export default function ({ getService, getPageObjects }: FtrProviderContext) { + const PageObjects = getPageObjects(['settings', 'common', 'header']); + const find = getService('find'); + + describe('"Create Index Pattern" wizard', async function () { + await PageObjects.settings.navigateTo(); + await PageObjects.settings.clickKibanaSettings(); + const cacheSetting = await find.byCssSelector('#data_views:cache_max_age-group'); + expect(cacheSetting).to.be(undefined); + }); +} diff --git a/x-pack/test_serverless/functional/test_suites/common/management/data_views/index.ts b/x-pack/test_serverless/functional/test_suites/common/management/data_views/index.ts index 6a512b6d6e273..cc14ab8d6d526 100644 --- a/x-pack/test_serverless/functional/test_suites/common/management/data_views/index.ts +++ b/x-pack/test_serverless/functional/test_suites/common/management/data_views/index.ts @@ -23,6 +23,7 @@ export default ({ getService, loadTestFile, getPageObject }: FtrProviderContext) await esArchiver.unload('test/functional/fixtures/es_archiver/makelogs'); }); + loadTestFile(require.resolve('./_cache')); loadTestFile(require.resolve('./serverless')); loadTestFile(require.resolve('./_data_view_create_delete')); loadTestFile(require.resolve('./_runtime_fields')); From 603708e4b47e5937f1cf4bc8d84e456b13e3ce35 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Fri, 5 Jan 2024 22:52:19 -0600 Subject: [PATCH 146/168] fix tests, improve buttons? --- .../edit_index_pattern/tabs/tabs.tsx | 31 ++++++++++++++----- test/functional/apps/management/index.ts | 2 +- .../common/management/data_views/index.ts | 2 +- 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/src/plugins/data_view_management/public/components/edit_index_pattern/tabs/tabs.tsx b/src/plugins/data_view_management/public/components/edit_index_pattern/tabs/tabs.tsx index 4927c1d1b5b58..ca60f2742da62 100644 --- a/src/plugins/data_view_management/public/components/edit_index_pattern/tabs/tabs.tsx +++ b/src/plugins/data_view_management/public/components/edit_index_pattern/tabs/tabs.tsx @@ -19,7 +19,6 @@ import { EuiTabbedContentTab, EuiSpacer, EuiFieldSearch, - EuiButton, EuiFilterSelectItem, FilterChecked, EuiToolTip, @@ -338,6 +337,7 @@ export const Tabs: React.FC = ({ ); const refreshRef = useRef(null); + const addFieldRef = useRef(null); const userEditPermission = dataViews.getCanSaveSync(); const getFilterSection = useCallback( @@ -445,6 +445,28 @@ export const Tabs: React.FC = ({ + {userEditPermission && ( + + {addFieldButtonLabel}

    }> + { + openFieldEditor(); + // clear tooltip focus + if (addFieldRef.current) { + addFieldRef.current.blur(); + } + }} + data-test-subj="addField" + iconType="plus" + aria-label={addFieldButtonLabel} + /> +     +     + )} + {refreshTooltip}

    }> = ({ />     - {userEditPermission && ( - - openFieldEditor()} data-test-subj="addField"> - {addFieldButtonLabel} - - - )} )} {type === TAB_SCRIPTED_FIELDS && scriptedFieldLanguages.length > 0 && ( diff --git a/test/functional/apps/management/index.ts b/test/functional/apps/management/index.ts index 7b4a0dd5af000..808a4aaccc9c2 100644 --- a/test/functional/apps/management/index.ts +++ b/test/functional/apps/management/index.ts @@ -21,7 +21,6 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { await esArchiver.unload('test/functional/fixtures/es_archiver/makelogs'); }); - loadTestFile(require.resolve('./data_views/_cache')); loadTestFile(require.resolve('./data_views/_create_index_pattern_wizard')); loadTestFile(require.resolve('./data_views/_data_view_create_delete')); loadTestFile(require.resolve('./data_views/_index_pattern_results_sort')); @@ -48,5 +47,6 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./data_views/_edit_field')); loadTestFile(require.resolve('./_files')); loadTestFile(require.resolve('./_data_view_field_filters')); + loadTestFile(require.resolve('./data_views/_cache')); }); } diff --git a/x-pack/test_serverless/functional/test_suites/common/management/data_views/index.ts b/x-pack/test_serverless/functional/test_suites/common/management/data_views/index.ts index cc14ab8d6d526..69fec71b2ad22 100644 --- a/x-pack/test_serverless/functional/test_suites/common/management/data_views/index.ts +++ b/x-pack/test_serverless/functional/test_suites/common/management/data_views/index.ts @@ -23,7 +23,6 @@ export default ({ getService, loadTestFile, getPageObject }: FtrProviderContext) await esArchiver.unload('test/functional/fixtures/es_archiver/makelogs'); }); - loadTestFile(require.resolve('./_cache')); loadTestFile(require.resolve('./serverless')); loadTestFile(require.resolve('./_data_view_create_delete')); loadTestFile(require.resolve('./_runtime_fields')); @@ -31,5 +30,6 @@ export default ({ getService, loadTestFile, getPageObject }: FtrProviderContext) loadTestFile(require.resolve('./_exclude_index_pattern')); loadTestFile(require.resolve('./_index_pattern_filter')); loadTestFile(require.resolve('./_edit_field')); + loadTestFile(require.resolve('./_cache')); }); }; From 937366bc3f829ed391fef737cf660c06bd2a1223 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Fri, 5 Jan 2024 23:40:56 -0600 Subject: [PATCH 147/168] fix functional tests --- test/functional/apps/management/data_views/_cache.ts | 5 ++++- .../test_suites/common/management/data_views/_cache.ts | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/test/functional/apps/management/data_views/_cache.ts b/test/functional/apps/management/data_views/_cache.ts index f384d253de79d..f6ec9dd427981 100644 --- a/test/functional/apps/management/data_views/_cache.ts +++ b/test/functional/apps/management/data_views/_cache.ts @@ -13,8 +13,11 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const PageObjects = getPageObjects(['settings', 'common', 'header']); const find = getService('find'); - describe('"Create Index Pattern" wizard', async function () { + before(async () => { await PageObjects.settings.navigateTo(); + }); + + describe('"Create Index Pattern" wizard', async function () { await PageObjects.settings.clickKibanaSettings(); const cacheSetting = await find.byCssSelector('#data_views:cache_max_age-group'); expect(cacheSetting).to.not.be(undefined); diff --git a/x-pack/test_serverless/functional/test_suites/common/management/data_views/_cache.ts b/x-pack/test_serverless/functional/test_suites/common/management/data_views/_cache.ts index 3696bf9965a7b..f654272623dc7 100644 --- a/x-pack/test_serverless/functional/test_suites/common/management/data_views/_cache.ts +++ b/x-pack/test_serverless/functional/test_suites/common/management/data_views/_cache.ts @@ -12,8 +12,11 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const PageObjects = getPageObjects(['settings', 'common', 'header']); const find = getService('find'); - describe('"Create Index Pattern" wizard', async function () { + before(async () => { await PageObjects.settings.navigateTo(); + }); + + describe('"Create Index Pattern" wizard', async function () { await PageObjects.settings.clickKibanaSettings(); const cacheSetting = await find.byCssSelector('#data_views:cache_max_age-group'); expect(cacheSetting).to.be(undefined); From a6bbc4037bee7881d1db9c59038a23afba8cb936 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Sat, 6 Jan 2024 00:26:28 -0600 Subject: [PATCH 148/168] fix functional tests --- .../apps/management/data_views/_cache.ts | 17 +++++++++-------- .../common/management/data_views/_cache.ts | 16 +++++++++------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/test/functional/apps/management/data_views/_cache.ts b/test/functional/apps/management/data_views/_cache.ts index f6ec9dd427981..f05a8b9bdd3a4 100644 --- a/test/functional/apps/management/data_views/_cache.ts +++ b/test/functional/apps/management/data_views/_cache.ts @@ -13,13 +13,14 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const PageObjects = getPageObjects(['settings', 'common', 'header']); const find = getService('find'); - before(async () => { - await PageObjects.settings.navigateTo(); - }); - - describe('"Create Index Pattern" wizard', async function () { - await PageObjects.settings.clickKibanaSettings(); - const cacheSetting = await find.byCssSelector('#data_views:cache_max_age-group'); - expect(cacheSetting).to.not.be(undefined); + describe('Data view field caps cache advanced setting', async function () { + before(async () => { + await PageObjects.settings.navigateTo(); + }); + it('should have cache setting', async () => { + await PageObjects.settings.clickKibanaSettings(); + const cacheSetting = await find.byCssSelector('#data_views:cache_max_age-group'); + expect(cacheSetting).to.not.be(undefined); + }); }); } diff --git a/x-pack/test_serverless/functional/test_suites/common/management/data_views/_cache.ts b/x-pack/test_serverless/functional/test_suites/common/management/data_views/_cache.ts index f654272623dc7..083fe2832537e 100644 --- a/x-pack/test_serverless/functional/test_suites/common/management/data_views/_cache.ts +++ b/x-pack/test_serverless/functional/test_suites/common/management/data_views/_cache.ts @@ -12,13 +12,15 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const PageObjects = getPageObjects(['settings', 'common', 'header']); const find = getService('find'); - before(async () => { - await PageObjects.settings.navigateTo(); - }); + describe('Data view field caps cache advanced setting', async function () { + before(async () => { + await PageObjects.settings.navigateTo(); + }); - describe('"Create Index Pattern" wizard', async function () { - await PageObjects.settings.clickKibanaSettings(); - const cacheSetting = await find.byCssSelector('#data_views:cache_max_age-group'); - expect(cacheSetting).to.be(undefined); + it('should not have cache setting', async () => { + await PageObjects.settings.clickKibanaSettings(); + const cacheSetting = await find.byCssSelector('#data_views:cache_max_age-group'); + expect(cacheSetting).to.be(undefined); + }); }); } From 15f877130328ae2bafc2630be0070d4dcfe3ba41 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Sat, 6 Jan 2024 10:54:29 -0600 Subject: [PATCH 149/168] fix functional tests --- test/functional/apps/management/data_views/_cache.ts | 2 +- .../test_suites/common/management/data_views/_cache.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/functional/apps/management/data_views/_cache.ts b/test/functional/apps/management/data_views/_cache.ts index f05a8b9bdd3a4..c62a78101a864 100644 --- a/test/functional/apps/management/data_views/_cache.ts +++ b/test/functional/apps/management/data_views/_cache.ts @@ -19,7 +19,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); it('should have cache setting', async () => { await PageObjects.settings.clickKibanaSettings(); - const cacheSetting = await find.byCssSelector('#data_views:cache_max_age-group'); + const cacheSetting = await find.byCssSelector('#data_views\\:cache_max_age-group'); expect(cacheSetting).to.not.be(undefined); }); }); diff --git a/x-pack/test_serverless/functional/test_suites/common/management/data_views/_cache.ts b/x-pack/test_serverless/functional/test_suites/common/management/data_views/_cache.ts index 083fe2832537e..fbfd8bf813796 100644 --- a/x-pack/test_serverless/functional/test_suites/common/management/data_views/_cache.ts +++ b/x-pack/test_serverless/functional/test_suites/common/management/data_views/_cache.ts @@ -19,7 +19,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { it('should not have cache setting', async () => { await PageObjects.settings.clickKibanaSettings(); - const cacheSetting = await find.byCssSelector('#data_views:cache_max_age-group'); + const cacheSetting = await find.byCssSelector('#data_views\\:cache_max_age-group'); expect(cacheSetting).to.be(undefined); }); }); From 290a3ca32664e3b9cb91e449c8d9c4c8fa15ac02 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Sat, 6 Jan 2024 13:29:58 -0600 Subject: [PATCH 150/168] fix functional tests --- test/functional/apps/management/data_views/_cache.ts | 1 - .../test_suites/common/management/data_views/_cache.ts | 1 - 2 files changed, 2 deletions(-) diff --git a/test/functional/apps/management/data_views/_cache.ts b/test/functional/apps/management/data_views/_cache.ts index c62a78101a864..44d384a7dda90 100644 --- a/test/functional/apps/management/data_views/_cache.ts +++ b/test/functional/apps/management/data_views/_cache.ts @@ -18,7 +18,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.settings.navigateTo(); }); it('should have cache setting', async () => { - await PageObjects.settings.clickKibanaSettings(); const cacheSetting = await find.byCssSelector('#data_views\\:cache_max_age-group'); expect(cacheSetting).to.not.be(undefined); }); diff --git a/x-pack/test_serverless/functional/test_suites/common/management/data_views/_cache.ts b/x-pack/test_serverless/functional/test_suites/common/management/data_views/_cache.ts index fbfd8bf813796..7d5607015495e 100644 --- a/x-pack/test_serverless/functional/test_suites/common/management/data_views/_cache.ts +++ b/x-pack/test_serverless/functional/test_suites/common/management/data_views/_cache.ts @@ -18,7 +18,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); it('should not have cache setting', async () => { - await PageObjects.settings.clickKibanaSettings(); const cacheSetting = await find.byCssSelector('#data_views\\:cache_max_age-group'); expect(cacheSetting).to.be(undefined); }); From dda0d98c82fafce8011cdd1cb2d79b060d4837b5 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Sat, 6 Jan 2024 21:02:12 -0600 Subject: [PATCH 151/168] fix functional tests --- test/functional/apps/management/data_views/_cache.ts | 1 + .../test_suites/common/management/data_views/_cache.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/test/functional/apps/management/data_views/_cache.ts b/test/functional/apps/management/data_views/_cache.ts index 44d384a7dda90..447e44052bb8d 100644 --- a/test/functional/apps/management/data_views/_cache.ts +++ b/test/functional/apps/management/data_views/_cache.ts @@ -16,6 +16,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { describe('Data view field caps cache advanced setting', async function () { before(async () => { await PageObjects.settings.navigateTo(); + await PageObjects.settings.clickKibanaSettings(); }); it('should have cache setting', async () => { const cacheSetting = await find.byCssSelector('#data_views\\:cache_max_age-group'); diff --git a/x-pack/test_serverless/functional/test_suites/common/management/data_views/_cache.ts b/x-pack/test_serverless/functional/test_suites/common/management/data_views/_cache.ts index 7d5607015495e..650864405088c 100644 --- a/x-pack/test_serverless/functional/test_suites/common/management/data_views/_cache.ts +++ b/x-pack/test_serverless/functional/test_suites/common/management/data_views/_cache.ts @@ -15,6 +15,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { describe('Data view field caps cache advanced setting', async function () { before(async () => { await PageObjects.settings.navigateTo(); + await PageObjects.settings.clickKibanaSettings(); }); it('should not have cache setting', async () => { From f751a4e6c691654ea3ddd006deb6cf232e81acc1 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Tue, 9 Jan 2024 10:56:27 +0100 Subject: [PATCH 152/168] Address review feedback --- .../src/hooks/use_existing_fields.ts | 2 +- .../public/data_views_service/loader.test.ts | 37 ++++++++++++++++++- .../lens/public/data_views_service/loader.ts | 7 ++-- .../datasources/form_based/datapanel.tsx | 4 +- 4 files changed, 43 insertions(+), 7 deletions(-) diff --git a/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts b/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts index e71d89d6b3626..cd4e9ef05206f 100644 --- a/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts +++ b/packages/kbn-unified-field-list/src/hooks/use_existing_fields.ts @@ -121,7 +121,7 @@ export const useExistingFieldsFetcher = ( setActiveRequests((value) => value + 1); - const hasRestrictions = Boolean(dataView?.getAggregationRestrictions?.()); + const hasRestrictions = Boolean(dataView.getAggregationRestrictions?.()); const info: ExistingFieldsInfo = { ...unknownInfo, numberOfFetches, diff --git a/x-pack/plugins/lens/public/data_views_service/loader.test.ts b/x-pack/plugins/lens/public/data_views_service/loader.test.ts index f91d236986b11..4c648a7782896 100644 --- a/x-pack/plugins/lens/public/data_views_service/loader.test.ts +++ b/x-pack/plugins/lens/public/data_views_service/loader.test.ts @@ -5,8 +5,13 @@ * 2.0. */ -import { DataViewsContract } from '@kbn/data-views-plugin/public'; -import { ensureIndexPattern, loadIndexPatternRefs, loadIndexPatterns } from './loader'; +import { DataViewsContract, DataViewField } from '@kbn/data-views-plugin/public'; +import { + ensureIndexPattern, + loadIndexPatternRefs, + loadIndexPatterns, + buildIndexPatternField, +} from './loader'; import { sampleIndexPatterns, mockDataViewsService } from './mocks'; import { documentField } from '../datasources/form_based/document_field'; @@ -313,4 +318,32 @@ describe('loader', () => { expect(onError).not.toHaveBeenCalled(); }); }); + + describe('buildIndexPatternField', () => { + it('should return a field with the correct name and derived parameters', async () => { + const field = buildIndexPatternField({ + name: 'foo', + displayName: 'Foo', + type: 'string', + aggregatable: true, + searchable: true, + } as DataViewField); + expect(field.name).toEqual('foo'); + expect(field.meta).toEqual(false); + expect(field.runtime).toEqual(false); + }); + it('should return return the right meta field value', async () => { + const field = buildIndexPatternField( + { + name: 'meta', + displayName: 'Meta', + type: 'string', + aggregatable: true, + searchable: true, + } as DataViewField, + new Set(['meta']) + ); + expect(field.meta).toEqual(true); + }); + }); }); diff --git a/x-pack/plugins/lens/public/data_views_service/loader.ts b/x-pack/plugins/lens/public/data_views_service/loader.ts index 00bd291ef9c24..8a52146991b8d 100644 --- a/x-pack/plugins/lens/public/data_views_service/loader.ts +++ b/x-pack/plugins/lens/public/data_views_service/loader.ts @@ -34,9 +34,10 @@ export function convertDataViewIntoLensIndexPattern( dataView: DataView, restrictionRemapper: (name: string) => string = onRestrictionMapping ): IndexPattern { + const metaKeys = new Set(dataView.metaFields); const newFields = dataView.fields .filter(isFieldLensCompatible) - .map((field) => buildIndexPatternField(field, dataView)) + .map((field) => buildIndexPatternField(field, metaKeys)) .concat(documentField); const { typeMeta, title, name, timeFieldName, fieldFormatMap } = dataView; @@ -80,9 +81,9 @@ export function convertDataViewIntoLensIndexPattern( export function buildIndexPatternField( field: DataViewField, - dataView: DataView | null + metaKeys?: Set ): IndexPatternField { - const meta = dataView?.metaFields.includes(field.name); + const meta = metaKeys ? metaKeys.has(field.name) : false; // Convert the getters on the index pattern service into plain JSON const base = { name: field.name, diff --git a/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx b/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx index 2674293da399a..b2db83cf0e7c6 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx +++ b/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx @@ -421,10 +421,12 @@ export const InnerFormBasedDataPanel = function InnerFormBasedDataPanel({ }; function getNewFieldsBySpec(spec: FieldSpec[], dataView: DataView | null) { + const metaKeys = dataView ? new Set(dataView.metaFields) : undefined; + return spec.reduce((result: IndexPatternField[], fieldSpec: FieldSpec) => { const field = new DataViewField(fieldSpec); if (isFieldLensCompatible(field)) { - result.push(buildIndexPatternField(field, dataView)); + result.push(buildIndexPatternField(field, metaKeys)); } return result; }, []); From d024027cec062a59ae8bbb0b6f15f2a7f00a9320 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Tue, 9 Jan 2024 11:24:09 +0100 Subject: [PATCH 153/168] Apply useLatest feedback --- .../datasources/form_based/datapanel.tsx | 21 +++++++------------ 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx b/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx index b2db83cf0e7c6..5d7e928c95594 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx +++ b/x-pack/plugins/lens/public/datasources/form_based/datapanel.tsx @@ -28,6 +28,7 @@ import { useGroupedFields, } from '@kbn/unified-field-list'; import { ChartsPluginSetup } from '@kbn/charts-plugin/public'; +import useLatest from 'react-use/lib/useLatest'; import { isFieldLensCompatible } from '@kbn/visualization-ui-components'; import { buildIndexPatternField } from '../../data_views_service/loader'; import type { @@ -277,7 +278,7 @@ export const InnerFormBasedDataPanel = function InnerFormBasedDataPanel({ }; }, []); - const refreshFieldList = useCallback(async () => { + const refreshFieldList = useLatest(async () => { if (currentIndexPattern) { const newlyMappedIndexPattern = await indexPatternService.loadIndexPatterns({ patterns: [currentIndexPattern.id], @@ -293,21 +294,13 @@ export const InnerFormBasedDataPanel = function InnerFormBasedDataPanel({ } // start a new session so all charts are refreshed data.search.session.start(); - }, [ - indexPatternService, - currentIndexPattern, - onIndexPatternRefresh, - frame.dataViews.indexPatterns, - data.search.session, - ]); + }); useEffect(() => { if (hasNewFields) { - refreshFieldList(); + refreshFieldList.current(); } - // Preventing a race condition, making sure refreshFieldList is just executed once when hasNewFields is true - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [hasNewFields]); + }, [hasNewFields, refreshFieldList]); const editField = useMemo( () => @@ -321,7 +314,7 @@ export const InnerFormBasedDataPanel = function InnerFormBasedDataPanel({ fieldName, onSave: () => { if (indexPatternInstance.isPersisted()) { - refreshFieldList(); + refreshFieldList.current(); refetchFieldsExistenceInfo(indexPatternInstance.id); } else { indexPatternService.replaceDataViewId(indexPatternInstance); @@ -353,7 +346,7 @@ export const InnerFormBasedDataPanel = function InnerFormBasedDataPanel({ fieldName, onDelete: () => { if (indexPatternInstance.isPersisted()) { - refreshFieldList(); + refreshFieldList.current(); refetchFieldsExistenceInfo(indexPatternInstance.id); } else { indexPatternService.replaceDataViewId(indexPatternInstance); From e628af78405c9715722297e7f9d5c2645e14123e Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Tue, 9 Jan 2024 14:20:04 -0600 Subject: [PATCH 154/168] better buttons, better software --- .../edit_index_pattern/tabs/tabs.tsx | 51 ++++++++----------- 1 file changed, 22 insertions(+), 29 deletions(-) diff --git a/src/plugins/data_view_management/public/components/edit_index_pattern/tabs/tabs.tsx b/src/plugins/data_view_management/public/components/edit_index_pattern/tabs/tabs.tsx index ca60f2742da62..5ef028a0e3720 100644 --- a/src/plugins/data_view_management/public/components/edit_index_pattern/tabs/tabs.tsx +++ b/src/plugins/data_view_management/public/components/edit_index_pattern/tabs/tabs.tsx @@ -22,7 +22,7 @@ import { EuiFilterSelectItem, FilterChecked, EuiToolTip, - EuiButtonIcon, + EuiButton, } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { fieldWildcardMatcher } from '@kbn/kibana-utils-plugin/public'; @@ -337,13 +337,12 @@ export const Tabs: React.FC = ({ ); const refreshRef = useRef(null); - const addFieldRef = useRef(null); const userEditPermission = dataViews.getCanSaveSync(); const getFilterSection = useCallback( (type: string) => { return ( - + = ({ - - {userEditPermission && ( - - {addFieldButtonLabel}

    }> - { - openFieldEditor(); - // clear tooltip focus - if (addFieldRef.current) { - addFieldRef.current.blur(); - } - }} - data-test-subj="addField" - iconType="plus" - aria-label={addFieldButtonLabel} - /> -     -     - )} - {refreshTooltip}

    }> - { refreshIndexPatternClick(); @@ -483,11 +459,28 @@ export const Tabs: React.FC = ({ data-test-subj="refreshDataViewButton" isLoading={isRefreshing} isDisabled={isRefreshing} - display="base" size="m" - /> + color="success" + > + Refresh +     + {userEditPermission && ( + + openFieldEditor()} + data-test-subj="addField" + iconType="plusInCircle" + aria-label={addFieldButtonLabel} + color="primary" + fill + > + Add field + + + )} )} {type === TAB_SCRIPTED_FIELDS && scriptedFieldLanguages.length > 0 && ( From 66fd722b473d26c2f3fc3677dfc7d88b490171d3 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Tue, 9 Jan 2024 14:31:06 -0600 Subject: [PATCH 155/168] fix functional test --- .../test_suites/common/management/data_views/_cache.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/test_serverless/functional/test_suites/common/management/data_views/_cache.ts b/x-pack/test_serverless/functional/test_suites/common/management/data_views/_cache.ts index 650864405088c..2d7285ff2e344 100644 --- a/x-pack/test_serverless/functional/test_suites/common/management/data_views/_cache.ts +++ b/x-pack/test_serverless/functional/test_suites/common/management/data_views/_cache.ts @@ -5,12 +5,11 @@ * 2.0. */ -import expect from '@kbn/expect'; import { FtrProviderContext } from '../../../../ftr_provider_context'; export default function ({ getService, getPageObjects }: FtrProviderContext) { const PageObjects = getPageObjects(['settings', 'common', 'header']); - const find = getService('find'); + const testSubjects = getService('testSubjects'); describe('Data view field caps cache advanced setting', async function () { before(async () => { @@ -19,8 +18,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); it('should not have cache setting', async () => { - const cacheSetting = await find.byCssSelector('#data_views\\:cache_max_age-group'); - expect(cacheSetting).to.be(undefined); + await testSubjects.missingOrFail( + 'advancedSetting-editField-data_views\\:cache_max_age-group' + ); }); }); } From 48d28c930f89a1272fc929c62131e719400b8458 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Tue, 9 Jan 2024 16:10:09 -0600 Subject: [PATCH 156/168] fix functional test --- test/functional/apps/management/data_views/_field_formatter.ts | 3 ++- .../apps/management/data_views/_index_pattern_filter.ts | 3 ++- .../apps/management/data_views/_index_pattern_results_sort.ts | 3 ++- .../apps/discover/feature_controls/discover_security.ts | 3 +++ .../apps/discover_ml_uptime/discover/search_source_alert.ts | 3 ++- .../common/discover_ml_uptime/discover/search_source_alert.ts | 3 ++- .../test_suites/common/management/data_views/_cache.ts | 1 - .../common/management/data_views/_index_pattern_filter.ts | 3 ++- .../test_suites/search/cases/attachment_framework.ts | 3 ++- 9 files changed, 17 insertions(+), 8 deletions(-) diff --git a/test/functional/apps/management/data_views/_field_formatter.ts b/test/functional/apps/management/data_views/_field_formatter.ts index dfcdc0b877581..adec04bf5bb87 100644 --- a/test/functional/apps/management/data_views/_field_formatter.ts +++ b/test/functional/apps/management/data_views/_field_formatter.ts @@ -465,7 +465,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.settings.navigateTo(); await PageObjects.settings.clickKibanaIndexPatterns(); await PageObjects.settings.clickIndexPatternByName(indexTitle); - await PageObjects.settings.refreshDataViewFieldList(); + // todo checking to see if this is needed after auto update code + // await PageObjects.settings.refreshDataViewFieldList(); }); afterEach(async () => { diff --git a/test/functional/apps/management/data_views/_index_pattern_filter.ts b/test/functional/apps/management/data_views/_index_pattern_filter.ts index 941451424dfb1..10221320be82b 100644 --- a/test/functional/apps/management/data_views/_index_pattern_filter.ts +++ b/test/functional/apps/management/data_views/_index_pattern_filter.ts @@ -170,7 +170,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.settings.clickIndexPatternLogstash(); - await PageObjects.settings.refreshDataViewFieldList(); + // todo checking to see if this is needed after auto update code + // await PageObjects.settings.refreshDataViewFieldList(); await testSubjects.existOrFail('dataViewMappingConflict'); diff --git a/test/functional/apps/management/data_views/_index_pattern_results_sort.ts b/test/functional/apps/management/data_views/_index_pattern_results_sort.ts index 8a94f96827a35..0172c2a67dc4a 100644 --- a/test/functional/apps/management/data_views/_index_pattern_results_sort.ts +++ b/test/functional/apps/management/data_views/_index_pattern_results_sort.ts @@ -68,7 +68,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { describe('field list pagination', function () { const EXPECTED_FIELD_COUNT = 85; it('makelogs data should have expected number of fields', async function () { - await testSubjects.click('refreshDataViewButton'); + // todo checking to see if this is needed after auto update code + // await testSubjects.click('refreshDataViewButton'); await retry.try(async function () { const TabCount = await PageObjects.settings.getFieldsTabCount(); expect(TabCount).to.be('' + EXPECTED_FIELD_COUNT); diff --git a/x-pack/test/functional/apps/discover/feature_controls/discover_security.ts b/x-pack/test/functional/apps/discover/feature_controls/discover_security.ts index 232a241691f4d..9409f8a3e6941 100644 --- a/x-pack/test/functional/apps/discover/feature_controls/discover_security.ts +++ b/x-pack/test/functional/apps/discover/feature_controls/discover_security.ts @@ -452,6 +452,8 @@ export default function (ctx: FtrProviderContext) { // swapping index patterns so we get an updated field list // this is necessary since we don't have access to data view management // nor can we force reload the browser in a test + // todo checking to see if this is needed after auto update code + /** await new Promise((resolve) => setTimeout(resolve, 5000)); await PageObjects.discover.selectIndexPattern('logstash-*'); await new Promise((resolve) => setTimeout(resolve, 5000)); @@ -462,6 +464,7 @@ export default function (ctx: FtrProviderContext) { await PageObjects.discover.selectIndexPattern('alias-logstash-discover'); await new Promise((resolve) => setTimeout(resolve, 5000)); + */ await PageObjects.discover.selectIndexPattern('logstash-*'); await PageObjects.header.waitUntilLoadingHasFinished(); await testSubjects.existOrFail('discoverNoResultsCheckIndices'); diff --git a/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts b/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts index bffa564fba774..a095bd2fa8dbc 100644 --- a/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts +++ b/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts @@ -399,7 +399,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); it('should navigate to alert results via link provided in notification', async () => { - await PageObjects.settings.refreshDataViewFieldList(OUTPUT_DATA_VIEW); + // todo checking to see if this is needed after auto update code + // await PageObjects.settings.refreshDataViewFieldList(OUTPUT_DATA_VIEW); await openAlertResults(RULE_NAME); await checkInitialRuleParamsState(SOURCE_DATA_VIEW); }); diff --git a/x-pack/test_serverless/functional/test_suites/common/discover_ml_uptime/discover/search_source_alert.ts b/x-pack/test_serverless/functional/test_suites/common/discover_ml_uptime/discover/search_source_alert.ts index d2e8f863d5fe9..ba4203a5cb2ac 100644 --- a/x-pack/test_serverless/functional/test_suites/common/discover_ml_uptime/discover/search_source_alert.ts +++ b/x-pack/test_serverless/functional/test_suites/common/discover_ml_uptime/discover/search_source_alert.ts @@ -447,7 +447,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); it('should navigate to alert results via link provided in notification', async () => { - await PageObjects.settings.refreshDataViewFieldList(OUTPUT_DATA_VIEW); + // todo checking to see if this is needed after auto update code + // await PageObjects.settings.refreshDataViewFieldList(OUTPUT_DATA_VIEW); await openAlertResults(RULE_NAME); await checkInitialRuleParamsState(SOURCE_DATA_VIEW); }); diff --git a/x-pack/test_serverless/functional/test_suites/common/management/data_views/_cache.ts b/x-pack/test_serverless/functional/test_suites/common/management/data_views/_cache.ts index 2d7285ff2e344..3050ef2456fbf 100644 --- a/x-pack/test_serverless/functional/test_suites/common/management/data_views/_cache.ts +++ b/x-pack/test_serverless/functional/test_suites/common/management/data_views/_cache.ts @@ -14,7 +14,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { describe('Data view field caps cache advanced setting', async function () { before(async () => { await PageObjects.settings.navigateTo(); - await PageObjects.settings.clickKibanaSettings(); }); it('should not have cache setting', async () => { diff --git a/x-pack/test_serverless/functional/test_suites/common/management/data_views/_index_pattern_filter.ts b/x-pack/test_serverless/functional/test_suites/common/management/data_views/_index_pattern_filter.ts index 9b8a51feac10f..b323a2b1b30c9 100644 --- a/x-pack/test_serverless/functional/test_suites/common/management/data_views/_index_pattern_filter.ts +++ b/x-pack/test_serverless/functional/test_suites/common/management/data_views/_index_pattern_filter.ts @@ -174,7 +174,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.settings.clickIndexPatternLogstash(); - await PageObjects.settings.refreshDataViewFieldList(); + // todo checking to see if this is needed after auto update code + // await PageObjects.settings.refreshDataViewFieldList(); await testSubjects.existOrFail('dataViewMappingConflict'); diff --git a/x-pack/test_serverless/functional/test_suites/search/cases/attachment_framework.ts b/x-pack/test_serverless/functional/test_suites/search/cases/attachment_framework.ts index e74262f60ac3e..f91b5dd7f0785 100644 --- a/x-pack/test_serverless/functional/test_suites/search/cases/attachment_framework.ts +++ b/x-pack/test_serverless/functional/test_suites/search/cases/attachment_framework.ts @@ -34,7 +34,8 @@ export default ({ getPageObject, getService }: FtrProviderContext) => { 'x-pack/test/functional/fixtures/kbn_archiver/lens/lens_basic.json' ); - await settings.refreshDataViewFieldList('default:all-data'); + // todo checking to see if this is needed after auto update code + // await settings.refreshDataViewFieldList('default:all-data'); await svlSearchNavigation.navigateToLandingPage(); From a40cc37c840400bd1de5c6356d08a1de026d64e2 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Tue, 9 Jan 2024 17:37:42 -0600 Subject: [PATCH 157/168] partial progress on fixing fields --- test/functional/apps/discover/group4/_field_list_new_fields.ts | 1 + test/functional/apps/management/data_views/_field_formatter.ts | 3 +-- x-pack/test/functional/apps/lens/group2/fields_list.ts | 1 + .../apps/discover_ml_uptime/discover/search_source_alert.ts | 3 +-- .../common/discover_ml_uptime/discover/search_source_alert.ts | 3 +-- .../common/management/data_views/_index_pattern_filter.ts | 3 +-- .../test_suites/search/cases/attachment_framework.ts | 3 +-- 7 files changed, 7 insertions(+), 10 deletions(-) diff --git a/test/functional/apps/discover/group4/_field_list_new_fields.ts b/test/functional/apps/discover/group4/_field_list_new_fields.ts index 3c24bcf613ae4..1c2058ef2cef4 100644 --- a/test/functional/apps/discover/group4/_field_list_new_fields.ts +++ b/test/functional/apps/discover/group4/_field_list_new_fields.ts @@ -37,6 +37,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); }); + // todo it('Check that new ingested fields are added to the available fields section', async function () { const initialPattern = 'my-index-'; await es.transport.request({ diff --git a/test/functional/apps/management/data_views/_field_formatter.ts b/test/functional/apps/management/data_views/_field_formatter.ts index adec04bf5bb87..dfcdc0b877581 100644 --- a/test/functional/apps/management/data_views/_field_formatter.ts +++ b/test/functional/apps/management/data_views/_field_formatter.ts @@ -465,8 +465,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.settings.navigateTo(); await PageObjects.settings.clickKibanaIndexPatterns(); await PageObjects.settings.clickIndexPatternByName(indexTitle); - // todo checking to see if this is needed after auto update code - // await PageObjects.settings.refreshDataViewFieldList(); + await PageObjects.settings.refreshDataViewFieldList(); }); afterEach(async () => { diff --git a/x-pack/test/functional/apps/lens/group2/fields_list.ts b/x-pack/test/functional/apps/lens/group2/fields_list.ts index 79baafe6100a6..b768c922c894b 100644 --- a/x-pack/test/functional/apps/lens/group2/fields_list.ts +++ b/x-pack/test/functional/apps/lens/group2/fields_list.ts @@ -260,6 +260,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); }); + // todo it('should show new fields Available fields', async () => { await es.transport.request({ path: '/field-update-test/_doc', diff --git a/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts b/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts index a095bd2fa8dbc..bffa564fba774 100644 --- a/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts +++ b/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover/search_source_alert.ts @@ -399,8 +399,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); it('should navigate to alert results via link provided in notification', async () => { - // todo checking to see if this is needed after auto update code - // await PageObjects.settings.refreshDataViewFieldList(OUTPUT_DATA_VIEW); + await PageObjects.settings.refreshDataViewFieldList(OUTPUT_DATA_VIEW); await openAlertResults(RULE_NAME); await checkInitialRuleParamsState(SOURCE_DATA_VIEW); }); diff --git a/x-pack/test_serverless/functional/test_suites/common/discover_ml_uptime/discover/search_source_alert.ts b/x-pack/test_serverless/functional/test_suites/common/discover_ml_uptime/discover/search_source_alert.ts index ba4203a5cb2ac..d2e8f863d5fe9 100644 --- a/x-pack/test_serverless/functional/test_suites/common/discover_ml_uptime/discover/search_source_alert.ts +++ b/x-pack/test_serverless/functional/test_suites/common/discover_ml_uptime/discover/search_source_alert.ts @@ -447,8 +447,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); it('should navigate to alert results via link provided in notification', async () => { - // todo checking to see if this is needed after auto update code - // await PageObjects.settings.refreshDataViewFieldList(OUTPUT_DATA_VIEW); + await PageObjects.settings.refreshDataViewFieldList(OUTPUT_DATA_VIEW); await openAlertResults(RULE_NAME); await checkInitialRuleParamsState(SOURCE_DATA_VIEW); }); diff --git a/x-pack/test_serverless/functional/test_suites/common/management/data_views/_index_pattern_filter.ts b/x-pack/test_serverless/functional/test_suites/common/management/data_views/_index_pattern_filter.ts index b323a2b1b30c9..9b8a51feac10f 100644 --- a/x-pack/test_serverless/functional/test_suites/common/management/data_views/_index_pattern_filter.ts +++ b/x-pack/test_serverless/functional/test_suites/common/management/data_views/_index_pattern_filter.ts @@ -174,8 +174,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.settings.clickIndexPatternLogstash(); - // todo checking to see if this is needed after auto update code - // await PageObjects.settings.refreshDataViewFieldList(); + await PageObjects.settings.refreshDataViewFieldList(); await testSubjects.existOrFail('dataViewMappingConflict'); diff --git a/x-pack/test_serverless/functional/test_suites/search/cases/attachment_framework.ts b/x-pack/test_serverless/functional/test_suites/search/cases/attachment_framework.ts index f91b5dd7f0785..e74262f60ac3e 100644 --- a/x-pack/test_serverless/functional/test_suites/search/cases/attachment_framework.ts +++ b/x-pack/test_serverless/functional/test_suites/search/cases/attachment_framework.ts @@ -34,8 +34,7 @@ export default ({ getPageObject, getService }: FtrProviderContext) => { 'x-pack/test/functional/fixtures/kbn_archiver/lens/lens_basic.json' ); - // todo checking to see if this is needed after auto update code - // await settings.refreshDataViewFieldList('default:all-data'); + await settings.refreshDataViewFieldList('default:all-data'); await svlSearchNavigation.navigateToLandingPage(); From 6342b8903bdf78f72513c6beb634e8ab24df801d Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Tue, 9 Jan 2024 17:53:26 -0600 Subject: [PATCH 158/168] cleanup and hopefully fix tests related to auto refresh --- .../field_existing/field_existing_utils.ts | 2 +- .../data_views/_index_pattern_results_sort.ts | 3 --- .../feature_controls/discover_security.ts | 16 ---------------- 3 files changed, 1 insertion(+), 20 deletions(-) diff --git a/packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts b/packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts index 41a1e53a33849..5cceb49cb5ecb 100644 --- a/packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts +++ b/packages/kbn-unified-field-list/src/services/field_existing/field_existing_utils.ts @@ -61,7 +61,7 @@ export async function fetchFieldExistence({ const newFields = existingFieldList.filter((field) => !dataView.getFieldByName(field.name)); // refresh the data view in case there are new fields if (newFields.length) { - await dataViewsService.refreshFields(dataView, false); + await dataViewsService.refreshFields(dataView, false, true); } const allFields = buildFieldList(dataView, metaFields); return { diff --git a/test/functional/apps/management/data_views/_index_pattern_results_sort.ts b/test/functional/apps/management/data_views/_index_pattern_results_sort.ts index 0172c2a67dc4a..71dc85eed1f0e 100644 --- a/test/functional/apps/management/data_views/_index_pattern_results_sort.ts +++ b/test/functional/apps/management/data_views/_index_pattern_results_sort.ts @@ -12,7 +12,6 @@ import { FtrProviderContext } from '../../../ftr_provider_context'; export default function ({ getService, getPageObjects }: FtrProviderContext) { const kibanaServer = getService('kibanaServer'); const retry = getService('retry'); - const testSubjects = getService('testSubjects'); const PageObjects = getPageObjects(['settings', 'common']); describe('index result field sort', function describeIndexTests() { @@ -68,8 +67,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { describe('field list pagination', function () { const EXPECTED_FIELD_COUNT = 85; it('makelogs data should have expected number of fields', async function () { - // todo checking to see if this is needed after auto update code - // await testSubjects.click('refreshDataViewButton'); await retry.try(async function () { const TabCount = await PageObjects.settings.getFieldsTabCount(); expect(TabCount).to.be('' + EXPECTED_FIELD_COUNT); diff --git a/x-pack/test/functional/apps/discover/feature_controls/discover_security.ts b/x-pack/test/functional/apps/discover/feature_controls/discover_security.ts index 9409f8a3e6941..d0173072a0879 100644 --- a/x-pack/test/functional/apps/discover/feature_controls/discover_security.ts +++ b/x-pack/test/functional/apps/discover/feature_controls/discover_security.ts @@ -449,22 +449,6 @@ export default function (ctx: FtrProviderContext) { await globalNav.badgeExistsOrFail('Read only'); // can't access logstash index directly - // swapping index patterns so we get an updated field list - // this is necessary since we don't have access to data view management - // nor can we force reload the browser in a test - // todo checking to see if this is needed after auto update code - /** - await new Promise((resolve) => setTimeout(resolve, 5000)); - await PageObjects.discover.selectIndexPattern('logstash-*'); - await new Promise((resolve) => setTimeout(resolve, 5000)); - await PageObjects.discover.selectIndexPattern('alias-logstash-discover'); - await new Promise((resolve) => setTimeout(resolve, 5000)); - await PageObjects.discover.selectIndexPattern('logstash-*'); - await new Promise((resolve) => setTimeout(resolve, 5000)); - await PageObjects.discover.selectIndexPattern('alias-logstash-discover'); - - await new Promise((resolve) => setTimeout(resolve, 5000)); - */ await PageObjects.discover.selectIndexPattern('logstash-*'); await PageObjects.header.waitUntilLoadingHasFinished(); await testSubjects.existOrFail('discoverNoResultsCheckIndices'); From 8e5ba7e3405fde1b1cea2ab0de46e9f5506ca886 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Tue, 9 Jan 2024 19:07:53 -0600 Subject: [PATCH 159/168] remove some comments --- test/functional/apps/discover/group4/_field_list_new_fields.ts | 1 - .../apps/management/data_views/_index_pattern_filter.ts | 3 --- x-pack/test/functional/apps/lens/group2/fields_list.ts | 1 - 3 files changed, 5 deletions(-) diff --git a/test/functional/apps/discover/group4/_field_list_new_fields.ts b/test/functional/apps/discover/group4/_field_list_new_fields.ts index 1c2058ef2cef4..3c24bcf613ae4 100644 --- a/test/functional/apps/discover/group4/_field_list_new_fields.ts +++ b/test/functional/apps/discover/group4/_field_list_new_fields.ts @@ -37,7 +37,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); }); - // todo it('Check that new ingested fields are added to the available fields section', async function () { const initialPattern = 'my-index-'; await es.transport.request({ diff --git a/test/functional/apps/management/data_views/_index_pattern_filter.ts b/test/functional/apps/management/data_views/_index_pattern_filter.ts index 10221320be82b..81ff2b450755d 100644 --- a/test/functional/apps/management/data_views/_index_pattern_filter.ts +++ b/test/functional/apps/management/data_views/_index_pattern_filter.ts @@ -170,9 +170,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.settings.clickIndexPatternLogstash(); - // todo checking to see if this is needed after auto update code - // await PageObjects.settings.refreshDataViewFieldList(); - await testSubjects.existOrFail('dataViewMappingConflict'); expect(await PageObjects.settings.getFieldTypes()).to.eql([ diff --git a/x-pack/test/functional/apps/lens/group2/fields_list.ts b/x-pack/test/functional/apps/lens/group2/fields_list.ts index b768c922c894b..79baafe6100a6 100644 --- a/x-pack/test/functional/apps/lens/group2/fields_list.ts +++ b/x-pack/test/functional/apps/lens/group2/fields_list.ts @@ -260,7 +260,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); }); - // todo it('should show new fields Available fields', async () => { await es.transport.request({ path: '/field-update-test/_doc', From b56cb260f2bff40504bf2b833ed6c5c822fa32e4 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Tue, 9 Jan 2024 19:10:57 -0600 Subject: [PATCH 160/168] remve unneeded change --- docs/concepts/data-views.asciidoc | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/docs/concepts/data-views.asciidoc b/docs/concepts/data-views.asciidoc index 2eba42aed3051..992741be518a7 100644 --- a/docs/concepts/data-views.asciidoc +++ b/docs/concepts/data-views.asciidoc @@ -172,14 +172,3 @@ WARNING: Deleting a {data-source} breaks all visualizations, saved searches, and . Find the {data-source} that you want to delete, and then click image:management/index-patterns/images/delete.png[Delete icon] in the *Actions* column. - -[float] -[[data-view-field-cache]] -=== {data-source} field cache - -The browser caches {data-source} field lists for increased performance. This is particularly impactful -for {data-sources} with a high field count that span a large number of indices and clusters. The field -list is updated every couple of minutes in typical {kib} usage. Alternatively, use the refresh button on the {data-source} -management detail page to get an updated field list. A force reload of {kib} has the same effect. - -The field list may be impacted by changes in indices and user permissions. \ No newline at end of file From 19c06b23b91d8c262f1867207029d652d15bd196 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Tue, 9 Jan 2024 20:34:27 -0600 Subject: [PATCH 161/168] functional test fix --- .../apps/management/data_views/_index_pattern_filter.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/functional/apps/management/data_views/_index_pattern_filter.ts b/test/functional/apps/management/data_views/_index_pattern_filter.ts index 81ff2b450755d..941451424dfb1 100644 --- a/test/functional/apps/management/data_views/_index_pattern_filter.ts +++ b/test/functional/apps/management/data_views/_index_pattern_filter.ts @@ -170,6 +170,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.settings.clickIndexPatternLogstash(); + await PageObjects.settings.refreshDataViewFieldList(); + await testSubjects.existOrFail('dataViewMappingConflict'); expect(await PageObjects.settings.getFieldTypes()).to.eql([ From 3bf8c03b1bae43409411921c2586e6ed59f22673 Mon Sep 17 00:00:00 2001 From: Matthew Kime Date: Wed, 10 Jan 2024 07:33:50 -0600 Subject: [PATCH 162/168] Update src/plugins/data_view_management/public/components/edit_index_pattern/index_header/index_header.tsx Co-authored-by: Julia Rechkunova --- .../components/edit_index_pattern/index_header/index_header.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/src/plugins/data_view_management/public/components/edit_index_pattern/index_header/index_header.tsx b/src/plugins/data_view_management/public/components/edit_index_pattern/index_header/index_header.tsx index 1d73ef615e26c..8b12ed59ed6fd 100644 --- a/src/plugins/data_view_management/public/components/edit_index_pattern/index_header/index_header.tsx +++ b/src/plugins/data_view_management/public/components/edit_index_pattern/index_header/index_header.tsx @@ -70,7 +70,6 @@ export const IndexHeader: React.FC = ({ {editTooltip} ), - , defaultIndex !== indexPattern.id && setDefault && canSave && indexPattern.isPersisted() && ( Date: Wed, 10 Jan 2024 07:37:44 -0600 Subject: [PATCH 163/168] translate some text, remove unneeded comma --- .../edit_index_pattern/index_header/index_header.tsx | 1 - .../public/components/edit_index_pattern/tabs/tabs.tsx | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/plugins/data_view_management/public/components/edit_index_pattern/index_header/index_header.tsx b/src/plugins/data_view_management/public/components/edit_index_pattern/index_header/index_header.tsx index 1d73ef615e26c..8b12ed59ed6fd 100644 --- a/src/plugins/data_view_management/public/components/edit_index_pattern/index_header/index_header.tsx +++ b/src/plugins/data_view_management/public/components/edit_index_pattern/index_header/index_header.tsx @@ -70,7 +70,6 @@ export const IndexHeader: React.FC = ({ {editTooltip} ), - , defaultIndex !== indexPattern.id && setDefault && canSave && indexPattern.isPersisted() && ( = ({ size="m" color="success" > - Refresh + {refreshAriaLabel} @@ -477,7 +477,7 @@ export const Tabs: React.FC = ({ color="primary" fill > - Add field + {addFieldButtonLabel} )} From 67dbffec384529bb69bbf616698d693447418753 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Fri, 12 Jan 2024 00:02:28 -0600 Subject: [PATCH 164/168] fix refresh button in mobile view --- .../public/components/edit_index_pattern/tabs/tabs.tsx | 1 + .../apps/discover/feature_controls/discover_security.ts | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/data_view_management/public/components/edit_index_pattern/tabs/tabs.tsx b/src/plugins/data_view_management/public/components/edit_index_pattern/tabs/tabs.tsx index d66da0160e729..426442bcfb031 100644 --- a/src/plugins/data_view_management/public/components/edit_index_pattern/tabs/tabs.tsx +++ b/src/plugins/data_view_management/public/components/edit_index_pattern/tabs/tabs.tsx @@ -461,6 +461,7 @@ export const Tabs: React.FC = ({ isDisabled={isRefreshing} size="m" color="success" + className="eui-fullWidth" > {refreshAriaLabel} diff --git a/x-pack/test/functional/apps/discover/feature_controls/discover_security.ts b/x-pack/test/functional/apps/discover/feature_controls/discover_security.ts index d0173072a0879..f8e4e1e6933db 100644 --- a/x-pack/test/functional/apps/discover/feature_controls/discover_security.ts +++ b/x-pack/test/functional/apps/discover/feature_controls/discover_security.ts @@ -31,7 +31,6 @@ export default function (ctx: FtrProviderContext) { 'spaceSelector', 'header', 'unifiedFieldList', - 'settings', ]); const testSubjects = getService('testSubjects'); const appsMenu = getService('appsMenu'); From 765152d628020f1c219d926392874d33dd642238 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Fri, 12 Jan 2024 00:11:34 -0600 Subject: [PATCH 165/168] restore doc change --- docs/concepts/data-views.asciidoc | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/concepts/data-views.asciidoc b/docs/concepts/data-views.asciidoc index 992741be518a7..2eba42aed3051 100644 --- a/docs/concepts/data-views.asciidoc +++ b/docs/concepts/data-views.asciidoc @@ -172,3 +172,14 @@ WARNING: Deleting a {data-source} breaks all visualizations, saved searches, and . Find the {data-source} that you want to delete, and then click image:management/index-patterns/images/delete.png[Delete icon] in the *Actions* column. + +[float] +[[data-view-field-cache]] +=== {data-source} field cache + +The browser caches {data-source} field lists for increased performance. This is particularly impactful +for {data-sources} with a high field count that span a large number of indices and clusters. The field +list is updated every couple of minutes in typical {kib} usage. Alternatively, use the refresh button on the {data-source} +management detail page to get an updated field list. A force reload of {kib} has the same effect. + +The field list may be impacted by changes in indices and user permissions. \ No newline at end of file From 416a7856aae3dbe47b5b8fa52481beb9a35871f3 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Sat, 13 Jan 2024 20:21:41 -0600 Subject: [PATCH 166/168] better caching, with test --- .../common/data_views/data_views.ts | 4 +--- .../data_views/data_views_api_client.test.ts | 21 +++++++++++++++++-- .../data_views/data_views_api_client.ts | 4 ++-- 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/src/plugins/data_views/common/data_views/data_views.ts b/src/plugins/data_views/common/data_views/data_views.ts index dfdb8025b182c..a244868c0614d 100644 --- a/src/plugins/data_views/common/data_views/data_views.ts +++ b/src/plugins/data_views/common/data_views/data_views.ts @@ -536,9 +536,7 @@ export class DataViewsService { ...options, pattern: indexPattern.title as string, allowHidden: - (indexPattern as DataViewSpec).allowHidden || - (indexPattern as DataView)?.getAllowHidden() || - undefined, + (indexPattern as DataViewSpec).allowHidden || (indexPattern as DataView)?.getAllowHidden(), }); private getFieldsAndIndicesForDataView = async ( diff --git a/src/plugins/data_views/public/data_views/data_views_api_client.test.ts b/src/plugins/data_views/public/data_views/data_views_api_client.test.ts index 6aada9616737b..172a3c2db738a 100644 --- a/src/plugins/data_views/public/data_views/data_views_api_client.test.ts +++ b/src/plugins/data_views/public/data_views/data_views_api_client.test.ts @@ -23,8 +23,25 @@ describe('IndexPatternsApiClient', () => { }); test('uses the right URI to fetch fields for wildcard', async function () { - await indexPatternsApiClient.getFieldsForWildcard({ pattern: 'blah' }); + await indexPatternsApiClient.getFieldsForWildcard({ pattern: 'blah', allowHidden: false }); - expect(fetchSpy).toHaveBeenCalledWith(expectedPath, expect.any(Object)); + expect(fetchSpy).toHaveBeenCalledWith(expectedPath, { + // not sure what asResponse is but the rest of the results are useful + asResponse: true, + headers: { + 'user-hash': '', + }, + query: { + allow_hidden: undefined, + allow_no_index: undefined, + apiVersion: '1', + fields: undefined, + include_unmapped: undefined, + meta_fields: undefined, + pattern: 'blah', + rollup_index: undefined, + type: undefined, + }, + }); }); }); diff --git a/src/plugins/data_views/public/data_views/data_views_api_client.ts b/src/plugins/data_views/public/data_views/data_views_api_client.ts index 29791d3567761..ae6796f53247a 100644 --- a/src/plugins/data_views/public/data_views/data_views_api_client.ts +++ b/src/plugins/data_views/public/data_views/data_views_api_client.ts @@ -55,7 +55,6 @@ export class DataViewsApiClient implements IDataViewsApiClient { ? this.http.post(url, { query, body, version, asResponse }) : this.http.fetch(url, { query, - version, ...cacheOptions, asResponse, headers: { 'user-hash': userHash }, @@ -104,7 +103,8 @@ export class DataViewsApiClient implements IDataViewsApiClient { allow_no_index: allowNoIndex, include_unmapped: includeUnmapped, fields, - allow_hidden: allowHidden, + // default to undefined to keep value out of URL params and improve caching + allow_hidden: allowHidden || undefined, ...versionQueryParam, }, indexFilter ? JSON.stringify({ index_filter: indexFilter }) : undefined, From 99356b310f8233c76d1c8d498f7d12544d2a6f79 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Sun, 14 Jan 2024 13:14:10 -0600 Subject: [PATCH 167/168] undo removal of version header for hasUserDataView --- .../data_views/public/data_views/data_views_api_client.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/plugins/data_views/public/data_views/data_views_api_client.ts b/src/plugins/data_views/public/data_views/data_views_api_client.ts index ae6796f53247a..65b418642b34f 100644 --- a/src/plugins/data_views/public/data_views/data_views_api_client.ts +++ b/src/plugins/data_views/public/data_views/data_views_api_client.ts @@ -55,6 +55,7 @@ export class DataViewsApiClient implements IDataViewsApiClient { ? this.http.post(url, { query, body, version, asResponse }) : this.http.fetch(url, { query, + version, ...cacheOptions, asResponse, headers: { 'user-hash': userHash }, From a9315ac6edc230d3393d2b738b21996d5bc76b2b Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Sun, 14 Jan 2024 15:35:37 -0600 Subject: [PATCH 168/168] fix test --- .../data_views/public/data_views/data_views_api_client.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/plugins/data_views/public/data_views/data_views_api_client.test.ts b/src/plugins/data_views/public/data_views/data_views_api_client.test.ts index 172a3c2db738a..66f025dc07835 100644 --- a/src/plugins/data_views/public/data_views/data_views_api_client.test.ts +++ b/src/plugins/data_views/public/data_views/data_views_api_client.test.ts @@ -34,7 +34,7 @@ describe('IndexPatternsApiClient', () => { query: { allow_hidden: undefined, allow_no_index: undefined, - apiVersion: '1', + apiVersion: '1', // version passed through query params for caching fields: undefined, include_unmapped: undefined, meta_fields: undefined, @@ -42,6 +42,7 @@ describe('IndexPatternsApiClient', () => { rollup_index: undefined, type: undefined, }, + version: '1', // version header }); }); });