diff --git a/.buildkite/pipelines/quality-gates/pipeline.tests-qa.yaml b/.buildkite/pipelines/quality-gates/pipeline.tests-qa.yaml index 5321f24ae6e3b..962da8da4d86e 100644 --- a/.buildkite/pipelines/quality-gates/pipeline.tests-qa.yaml +++ b/.buildkite/pipelines/quality-gates/pipeline.tests-qa.yaml @@ -21,6 +21,8 @@ steps: build: env: ENVIRONMENT: ${ENVIRONMENT} + EC_ENV: qa + EC_REGION: aws-eu-west-1 message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-qa.yaml)" - group: ":female-detective: Security Solution Tests" diff --git a/.buildkite/pipelines/quality-gates/pipeline.tests-staging.yaml b/.buildkite/pipelines/quality-gates/pipeline.tests-staging.yaml index 6a5edc3a97073..42fa2b34ea84f 100644 --- a/.buildkite/pipelines/quality-gates/pipeline.tests-staging.yaml +++ b/.buildkite/pipelines/quality-gates/pipeline.tests-staging.yaml @@ -21,6 +21,16 @@ steps: NAME_PREFIX: ci_test_kibana-promotion_ message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-staging.yaml)" + - label: ":pipeline::kibana::seedling: Trigger Kibana Serverless Tests for ${ENVIRONMENT}" + trigger: appex-qa-serverless-kibana-ftr-tests # https://buildkite.com/elastic/appex-qa-serverless-kibana-ftr-tests + soft_fail: true # Remove this before release or when tests stabilize + build: + env: + ENVIRONMENT: ${ENVIRONMENT} + EC_ENV: staging + EC_REGION: aws-us-east-1 + message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-staging.yaml)" + - wait: ~ - label: ":judge::seedling: Trigger Manual Tests Phase" diff --git a/packages/kbn-alerts-as-data-utils/src/field_maps/types.ts b/packages/kbn-alerts-as-data-utils/src/field_maps/types.ts index 04f9d045f6e28..0a0b68a2f26e6 100644 --- a/packages/kbn-alerts-as-data-utils/src/field_maps/types.ts +++ b/packages/kbn-alerts-as-data-utils/src/field_maps/types.ts @@ -34,6 +34,7 @@ export interface EcsMetadata { scaling_factor?: number; short: string; type: string; + properties?: Record; } export interface FieldMap { @@ -50,5 +51,6 @@ export interface FieldMap { path?: string; scaling_factor?: number; dynamic?: boolean | 'strict'; + properties?: Record; }; } diff --git a/packages/kbn-alerts-as-data-utils/src/schemas/create_schema_from_field_map.ts b/packages/kbn-alerts-as-data-utils/src/schemas/create_schema_from_field_map.ts index 99bbb502e1011..a0599d85fab33 100644 --- a/packages/kbn-alerts-as-data-utils/src/schemas/create_schema_from_field_map.ts +++ b/packages/kbn-alerts-as-data-utils/src/schemas/create_schema_from_field_map.ts @@ -198,6 +198,7 @@ const generateSchemaLines = ({ break; case 'float': case 'integer': + case 'double': lineWriter.addLine(`${keyToWrite}: ${getSchemaDefinition('schemaNumber', isArray)},`); break; case 'boolean': diff --git a/packages/kbn-alerts-as-data-utils/src/schemas/generated/ml_anomaly_detection_schema.ts b/packages/kbn-alerts-as-data-utils/src/schemas/generated/ml_anomaly_detection_schema.ts new file mode 100644 index 0000000000000..2e5912bca84c2 --- /dev/null +++ b/packages/kbn-alerts-as-data-utils/src/schemas/generated/ml_anomaly_detection_schema.ts @@ -0,0 +1,120 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +// ---------------------------------- WARNING ---------------------------------- +// this file was generated, and should not be edited by hand +// ---------------------------------- WARNING ---------------------------------- +import * as rt from 'io-ts'; +import { Either } from 'fp-ts/lib/Either'; +import { AlertSchema } from './alert_schema'; +const ISO_DATE_PATTERN = /^d{4}-d{2}-d{2}Td{2}:d{2}:d{2}.d{3}Z$/; +export const IsoDateString = new rt.Type( + 'IsoDateString', + rt.string.is, + (input, context): Either => { + if (typeof input === 'string' && ISO_DATE_PATTERN.test(input)) { + return rt.success(input); + } else { + return rt.failure(input, context); + } + }, + rt.identity +); +export type IsoDateStringC = typeof IsoDateString; +export const schemaDate = IsoDateString; +export const schemaDateArray = rt.array(IsoDateString); +export const schemaDateRange = rt.partial({ + gte: schemaDate, + lte: schemaDate, +}); +export const schemaDateRangeArray = rt.array(schemaDateRange); +export const schemaUnknown = rt.unknown; +export const schemaUnknownArray = rt.array(rt.unknown); +export const schemaString = rt.string; +export const schemaStringArray = rt.array(schemaString); +export const schemaNumber = rt.number; +export const schemaNumberArray = rt.array(schemaNumber); +export const schemaStringOrNumber = rt.union([schemaString, schemaNumber]); +export const schemaStringOrNumberArray = rt.array(schemaStringOrNumber); +export const schemaBoolean = rt.boolean; +export const schemaBooleanArray = rt.array(schemaBoolean); +const schemaGeoPointCoords = rt.type({ + type: schemaString, + coordinates: schemaNumberArray, +}); +const schemaGeoPointString = schemaString; +const schemaGeoPointLatLon = rt.type({ + lat: schemaNumber, + lon: schemaNumber, +}); +const schemaGeoPointLocation = rt.type({ + location: schemaNumberArray, +}); +const schemaGeoPointLocationString = rt.type({ + location: schemaString, +}); +export const schemaGeoPoint = rt.union([ + schemaGeoPointCoords, + schemaGeoPointString, + schemaGeoPointLatLon, + schemaGeoPointLocation, + schemaGeoPointLocationString, +]); +export const schemaGeoPointArray = rt.array(schemaGeoPoint); +// prettier-ignore +const MlAnomalyDetectionAlertRequired = rt.type({ + kibana: rt.type({ + alert: rt.type({ + job_id: schemaString, + }), + }), +}); +const MlAnomalyDetectionAlertOptional = rt.partial({ + kibana: rt.partial({ + alert: rt.partial({ + anomaly_score: schemaNumber, + anomaly_timestamp: schemaDate, + is_interim: schemaBoolean, + top_influencers: rt.array( + rt.partial({ + influencer_field_name: schemaString, + influencer_field_value: schemaString, + influencer_score: schemaNumber, + initial_influencer_score: schemaNumber, + is_interim: schemaBoolean, + job_id: schemaString, + timestamp: schemaDate, + }) + ), + top_records: rt.array( + rt.partial({ + actual: schemaNumber, + by_field_name: schemaString, + by_field_value: schemaString, + detector_index: schemaNumber, + field_name: schemaString, + function: schemaString, + initial_record_score: schemaNumber, + is_interim: schemaBoolean, + job_id: schemaString, + over_field_name: schemaString, + over_field_value: schemaString, + partition_field_name: schemaString, + partition_field_value: schemaString, + record_score: schemaNumber, + timestamp: schemaDate, + typical: schemaNumber, + }) + ), + }), + }), +}); + +// prettier-ignore +export const MlAnomalyDetectionAlertSchema = rt.intersection([MlAnomalyDetectionAlertRequired, MlAnomalyDetectionAlertOptional, AlertSchema]); +// prettier-ignore +export type MlAnomalyDetectionAlert = rt.TypeOf; diff --git a/packages/kbn-alerts-as-data-utils/src/schemas/index.ts b/packages/kbn-alerts-as-data-utils/src/schemas/index.ts index 77d9476d2034b..28da937087cf1 100644 --- a/packages/kbn-alerts-as-data-utils/src/schemas/index.ts +++ b/packages/kbn-alerts-as-data-utils/src/schemas/index.ts @@ -13,6 +13,7 @@ import type { ObservabilityMetricsAlert } from './generated/observability_metric import type { ObservabilitySloAlert } from './generated/observability_slo_schema'; import type { ObservabilityUptimeAlert } from './generated/observability_uptime_schema'; import type { SecurityAlert } from './generated/security_schema'; +import type { MlAnomalyDetectionAlert } from './generated/ml_anomaly_detection_schema'; export * from './create_schema_from_field_map'; @@ -24,6 +25,7 @@ export type { ObservabilitySloAlert } from './generated/observability_slo_schema export type { ObservabilityUptimeAlert } from './generated/observability_uptime_schema'; export type { SecurityAlert } from './generated/security_schema'; export type { StackAlert } from './generated/stack_schema'; +export type { MlAnomalyDetectionAlert } from './generated/ml_anomaly_detection_schema'; export type AADAlert = | Alert @@ -32,4 +34,5 @@ export type AADAlert = | ObservabilityMetricsAlert | ObservabilitySloAlert | ObservabilityUptimeAlert - | SecurityAlert; + | SecurityAlert + | MlAnomalyDetectionAlert; diff --git a/packages/kbn-content-management-utils/src/saved_object_content_storage.test.ts b/packages/kbn-content-management-utils/src/saved_object_content_storage.test.ts new file mode 100644 index 0000000000000..2268f279ae7ed --- /dev/null +++ b/packages/kbn-content-management-utils/src/saved_object_content_storage.test.ts @@ -0,0 +1,600 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { SOContentStorage } from './saved_object_content_storage'; +import { CMCrudTypes } from './types'; +import { loggerMock, MockedLogger } from '@kbn/logging-mocks'; + +import { schema } from '@kbn/config-schema'; +import type { + ContentManagementServicesDefinition as ServicesDefinition, + Version, +} from '@kbn/object-versioning'; +import { getContentManagmentServicesTransforms } from '@kbn/object-versioning'; +import { savedObjectSchema, objectTypeToGetResultSchema, createResultSchema } from './schema'; + +import { coreMock } from '@kbn/core/server/mocks'; +import type { SavedObject } from '@kbn/core/server'; + +const testAttributesSchema = schema.object( + { + title: schema.string(), + description: schema.string(), + }, + { unknowns: 'forbid' } +); + +const testSavedObjectSchema = savedObjectSchema(testAttributesSchema); + +export const serviceDefinition: ServicesDefinition = { + get: { + out: { + result: { + schema: objectTypeToGetResultSchema(testSavedObjectSchema), + }, + }, + }, + create: { + out: { + result: { + schema: createResultSchema(testSavedObjectSchema), + }, + }, + }, + update: { + out: { + result: { + schema: createResultSchema(testSavedObjectSchema), + }, + }, + }, + search: { + out: { + result: { + schema: schema.object({ hits: schema.arrayOf(testSavedObjectSchema) }), + }, + }, + }, + mSearch: { + out: { + result: { + schema: testSavedObjectSchema, + }, + }, + }, +}; + +export const cmServicesDefinition: { [version: Version]: ServicesDefinition } = { + 1: serviceDefinition, +}; + +const transforms = getContentManagmentServicesTransforms(cmServicesDefinition, 1); + +class TestSOContentStorage extends SOContentStorage { + constructor({ + throwOnResultValidationError, + logger, + }: { throwOnResultValidationError?: boolean; logger?: MockedLogger } = {}) { + super({ + savedObjectType: 'test', + cmServicesDefinition, + allowedSavedObjectAttributes: ['title', 'description'], + logger: logger ?? loggerMock.create(), + throwOnResultValidationError: throwOnResultValidationError ?? false, + enableMSearch: true, + }); + } +} + +const setup = ({ storage }: { storage?: TestSOContentStorage } = {}) => { + storage = storage ?? new TestSOContentStorage(); + const requestHandlerCoreContext = coreMock.createRequestHandlerContext(); + + const requestHandlerContext = { + core: Promise.resolve(requestHandlerCoreContext), + resolve: jest.fn(), + }; + + return { + get: (mockSavedObject: SavedObject) => { + requestHandlerCoreContext.savedObjects.client.resolve.mockResolvedValue({ + saved_object: mockSavedObject, + outcome: 'exactMatch', + }); + + return storage!.get( + { + requestHandlerContext, + version: { + request: 1, + latest: 1, + }, + utils: { + getTransforms: () => transforms, + }, + }, + mockSavedObject.id + ); + }, + create: (mockSavedObject: SavedObject<{}>) => { + requestHandlerCoreContext.savedObjects.client.create.mockResolvedValue(mockSavedObject); + + return storage!.create( + { + requestHandlerContext, + version: { + request: 1, + latest: 1, + }, + utils: { + getTransforms: () => transforms, + }, + }, + mockSavedObject.attributes, + {} + ); + }, + update: (mockSavedObject: SavedObject<{}>) => { + requestHandlerCoreContext.savedObjects.client.update.mockResolvedValue(mockSavedObject); + + return storage!.update( + { + requestHandlerContext, + version: { + request: 1, + latest: 1, + }, + utils: { + getTransforms: () => transforms, + }, + }, + mockSavedObject.id, + mockSavedObject.attributes, + {} + ); + }, + search: (mockSavedObject: SavedObject<{}>) => { + requestHandlerCoreContext.savedObjects.client.find.mockResolvedValue({ + saved_objects: [{ ...mockSavedObject, score: 100 }], + total: 1, + per_page: 10, + page: 1, + }); + + return storage!.search( + { + requestHandlerContext, + version: { + request: 1, + latest: 1, + }, + utils: { + getTransforms: () => transforms, + }, + }, + {}, + {} + ); + }, + mSearch: async (mockSavedObject: SavedObject<{}>) => { + return storage!.mSearch!.toItemResult( + { + requestHandlerContext, + version: { + request: 1, + latest: 1, + }, + utils: { + getTransforms: () => transforms, + }, + }, + { ...mockSavedObject, score: 100 } + ); + }, + }; +}; + +describe('get', () => { + test('returns the storage get() result', async () => { + const get = setup().get; + + const testSavedObject = { + id: 'id', + type: 'test', + references: [], + attributes: { + title: 'title', + description: 'description', + }, + }; + + const result = await get(testSavedObject); + + expect(result).toEqual({ item: testSavedObject, meta: { outcome: 'exactMatch' } }); + }); + + test('filters out unknown attributes', async () => { + const get = setup().get; + + const testSavedObject = { + id: 'id', + type: 'test', + references: [], + attributes: { + title: 'title', + description: 'description', + unknown: 'unknown', + }, + }; + + const result = await get(testSavedObject); + expect(result.item.attributes).not.toHaveProperty('unknown'); + }); + + test('throws response validation error', async () => { + const get = setup({ + storage: new TestSOContentStorage({ throwOnResultValidationError: true }), + }).get; + + const testSavedObject = { + id: 'id', + type: 'test', + references: [], + attributes: { + title: 'title', + description: null, + }, + }; + + await expect(get(testSavedObject)).rejects.toThrowErrorMatchingInlineSnapshot( + `"Invalid response. [item.attributes.description]: expected value of type [string] but got [null]"` + ); + }); + + test('logs response validation error', async () => { + const logger = loggerMock.create(); + const get = setup({ + storage: new TestSOContentStorage({ throwOnResultValidationError: false, logger }), + }).get; + + const testSavedObject = { + id: 'id', + type: 'test', + references: [], + attributes: { + title: 'title', + description: null, + }, + }; + + await expect(get(testSavedObject)).resolves.toBeDefined(); + expect(logger.warn).toBeCalledWith( + `Invalid response. [item.attributes.description]: expected value of type [string] but got [null]` + ); + }); +}); + +describe('create', () => { + test('returns the storage create() result', async () => { + const create = setup().create; + + const testSavedObject = { + id: 'id', + type: 'test', + references: [], + attributes: { + title: 'title', + description: 'description', + }, + }; + + const result = await create(testSavedObject); + + expect(result).toEqual({ item: testSavedObject }); + }); + + test('filters out unknown attributes', async () => { + const create = setup().create; + + const testSavedObject = { + id: 'id', + type: 'test', + references: [], + attributes: { + title: 'title', + description: 'description', + unknown: 'unknown', + }, + }; + + const result = await create(testSavedObject); + expect(result.item.attributes).not.toHaveProperty('unknown'); + }); + + test('throws response validation error', async () => { + const create = setup({ + storage: new TestSOContentStorage({ throwOnResultValidationError: true }), + }).create; + + const testSavedObject = { + id: 'id', + type: 'test', + references: [], + attributes: { + title: 'title', + description: null, + }, + }; + + await expect(create(testSavedObject)).rejects.toThrowErrorMatchingInlineSnapshot( + `"Invalid response. [item.attributes.description]: expected value of type [string] but got [null]"` + ); + }); + + test('logs response validation error', async () => { + const logger = loggerMock.create(); + const create = setup({ + storage: new TestSOContentStorage({ throwOnResultValidationError: false, logger }), + }).create; + + const testSavedObject = { + id: 'id', + type: 'test', + references: [], + attributes: { + title: 'title', + description: null, + }, + }; + + await expect(create(testSavedObject)).resolves.toBeDefined(); + expect(logger.warn).toBeCalledWith( + `Invalid response. [item.attributes.description]: expected value of type [string] but got [null]` + ); + }); +}); + +describe('update', () => { + test('returns the storage update() result', async () => { + const update = setup().update; + + const testSavedObject = { + id: 'id', + type: 'test', + references: [], + attributes: { + title: 'title', + description: 'description', + }, + }; + + const result = await update(testSavedObject); + + expect(result).toEqual({ item: testSavedObject }); + }); + + test('filters out unknown attributes', async () => { + const update = setup().update; + + const testSavedObject = { + id: 'id', + type: 'test', + references: [], + attributes: { + title: 'title', + description: 'description', + unknown: 'unknown', + }, + }; + + const result = await update(testSavedObject); + expect(result.item.attributes).not.toHaveProperty('unknown'); + }); + + test('throws response validation error', async () => { + const update = setup({ + storage: new TestSOContentStorage({ throwOnResultValidationError: true }), + }).update; + + const testSavedObject = { + id: 'id', + type: 'test', + references: [], + attributes: { + title: 'title', + description: null, + }, + }; + + await expect(update(testSavedObject)).rejects.toThrowErrorMatchingInlineSnapshot( + `"Invalid response. [item.attributes.description]: expected value of type [string] but got [null]"` + ); + }); + + test('logs response validation error', async () => { + const logger = loggerMock.create(); + const update = setup({ + storage: new TestSOContentStorage({ throwOnResultValidationError: false, logger }), + }).update; + + const testSavedObject = { + id: 'id', + type: 'test', + references: [], + attributes: { + title: 'title', + description: null, + }, + }; + + await expect(update(testSavedObject)).resolves.toBeDefined(); + expect(logger.warn).toBeCalledWith( + `Invalid response. [item.attributes.description]: expected value of type [string] but got [null]` + ); + }); +}); + +describe('search', () => { + test('returns the storage search() result', async () => { + const search = setup().search; + + const testSavedObject = { + id: 'id', + type: 'test', + references: [], + attributes: { + title: 'title', + description: 'description', + }, + }; + + const result = await search(testSavedObject); + + expect(result).toEqual({ hits: [testSavedObject], pagination: { total: 1 } }); + }); + + test('filters out unknown attributes', async () => { + const search = setup().search; + + const testSavedObject = { + id: 'id', + type: 'test', + references: [], + attributes: { + title: 'title', + description: 'description', + unknown: 'unknown', + }, + }; + + const result = await search(testSavedObject); + expect(result.hits[0].attributes).not.toHaveProperty('unknown'); + }); + + test('throws response validation error', async () => { + const search = setup({ + storage: new TestSOContentStorage({ throwOnResultValidationError: true }), + }).search; + + const testSavedObject = { + id: 'id', + type: 'test', + references: [], + attributes: { + title: 'title', + description: null, + }, + }; + + await expect(search(testSavedObject)).rejects.toThrowErrorMatchingInlineSnapshot( + `"Invalid response. [hits.0.attributes.description]: expected value of type [string] but got [null]"` + ); + }); + + test('logs response validation error', async () => { + const logger = loggerMock.create(); + const update = setup({ + storage: new TestSOContentStorage({ throwOnResultValidationError: false, logger }), + }).search; + + const testSavedObject = { + id: 'id', + type: 'test', + references: [], + attributes: { + title: 'title', + description: null, + }, + }; + + await expect(update(testSavedObject)).resolves.toBeDefined(); + expect(logger.warn).toBeCalledWith( + `Invalid response. [hits.0.attributes.description]: expected value of type [string] but got [null]` + ); + }); +}); + +describe('mSearch', () => { + test('returns the storage mSearch() result', async () => { + const mSearch = setup().mSearch; + + const testSavedObject = { + id: 'id', + type: 'test', + references: [], + attributes: { + title: 'title', + description: 'description', + }, + }; + + const result = await mSearch(testSavedObject); + + expect(result).toEqual(testSavedObject); + }); + + test('filters out unknown attributes', async () => { + const mSearch = setup().mSearch; + + const testSavedObject = { + id: 'id', + type: 'test', + references: [], + attributes: { + title: 'title', + description: 'description', + unknown: 'unknown', + }, + }; + + const result = await mSearch(testSavedObject); + expect(result.attributes).not.toHaveProperty('unknown'); + }); + + test('throws response validation error', async () => { + const mSearch = setup({ + storage: new TestSOContentStorage({ throwOnResultValidationError: true }), + }).mSearch; + + const testSavedObject = { + id: 'id', + type: 'test', + references: [], + attributes: { + title: 'title', + description: null, + }, + }; + + await expect(mSearch(testSavedObject)).rejects.toThrowErrorMatchingInlineSnapshot( + `"Invalid response. [attributes.description]: expected value of type [string] but got [null]"` + ); + }); + + test('logs response validation error', async () => { + const logger = loggerMock.create(); + const mSearch = setup({ + storage: new TestSOContentStorage({ throwOnResultValidationError: false, logger }), + }).mSearch; + + const testSavedObject = { + id: 'id', + type: 'test', + references: [], + attributes: { + title: 'title', + description: null, + }, + }; + + await expect(mSearch(testSavedObject)).resolves.toBeDefined(); + expect(logger.warn).toBeCalledWith( + 'Invalid response. [attributes.description]: expected value of type [string] but got [null]' + ); + }); +}); diff --git a/packages/kbn-content-management-utils/src/saved_object_content_storage.ts b/packages/kbn-content-management-utils/src/saved_object_content_storage.ts index 70cf7c9775863..8ff22a0d9be02 100644 --- a/packages/kbn-content-management-utils/src/saved_object_content_storage.ts +++ b/packages/kbn-content-management-utils/src/saved_object_content_storage.ts @@ -21,6 +21,7 @@ import type { SavedObjectsUpdateOptions, SavedObjectsFindResult, } from '@kbn/core-saved-objects-api-server'; +import type { Logger } from '@kbn/logging'; import { pick } from 'lodash'; import type { CMCrudTypes, @@ -138,6 +139,9 @@ export interface SOContentStorageConstrutorParams { searchArgsToSOFindOptions?: SearchArgsToSOFindOptions; enableMSearch?: boolean; mSearchAdditionalSearchFields?: string[]; + + logger: Logger; + throwOnResultValidationError: boolean; } export abstract class SOContentStorage @@ -157,7 +161,11 @@ export abstract class SOContentStorage enableMSearch, allowedSavedObjectAttributes, mSearchAdditionalSearchFields, + logger, + throwOnResultValidationError, }: SOContentStorageConstrutorParams) { + this.logger = logger; + this.throwOnResultValidationError = throwOnResultValidationError ?? false; this.savedObjectType = savedObjectType; this.cmServicesDefinition = cmServicesDefinition; this.createArgsToSoCreateOptions = @@ -174,16 +182,29 @@ export abstract class SOContentStorage toItemResult: (ctx: StorageContext, savedObject: SavedObjectsFindResult): Types['Item'] => { const transforms = ctx.utils.getTransforms(this.cmServicesDefinition); + const contentItem = savedObjectToItem( + savedObject as SavedObjectsFindResult, + this.allowedSavedObjectAttributes, + false + ); + + const validationError = transforms.mSearch.out.result.validate(contentItem); + if (validationError) { + if (this.throwOnResultValidationError) { + throw Boom.badRequest(`Invalid response. ${validationError.message}`); + } else { + this.logger.warn(`Invalid response. ${validationError.message}`); + } + } + // Validate DB response and DOWN transform to the request version const { value, error: resultError } = transforms.mSearch.out.result.down< Types['Item'], Types['Item'] >( - savedObjectToItem( - savedObject as SavedObjectsFindResult, - this.allowedSavedObjectAttributes, - false - ) + contentItem, + undefined, // do not override version + { validate: false } // validation is done above ); if (resultError) { @@ -196,6 +217,8 @@ export abstract class SOContentStorage } } + private throwOnResultValidationError: boolean; + private logger: Logger; private savedObjectType: SOContentStorageConstrutorParams['savedObjectType']; private cmServicesDefinition: SOContentStorageConstrutorParams['cmServicesDefinition']; private createArgsToSoCreateOptions: CreateArgsToSoCreateOptions; @@ -230,11 +253,24 @@ export abstract class SOContentStorage }, }; + const validationError = transforms.get.out.result.validate(response); + if (validationError) { + if (this.throwOnResultValidationError) { + throw Boom.badRequest(`Invalid response. ${validationError.message}`); + } else { + this.logger.warn(`Invalid response. ${validationError.message}`); + } + } + // Validate DB response and DOWN transform to the request version const { value, error: resultError } = transforms.get.out.result.down< Types['GetOut'], Types['GetOut'] - >(response); + >( + response, + undefined, // do not override version + { validate: false } // validation is done above + ); if (resultError) { throw Boom.badRequest(`Invalid response. ${resultError.message}`); @@ -282,13 +318,28 @@ export abstract class SOContentStorage createOptions ); + const result = { + item: savedObjectToItem(savedObject, this.allowedSavedObjectAttributes, false), + }; + + const validationError = transforms.create.out.result.validate(result); + if (validationError) { + if (this.throwOnResultValidationError) { + throw Boom.badRequest(`Invalid response. ${validationError.message}`); + } else { + this.logger.warn(`Invalid response. ${validationError.message}`); + } + } + // Validate DB response and DOWN transform to the request version const { value, error: resultError } = transforms.create.out.result.down< Types['CreateOut'], Types['CreateOut'] - >({ - item: savedObjectToItem(savedObject, this.allowedSavedObjectAttributes, false), - }); + >( + result, + undefined, // do not override version + { validate: false } // validation is done above + ); if (resultError) { throw Boom.badRequest(`Invalid response. ${resultError.message}`); @@ -333,13 +384,28 @@ export abstract class SOContentStorage updateOptions ); + const result = { + item: savedObjectToItem(partialSavedObject, this.allowedSavedObjectAttributes, true), + }; + + const validationError = transforms.update.out.result.validate(result); + if (validationError) { + if (this.throwOnResultValidationError) { + throw Boom.badRequest(`Invalid response. ${validationError.message}`); + } else { + this.logger.warn(`Invalid response. ${validationError.message}`); + } + } + // Validate DB response and DOWN transform to the request version const { value, error: resultError } = transforms.update.out.result.down< Types['UpdateOut'], Types['UpdateOut'] - >({ - item: savedObjectToItem(partialSavedObject, this.allowedSavedObjectAttributes, true), - }); + >( + result, + undefined, // do not override version + { validate: false } // validation is done above + ); if (resultError) { throw Boom.badRequest(`Invalid response. ${resultError.message}`); @@ -382,20 +448,34 @@ export abstract class SOContentStorage options: optionsToLatest, }); // Execute the query in the DB - const response = await soClient.find(soQuery); + const soResponse = await soClient.find(soQuery); + const response = { + hits: soResponse.saved_objects.map((so) => + savedObjectToItem(so, this.allowedSavedObjectAttributes, false) + ), + pagination: { + total: soResponse.total, + }, + }; + + const validationError = transforms.search.out.result.validate(response); + if (validationError) { + if (this.throwOnResultValidationError) { + throw Boom.badRequest(`Invalid response. ${validationError.message}`); + } else { + this.logger.warn(`Invalid response. ${validationError.message}`); + } + } // Validate the response and DOWN transform to the request version const { value, error: resultError } = transforms.search.out.result.down< Types['SearchOut'], Types['SearchOut'] - >({ - hits: response.saved_objects.map((so) => - savedObjectToItem(so, this.allowedSavedObjectAttributes, false) - ), - pagination: { - total: response.total, - }, - }); + >( + response, + undefined, // do not override version + { validate: false } // validation is done above + ); if (resultError) { throw Boom.badRequest(`Invalid response. ${resultError.message}`); diff --git a/packages/kbn-content-management-utils/tsconfig.json b/packages/kbn-content-management-utils/tsconfig.json index 5a6f68e03a64e..dd279ed3f5284 100644 --- a/packages/kbn-content-management-utils/tsconfig.json +++ b/packages/kbn-content-management-utils/tsconfig.json @@ -21,5 +21,8 @@ "@kbn/core-saved-objects-api-server", "@kbn/config-schema", "@kbn/object-versioning", + "@kbn/logging", + "@kbn/logging-mocks", + "@kbn/core", ] } diff --git a/src/plugins/dashboard/server/content_management/dashboard_storage.ts b/src/plugins/dashboard/server/content_management/dashboard_storage.ts index fbbfa0ef26a47..4391aeaa90563 100644 --- a/src/plugins/dashboard/server/content_management/dashboard_storage.ts +++ b/src/plugins/dashboard/server/content_management/dashboard_storage.ts @@ -8,6 +8,7 @@ import { SOContentStorage, tagsToFindOptions } from '@kbn/content-management-utils'; import { SavedObjectsFindOptions } from '@kbn/core-saved-objects-api-server'; +import type { Logger } from '@kbn/logging'; import { CONTENT_ID } from '../../common/content_management'; import { cmServicesDefinition } from '../../common/content_management/cm_services'; @@ -31,7 +32,13 @@ const searchArgsToSOFindOptions = ( }; export class DashboardStorage extends SOContentStorage { - constructor() { + constructor({ + logger, + throwOnResultValidationError, + }: { + logger: Logger; + throwOnResultValidationError: boolean; + }) { super({ savedObjectType: CONTENT_ID, cmServicesDefinition, @@ -50,6 +57,8 @@ export class DashboardStorage extends SOContentStorage { 'timeTo', 'title', ], + logger, + throwOnResultValidationError, }); } } diff --git a/src/plugins/dashboard/server/plugin.ts b/src/plugins/dashboard/server/plugin.ts index 8a68d406d16c9..e1626c2e72108 100644 --- a/src/plugins/dashboard/server/plugin.ts +++ b/src/plugins/dashboard/server/plugin.ts @@ -45,7 +45,7 @@ export class DashboardPlugin { private readonly logger: Logger; - constructor(initializerContext: PluginInitializerContext) { + constructor(private initializerContext: PluginInitializerContext) { this.logger = initializerContext.logger.get(); } @@ -62,7 +62,10 @@ export class DashboardPlugin plugins.contentManagement.register({ id: CONTENT_ID, - storage: new DashboardStorage(), + storage: new DashboardStorage({ + throwOnResultValidationError: this.initializerContext.env.mode.dev, + logger: this.logger.get('storage'), + }), version: { latest: LATEST_VERSION, }, diff --git a/src/plugins/dashboard/tsconfig.json b/src/plugins/dashboard/tsconfig.json index 79aa7716b0160..82c71e7743ff2 100644 --- a/src/plugins/dashboard/tsconfig.json +++ b/src/plugins/dashboard/tsconfig.json @@ -67,7 +67,8 @@ "@kbn/serverless", "@kbn/no-data-page-plugin", "@kbn/react-kibana-mount", - "@kbn/core-lifecycle-browser" + "@kbn/core-lifecycle-browser", + "@kbn/logging" ], "exclude": ["target/**/*"] } diff --git a/src/plugins/data_views/server/content_management/data_views_storage.ts b/src/plugins/data_views/server/content_management/data_views_storage.ts index bd33b43a45bc1..35a9151f3f69e 100644 --- a/src/plugins/data_views/server/content_management/data_views_storage.ts +++ b/src/plugins/data_views/server/content_management/data_views_storage.ts @@ -7,13 +7,20 @@ */ import { SOContentStorage } from '@kbn/content-management-utils'; +import type { Logger } from '@kbn/logging'; import type { DataViewCrudTypes } from '../../common/content_management'; import { DataViewSOType } from '../../common/content_management'; import { cmServicesDefinition } from '../../common/content_management/cm_services'; export class DataViewsStorage extends SOContentStorage { - constructor() { + constructor({ + logger, + throwOnResultValidationError, + }: { + logger: Logger; + throwOnResultValidationError: boolean; + }) { super({ savedObjectType: DataViewSOType, cmServicesDefinition, @@ -32,6 +39,8 @@ export class DataViewsStorage extends SOContentStorage { 'name', ], mSearchAdditionalSearchFields: ['name'], + logger, + throwOnResultValidationError, }); } } diff --git a/src/plugins/data_views/server/plugin.ts b/src/plugins/data_views/server/plugin.ts index 2be269b1a7636..bbcc5dafc81c2 100644 --- a/src/plugins/data_views/server/plugin.ts +++ b/src/plugins/data_views/server/plugin.ts @@ -61,7 +61,10 @@ export class DataViewsServerPlugin contentManagement.register({ id: DATA_VIEW_SAVED_OBJECT_TYPE, - storage: new DataViewsStorage(), + storage: new DataViewsStorage({ + throwOnResultValidationError: this.initializerContext.env.mode.dev, + logger: this.logger.get('storage'), + }), version: { latest: LATEST_VERSION, }, diff --git a/src/plugins/data_views/tsconfig.json b/src/plugins/data_views/tsconfig.json index 558d22ec5b41f..e5613323bc222 100644 --- a/src/plugins/data_views/tsconfig.json +++ b/src/plugins/data_views/tsconfig.json @@ -32,6 +32,7 @@ "@kbn/content-management-utils", "@kbn/object-versioning", "@kbn/core-saved-objects-server", + "@kbn/logging", ], "exclude": [ "target/**/*", diff --git a/src/plugins/saved_search/server/content_management/saved_search_storage.ts b/src/plugins/saved_search/server/content_management/saved_search_storage.ts index 9d13d52db0271..797430a159159 100644 --- a/src/plugins/saved_search/server/content_management/saved_search_storage.ts +++ b/src/plugins/saved_search/server/content_management/saved_search_storage.ts @@ -7,13 +7,20 @@ */ import { SOContentStorage } from '@kbn/content-management-utils'; +import type { Logger } from '@kbn/logging'; import type { SavedSearchCrudTypes } from '../../common/content_management'; import { SavedSearchType } from '../../common/content_management'; import { cmServicesDefinition } from '../../common/content_management/cm_services'; export class SavedSearchStorage extends SOContentStorage { - constructor() { + constructor({ + logger, + throwOnResultValidationError, + }: { + logger: Logger; + throwOnResultValidationError: boolean; + }) { super({ savedObjectType: SavedSearchType, cmServicesDefinition, @@ -37,6 +44,8 @@ export class SavedSearchStorage extends SOContentStorage { 'rowsPerPage', 'breakdownField', ], + logger, + throwOnResultValidationError, }); } } diff --git a/src/plugins/saved_search/server/index.ts b/src/plugins/saved_search/server/index.ts index b125cf3d1fe52..056de3732b474 100644 --- a/src/plugins/saved_search/server/index.ts +++ b/src/plugins/saved_search/server/index.ts @@ -6,8 +6,10 @@ * Side Public License, v 1. */ +import type { PluginInitializerContext } from '@kbn/core-plugins-server'; import { SavedSearchServerPlugin } from './plugin'; export { getSavedSearch } from './services/saved_searches'; -export const plugin = () => new SavedSearchServerPlugin(); +export const plugin = (initContext: PluginInitializerContext) => + new SavedSearchServerPlugin(initContext); diff --git a/src/plugins/saved_search/server/plugin.ts b/src/plugins/saved_search/server/plugin.ts index 0f3e41894ff22..d09775442fd08 100644 --- a/src/plugins/saved_search/server/plugin.ts +++ b/src/plugins/saved_search/server/plugin.ts @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -import { CoreSetup, CoreStart, Plugin } from '@kbn/core/server'; +import { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from '@kbn/core/server'; import { StartServicesAccessor } from '@kbn/core/server'; import type { PluginSetup as DataPluginSetup, @@ -37,13 +37,18 @@ export interface SavedSearchServerStartDeps { export class SavedSearchServerPlugin implements Plugin { + constructor(private initializerContext: PluginInitializerContext) {} + public setup( core: CoreSetup, { data, contentManagement, expressions }: SavedSearchPublicSetupDependencies ) { contentManagement.register({ id: SavedSearchType, - storage: new SavedSearchStorage(), + storage: new SavedSearchStorage({ + throwOnResultValidationError: this.initializerContext.env.mode.dev, + logger: this.initializerContext.logger.get('storage'), + }), version: { latest: LATEST_VERSION, }, diff --git a/src/plugins/saved_search/tsconfig.json b/src/plugins/saved_search/tsconfig.json index 491461c2efc5a..7ed2cb4e82119 100644 --- a/src/plugins/saved_search/tsconfig.json +++ b/src/plugins/saved_search/tsconfig.json @@ -29,6 +29,8 @@ "@kbn/saved-objects-plugin", "@kbn/es-query", "@kbn/discover-utils", + "@kbn/logging", + "@kbn/core-plugins-server", ], "exclude": [ "target/**/*", diff --git a/x-pack/performance/journeys/many_fields_lens_editor.ts b/x-pack/performance/journeys/many_fields_lens_editor.ts index 18af2111115ed..8ad343311b350 100644 --- a/x-pack/performance/journeys/many_fields_lens_editor.ts +++ b/x-pack/performance/journeys/many_fields_lens_editor.ts @@ -9,6 +9,8 @@ import { Journey } from '@kbn/journeys'; import { subj } from '@kbn/test-subj-selector'; export const journey = new Journey({ + // Failing: See https://github.com/elastic/kibana/issues/167496 + skipped: true, kbnArchives: ['x-pack/performance/kbn_archives/lens_many_fields'], esArchives: ['test/functional/fixtures/es_archiver/stress_test'], }) diff --git a/x-pack/plugins/alerting/common/alert_schema/context_to_schema_name.ts b/x-pack/plugins/alerting/common/alert_schema/context_to_schema_name.ts index 52435b05dbaff..e2268f5e1429f 100644 --- a/x-pack/plugins/alerting/common/alert_schema/context_to_schema_name.ts +++ b/x-pack/plugins/alerting/common/alert_schema/context_to_schema_name.ts @@ -9,7 +9,7 @@ import { capitalize } from 'lodash'; export const contextToSchemaName = (context: string) => { return `${context - .split('.') + .split(/[.\-]/) .map((part: string) => capitalize(part)) .join('')}Alert`; }; diff --git a/x-pack/plugins/alerting/server/alerts_client/alerts_client.test.ts b/x-pack/plugins/alerting/server/alerts_client/alerts_client.test.ts index a5378245b7a02..6fc742e344758 100644 --- a/x-pack/plugins/alerting/server/alerts_client/alerts_client.test.ts +++ b/x-pack/plugins/alerting/server/alerts_client/alerts_client.test.ts @@ -1299,7 +1299,7 @@ describe('Alerts Client', () => { expect(clusterClient.bulk).toHaveBeenCalled(); expect(logger.error).toHaveBeenCalledWith( - `Error writing 1 out of 2 alerts - [{\"type\":\"action_request_validation_exception\",\"reason\":\"Validation Failed: 1: index is missing;2: type is missing;\"}]` + `Error writing alerts: 1 successful, 0 conflicts, 1 errors: Validation Failed: 1: index is missing;2: type is missing;` ); }); diff --git a/x-pack/plugins/alerting/server/alerts_client/alerts_client.ts b/x-pack/plugins/alerting/server/alerts_client/alerts_client.ts index 8164989761af7..eec5d3c5595bd 100644 --- a/x-pack/plugins/alerting/server/alerts_client/alerts_client.ts +++ b/x-pack/plugins/alerting/server/alerts_client/alerts_client.ts @@ -55,6 +55,7 @@ import { getContinualAlertsQuery, } from './lib'; import { isValidAlertIndexName } from '../alerts_service'; +import { resolveAlertConflicts } from './lib/alert_conflict_resolver'; // Term queries can take up to 10,000 terms const CHUNK_SIZE = 10000; @@ -467,15 +468,17 @@ export class AlertsClient< // If there were individual indexing errors, they will be returned in the success response if (response && response.errors) { - const errorsInResponse = (response.items ?? []) - .map((item) => item?.index?.error || item?.create?.error) - .filter((item) => item != null); - - this.options.logger.error( - `Error writing ${errorsInResponse.length} out of ${ - alertsToIndex.length - } alerts - ${JSON.stringify(errorsInResponse)}` - ); + await resolveAlertConflicts({ + logger: this.options.logger, + esClient, + bulkRequest: { + refresh: 'wait_for', + index: this.indexTemplateAndPattern.alias, + require_alias: !this.isUsingDataStreams(), + operations: bulkBody, + }, + bulkResponse: response, + }); } } catch (err) { this.options.logger.error( diff --git a/x-pack/plugins/alerting/server/alerts_client/lib/alert_conflict_resolver.test.ts b/x-pack/plugins/alerting/server/alerts_client/lib/alert_conflict_resolver.test.ts new file mode 100644 index 0000000000000..ffa2adc96f54f --- /dev/null +++ b/x-pack/plugins/alerting/server/alerts_client/lib/alert_conflict_resolver.test.ts @@ -0,0 +1,307 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { loggingSystemMock } from '@kbn/core/server/mocks'; +import { elasticsearchServiceMock } from '@kbn/core/server/mocks'; +import { + BulkRequest, + BulkResponse, + BulkResponseItem, + BulkOperationType, +} from '@elastic/elasticsearch/lib/api/types'; + +import { resolveAlertConflicts } from './alert_conflict_resolver'; + +const logger = loggingSystemMock.create().get(); +const esClient = elasticsearchServiceMock.createElasticsearchClient(); + +const alertDoc = { + event: { action: 'active' }, + kibana: { + alert: { + status: 'untracked', + workflow_status: 'a-ok!', + workflow_tags: ['fee', 'fi', 'fo', 'fum'], + case_ids: ['123', '456', '789'], + }, + }, +}; + +describe('alert_conflict_resolver', () => { + beforeEach(() => { + jest.resetAllMocks(); + }); + + describe('handles errors gracefully', () => { + test('when mget fails', async () => { + const { bulkRequest, bulkResponse } = getReqRes('ic'); + + esClient.mget.mockRejectedValueOnce(new Error('mget failed')); + + await resolveAlertConflicts({ logger, esClient, bulkRequest, bulkResponse }); + + expect(logger.error).toHaveBeenNthCalledWith( + 2, + 'Error resolving alert conflicts: mget failed' + ); + }); + + test('when bulk fails', async () => { + const { bulkRequest, bulkResponse } = getReqRes('ic'); + + esClient.mget.mockResolvedValueOnce({ + docs: [getMGetResDoc(0, alertDoc)], + }); + esClient.bulk.mockRejectedValueOnce(new Error('bulk failed')); + + await resolveAlertConflicts({ logger, esClient, bulkRequest, bulkResponse }); + + expect(logger.error).toHaveBeenNthCalledWith( + 2, + 'Error resolving alert conflicts: bulk failed' + ); + }); + }); + + describe('is successful with', () => { + test('no bulk results', async () => { + const { bulkRequest, bulkResponse } = getReqRes(''); + await resolveAlertConflicts({ logger, esClient, bulkRequest, bulkResponse }); + expect(logger.error).not.toHaveBeenCalled(); + }); + + test('no errors in bulk results', async () => { + const { bulkRequest, bulkResponse } = getReqRes('c is is c is'); + await resolveAlertConflicts({ logger, esClient, bulkRequest, bulkResponse }); + expect(logger.error).not.toHaveBeenCalled(); + }); + + test('one conflicted doc', async () => { + const { bulkRequest, bulkResponse } = getReqRes('ic'); + + esClient.mget.mockResolvedValueOnce({ + docs: [getMGetResDoc(0, alertDoc)], + }); + + esClient.bulk.mockResolvedValueOnce({ + errors: false, + took: 0, + items: [getBulkResItem(0)], + }); + + await resolveAlertConflicts({ logger, esClient, bulkRequest, bulkResponse }); + + expect(logger.error).toHaveBeenNthCalledWith( + 1, + `Error writing alerts: 0 successful, 1 conflicts, 0 errors: ` + ); + expect(logger.info).toHaveBeenNthCalledWith(1, `Retrying bulk update of 1 conflicted alerts`); + expect(logger.info).toHaveBeenNthCalledWith( + 2, + `Retried bulk update of 1 conflicted alerts succeeded` + ); + }); + + test('one conflicted doc amonst other successes and errors', async () => { + const { bulkRequest, bulkResponse } = getReqRes('is c ic ie'); + + esClient.mget.mockResolvedValueOnce({ + docs: [getMGetResDoc(2, alertDoc)], + }); + + esClient.bulk.mockResolvedValueOnce({ + errors: false, + took: 0, + items: [getBulkResItem(2)], + }); + + await resolveAlertConflicts({ logger, esClient, bulkRequest, bulkResponse }); + + expect(logger.error).toHaveBeenNthCalledWith( + 1, + `Error writing alerts: 2 successful, 1 conflicts, 1 errors: hallo` + ); + expect(logger.info).toHaveBeenNthCalledWith(1, `Retrying bulk update of 1 conflicted alerts`); + expect(logger.info).toHaveBeenNthCalledWith( + 2, + `Retried bulk update of 1 conflicted alerts succeeded` + ); + }); + + test('multiple conflicted doc amonst other successes and errors', async () => { + const { bulkRequest, bulkResponse } = getReqRes('is c ic ic ie ic'); + + esClient.mget.mockResolvedValueOnce({ + docs: [getMGetResDoc(2, alertDoc), getMGetResDoc(3, alertDoc), getMGetResDoc(5, alertDoc)], + }); + + esClient.bulk.mockResolvedValueOnce({ + errors: false, + took: 0, + items: [getBulkResItem(2), getBulkResItem(3), getBulkResItem(5)], + }); + + await resolveAlertConflicts({ logger, esClient, bulkRequest, bulkResponse }); + + expect(logger.error).toHaveBeenNthCalledWith( + 1, + `Error writing alerts: 2 successful, 3 conflicts, 1 errors: hallo` + ); + expect(logger.info).toHaveBeenNthCalledWith(1, `Retrying bulk update of 3 conflicted alerts`); + expect(logger.info).toHaveBeenNthCalledWith( + 2, + `Retried bulk update of 3 conflicted alerts succeeded` + ); + }); + }); +}); + +function getBulkResItem(id: number) { + return { + index: { + _index: `index-${id}`, + _id: `id-${id}`, + _seq_no: 18, + _primary_term: 1, + status: 200, + }, + }; +} + +function getMGetResDoc(id: number, doc: unknown) { + return { + _index: `index-${id}}`, + _id: `id-${id}`, + _seq_no: 18, + _primary_term: 1, + found: true, + _source: doc, + }; +} + +interface GetReqResResult { + bulkRequest: BulkRequest; + bulkResponse: BulkResponse; +} + +/** + * takes as input a string of c, is, ic, ie tokens and builds appropriate + * bulk request and response objects to use in the tests: + * - c: create, ignored by the resolve logic + * - is: index with success + * - ic: index with conflict + * - ie: index with error but not conflict + */ +function getReqRes(bulkOps: string): GetReqResResult { + const ops = bulkOps.trim().split(/\s+/g); + + const bulkRequest = getBulkRequest(); + const bulkResponse = getBulkResponse(); + + bulkRequest.operations = []; + bulkResponse.items = []; + bulkResponse.errors = false; + + if (ops[0] === '') return { bulkRequest, bulkResponse }; + + const createOp = { create: {} }; + + let id = 0; + for (const op of ops) { + id++; + switch (op) { + // create, ignored by the resolve logic + case 'c': + bulkRequest.operations.push(createOp, alertDoc); + bulkResponse.items.push(getResponseItem('create', id, false, 200)); + break; + + // index with success + case 'is': + bulkRequest.operations.push(getIndexOp(id), alertDoc); + bulkResponse.items.push(getResponseItem('index', id, false, 200)); + break; + + // index with conflict + case 'ic': + bulkResponse.errors = true; + bulkRequest.operations.push(getIndexOp(id), alertDoc); + bulkResponse.items.push(getResponseItem('index', id, true, 409)); + break; + + // index with error but not conflict + case 'ie': + bulkResponse.errors = true; + bulkRequest.operations.push(getIndexOp(id), alertDoc); + bulkResponse.items.push(getResponseItem('index', id, true, 418)); // I'm a teapot + break; + + // developer error + default: + throw new Error('bad input'); + } + } + + return { bulkRequest, bulkResponse }; +} + +function getBulkRequest(): BulkRequest { + return { + refresh: 'wait_for', + index: 'some-index', + require_alias: true, + operations: [], + }; +} + +function getIndexOp(id: number) { + return { + index: { + _id: `id-${id}`, + _index: `index-${id}`, + if_seq_no: 17, + if_primary_term: 1, + require_alias: false, + }, + }; +} + +function getBulkResponse(): BulkResponse { + return { + errors: false, + took: 0, + items: [], + }; +} + +function getResponseItem( + type: BulkOperationType, + id: number, + error: boolean, + status: number +): Partial> { + if (error) { + return { + [type]: { + _index: `index-${id}`, + _id: `id-${id}`, + error: { reason: 'hallo' }, + status, + }, + }; + } + + return { + [type]: { + _index: `index-${id}`, + _id: `id-${id}`, + _seq_no: 18, + _primary_term: 1, + status: 200, + }, + }; +} diff --git a/x-pack/plugins/alerting/server/alerts_client/lib/alert_conflict_resolver.ts b/x-pack/plugins/alerting/server/alerts_client/lib/alert_conflict_resolver.ts new file mode 100644 index 0000000000000..223070c0e7245 --- /dev/null +++ b/x-pack/plugins/alerting/server/alerts_client/lib/alert_conflict_resolver.ts @@ -0,0 +1,288 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + BulkRequest, + BulkResponse, + BulkOperationContainer, + MgetResponseItem, +} from '@elastic/elasticsearch/lib/api/types'; + +import { Logger, ElasticsearchClient } from '@kbn/core/server'; +import { + ALERT_STATUS, + ALERT_STATUS_ACTIVE, + ALERT_STATUS_RECOVERED, + ALERT_WORKFLOW_STATUS, + ALERT_WORKFLOW_TAGS, + ALERT_CASE_IDS, +} from '@kbn/rule-data-utils'; + +import { set } from '@kbn/safer-lodash-set'; +import { zip, get } from 'lodash'; + +// these fields are the one's we'll refresh from the fresh mget'd docs +const REFRESH_FIELDS_ALWAYS = [ALERT_WORKFLOW_STATUS, ALERT_WORKFLOW_TAGS, ALERT_CASE_IDS]; +const REFRESH_FIELDS_CONDITIONAL = [ALERT_STATUS]; +const REFRESH_FIELDS_ALL = [...REFRESH_FIELDS_ALWAYS, ...REFRESH_FIELDS_CONDITIONAL]; + +export interface ResolveAlertConflictsParams { + esClient: ElasticsearchClient; + logger: Logger; + bulkRequest: BulkRequest; + bulkResponse: BulkResponse; +} + +interface NormalizedBulkRequest { + op: BulkOperationContainer; + doc: unknown; +} + +// wrapper to catch anything thrown; current usage of this function is +// to replace just logging that the error occurred, so we don't want +// to cause _more_ errors ... +export async function resolveAlertConflicts(params: ResolveAlertConflictsParams): Promise { + const { logger } = params; + try { + await resolveAlertConflicts_(params); + } catch (err) { + logger.error(`Error resolving alert conflicts: ${err.message}`); + } +} + +async function resolveAlertConflicts_(params: ResolveAlertConflictsParams): Promise { + const { logger, esClient, bulkRequest, bulkResponse } = params; + if (bulkRequest.operations && bulkRequest.operations?.length === 0) return; + if (bulkResponse.items && bulkResponse.items?.length === 0) return; + + // get numbers for a summary log message + const { success, errors, conflicts, messages } = getResponseStats(bulkResponse); + if (conflicts === 0 && errors === 0) return; + + const allMessages = messages.join('; '); + logger.error( + `Error writing alerts: ${success} successful, ${conflicts} conflicts, ${errors} errors: ${allMessages}` + ); + + // get a new bulk request for just conflicted docs + const conflictRequest = getConflictRequest(bulkRequest, bulkResponse); + if (conflictRequest.length === 0) return; + + // get the fresh versions of those docs + const freshDocs = await getFreshDocs(esClient, conflictRequest); + + // update the OCC and refresh-able fields + await updateOCC(conflictRequest, freshDocs); + await refreshFieldsInDocs(conflictRequest, freshDocs); + + logger.info(`Retrying bulk update of ${conflictRequest.length} conflicted alerts`); + const mbrResponse = await makeBulkRequest(params.esClient, params.bulkRequest, conflictRequest); + + if (mbrResponse.bulkResponse?.items.length !== conflictRequest.length) { + const actual = mbrResponse.bulkResponse?.items.length; + const expected = conflictRequest.length; + logger.error( + `Unexpected number of bulk response items retried; expecting ${expected}, retried ${actual}` + ); + return; + } + + if (mbrResponse.error) { + const index = bulkRequest.index || 'unknown index'; + logger.error( + `Error writing ${conflictRequest.length} alerts to ${index} - ${mbrResponse.error.message}` + ); + return; + } + + if (mbrResponse.errors === 0) { + logger.info(`Retried bulk update of ${conflictRequest.length} conflicted alerts succeeded`); + } else { + logger.error( + `Retried bulk update of ${conflictRequest.length} conflicted alerts still had ${mbrResponse.errors} conflicts` + ); + } +} + +interface MakeBulkRequestResponse { + bulkRequest: BulkRequest; + bulkResponse?: BulkResponse; + errors: number; + error?: Error; +} + +// make the bulk request to fix conflicts +async function makeBulkRequest( + esClient: ElasticsearchClient, + bulkRequest: BulkRequest, + conflictRequest: NormalizedBulkRequest[] +): Promise { + const operations = conflictRequest.map((req) => [req.op, req.doc]).flat(); + // just replace the operations from the original request + const updatedBulkRequest = { ...bulkRequest, operations }; + + const bulkResponse = await esClient.bulk(updatedBulkRequest); + + const errors = bulkResponse.items.filter((item) => item.index?.error).length; + return { bulkRequest, bulkResponse, errors }; +} + +/** Update refreshable fields in the conflict requests. */ +async function refreshFieldsInDocs( + conflictRequests: NormalizedBulkRequest[], + freshResponses: MgetResponseItem[] +) { + for (const [conflictRequest, freshResponse] of zip(conflictRequests, freshResponses)) { + if (!conflictRequest?.op.index || !freshResponse) continue; + + // @ts-expect-error @elastic/elasticsearch _source is not in the type! + const freshDoc = freshResponse._source; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const conflictDoc = conflictRequest.doc as Record; + if (!freshDoc || !conflictDoc) continue; + + for (const refreshField of REFRESH_FIELDS_ALWAYS) { + const val = get(freshDoc, refreshField); + set(conflictDoc, refreshField, val); + } + + // structured this way to make sure all conditional refresh + // fields are listed in REFRESH_FIELDS_CONDITIONAL when we mget + for (const refreshField of REFRESH_FIELDS_CONDITIONAL) { + switch (refreshField) { + // hamdling for kibana.alert.status: overwrite conflict doc + // with fresh version if it's not active or recovered (ie, untracked) + case ALERT_STATUS: + const freshStatus = get(freshDoc, ALERT_STATUS); + + if (freshStatus !== ALERT_STATUS_ACTIVE && freshStatus !== ALERT_STATUS_RECOVERED) { + set(conflictDoc, ALERT_STATUS, freshStatus); + } + break; + } + } + } +} + +/** Update the OCC info in the conflict request with the fresh info. */ +async function updateOCC(conflictRequests: NormalizedBulkRequest[], freshDocs: MgetResponseItem[]) { + for (const [req, freshDoc] of zip(conflictRequests, freshDocs)) { + if (!req?.op.index || !freshDoc) continue; + + // @ts-expect-error @elastic/elasticsearch _seq_no is not in the type! + const seqNo: number | undefined = freshDoc._seq_no; + // @ts-expect-error @elastic/elasticsearch _primary_term is not in the type! + const primaryTerm: number | undefined = freshDoc._primary_term; + + if (seqNo === undefined) throw new Error('Unexpected undefined seqNo'); + if (primaryTerm === undefined) throw new Error('Unexpected undefined primaryTerm'); + + req.op.index.if_seq_no = seqNo; + req.op.index.if_primary_term = primaryTerm; + } +} + +/** Get the latest version of the conflicted docs, with fields to refresh. */ +async function getFreshDocs( + esClient: ElasticsearchClient, + conflictRequests: NormalizedBulkRequest[] +): Promise { + const docs: Array<{ _id: string; _index: string }> = []; + + conflictRequests.forEach((req) => { + const [id, index] = [req.op.index?._id, req.op.index?._index]; + if (!id || !index) return; + + docs.push({ _id: id, _index: index }); + }); + + const mgetRes = await esClient.mget({ docs, _source_includes: REFRESH_FIELDS_ALL }); + + if (mgetRes.docs.length !== docs.length) { + throw new Error( + `Unexpected number of mget response docs; expected ${docs.length}, got ${mgetRes.docs.length}` + ); + } + + return mgetRes.docs; +} + +/** Return the bulk request, filtered to those requests that had conflicts. */ +function getConflictRequest( + bulkRequest: BulkRequest, + bulkResponse: BulkResponse +): NormalizedBulkRequest[] { + // first "normalize" the request from it's non-linear form + const request = normalizeRequest(bulkRequest); + + // maybe we didn't unwind it right ... + if (request.length !== bulkResponse.items.length) { + throw new Error('Unexpected number of bulk response items'); + } + + if (request.length === 0) return []; + + // we only want op: index where the status was 409 / conflict + const conflictRequest = zip(request, bulkResponse.items) + .filter(([_, res]) => res?.index?.status === 409) + .map(([req, _]) => req!); + + return conflictRequest; +} + +/** Convert a bulk request (op | doc)[] to an array of { op, doc }[] */ +function normalizeRequest(bulkRequest: BulkRequest) { + if (!bulkRequest.operations) return []; + const result: NormalizedBulkRequest[] = []; + + let index = 0; + while (index < bulkRequest.operations.length) { + // the "op" data + const op = bulkRequest.operations[index] as BulkOperationContainer; + + // now the "doc" data, if there is any (none for delete) + if (op.create || op.index || op.update) { + index++; + const doc = bulkRequest.operations[index]; + result.push({ op, doc }); + } else if (op.delete) { + // no doc for delete op + } else { + throw new Error(`Unsupported bulk operation: ${JSON.stringify(op)}`); + } + + index++; + } + + return result; +} + +interface ResponseStatsResult { + success: number; + conflicts: number; + errors: number; + messages: string[]; +} + +// generate a summary of the original bulk request attempt, for logging +function getResponseStats(bulkResponse: BulkResponse): ResponseStatsResult { + const stats: ResponseStatsResult = { success: 0, conflicts: 0, errors: 0, messages: [] }; + for (const item of bulkResponse.items) { + const op = item.create || item.index || item.update || item.delete; + if (op?.error) { + if (op?.status === 409 && op === item.index) { + stats.conflicts++; + } else { + stats.errors++; + stats.messages.push(op?.error?.reason || 'no bulk reason provided'); + } + } else { + stats.success++; + } + } + return stats; +} diff --git a/x-pack/plugins/fleet/common/experimental_features.ts b/x-pack/plugins/fleet/common/experimental_features.ts index 39d7b998953fb..e9d7184928046 100644 --- a/x-pack/plugins/fleet/common/experimental_features.ts +++ b/x-pack/plugins/fleet/common/experimental_features.ts @@ -20,7 +20,7 @@ export const allowedExperimentalValues = Object.freeze({ showIntegrationsSubcategories: true, agentFqdnMode: true, showExperimentalShipperOptions: false, - agentTamperProtectionEnabled: false, + agentTamperProtectionEnabled: true, secretsStorage: true, kafkaOutput: true, }); diff --git a/x-pack/plugins/fleet/common/openapi/bundled.json b/x-pack/plugins/fleet/common/openapi/bundled.json index dfa970fb4b43c..a4604a7d7427b 100644 --- a/x-pack/plugins/fleet/common/openapi/bundled.json +++ b/x-pack/plugins/fleet/common/openapi/bundled.json @@ -5415,6 +5415,156 @@ } ] } + }, + "/uninstall_tokens": { + "get": { + "summary": "List metadata for latest uninstall tokens per agent policy", + "tags": [ + "Uninstall tokens" + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "policy_id": { + "type": "string" + }, + "created_at": { + "type": "string" + } + }, + "required": [ + "id", + "policy_id", + "created_at" + ] + } + }, + "total": { + "type": "number" + }, + "page": { + "type": "number" + }, + "perPage": { + "type": "number" + } + }, + "required": [ + "items", + "total", + "page", + "perPage" + ] + } + } + } + }, + "400": { + "$ref": "#/components/responses/error" + } + }, + "operationId": "get-uninstall-tokens", + "parameters": [ + { + "name": "perPage", + "in": "query", + "description": "The number of items to return", + "required": false, + "schema": { + "type": "integer", + "default": 20, + "minimum": 5 + } + }, + { + "$ref": "#/components/parameters/page_index" + }, + { + "name": "policyId", + "in": "query", + "description": "Partial match filtering for policy IDs", + "required": false, + "schema": { + "type": "string" + } + } + ] + } + }, + "/uninstall_tokens/{uninstallTokenId}": { + "get": { + "summary": "Get one decrypted uninstall token by its ID", + "tags": [ + "Uninstall tokens" + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "item": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "token": { + "type": "string" + }, + "policy_id": { + "type": "string" + }, + "created_at": { + "type": "string" + } + }, + "required": [ + "id", + "token", + "policy_id", + "created_at" + ] + } + }, + "required": [ + "item" + ] + } + } + } + }, + "400": { + "$ref": "#/components/responses/error" + } + }, + "operationId": "get-uninstall-token", + "parameters": [ + { + "name": "uninstallTokenId", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ] + } } }, "components": { diff --git a/x-pack/plugins/fleet/common/openapi/bundled.yaml b/x-pack/plugins/fleet/common/openapi/bundled.yaml index a996c3403810d..be132c9f19e48 100644 --- a/x-pack/plugins/fleet/common/openapi/bundled.yaml +++ b/x-pack/plugins/fleet/common/openapi/bundled.yaml @@ -3369,6 +3369,104 @@ paths: name: enrolToken in: query required: false + /uninstall_tokens: + get: + summary: List metadata for latest uninstall tokens per agent policy + tags: + - Uninstall tokens + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + items: + type: array + items: + type: object + properties: + id: + type: string + policy_id: + type: string + created_at: + type: string + required: + - id + - policy_id + - created_at + total: + type: number + page: + type: number + perPage: + type: number + required: + - items + - total + - page + - perPage + '400': + $ref: '#/components/responses/error' + operationId: get-uninstall-tokens + parameters: + - name: perPage + in: query + description: The number of items to return + required: false + schema: + type: integer + default: 20 + minimum: 5 + - $ref: '#/components/parameters/page_index' + - name: policyId + in: query + description: Partial match filtering for policy IDs + required: false + schema: + type: string + /uninstall_tokens/{uninstallTokenId}: + get: + summary: Get one decrypted uninstall token by its ID + tags: + - Uninstall tokens + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + item: + type: object + properties: + id: + type: string + token: + type: string + policy_id: + type: string + created_at: + type: string + required: + - id + - token + - policy_id + - created_at + required: + - item + '400': + $ref: '#/components/responses/error' + operationId: get-uninstall-token + parameters: + - name: uninstallTokenId + in: path + required: true + schema: + type: string components: securitySchemes: basicAuth: diff --git a/x-pack/plugins/fleet/common/openapi/entrypoint.yaml b/x-pack/plugins/fleet/common/openapi/entrypoint.yaml index 443caa36feadc..b8a7e024f3c4e 100644 --- a/x-pack/plugins/fleet/common/openapi/entrypoint.yaml +++ b/x-pack/plugins/fleet/common/openapi/entrypoint.yaml @@ -164,6 +164,13 @@ paths: # K8s /kubernetes: $ref: paths/kubernetes.yaml + + # Uninstall tokens + /uninstall_tokens: + $ref: paths/uninstall_tokens.yaml + /uninstall_tokens/{uninstallTokenId}: + $ref: paths/uninstall_tokens@{uninstall_token_id}.yaml + components: securitySchemes: basicAuth: diff --git a/x-pack/plugins/fleet/common/openapi/paths/uninstall_tokens.yaml b/x-pack/plugins/fleet/common/openapi/paths/uninstall_tokens.yaml new file mode 100644 index 0000000000000..daa6727007b2d --- /dev/null +++ b/x-pack/plugins/fleet/common/openapi/paths/uninstall_tokens.yaml @@ -0,0 +1,57 @@ +get: + summary: List metadata for latest uninstall tokens per agent policy + tags: + - Uninstall tokens + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + items: + type: array + items: + type: object + properties: + id: + type: string + policy_id: + type: string + created_at: + type: string + required: + - id + - policy_id + - created_at + total: + type: number + page: + type: number + perPage: + type: number + required: + - items + - total + - page + - perPage + '400': + $ref: ../components/responses/error.yaml + operationId: get-uninstall-tokens + parameters: + - name: perPage + in: query + description: The number of items to return + required: false + schema: + type: integer + default: 20 + minimum: 5 + - $ref: ../components/parameters/page_index.yaml + - name: policyId + in: query + description: Partial match filtering for policy IDs + required: false + schema: + type: string diff --git a/x-pack/plugins/fleet/common/openapi/paths/uninstall_tokens@{uninstall_token_id}.yaml b/x-pack/plugins/fleet/common/openapi/paths/uninstall_tokens@{uninstall_token_id}.yaml new file mode 100644 index 0000000000000..549a2c61f542d --- /dev/null +++ b/x-pack/plugins/fleet/common/openapi/paths/uninstall_tokens@{uninstall_token_id}.yaml @@ -0,0 +1,39 @@ +get: + summary: Get one decrypted uninstall token by its ID + tags: + - Uninstall tokens + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + item: + type: object + properties: + id: + type: string + token: + type: string + policy_id: + type: string + created_at: + type: string + required: + - id + - token + - policy_id + - created_at + required: + - item + '400': + $ref: ../components/responses/error.yaml + operationId: get-uninstall-token + parameters: + - name: uninstallTokenId + in: path + required: true + schema: + type: string diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/components/agent_policy_advanced_fields/index.test.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/components/agent_policy_advanced_fields/index.test.tsx index ceea82434e1e1..15f4fc928eada 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/components/agent_policy_advanced_fields/index.test.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/components/agent_policy_advanced_fields/index.test.tsx @@ -13,10 +13,6 @@ import type { RenderResult } from '@testing-library/react'; import { createFleetTestRendererMock } from '../../../../../../mock'; import type { TestRenderer } from '../../../../../../mock'; -import { allowedExperimentalValues } from '../../../../../../../common/experimental_features'; - -import { ExperimentalFeaturesService } from '../../../../../../services/experimental_features'; - import { createAgentPolicyMock, createPackagePolicyMock } from '../../../../../../../common/mocks'; import type { AgentPolicy, NewAgentPolicy } from '../../../../../../../common/types'; @@ -51,13 +47,6 @@ describe('Agent policy advanced options content', () => { newAgentPolicy = false, packagePolicy = [createPackagePolicyMock()], } = {}) => { - // remove when feature flag is removed - ExperimentalFeaturesService.init({ - ...allowedExperimentalValues, - // @ts-expect-error ts upgrade v4.7.4 - agentTamperProtectionEnabled: true, - }); - if (newAgentPolicy) { mockAgentPolicy = generateNewAgentPolicyWithDefaults(); } else { diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/list_page/index.test.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/list_page/index.test.tsx index 97ec62dce1d87..e2e9c11192b0e 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/list_page/index.test.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/list_page/index.test.tsx @@ -9,8 +9,6 @@ import React from 'react'; import type { RenderResult } from '@testing-library/react'; import { fireEvent, waitFor } from '@testing-library/react'; -import { allowedExperimentalValues } from '../../../../../../common/experimental_features'; -import { ExperimentalFeaturesService } from '../../../../../services'; import { createFleetTestRendererMock } from '../../../../../mock'; import type { GetAgentPoliciesResponse } from '../../../../../../common'; @@ -37,13 +35,6 @@ describe('AgentPolicyListPage', () => { const render = () => { const renderer = createFleetTestRendererMock(); - // todo: this can be removed when agentTamperProtectionEnabled feature flag is enabled/deleted - ExperimentalFeaturesService.init({ - ...allowedExperimentalValues, - // @ts-expect-error ts upgrade v4.7.4 - agentTamperProtectionEnabled: true, - }); - return renderer.render(); }; diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_list_page/index.test.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_list_page/index.test.tsx index 49b1e74014341..e276023a27674 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_list_page/index.test.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_list_page/index.test.tsx @@ -10,8 +10,6 @@ import React from 'react'; import type { RenderResult } from '@testing-library/react'; import { act, fireEvent, waitFor } from '@testing-library/react'; -import { allowedExperimentalValues } from '../../../../../../common/experimental_features'; -import { ExperimentalFeaturesService } from '../../../../../services'; import type { GetAgentPoliciesResponse } from '../../../../../../common'; import { createFleetTestRendererMock } from '../../../../../mock'; import { sendGetAgents, sendGetAgentStatus } from '../../../hooks'; @@ -290,13 +288,6 @@ describe('agent_list_page', () => { const renderer = createFleetTestRendererMock(); - // todo: this can be removed when agentTamperProtectionEnabled feature flag is enabled/deleted - ExperimentalFeaturesService.init({ - ...allowedExperimentalValues, - // @ts-expect-error ts upgrade v4.7.4 - agentTamperProtectionEnabled: true, - }); - renderResult = renderer.render(); await waitFor(() => { diff --git a/x-pack/plugins/fleet/server/integration_tests/cloud_preconfiguration.test.ts b/x-pack/plugins/fleet/server/integration_tests/cloud_preconfiguration.test.ts index 139f07fb999b3..ae19383f37216 100644 --- a/x-pack/plugins/fleet/server/integration_tests/cloud_preconfiguration.test.ts +++ b/x-pack/plugins/fleet/server/integration_tests/cloud_preconfiguration.test.ts @@ -160,9 +160,6 @@ describe('Fleet preconfiguration reset', () => { input['apm-server'].rum.source_mapping.elasticsearch.api_key = ''; } }); - data.agent.protection.signing_key = ''; - data.signed.data = ''; - data.signed.signature = ''; expect(data).toEqual( expect.objectContaining({ @@ -178,8 +175,8 @@ describe('Fleet preconfiguration reset', () => { }, protection: { enabled: false, - signing_key: '', - uninstall_token_hash: '', + signing_key: data.agent.protection.signing_key, + uninstall_token_hash: data.agent.protection.uninstall_token_hash, }, }, id: 'policy-elastic-agent-on-cloud', @@ -337,10 +334,7 @@ describe('Fleet preconfiguration reset', () => { }, revision: 5, secret_references: [], - signed: { - data: '', - signature: '', - }, + signed: data.signed, }) ); }); diff --git a/x-pack/plugins/fleet/server/mocks/index.ts b/x-pack/plugins/fleet/server/mocks/index.ts index 1e3d0e5c52b0a..adc0ecb1931b4 100644 --- a/x-pack/plugins/fleet/server/mocks/index.ts +++ b/x-pack/plugins/fleet/server/mocks/index.ts @@ -71,7 +71,6 @@ export const createAppContextStartContractMock = ( securitySetup: securityMock.createSetup(), securityStart: securityMock.createStart(), logger: loggingSystemMock.create().get(), - // @ts-expect-error ts upgrade v4.7.4 experimentalFeatures: { agentTamperProtectionEnabled: true, diagnosticFileUploadEnabled: true, diff --git a/x-pack/plugins/fleet/server/routes/uninstall_token/handlers.test.ts b/x-pack/plugins/fleet/server/routes/uninstall_token/handlers.test.ts index 3767c9a8d66ee..96bda0ed31ae8 100644 --- a/x-pack/plugins/fleet/server/routes/uninstall_token/handlers.test.ts +++ b/x-pack/plugins/fleet/server/routes/uninstall_token/handlers.test.ts @@ -184,7 +184,8 @@ describe('uninstall token handlers', () => { }); }); - describe('Agent Tamper Protection feature flag', () => { + // TODO: remove it when agentTamperProtectionEnabled FF is removed + describe.skip('Agent Tamper Protection feature flag', () => { let config: { enableExperimental: string[] }; let fakeRouter: jest.Mocked>; let fleetAuthzRouter: FleetAuthzRouter; diff --git a/x-pack/plugins/lens/server/content_management/lens_storage.ts b/x-pack/plugins/lens/server/content_management/lens_storage.ts index 72f78472356e5..3894ef20af30c 100644 --- a/x-pack/plugins/lens/server/content_management/lens_storage.ts +++ b/x-pack/plugins/lens/server/content_management/lens_storage.ts @@ -9,6 +9,7 @@ import type { SavedObjectsFindOptions } from '@kbn/core-saved-objects-api-server import type { StorageContext } from '@kbn/content-management-plugin/server'; import { SOContentStorage, tagsToFindOptions } from '@kbn/content-management-utils'; import type { SavedObject, SavedObjectReference } from '@kbn/core-saved-objects-api-server'; +import type { Logger } from '@kbn/logging'; import { CONTENT_ID, @@ -82,13 +83,20 @@ function savedObjectToLensSavedObject( } export class LensStorage extends SOContentStorage { - constructor() { + constructor( + private params: { + logger: Logger; + throwOnResultValidationError: boolean; + } + ) { super({ savedObjectType: CONTENT_ID, cmServicesDefinition, searchArgsToSOFindOptions, enableMSearch: true, allowedSavedObjectAttributes: ['title', 'description', 'visualizationType', 'state'], + logger: params.logger, + throwOnResultValidationError: params.throwOnResultValidationError, }); } @@ -134,13 +142,28 @@ export class LensStorage extends SOContentStorage { ...optionsToLatest, }); + const result = { + item: savedObjectToLensSavedObject(savedObject), + }; + + const validationError = transforms.update.out.result.validate(result); + if (validationError) { + if (this.params.throwOnResultValidationError) { + throw Boom.badRequest(`Invalid response. ${validationError.message}`); + } else { + this.params.logger.warn(`Invalid response. ${validationError.message}`); + } + } + // Validate DB response and DOWN transform to the request version const { value, error: resultError } = transforms.update.out.result.down< LensCrudTypes['UpdateOut'], LensCrudTypes['UpdateOut'] - >({ - item: savedObjectToLensSavedObject(savedObject), - }); + >( + result, + undefined, // do not override version + { validate: false } // validation is done above + ); if (resultError) { throw Boom.badRequest(`Invalid response. ${resultError.message}`); diff --git a/x-pack/plugins/lens/server/index.ts b/x-pack/plugins/lens/server/index.ts index 4140c5de37b3b..6b9f823c3bbc6 100644 --- a/x-pack/plugins/lens/server/index.ts +++ b/x-pack/plugins/lens/server/index.ts @@ -5,10 +5,10 @@ * 2.0. */ +import type { PluginInitializerContext } from '@kbn/core-plugins-server'; import { LensServerPlugin } from './plugin'; - export type { LensServerPluginSetup } from './plugin'; -export const plugin = () => new LensServerPlugin(); +export const plugin = (initContext: PluginInitializerContext) => new LensServerPlugin(initContext); export type { LensDocShape715 } from './migrations/types'; diff --git a/x-pack/plugins/lens/server/plugin.tsx b/x-pack/plugins/lens/server/plugin.tsx index c811058511fb1..c7584474dfc2b 100644 --- a/x-pack/plugins/lens/server/plugin.tsx +++ b/x-pack/plugins/lens/server/plugin.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import { Plugin, CoreSetup, CoreStart } from '@kbn/core/server'; +import { Plugin, CoreSetup, CoreStart, PluginInitializerContext } from '@kbn/core/server'; import { PluginStart as DataViewsServerPluginStart } from '@kbn/data-views-plugin/server'; import { PluginStart as DataPluginStart, @@ -64,7 +64,7 @@ export interface LensServerPluginSetup { export class LensServerPlugin implements Plugin { private customVisualizationMigrations: CustomVisualizationMigrations = {}; - constructor() {} + constructor(private initializerContext: PluginInitializerContext) {} setup(core: CoreSetup, plugins: PluginSetupContract) { const getFilterMigrations = plugins.data.query.filterManager.getAllMigrations.bind( @@ -79,7 +79,10 @@ export class LensServerPlugin implements Plugin { - constructor() { + constructor({ + logger, + throwOnResultValidationError, + }: { + logger: Logger; + throwOnResultValidationError: boolean; + }) { super({ savedObjectType: CONTENT_ID, cmServicesDefinition, @@ -40,6 +47,8 @@ export class MapsStorage extends SOContentStorage { 'layerListJSON', 'uiStateJSON', ], + logger, + throwOnResultValidationError, }); } } diff --git a/x-pack/plugins/maps/server/plugin.ts b/x-pack/plugins/maps/server/plugin.ts index 8945b7d034377..dffd3e8a23aaa 100644 --- a/x-pack/plugins/maps/server/plugin.ts +++ b/x-pack/plugins/maps/server/plugin.ts @@ -204,7 +204,10 @@ export class MapsPlugin implements Plugin { contentManagement.register({ id: CONTENT_ID, - storage: new MapsStorage(), + storage: new MapsStorage({ + throwOnResultValidationError: this._initializerContext.env.mode.dev, + logger: this._logger.get('storage'), + }), version: { latest: LATEST_VERSION, }, diff --git a/x-pack/plugins/maps/tsconfig.json b/x-pack/plugins/maps/tsconfig.json index 34066a8b8d538..364a6d24473d6 100644 --- a/x-pack/plugins/maps/tsconfig.json +++ b/x-pack/plugins/maps/tsconfig.json @@ -73,6 +73,7 @@ "@kbn/content-management-table-list-view-table", "@kbn/content-management-table-list-view", "@kbn/serverless", + "@kbn/logging", ], "exclude": [ "target/**/*", diff --git a/x-pack/plugins/ml/common/types/alerts.ts b/x-pack/plugins/ml/common/types/alerts.ts index 267096e105ef6..12bae1a9d3d16 100644 --- a/x-pack/plugins/ml/common/types/alerts.ts +++ b/x-pack/plugins/ml/common/types/alerts.ts @@ -44,6 +44,35 @@ interface BaseAnomalyAlertDoc { unique_key: string; } +export interface TopRecordAADDoc { + job_id: string; + record_score: number; + initial_record_score: number; + timestamp: number; + is_interim: boolean; + function: string; + field_name?: string; + by_field_name?: string; + by_field_value?: string | number; + over_field_name?: string; + over_field_value?: string | number; + partition_field_name?: string; + partition_field_value?: string | number; + typical: number[]; + actual: number[]; + detector_index: number; +} + +export interface TopInfluencerAADDoc { + job_id: string; + influencer_score: number; + initial_influencer_score: number; + is_interim: boolean; + timestamp: number; + influencer_field_name: string; + influencer_field_value: string | number; +} + export interface RecordAnomalyAlertDoc extends BaseAnomalyAlertDoc { result_type: typeof ML_ANOMALY_RESULT_TYPE.RECORD; function: string; diff --git a/x-pack/plugins/ml/server/lib/alerts/alerting_service.ts b/x-pack/plugins/ml/server/lib/alerts/alerting_service.ts index 18be37a187c44..dc85428a63386 100644 --- a/x-pack/plugins/ml/server/lib/alerts/alerting_service.ts +++ b/x-pack/plugins/ml/server/lib/alerts/alerting_service.ts @@ -9,7 +9,7 @@ import Boom from '@hapi/boom'; import { i18n } from '@kbn/i18n'; import rison from '@kbn/rison'; import type { Duration } from 'moment/moment'; -import { memoize } from 'lodash'; +import { memoize, pick } from 'lodash'; import { FIELD_FORMAT_IDS, type IFieldFormat, @@ -24,6 +24,7 @@ import { ML_ANOMALY_RESULT_TYPE, } from '@kbn/ml-anomaly-utils'; import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common'; +import { ALERT_REASON, ALERT_URL } from '@kbn/rule-data-utils'; import type { MlClient } from '../ml_client'; import type { MlAnomalyDetectionAlertParams, @@ -36,8 +37,13 @@ import type { PreviewResultsKeys, RecordAnomalyAlertDoc, TopHitsResultsKeys, + TopInfluencerAADDoc, + TopRecordAADDoc, } from '../../../common/types/alerts'; -import type { AnomalyDetectionAlertContext } from './register_anomaly_detection_alert_type'; +import type { + AnomalyDetectionAlertContext, + AnomalyDetectionAlertPayload, +} from './register_anomaly_detection_alert_type'; import { resolveMaxTimeInterval } from '../../../common/util/job_utils'; import { getTopNBuckets, resolveLookbackInterval } from '../../../common/util/alerts'; import type { DatafeedsService } from '../../models/job_service/datafeeds'; @@ -391,12 +397,89 @@ export function alertingServiceProvider( return alertInstanceKey; }; + /** + * Returns a callback for formatting elasticsearch aggregation response + * to the alert-as-data document. + * @param resultType + */ + const getResultsToPayloadFormatter = ( + resultType: MlAnomalyResultType, + useInitialScore: boolean = false + ) => { + const resultsLabel = getAggResultsLabel(resultType); + + return ( + v: AggResultsResponse + ): Omit | undefined => { + const aggTypeResults = v[resultsLabel.aggGroupLabel]; + if (aggTypeResults.doc_count === 0) { + return; + } + const requestedAnomalies = aggTypeResults[resultsLabel.topHitsLabel].hits.hits; + const topAnomaly = requestedAnomalies[0]; + const timestamp = topAnomaly._source.timestamp; + + return { + [ALERT_REASON]: i18n.translate( + 'xpack.ml.alertTypes.anomalyDetectionAlertingRule.alertMessage', + { + defaultMessage: + 'Alerts are raised based on real-time scores. Remember that scores may be adjusted over time as data continues to be analyzed.', + } + ), + job_id: [...new Set(requestedAnomalies.map((h) => h._source.job_id))][0], + is_interim: requestedAnomalies.some((h) => h._source.is_interim), + anomaly_timestamp: timestamp, + anomaly_score: topAnomaly._source[getScoreFields(resultType, useInitialScore)], + top_records: v.record_results.top_record_hits.hits.hits.map((h) => { + const { actual, typical } = getTypicalAndActualValues(h._source); + return pick( + { + ...h._source, + typical, + actual, + }, + [ + 'job_id', + 'record_score', + 'initial_record_score', + 'detector_index', + 'is_interim', + 'timestamp', + 'partition_field_name', + 'partition_field_value', + 'function', + 'actual', + 'typical', + ] + ) as TopRecordAADDoc; + }) as TopRecordAADDoc[], + top_influencers: v.influencer_results.top_influencer_hits.hits.hits.map((influencerDoc) => { + return pick( + { + ...influencerDoc._source, + }, + [ + 'job_id', + 'influencer_field_name', + 'influencer_field_value', + 'influencer_score', + 'initial_influencer_score', + 'is_interim', + 'timestamp', + ] + ) as TopInfluencerAADDoc; + }) as TopInfluencerAADDoc[], + }; + }; + }; + /** * Returns a callback for formatting elasticsearch aggregation response * to the alert context. * @param resultType */ - const getResultsFormatter = ( + const getResultsToContextFormatter = ( resultType: MlAnomalyResultType, useInitialScore: boolean = false, formatters: FieldFormatters @@ -468,7 +551,7 @@ export function alertingServiceProvider( * @param previewTimeInterval - Relative time interval to test the alert condition * @param checkIntervalGap - Interval between alert executions */ - const fetchAnomalies = async ( + const fetchPreviewResults = async ( params: MlAnomalyDetectionAlertParams, previewTimeInterval?: string, checkIntervalGap?: Duration @@ -570,7 +653,7 @@ export function alertingServiceProvider( const fieldsFormatters = await getFormatters(datafeeds![0]!.indices[0]); - const formatter = getResultsFormatter( + const formatter = getResultsToContextFormatter( params.resultType, !!previewTimeInterval, fieldsFormatters @@ -660,7 +743,7 @@ export function alertingServiceProvider( */ const fetchResult = async ( params: AnomalyESQueryParams - ): Promise => { + ): Promise => { const { resultType, jobIds, @@ -670,7 +753,6 @@ export function alertingServiceProvider( anomalyScoreField, includeInterimResults, anomalyScoreThreshold, - indexPattern, } = params; const requestBody = { @@ -761,9 +843,44 @@ export function alertingServiceProvider( prev.max_score.value > current.max_score.value ? prev : current ); + return topResult; + }; + + const getFormatted = async ( + indexPattern: string, + resultType: MlAnomalyDetectionAlertParams['resultType'], + spaceId: string, + value: AggResultsResponse + ): Promise< + | { payload: AnomalyDetectionAlertPayload; context: AnomalyDetectionAlertContext; name: string } + | undefined + > => { const formatters = await getFormatters(indexPattern); - return getResultsFormatter(params.resultType, false, formatters)(topResult); + const context = getResultsToContextFormatter(resultType, false, formatters)(value); + const payload = getResultsToPayloadFormatter(resultType, false)(value); + + if (!context || !payload) return; + + const anomalyExplorerUrl = buildExplorerUrl( + context.jobIds, + { from: context.bucketRange.start, to: context.bucketRange.end }, + resultType, + spaceId, + context + ); + + return { + payload: { + ...payload, + [ALERT_URL]: anomalyExplorerUrl, + }, + context: { + ...context, + anomalyExplorerUrl, + }, + name: context.alertInstanceKey, + }; }; return { @@ -777,7 +894,13 @@ export function alertingServiceProvider( params: MlAnomalyDetectionAlertParams, spaceId: string ): Promise< - { context: AnomalyDetectionAlertContext; name: string; isHealthy: boolean } | undefined + | { + payload: AnomalyDetectionAlertPayload; + context: AnomalyDetectionAlertContext; + name: string; + isHealthy: boolean; + } + | undefined > => { const queryParams = await getQueryParams(params); @@ -787,50 +910,57 @@ export function alertingServiceProvider( const result = await fetchResult(queryParams); - if (result) { - const anomalyExplorerUrl = buildExplorerUrl( - result.jobIds, - { from: result.bucketRange.start, to: result.bucketRange.end }, - params.resultType, - spaceId, - result + const formattedResult = result + ? await getFormatted(queryParams.indexPattern, queryParams.resultType, spaceId, result) + : undefined; + + if (!formattedResult) { + // If no anomalies found, report as recovered. + + const url = buildExplorerUrl( + queryParams.jobIds, + { + from: `now-${queryParams.lookBackTimeInterval}`, + to: 'now', + mode: 'relative', + }, + queryParams.resultType, + spaceId ); - const executionResult = { - ...result, - anomalyExplorerUrl, - }; + const message = i18n.translate( + 'xpack.ml.alertTypes.anomalyDetectionAlertingRule.recoveredMessage', + { + defaultMessage: + 'No anomalies have been found in the past {lookbackInterval} that exceed the severity threshold of {severity}.', + values: { + severity: queryParams.anomalyScoreThreshold, + lookbackInterval: queryParams.lookBackTimeInterval, + }, + } + ); - return { context: executionResult, name: result.alertInstanceKey, isHealthy: false }; + return { + name: '', + isHealthy: true, + payload: { + [ALERT_URL]: url, + [ALERT_REASON]: message, + job_id: queryParams.jobIds[0], + }, + context: { + anomalyExplorerUrl: url, + jobIds: queryParams.jobIds, + message, + } as AnomalyDetectionAlertContext, + }; } return { - name: '', - isHealthy: true, - context: { - anomalyExplorerUrl: buildExplorerUrl( - queryParams.jobIds, - { - from: `now-${queryParams.lookBackTimeInterval}`, - to: 'now', - mode: 'relative', - }, - queryParams.resultType, - spaceId - ), - jobIds: queryParams.jobIds, - message: i18n.translate( - 'xpack.ml.alertTypes.anomalyDetectionAlertingRule.recoveredMessage', - { - defaultMessage: - 'No anomalies have been found in the past {lookbackInterval} that exceed the severity threshold of {severity}.', - values: { - severity: queryParams.anomalyScoreThreshold, - lookbackInterval: queryParams.lookBackTimeInterval, - }, - } - ), - } as AnomalyDetectionAlertContext, + context: formattedResult.context, + payload: formattedResult.payload, + name: formattedResult.name, + isHealthy: false, }; }, /** @@ -844,16 +974,16 @@ export function alertingServiceProvider( timeRange, sampleSize, }: MlAnomalyDetectionAlertPreviewRequest): Promise => { - const res = await fetchAnomalies(alertParams, timeRange); + const previewResults = await fetchPreviewResults(alertParams, timeRange); - if (!res) { + if (!previewResults) { throw Boom.notFound(`No results found`); } return { // sum of all alert responses within the time range - count: res.length, - results: res.slice(0, sampleSize), + count: previewResults.length, + results: previewResults.slice(0, sampleSize), }; }, }; diff --git a/x-pack/plugins/ml/server/lib/alerts/register_anomaly_detection_alert_type.ts b/x-pack/plugins/ml/server/lib/alerts/register_anomaly_detection_alert_type.ts index 2a1f19b48802e..2935643348f76 100644 --- a/x-pack/plugins/ml/server/lib/alerts/register_anomaly_detection_alert_type.ts +++ b/x-pack/plugins/ml/server/lib/alerts/register_anomaly_detection_alert_type.ts @@ -11,8 +11,15 @@ import type { ActionGroup, AlertInstanceContext, AlertInstanceState, + RuleTypeParams, RuleTypeState, + RecoveredActionGroupId, } from '@kbn/alerting-plugin/common'; +import { IRuleTypeAlerts, RuleExecutorOptions } from '@kbn/alerting-plugin/server'; +import { ALERT_NAMESPACE, ALERT_REASON, ALERT_URL } from '@kbn/rule-data-utils'; +import { MlAnomalyDetectionAlert } from '@kbn/alerts-as-data-utils'; +import { ES_FIELD_TYPES } from '@kbn/field-types'; +import { expandFlattenedAlert } from '@kbn/alerting-plugin/server/alerts_client/lib'; import { ML_ALERT_TYPES } from '../../../common/constants/alerts'; import { PLUGIN_ID } from '../../../common/constants/app'; import { MINIMUM_FULL_LICENSE } from '../../../common/license'; @@ -36,6 +43,19 @@ export type AnomalyDetectionAlertBaseContext = AlertInstanceContext & { message: string; }; +// Flattened alert payload for alert-as-data +export type AnomalyDetectionAlertPayload = { + job_id: string; + anomaly_score?: number; + is_interim?: boolean; + anomaly_timestamp?: number; + top_records?: any; + top_influencers?: any; +} & { + [ALERT_URL]: string; + [ALERT_REASON]: string; +}; + export type AnomalyDetectionAlertContext = AnomalyDetectionAlertBaseContext & { timestampIso8601: string; timestamp: number; @@ -45,10 +65,88 @@ export type AnomalyDetectionAlertContext = AnomalyDetectionAlertBaseContext & { topInfluencers?: InfluencerAnomalyAlertDoc[]; }; +export type ExecutorOptions

= RuleExecutorOptions< + P, + RuleTypeState, + {}, + AnomalyDetectionAlertContext, + typeof ANOMALY_SCORE_MATCH_GROUP_ID, + MlAnomalyDetectionAlert +>; + export const ANOMALY_SCORE_MATCH_GROUP_ID = 'anomaly_score_match'; export type AnomalyScoreMatchGroupId = typeof ANOMALY_SCORE_MATCH_GROUP_ID; +export const ANOMALY_DETECTION_AAD_INDEX_NAME = 'ml.anomaly-detection'; + +const ML_ALERT_NAMESPACE = ALERT_NAMESPACE; + +export const ALERT_ANOMALY_DETECTION_JOB_ID = `${ML_ALERT_NAMESPACE}.job_id` as const; + +export const ALERT_ANOMALY_SCORE = `${ML_ALERT_NAMESPACE}.anomaly_score` as const; +export const ALERT_ANOMALY_IS_INTERIM = `${ML_ALERT_NAMESPACE}.is_interim` as const; +export const ALERT_ANOMALY_TIMESTAMP = `${ML_ALERT_NAMESPACE}.anomaly_timestamp` as const; + +export const ALERT_TOP_RECORDS = `${ML_ALERT_NAMESPACE}.top_records` as const; +export const ALERT_TOP_INFLUENCERS = `${ML_ALERT_NAMESPACE}.top_influencers` as const; + +export const ANOMALY_DETECTION_AAD_CONFIG: IRuleTypeAlerts = { + context: ANOMALY_DETECTION_AAD_INDEX_NAME, + mappings: { + fieldMap: { + [ALERT_ANOMALY_DETECTION_JOB_ID]: { + type: ES_FIELD_TYPES.KEYWORD, + array: false, + required: true, + }, + [ALERT_ANOMALY_SCORE]: { type: ES_FIELD_TYPES.DOUBLE, array: false, required: false }, + [ALERT_ANOMALY_IS_INTERIM]: { type: ES_FIELD_TYPES.BOOLEAN, array: false, required: false }, + [ALERT_ANOMALY_TIMESTAMP]: { type: ES_FIELD_TYPES.DATE, array: false, required: false }, + [ALERT_TOP_RECORDS]: { + type: ES_FIELD_TYPES.OBJECT, + array: true, + required: false, + dynamic: false, + properties: { + job_id: { type: ES_FIELD_TYPES.KEYWORD }, + record_score: { type: ES_FIELD_TYPES.DOUBLE }, + initial_record_score: { type: ES_FIELD_TYPES.DOUBLE }, + detector_index: { type: ES_FIELD_TYPES.INTEGER }, + is_interim: { type: ES_FIELD_TYPES.BOOLEAN }, + timestamp: { type: ES_FIELD_TYPES.DATE }, + partition_field_name: { type: ES_FIELD_TYPES.KEYWORD }, + partition_field_value: { type: ES_FIELD_TYPES.KEYWORD }, + over_field_name: { type: ES_FIELD_TYPES.KEYWORD }, + over_field_value: { type: ES_FIELD_TYPES.KEYWORD }, + by_field_name: { type: ES_FIELD_TYPES.KEYWORD }, + by_field_value: { type: ES_FIELD_TYPES.KEYWORD }, + function: { type: ES_FIELD_TYPES.KEYWORD }, + typical: { type: ES_FIELD_TYPES.DOUBLE }, + actual: { type: ES_FIELD_TYPES.DOUBLE }, + field_name: { type: ES_FIELD_TYPES.KEYWORD }, + }, + }, + [ALERT_TOP_INFLUENCERS]: { + type: ES_FIELD_TYPES.OBJECT, + array: true, + required: false, + dynamic: false, + properties: { + job_id: { type: ES_FIELD_TYPES.KEYWORD }, + influencer_field_name: { type: ES_FIELD_TYPES.KEYWORD }, + influencer_field_value: { type: ES_FIELD_TYPES.KEYWORD }, + influencer_score: { type: ES_FIELD_TYPES.DOUBLE }, + initial_influencer_score: { type: ES_FIELD_TYPES.DOUBLE }, + is_interim: { type: ES_FIELD_TYPES.BOOLEAN }, + timestamp: { type: ES_FIELD_TYPES.DATE }, + }, + }, + }, + }, + shouldWrite: true, +}; + export const THRESHOLD_MET_GROUP: ActionGroup = { id: ANOMALY_SCORE_MATCH_GROUP_ID, name: i18n.translate('xpack.ml.anomalyDetectionAlert.actionGroupName', { @@ -66,7 +164,9 @@ export function registerAnomalyDetectionAlertType({ RuleTypeState, AlertInstanceState, AnomalyDetectionAlertContext, - AnomalyScoreMatchGroupId + AnomalyScoreMatchGroupId, + RecoveredActionGroupId, + MlAnomalyDetectionAlert >({ id: ML_ALERT_TYPES.ANOMALY_DETECTION, name: i18n.translate('xpack.ml.anomalyDetectionAlert.name', { @@ -140,29 +240,62 @@ export function registerAnomalyDetectionAlertType({ minimumLicenseRequired: MINIMUM_FULL_LICENSE, isExportable: true, doesSetRecoveryContext: true, - async executor({ services, params, spaceId }) { + executor: async ({ + services, + params, + spaceId, + }: ExecutorOptions) => { const fakeRequest = {} as KibanaRequest; const { execute } = mlSharedServices.alertingServiceProvider( services.savedObjectsClient, fakeRequest ); + + const { alertsClient } = services; + if (!alertsClient) return { state: {} }; + const executionResult = await execute(params, spaceId); - if (executionResult && !executionResult.isHealthy) { - const alertInstanceName = executionResult.name; - const alertInstance = services.alertFactory.create(alertInstanceName); - alertInstance.scheduleActions(ANOMALY_SCORE_MATCH_GROUP_ID, executionResult.context); + if (!executionResult) return { state: {} }; + + const { isHealthy, name, context, payload } = executionResult; + + if (!isHealthy) { + alertsClient.report({ + id: name, + actionGroup: ANOMALY_SCORE_MATCH_GROUP_ID, + context, + payload: expandFlattenedAlert({ + [ALERT_URL]: payload[ALERT_URL], + [ALERT_REASON]: payload[ALERT_REASON], + [ALERT_ANOMALY_DETECTION_JOB_ID]: payload.job_id, + [ALERT_ANOMALY_SCORE]: payload.anomaly_score, + [ALERT_ANOMALY_IS_INTERIM]: payload.is_interim, + [ALERT_ANOMALY_TIMESTAMP]: payload.anomaly_timestamp, + [ALERT_TOP_RECORDS]: payload.top_records, + [ALERT_TOP_INFLUENCERS]: payload.top_influencers, + }), + }); } // Set context for recovered alerts - const { getRecoveredAlerts } = services.alertFactory.done(); - for (const recoveredAlert of getRecoveredAlerts()) { - if (!!executionResult?.isHealthy) { - recoveredAlert.setContext(executionResult.context); + for (const recoveredAlert of alertsClient.getRecoveredAlerts()) { + if (isHealthy) { + const alertId = recoveredAlert.alert.getId(); + alertsClient.setAlertData({ + id: alertId, + context, + payload: expandFlattenedAlert({ + [ALERT_URL]: payload[ALERT_URL], + [ALERT_REASON]: payload[ALERT_REASON], + [ALERT_ANOMALY_DETECTION_JOB_ID]: payload.job_id, + }), + }); } } return { state: {} }; }, + alerts: ANOMALY_DETECTION_AAD_CONFIG, }); } diff --git a/x-pack/plugins/ml/tsconfig.json b/x-pack/plugins/ml/tsconfig.json index 7962e2dd27296..4a0077770808a 100644 --- a/x-pack/plugins/ml/tsconfig.json +++ b/x-pack/plugins/ml/tsconfig.json @@ -106,5 +106,7 @@ "@kbn/react-kibana-mount", "@kbn/core-http-browser", "@kbn/data-view-editor-plugin", + "@kbn/rule-data-utils", + "@kbn/alerts-as-data-utils", ], } diff --git a/x-pack/test/alerting_api_integration/common/plugins/alerts/server/alert_types.ts b/x-pack/test/alerting_api_integration/common/plugins/alerts/server/alert_types.ts index 5003acd160f29..72b3b7b34476f 100644 --- a/x-pack/test/alerting_api_integration/common/plugins/alerts/server/alert_types.ts +++ b/x-pack/test/alerting_api_integration/common/plugins/alerts/server/alert_types.ts @@ -7,7 +7,7 @@ import { v4 as uuidv4 } from 'uuid'; import { Logger } from '@kbn/logging'; -import { CoreSetup } from '@kbn/core/server'; +import { CoreSetup, ElasticsearchClient } from '@kbn/core/server'; import { schema, TypeOf } from '@kbn/config-schema'; import { curry, range, times } from 'lodash'; import { @@ -941,6 +941,136 @@ function getAlwaysFiringAlertAsDataRuleType( }); } +function getWaitingRuleType(logger: Logger) { + const ParamsType = schema.object({ + source: schema.string(), + alerts: schema.number(), + }); + type ParamsType = TypeOf; + interface State extends RuleTypeState { + runCount?: number; + } + const id = 'test.waitingRule'; + + const result: RuleType< + ParamsType, + never, + State, + {}, + {}, + 'default', + 'recovered', + { runCount: number } + > = { + id, + name: 'Test: Rule that waits for a signal before finishing', + actionGroups: [{ id: 'default', name: 'Default' }], + producer: 'alertsFixture', + defaultActionGroupId: 'default', + minimumLicenseRequired: 'basic', + isExportable: true, + doesSetRecoveryContext: true, + validate: { params: ParamsType }, + alerts: { + context: id.toLowerCase(), + shouldWrite: true, + mappings: { + fieldMap: { + runCount: { required: false, type: 'long' }, + }, + }, + }, + async executor(alertExecutorOptions) { + const { services, state, params } = alertExecutorOptions; + const { source, alerts } = params; + + const alertsClient = services.alertsClient; + if (!alertsClient) throw new Error(`Expected alertsClient!`); + + const runCount = (state.runCount || 0) + 1; + const es = services.scopedClusterClient.asInternalUser; + + await sendSignal(logger, es, id, source, `rule-starting-${runCount}`); + await waitForSignal(logger, es, id, source, `rule-complete-${runCount}`); + + for (let i = 0; i < alerts; i++) { + alertsClient.report({ + id: `alert-${i}`, + actionGroup: 'default', + payload: { runCount }, + }); + } + + return { state: { runCount } }; + }, + }; + + return result; +} + +async function sendSignal( + logger: Logger, + es: ElasticsearchClient, + id: string, + source: string, + reference: string +) { + logger.info(`rule type ${id} sending signal ${reference}`); + await es.index({ index: ES_TEST_INDEX_NAME, refresh: 'true', body: { source, reference } }); +} + +async function waitForSignal( + logger: Logger, + es: ElasticsearchClient, + id: string, + source: string, + reference: string +) { + let docs: unknown[] = []; + for (let attempt = 0; attempt < 20; attempt++) { + docs = await getSignalDocs(es, source, reference); + if (docs.length > 0) { + logger.info(`rule type ${id} received signal ${reference}`); + break; + } + + logger.info(`rule type ${id} waiting for signal ${reference}`); + await new Promise((resolve) => setTimeout(resolve, 1000)); + } + if (docs.length === 0) { + throw new Error(`Expected to find docs with source ${source}`); + } +} + +async function getSignalDocs(es: ElasticsearchClient, source: string, reference: string) { + const body = { + query: { + bool: { + must: [ + { + term: { + source, + }, + }, + { + term: { + reference, + }, + }, + ], + }, + }, + }; + const params = { + index: ES_TEST_INDEX_NAME, + size: 1000, + _source: false, + body, + }; + const result = await es.search(params, { meta: true }); + return result?.body?.hits?.hits || []; +} + export function defineAlertTypes( core: CoreSetup, { alerting, ruleRegistry }: Pick, @@ -1162,4 +1292,5 @@ export function defineAlertTypes( alerting.registerType(getAlwaysFiringAlertAsDataRuleType(logger, { ruleRegistry })); alerting.registerType(getPatternFiringAutoRecoverFalseAlertType()); alerting.registerType(getPatternFiringAlertsAsDataRuleType()); + alerting.registerType(getWaitingRuleType(logger)); } diff --git a/x-pack/test/alerting_api_integration/common/plugins/alerts/server/plugin.ts b/x-pack/test/alerting_api_integration/common/plugins/alerts/server/plugin.ts index 0809a4a5b71c7..7a257d214f26a 100644 --- a/x-pack/test/alerting_api_integration/common/plugins/alerts/server/plugin.ts +++ b/x-pack/test/alerting_api_integration/common/plugins/alerts/server/plugin.ts @@ -88,6 +88,7 @@ export class FixturePlugin implements Plugin { @@ -154,6 +168,20 @@ export default function alertTests({ getService }: FtrProviderContext) { ); } + async function waitForAAD(numDocs: number): Promise { + return await retry.try(async () => { + const searchResult = await es.search({ index: AAD_INDEX, size: 1000 }); + + // @ts-expect-error doesn't handle total: number + const value = searchResult.hits.total.value?.value || searchResult.hits.total.value; + if (value < numDocs) { + // @ts-expect-error doesn't handle total: number + throw new Error(`Expected ${numDocs} but received ${searchResult.hits.total.value}.`); + } + return searchResult.hits.hits; + }); + } + async function createAlert({ name, ...params diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/alerts_as_data/alerts_as_data_conflicts.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/alerts_as_data/alerts_as_data_conflicts.ts new file mode 100644 index 0000000000000..c0243dbd482fd --- /dev/null +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/alerts_as_data/alerts_as_data_conflicts.ts @@ -0,0 +1,284 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { Client } from '@elastic/elasticsearch'; +import { SearchHit } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import type { Alert } from '@kbn/alerts-as-data-utils'; +import { ESTestIndexTool } from '@kbn/alerting-api-integration-helpers'; +import { basename } from 'node:path'; +import { v4 as uuidv4 } from 'uuid'; +import { get, omit } from 'lodash'; +import { FtrProviderContext } from '../../../../../common/ftr_provider_context'; +import { Spaces } from '../../../../scenarios'; +import { getTestRuleData, getUrlPrefix, ObjectRemover } from '../../../../../common/lib'; + +type AlertDoc = Alert & { runCount: number }; + +// sort results of a search of alert docs by alert instance id +function sortAlertDocsByInstanceId(a: SearchHit, b: SearchHit) { + return a._source!.kibana.alert.instance.id.localeCompare(b._source!.kibana.alert.instance.id); +} + +// eslint-disable-next-line import/no-default-export +export default function createAlertsAsDataInstallResourcesTest({ getService }: FtrProviderContext) { + const es = getService('es'); + const retry = getService('retry'); + const supertestWithoutAuth = getService('supertestWithoutAuth'); + const objectRemover = new ObjectRemover(supertestWithoutAuth); + const esTestIndexTool = new ESTestIndexTool(es, retry); + + describe('document conflicts during rule execution', () => { + before(async () => { + await esTestIndexTool.destroy(); + await esTestIndexTool.setup(); + }); + + after(async () => { + await objectRemover.removeAll(); + await esTestIndexTool.destroy(); + }); + + const ruleType = 'test.waitingRule'; + const aadIndex = `.alerts-${ruleType.toLowerCase()}.alerts-default`; + + describe(`should be handled for alerting framework based AaD`, () => { + it('for a single conflicted alert', async () => { + const source = uuidv4(); + const count = 1; + const params = { source, alerts: count }; + const createdRule = await supertestWithoutAuth + .post(`${getUrlPrefix(Spaces.space1.id)}/api/alerting/rule`) + .set('kbn-xsrf', 'foo') + .send( + getTestRuleData({ + name: `${basename(__filename)} ${ruleType} ${source}}`, + rule_type_id: ruleType, + schedule: { interval: '1s' }, + throttle: null, + params, + actions: [], + }) + ); + + if (createdRule.status !== 200) { + log(`error creating rule: ${JSON.stringify(createdRule, null, 4)}`); + } + expect(createdRule.status).to.eql(200); + + const ruleId = createdRule.body.id; + objectRemover.add(Spaces.space1.id, ruleId, 'rule', 'alerting'); + + // this rule type uses esTextIndexTool documents to communicate + // with the rule executor. Once the rule starts executing, it + // "sends" `rule-starting-`, which this code waits for. It + // then updates the alert doc, and "sends" `rule-complete-`. + // which the rule executor is waiting on, to complete the rule + // execution. + log(`signal the rule to finish the first run`); + await esTestIndexTool.indexDoc(source, 'rule-complete-1'); + + log(`wait for the first alert doc to be created`); + const initialDocs = await waitForAlertDocs(aadIndex, ruleId, count); + expect(initialDocs.length).to.eql(count); + + log(`wait for the start of the next execution`); + await esTestIndexTool.waitForDocs(source, 'rule-starting-2'); + + log(`ad-hoc update the alert doc`); + await adHocUpdate(es, aadIndex, initialDocs[0]._id); + + log(`signal the rule to finish`); + await esTestIndexTool.indexDoc(source, 'rule-complete-2'); + + log(`wait for the start of the next execution`); + await esTestIndexTool.waitForDocs(source, 'rule-starting-3'); + + log(`get the updated alert doc`); + const updatedDocs = await waitForAlertDocs(aadIndex, ruleId, count); + expect(updatedDocs.length).to.eql(1); + + log(`signal the rule to finish, then delete it`); + await esTestIndexTool.indexDoc(source, 'rule-complete-3'); + await objectRemover.removeAll(); + + // compare the initial and updated alert docs + compareAlertDocs(initialDocs[0], updatedDocs[0], true); + }); + + it('for a mix of successful and conflicted alerts', async () => { + const source = uuidv4(); + const count = 5; + const params = { source, alerts: count }; + const createdRule = await supertestWithoutAuth + .post(`${getUrlPrefix(Spaces.space1.id)}/api/alerting/rule`) + .set('kbn-xsrf', 'foo') + .send( + getTestRuleData({ + name: `${basename(__filename)} ${ruleType} ${source}}`, + rule_type_id: ruleType, + schedule: { interval: '1s' }, + throttle: null, + params, + actions: [], + }) + ); + + if (createdRule.status !== 200) { + log(`error creating rule: ${JSON.stringify(createdRule, null, 4)}`); + } + expect(createdRule.status).to.eql(200); + + const ruleId = createdRule.body.id; + objectRemover.add(Spaces.space1.id, ruleId, 'rule', 'alerting'); + + log(`signal the rule to finish the first run`); + await esTestIndexTool.indexDoc(source, 'rule-complete-1'); + + log(`wait for the first alert doc to be created`); + const initialDocs = await waitForAlertDocs(aadIndex, ruleId, count); + initialDocs.sort(sortAlertDocsByInstanceId); + expect(initialDocs.length).to.eql(5); + + log(`wait for the start of the next execution`); + await esTestIndexTool.waitForDocs(source, 'rule-starting-2'); + + log(`ad-hoc update the 2nd and 4th alert docs`); + await adHocUpdate(es, aadIndex, initialDocs[1]._id); + await adHocUpdate(es, aadIndex, initialDocs[3]._id); + + log(`signal the rule to finish`); + await esTestIndexTool.indexDoc(source, 'rule-complete-2'); + + log(`wait for the start of the next execution`); + await esTestIndexTool.waitForDocs(source, 'rule-starting-3'); + + log(`get the updated alert doc`); + const updatedDocs = await waitForAlertDocs(aadIndex, ruleId, count); + updatedDocs.sort(sortAlertDocsByInstanceId); + expect(updatedDocs.length).to.eql(5); + + log(`signal the rule to finish, then delete it`); + await esTestIndexTool.indexDoc(source, 'rule-complete-3'); + await objectRemover.removeAll(); + + // compare the initial and updated alert docs + compareAlertDocs(initialDocs[0], updatedDocs[0], false); + compareAlertDocs(initialDocs[1], updatedDocs[1], true); + compareAlertDocs(initialDocs[2], updatedDocs[2], false); + compareAlertDocs(initialDocs[3], updatedDocs[3], true); + compareAlertDocs(initialDocs[4], updatedDocs[4], false); + }); + }); + }); + + // waits for a specified number of alert documents + async function waitForAlertDocs( + index: string, + ruleId: string, + count: number = 1 + ): Promise>> { + return await retry.try(async () => { + const searchResult = await es.search({ + index, + size: count, + body: { + query: { + bool: { + must: [{ term: { 'kibana.alert.rule.uuid': ruleId } }], + }, + }, + }, + }); + + const docs = searchResult.hits.hits as Array>; + if (docs.length < count) throw new Error(`only ${docs.length} out of ${count} docs found`); + + return docs; + }); + } +} + +// general comparator for initial / updated alert documents +function compareAlertDocs( + initialDoc: SearchHit, + updatedDoc: SearchHit, + conflicted: boolean +) { + // ensure both rule run updates and other updates persisted + if (!initialDoc) throw new Error('not enough initial docs'); + if (!updatedDoc) throw new Error('not enough updated docs'); + + const initialAlert = initialDoc._source!; + const updatedAlert = updatedDoc._source!; + + expect(initialAlert.runCount).to.be.greaterThan(0); + expect(updatedAlert.runCount).not.to.eql(-1); + expect(updatedAlert.runCount).to.be.greaterThan(initialAlert.runCount); + + if (conflicted) { + expect(get(updatedAlert, 'kibana.alert.case_ids')).to.eql( + get(DocUpdate, 'kibana.alert.case_ids') + ); + expect(get(updatedAlert, 'kibana.alert.workflow_tags')).to.eql( + get(DocUpdate, 'kibana.alert.workflow_tags') + ); + expect(get(updatedAlert, 'kibana.alert.workflow_status')).to.eql( + get(DocUpdate, 'kibana.alert.workflow_status') + ); + + expect(get(initialAlert, 'kibana.alert.status')).to.be('active'); + expect(get(updatedAlert, 'kibana.alert.status')).to.be('untracked'); + } + + const initial = omit(initialAlert, SkipFields); + const updated = omit(updatedAlert, SkipFields); + + expect(initial).to.eql(updated); +} + +// perform an adhoc update to an alert doc +async function adHocUpdate(es: Client, index: string, id: string) { + const body = { doc: DocUpdate }; + await es.update({ index, id, body, refresh: true }); +} + +// we'll do the adhoc updates with this data +const DocUpdate = { + runCount: -1, // rule-specific field, will be overwritten by rule execution + kibana: { + alert: { + action_group: 'not-the-default', // will be overwritten by rule execution + // below are all fields that will NOT be overwritten by rule execution + workflow_status: 'a-ok!', + workflow_tags: ['fee', 'fi', 'fo', 'fum'], + case_ids: ['123', '456', '789'], + status: 'untracked', + }, + }, +}; + +const SkipFields = [ + // dynamically changing fields we have no control over + '@timestamp', + 'event.action', + 'kibana.alert.duration.us', + 'kibana.alert.flapping_history', + 'kibana.alert.rule.execution.uuid', + + // fields under our control we test separately + 'runCount', + 'kibana.alert.status', + 'kibana.alert.case_ids', + 'kibana.alert.workflow_tags', + 'kibana.alert.workflow_status', +]; + +function log(message: string) { + // eslint-disable-next-line no-console + console.log(`${new Date().toISOString()} ${message}`); +} diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/alerts_as_data/index.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/alerts_as_data/index.ts index 9156fb9e8ec37..20342e053016d 100644 --- a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/alerts_as_data/index.ts +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/alerts_as_data/index.ts @@ -13,5 +13,6 @@ export default function alertsAsDataTests({ loadTestFile }: FtrProviderContext) loadTestFile(require.resolve('./install_resources')); loadTestFile(require.resolve('./alerts_as_data')); loadTestFile(require.resolve('./alerts_as_data_flapping')); + loadTestFile(require.resolve('./alerts_as_data_conflicts')); }); } diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/generate_alert_schemas.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/generate_alert_schemas.ts index 478a9b17a21f5..51b1adc7526a9 100644 --- a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/generate_alert_schemas.ts +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/generate_alert_schemas.ts @@ -71,7 +71,7 @@ export default function checkAlertSchemasTest({ getService }: FtrProviderContext createSchemaFromFieldMap({ outputFile: `schemas/generated/${alertsDefinition.context.replaceAll( - '.', + /[.\-]/g, '_' )}_schema.ts`, fieldMap: alertsDefinition.mappings.fieldMap, diff --git a/x-pack/test/functional/apps/dashboard/group3/reporting/screenshots.ts b/x-pack/test/functional/apps/dashboard/group3/reporting/screenshots.ts index 490ba84c8496c..6ecaa84c96974 100644 --- a/x-pack/test/functional/apps/dashboard/group3/reporting/screenshots.ts +++ b/x-pack/test/functional/apps/dashboard/group3/reporting/screenshots.ts @@ -226,7 +226,7 @@ export default function ({ updateBaselines ); - expect(percentDiff).to.be.lessThan(0.03); + expect(percentDiff).to.be.lessThan(0.035); }); }); }); diff --git a/x-pack/test/functional/apps/lens/group6/error_handling.ts b/x-pack/test/functional/apps/lens/group6/error_handling.ts index f268e829ca5fb..ccdb193b30951 100644 --- a/x-pack/test/functional/apps/lens/group6/error_handling.ts +++ b/x-pack/test/functional/apps/lens/group6/error_handling.ts @@ -142,7 +142,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const errorMessages = await Promise.all(failureElements.map((el) => el.getVisibleText())); expect(errorMessages).to.eql([ - 'Bad Request', + 'Visualization type not found.', 'The visualization type lnsUNKNOWN could not be resolved.', 'Could not find datasource for the visualization', ]); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rules_table/rules_table_filtering.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rules_table/rules_table_filtering.cy.ts index 537bd67d98aae..ff73d4c5775a7 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rules_table/rules_table_filtering.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rules_table/rules_table_filtering.cy.ts @@ -29,9 +29,7 @@ import { import { disableAutoRefresh } from '../../../../tasks/alerts_detection_rules'; import { getNewRule } from '../../../../objects/rule'; -// TODO: https://github.com/elastic/kibana/issues/161540 -// Flaky in serverless tests -describe('Rules table: filtering', { tags: ['@ess', '@serverless', '@skipInServerless'] }, () => { +describe('Rules table: filtering', { tags: ['@ess', '@serverless'] }, () => { before(() => { cleanKibana(); }); @@ -44,11 +42,8 @@ describe('Rules table: filtering', { tags: ['@ess', '@serverless', '@skipInServe cy.task('esArchiverResetKibana'); }); - // TODO: https://github.com/elastic/kibana/issues/161540 - describe.skip('Last response filter', () => { - // Flaky in serverless tests - // @brokenInServerless tag is not working so a skip was needed - it('Filters rules by last response', { tags: ['@brokenInServerless'] }, function () { + describe('Last response filter', () => { + it('Filters rules by last response', function () { deleteIndex('test_index'); createIndex('test_index', { diff --git a/x-pack/test/security_solution_cypress/cypress/tasks/api_calls/elasticsearch.ts b/x-pack/test/security_solution_cypress/cypress/tasks/api_calls/elasticsearch.ts index e5edbaf65bd0a..dd7c1a71048f2 100644 --- a/x-pack/test/security_solution_cypress/cypress/tasks/api_calls/elasticsearch.ts +++ b/x-pack/test/security_solution_cypress/cypress/tasks/api_calls/elasticsearch.ts @@ -9,8 +9,12 @@ import { rootRequest } from '../common'; export const deleteIndex = (index: string) => { rootRequest({ method: 'DELETE', - url: `${Cypress.env('ELASTICSEARCH_URL')}/${index}?refresh=wait_for`, - headers: { 'kbn-xsrf': 'cypress-creds', 'x-elastic-internal-origin': 'security-solution' }, + url: `${Cypress.env('ELASTICSEARCH_URL')}/${index}`, + headers: { + 'kbn-xsrf': 'cypress-creds', + 'x-elastic-internal-origin': 'security-solution', + 'elastic-api-version': '2023-10-31', + }, failOnStatusCode: false, }); }; @@ -19,7 +23,11 @@ export const deleteDataStream = (dataStreamName: string) => { rootRequest({ method: 'DELETE', url: `${Cypress.env('ELASTICSEARCH_URL')}/_data_stream/${dataStreamName}`, - headers: { 'kbn-xsrf': 'cypress-creds', 'x-elastic-internal-origin': 'security-solution' }, + headers: { + 'kbn-xsrf': 'cypress-creds', + 'x-elastic-internal-origin': 'security-solution', + 'elastic-api-version': '2023-10-31', + }, failOnStatusCode: false, }); }; @@ -30,6 +38,11 @@ export const deleteAllDocuments = (target: string) => url: `${Cypress.env( 'ELASTICSEARCH_URL' )}/${target}/_delete_by_query?conflicts=proceed&scroll_size=10000&refresh`, + headers: { + 'kbn-xsrf': 'cypress-creds', + 'x-elastic-internal-origin': 'security-solution', + 'elastic-api-version': '2023-10-31', + }, body: { query: { match_all: {}, @@ -41,6 +54,11 @@ export const createIndex = (indexName: string, properties: Record({ method: 'GET', url: `${Cypress.env('ELASTICSEARCH_URL')}/${index}/_search`, - headers: { 'kbn-xsrf': 'cypress-creds', 'x-elastic-internal-origin': 'security-solution' }, + headers: { + 'kbn-xsrf': 'cypress-creds', + 'x-elastic-internal-origin': 'security-solution', + 'elastic-api-version': '2023-10-31', + }, failOnStatusCode: false, }).then((response) => { if (response.status !== 200) { @@ -80,7 +107,11 @@ export const refreshIndex = (index: string) => { rootRequest({ method: 'POST', url: `${Cypress.env('ELASTICSEARCH_URL')}/${index}/_refresh`, - headers: { 'kbn-xsrf': 'cypress-creds', 'x-elastic-internal-origin': 'security-solution' }, + headers: { + 'kbn-xsrf': 'cypress-creds', + 'x-elastic-internal-origin': 'security-solution', + 'elastic-api-version': '2023-10-31', + }, failOnStatusCode: false, }).then((response) => { if (response.status !== 200) {