diff --git a/.buildkite/ftr_oblt_stateful_configs.yml b/.buildkite/ftr_oblt_stateful_configs.yml index 4edf75f385816..55f3c1b603fca 100644 --- a/.buildkite/ftr_oblt_stateful_configs.yml +++ b/.buildkite/ftr_oblt_stateful_configs.yml @@ -30,6 +30,7 @@ enabled: - x-pack/test/api_integration/apis/synthetics/config.ts - x-pack/test/api_integration/apis/slos/config.ts - x-pack/test/api_integration/apis/uptime/config.ts + - x-pack/test/api_integration/apis/entity_manager/config.ts - x-pack/test/apm_api_integration/basic/config.ts - x-pack/test/apm_api_integration/cloud/config.ts - x-pack/test/apm_api_integration/rules/config.ts @@ -48,4 +49,3 @@ enabled: - x-pack/test/observability_ai_assistant_functional/enterprise/config.ts - x-pack/test/profiling_api_integration/cloud/config.ts - x-pack/test/functional/apps/apm/config.ts - \ No newline at end of file diff --git a/docs/developer/plugin-list.asciidoc b/docs/developer/plugin-list.asciidoc index aff3a490c8d9b..00b2bd8b6a624 100644 --- a/docs/developer/plugin-list.asciidoc +++ b/docs/developer/plugin-list.asciidoc @@ -487,7 +487,7 @@ The plugin exposes the static DefaultEditorController class to consume. |{kib-repo}blob/{branch}/x-pack/plugins/cloud/README.md[cloud] -|The cloud plugin adds Cloud-specific features to Kibana. +|The cloud plugin exposes Cloud-specific metadata to Kibana. |{kib-repo}blob/{branch}/x-pack/plugins/cloud_integrations/cloud_chat/README.md[cloudChat] diff --git a/packages/core/http/core-http-router-server-internal/src/response_adapter.ts b/packages/core/http/core-http-router-server-internal/src/response_adapter.ts index a5cc1ca1f8d96..1ddb5572d01f7 100644 --- a/packages/core/http/core-http-router-server-internal/src/response_adapter.ts +++ b/packages/core/http/core-http-router-server-internal/src/response_adapter.ts @@ -115,7 +115,7 @@ export class HapiResponseAdapter { return response; } - private toError(kibanaResponse: KibanaResponse) { + private toError(kibanaResponse: KibanaResponse) { const { payload } = kibanaResponse; // Special case for when we are proxying requests and want to enable streaming back error responses opaquely. @@ -153,7 +153,12 @@ function getErrorMessage(payload?: ResponseError): string { if (!payload) { throw new Error('expected error message to be provided'); } - if (typeof payload === 'string') return payload; + if (typeof payload === 'string') { + return payload; + } + if (isStreamOrBuffer(payload)) { + throw new Error(`can't resolve error message from stream or buffer`); + } // for ES response errors include nested error reason message. it doesn't contain sensitive data. if (isElasticsearchResponseError(payload)) { return `[${payload.message}]: ${ @@ -164,6 +169,10 @@ function getErrorMessage(payload?: ResponseError): string { return getErrorMessage(payload.message); } +function isStreamOrBuffer(payload: ResponseError): payload is stream.Stream | Buffer { + return Buffer.isBuffer(payload) || stream.isReadable(payload as stream.Readable); +} + function getErrorAttributes(payload?: ResponseError): ResponseErrorAttributes | undefined { return typeof payload === 'object' && 'attributes' in payload ? payload.attributes : undefined; } diff --git a/packages/core/http/core-http-server/src/router/response.ts b/packages/core/http/core-http-server/src/router/response.ts index 385a03f518e7f..7e318f443a1cf 100644 --- a/packages/core/http/core-http-server/src/router/response.ts +++ b/packages/core/http/core-http-server/src/router/response.ts @@ -39,6 +39,8 @@ export type ResponseErrorAttributes = Record; */ export type ResponseError = | string + | Buffer + | Stream | Error | { message: string | Error; diff --git a/packages/core/http/core-http-server/src/router/response_factory.ts b/packages/core/http/core-http-server/src/router/response_factory.ts index f7c763da024bf..c4455be73e16f 100644 --- a/packages/core/http/core-http-server/src/router/response_factory.ts +++ b/packages/core/http/core-http-server/src/router/response_factory.ts @@ -6,7 +6,6 @@ * Side Public License, v 1. */ -import type { Stream } from 'stream'; import type { CustomHttpResponseOptions, HttpResponseOptions, @@ -139,7 +138,7 @@ export interface KibanaErrorResponseFactory { * Creates an error response with defined status code and payload. * @param options - {@link CustomHttpResponseOptions} configures HTTP response headers, error message and other error details to pass to the client */ - customError(options: CustomHttpResponseOptions): IKibanaResponse; + customError(options: CustomHttpResponseOptions): IKibanaResponse; } /** diff --git a/packages/kbn-analytics/README.md b/packages/kbn-analytics/README.md new file mode 100644 index 0000000000000..7cd705ea223fd --- /dev/null +++ b/packages/kbn-analytics/README.md @@ -0,0 +1,12 @@ +# `@kbn/analytics` + +> [!NOTE] +> The term _analytics_ here refers to _Usage Analytics_, and should not be confused with the Kibana (Data) Analytics tools. + +> [!IMPORTANT] +> This package is exclusively used by the plugin `usage_collection` and it's not expected to be used elsewhere. +> If you are still here for _Usage Analytics_, you might be looking for [core-analytics](../core/analytics), the [EBT packages](../analytics). + +This package implements the report that batches updates from Application Usage, UI Counters, and User Agent. +It defines the contract of the report, and the strategy to ship it to the server. + diff --git a/test/plugin_functional/plugins/core_http/server/plugin.ts b/test/plugin_functional/plugins/core_http/server/plugin.ts index 534d55a97bdfe..c5e8dd01847be 100644 --- a/test/plugin_functional/plugins/core_http/server/plugin.ts +++ b/test/plugin_functional/plugins/core_http/server/plugin.ts @@ -6,6 +6,7 @@ * Side Public License, v 1. */ +import { Readable } from 'stream'; import type { Plugin, CoreSetup } from '@kbn/core/server'; export class CoreHttpPlugin implements Plugin { @@ -87,6 +88,32 @@ export class CoreHttpPlugin implements Plugin { }, }); }); + + router.get( + { + path: '/api/core_http/error_stream', + validate: false, + }, + async (ctx, req, res) => { + return res.customError({ + body: Readable.from(['error stream'], { objectMode: false }), + statusCode: 501, + }); + } + ); + + router.get( + { + path: '/api/core_http/error_buffer', + validate: false, + }, + async (ctx, req, res) => { + return res.customError({ + body: Buffer.from('error buffer', 'utf8'), + statusCode: 501, + }); + } + ); } public start() {} diff --git a/test/plugin_functional/test_suites/core_plugins/error_response.ts b/test/plugin_functional/test_suites/core_plugins/error_response.ts new file mode 100644 index 0000000000000..0a87b4c9a6bd4 --- /dev/null +++ b/test/plugin_functional/test_suites/core_plugins/error_response.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import expect from '@kbn/expect'; +import '@kbn/core-provider-plugin/types'; +import { PluginFunctionalProviderContext } from '../../services'; + +export default function ({ getService }: PluginFunctionalProviderContext) { + const supertest = getService('supertest'); + + // routes defined in the `core_http` test plugin + describe('Custom errors', () => { + it('can serve an error response from stream', async () => { + await supertest + .get('/api/core_http/error_stream') + .expect(501) + .then((response) => { + const res = response.body.toString(); + expect(res).to.eql('error stream'); + }); + }); + + it('can serve an error response from buffer', async () => { + await supertest + .get('/api/core_http/error_buffer') + .expect(501) + .then((response) => { + const res = response.body.toString(); + expect(res).to.eql('error buffer'); + }); + }); + }); +} diff --git a/test/plugin_functional/test_suites/core_plugins/index.ts b/test/plugin_functional/test_suites/core_plugins/index.ts index 07e258e34e3f1..5e3d969bb0277 100644 --- a/test/plugin_functional/test_suites/core_plugins/index.ts +++ b/test/plugin_functional/test_suites/core_plugins/index.ts @@ -26,5 +26,6 @@ export default function ({ loadTestFile }: PluginFunctionalProviderContext) { loadTestFile(require.resolve('./http')); loadTestFile(require.resolve('./http_versioned')); loadTestFile(require.resolve('./dynamic_contract_resolving')); + loadTestFile(require.resolve('./error_response')); }); } diff --git a/test/plugin_functional/test_suites/core_plugins/rendering.ts b/test/plugin_functional/test_suites/core_plugins/rendering.ts index a5b5cba1469ba..27addbfc274a2 100644 --- a/test/plugin_functional/test_suites/core_plugins/rendering.ts +++ b/test/plugin_functional/test_suites/core_plugins/rendering.ts @@ -246,6 +246,7 @@ export default function ({ getService }: PluginFunctionalProviderContext) { 'xpack.cloud_integrations.full_story.eventTypesAllowlist (array)', 'xpack.cloud_integrations.full_story.pageVarsDebounceTime (duration)', 'xpack.cloud.id (string)', + 'xpack.cloud.organization_id (string)', 'xpack.cloud.organization_url (string)', 'xpack.cloud.billing_url (string)', 'xpack.cloud.profile_url (string)', @@ -256,6 +257,7 @@ export default function ({ getService }: PluginFunctionalProviderContext) { 'xpack.cloud.serverless.project_id (string)', 'xpack.cloud.serverless.project_name (string)', 'xpack.cloud.serverless.project_type (string)', + 'xpack.cloud.serverless.orchestrator_target (string)', 'xpack.cloud.onboarding.default_solution (string)', 'xpack.discoverEnhanced.actions.exploreDataInChart.enabled (boolean)', 'xpack.discoverEnhanced.actions.exploreDataInContextMenu.enabled (boolean)', diff --git a/x-pack/packages/kbn-entities-schema/src/schema/__snapshots__/common.test.ts.snap b/x-pack/packages/kbn-entities-schema/src/schema/__snapshots__/common.test.ts.snap new file mode 100644 index 0000000000000..4067947f7ddcf --- /dev/null +++ b/x-pack/packages/kbn-entities-schema/src/schema/__snapshots__/common.test.ts.snap @@ -0,0 +1,132 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`schemas metadataSchema should error on empty string 1`] = ` +Object { + "error": [ZodError: [ + { + "path": [ + "source" + ], + "code": "custom", + "message": "source should not be empty" + }, + { + "path": [ + "destination" + ], + "code": "custom", + "message": "destination should not be empty" + } +]], + "success": false, +} +`; + +exports[`schemas metadataSchema should error on empty string for destination 1`] = ` +Object { + "error": [ZodError: [ + { + "path": [ + "destination" + ], + "code": "custom", + "message": "destination should not be empty" + } +]], + "success": false, +} +`; + +exports[`schemas metadataSchema should error on empty string for source 1`] = ` +Object { + "error": [ZodError: [ + { + "path": [ + "source" + ], + "code": "custom", + "message": "source should not be empty" + }, + { + "path": [ + "destination" + ], + "code": "custom", + "message": "destination should not be empty" + } +]], + "success": false, +} +`; + +exports[`schemas metadataSchema should error when limit is too low 1`] = ` +Object { + "error": [ZodError: [ + { + "path": [ + "limit" + ], + "code": "custom", + "message": "limit should be greater than 1" + } +]], + "success": false, +} +`; + +exports[`schemas metadataSchema should parse successfully with a source and desitination 1`] = ` +Object { + "data": Object { + "destination": "hostName", + "limit": 1000, + "source": "host.name", + }, + "success": true, +} +`; + +exports[`schemas metadataSchema should parse successfully with an valid string 1`] = ` +Object { + "data": Object { + "destination": "host.name", + "limit": 1000, + "source": "host.name", + }, + "success": true, +} +`; + +exports[`schemas metadataSchema should parse successfully with just a source 1`] = ` +Object { + "data": Object { + "destination": "host.name", + "limit": 1000, + "source": "host.name", + }, + "success": true, +} +`; + +exports[`schemas metadataSchema should parse successfully with valid object 1`] = ` +Object { + "data": Object { + "destination": "hostName", + "limit": 1000, + "source": "host.name", + }, + "success": true, +} +`; + +exports[`schemas semVerSchema should not validate with 0.9 1`] = ` +Object { + "error": [ZodError: [ + { + "code": "custom", + "message": "The string does use the Semantic Versioning (Semver) format of {major}.{minor}.{patch} (e.g., 1.0.0), ensure each part contains only digits.", + "path": [] + } +]], + "success": false, +} +`; diff --git a/x-pack/packages/kbn-entities-schema/src/schema/common.test.ts b/x-pack/packages/kbn-entities-schema/src/schema/common.test.ts new file mode 100644 index 0000000000000..c03bff2db74c0 --- /dev/null +++ b/x-pack/packages/kbn-entities-schema/src/schema/common.test.ts @@ -0,0 +1,106 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SafeParseSuccess } from 'zod'; +import { durationSchema, metadataSchema, semVerSchema } from './common'; +import moment from 'moment'; + +describe('schemas', () => { + describe('metadataSchema', () => { + it('should error on empty string', () => { + const result = metadataSchema.safeParse(''); + expect(result.success).toBeFalsy(); + expect(result).toMatchSnapshot(); + }); + it('should error on empty string for source', () => { + const result = metadataSchema.safeParse({ source: '' }); + expect(result.success).toBeFalsy(); + expect(result).toMatchSnapshot(); + }); + it('should error on empty string for destination', () => { + const result = metadataSchema.safeParse({ source: 'host.name', destination: '', limit: 10 }); + expect(result.success).toBeFalsy(); + expect(result).toMatchSnapshot(); + }); + it('should error when limit is too low', () => { + const result = metadataSchema.safeParse({ + source: 'host.name', + destination: 'host.name', + limit: 0, + }); + expect(result.success).toBeFalsy(); + expect(result).toMatchSnapshot(); + }); + it('should parse successfully with an valid string', () => { + const result = metadataSchema.safeParse('host.name'); + expect(result.success).toBeTruthy(); + expect(result).toMatchSnapshot(); + }); + it('should parse successfully with just a source', () => { + const result = metadataSchema.safeParse({ source: 'host.name' }); + expect(result.success).toBeTruthy(); + expect(result).toMatchSnapshot(); + }); + it('should parse successfully with a source and desitination', () => { + const result = metadataSchema.safeParse({ source: 'host.name', destination: 'hostName' }); + expect(result.success).toBeTruthy(); + expect(result).toMatchSnapshot(); + }); + it('should parse successfully with valid object', () => { + const result = metadataSchema.safeParse({ + source: 'host.name', + destination: 'hostName', + size: 1, + }); + expect(result.success).toBeTruthy(); + expect(result).toMatchSnapshot(); + }); + }); + describe('durationSchema', () => { + it('should work with 1m', () => { + const result = durationSchema.safeParse('1m'); + expect(result.success).toBeTruthy(); + expect((result as SafeParseSuccess).data.toJSON()).toBe('1m'); + expect((result as SafeParseSuccess).data.asSeconds()).toEqual(60); + }); + it('should work with 10s', () => { + const result = durationSchema.safeParse('10s'); + expect(result.success).toBeTruthy(); + expect((result as SafeParseSuccess).data.toJSON()).toBe('10s'); + expect((result as SafeParseSuccess).data.asSeconds()).toEqual(10); + }); + it('should work with 999h', () => { + const result = durationSchema.safeParse('999h'); + expect(result.success).toBeTruthy(); + expect((result as SafeParseSuccess).data.toJSON()).toBe('999h'); + expect((result as SafeParseSuccess).data.asSeconds()).toEqual(999 * 60 * 60); + }); + it('should work with 90d', () => { + const result = durationSchema.safeParse('90d'); + expect(result.success).toBeTruthy(); + expect((result as SafeParseSuccess).data.toJSON()).toBe('90d'); + expect((result as SafeParseSuccess).data.asSeconds()).toEqual( + 90 * 24 * 60 * 60 + ); + }); + it('should not work with 1ms', () => { + const result = durationSchema.safeParse('1ms'); + expect(result.success).toBeFalsy(); + }); + }); + describe('semVerSchema', () => { + it('should validate with 999.999.999', () => { + const result = semVerSchema.safeParse('999.999.999'); + expect(result.success).toBeTruthy(); + }); + it('should not validate with 0.9', () => { + const result = semVerSchema.safeParse('0.9'); + expect(result.success).toBeFalsy(); + expect(result).toMatchSnapshot(); + }); + }); +}); diff --git a/x-pack/packages/kbn-entities-schema/src/schema/common.ts b/x-pack/packages/kbn-entities-schema/src/schema/common.ts index df6d20ef1d44b..6576a1c650a10 100644 --- a/x-pack/packages/kbn-entities-schema/src/schema/common.ts +++ b/x-pack/packages/kbn-entities-schema/src/schema/common.ts @@ -45,7 +45,7 @@ export const docCountMetricSchema = z.object({ export const durationSchema = z .string() - .regex(/\d+[m|d|s|h]/) + .regex(/^\d+[m|d|s|h]$/) .transform((val: string) => { const parts = val.match(/(\d+)([m|s|h|d])/); if (parts === null) { @@ -93,7 +93,30 @@ export const metadataSchema = z destination: metadata.destination ?? metadata.source, limit: metadata.limit ?? 1000, })) - .or(z.string().transform((value) => ({ source: value, destination: value, limit: 1000 }))); + .or(z.string().transform((value) => ({ source: value, destination: value, limit: 1000 }))) + .superRefine((value, ctx) => { + if (value.limit < 1) { + ctx.addIssue({ + path: ['limit'], + code: z.ZodIssueCode.custom, + message: 'limit should be greater than 1', + }); + } + if (value.source.length === 0) { + ctx.addIssue({ + path: ['source'], + code: z.ZodIssueCode.custom, + message: 'source should not be empty', + }); + } + if (value.destination.length === 0) { + ctx.addIssue({ + path: ['destination'], + code: z.ZodIssueCode.custom, + message: 'destination should not be empty', + }); + } + }); export const identityFieldsSchema = z .object({ diff --git a/x-pack/packages/kbn-entities-schema/src/schema/entity_definition.ts b/x-pack/packages/kbn-entities-schema/src/schema/entity_definition.ts index d433cc473a538..c297a2d5542ae 100644 --- a/x-pack/packages/kbn-entities-schema/src/schema/entity_definition.ts +++ b/x-pack/packages/kbn-entities-schema/src/schema/entity_definition.ts @@ -40,6 +40,9 @@ export const entityDefinitionSchema = z.object({ syncField: z.optional(z.string()), syncDelay: z.optional(z.string()), frequency: z.optional(z.string()), + backfillSyncDelay: z.optional(z.string()), + backfillLookbackPeriod: z.optional(durationSchema), + backfillFrequency: z.optional(z.string()), }) ), }), diff --git a/x-pack/plugins/cloud/README.md b/x-pack/plugins/cloud/README.md index 00aa160fb3600..6878c72eb4c5f 100644 --- a/x-pack/plugins/cloud/README.md +++ b/x-pack/plugins/cloud/README.md @@ -1,3 +1,3 @@ # `cloud` plugin -The `cloud` plugin adds Cloud-specific features to Kibana. \ No newline at end of file +The `cloud` plugin exposes Cloud-specific metadata to Kibana. \ No newline at end of file diff --git a/x-pack/plugins/cloud/common/register_cloud_deployment_id_analytics_context.ts b/x-pack/plugins/cloud/common/register_cloud_deployment_id_analytics_context.ts index 204b940c45cd5..e4c5a88a847c4 100644 --- a/x-pack/plugins/cloud/common/register_cloud_deployment_id_analytics_context.ts +++ b/x-pack/plugins/cloud/common/register_cloud_deployment_id_analytics_context.ts @@ -11,12 +11,14 @@ import { parseDeploymentIdFromDeploymentUrl } from './parse_deployment_id_from_d export interface CloudDeploymentMetadata { id?: string; + organization_id?: string; trial_end_date?: string; is_elastic_staff_owned?: boolean; deployment_url?: string; serverless?: { project_id?: string; project_type?: string; + orchestrator_target?: string; }; } @@ -29,26 +31,40 @@ export function registerCloudDeploymentMetadataAnalyticsContext( } const { id: cloudId, + organization_id: organizationId, trial_end_date: cloudTrialEndDate, is_elastic_staff_owned: cloudIsElasticStaffOwned, - serverless: { project_id: projectId, project_type: projectType } = {}, + serverless: { + project_id: projectId, + project_type: projectType, + orchestrator_target: orchestratorTarget, + } = {}, } = cloudMetadata; analytics.registerContextProvider({ name: 'Cloud Deployment Metadata', context$: of({ cloudId, + organizationId, deploymentId: parseDeploymentIdFromDeploymentUrl(cloudMetadata.deployment_url), cloudTrialEndDate, cloudIsElasticStaffOwned, projectId, projectType, + orchestratorTarget, }), schema: { cloudId: { type: 'keyword', _meta: { description: 'The Cloud ID' }, }, + organizationId: { + type: 'keyword', + _meta: { + description: 'The Elastic Cloud Organization ID that owns the deployment/project', + optional: true, + }, + }, deploymentId: { type: 'keyword', _meta: { description: 'The Deployment ID', optional: true }, @@ -72,6 +88,13 @@ export function registerCloudDeploymentMetadataAnalyticsContext( type: 'keyword', _meta: { description: 'The Serverless Project type', optional: true }, }, + orchestratorTarget: { + type: 'keyword', + _meta: { + description: 'The Orchestrator Target where it is deployed (canary/non-canary)', + optional: true, + }, + }, }, }); } diff --git a/x-pack/plugins/cloud/public/plugin.tsx b/x-pack/plugins/cloud/public/plugin.tsx index 0d071900418c3..d2671b18e4d68 100644 --- a/x-pack/plugins/cloud/public/plugin.tsx +++ b/x-pack/plugins/cloud/public/plugin.tsx @@ -21,6 +21,7 @@ import { getSupportUrl } from './utils'; export interface CloudConfigType { id?: string; + organization_id?: string; cname?: string; base_url?: string; profile_url?: string; @@ -40,6 +41,7 @@ export interface CloudConfigType { project_id: string; project_name?: string; project_type?: string; + orchestrator_target?: string; }; } @@ -89,6 +91,7 @@ export class CloudPlugin implements Plugin { return { cloudId: id, + organizationId: this.config.organization_id, deploymentId: parseDeploymentIdFromDeploymentUrl(this.config.deployment_url), cname, baseUrl, @@ -108,6 +111,7 @@ export class CloudPlugin implements Plugin { projectId: this.config.serverless?.project_id, projectName: this.config.serverless?.project_name, projectType: this.config.serverless?.project_type, + orchestratorTarget: this.config.serverless?.orchestrator_target, }, registerCloudService: (contextProvider) => { this.contextProviders.push(contextProvider); diff --git a/x-pack/plugins/cloud/public/types.ts b/x-pack/plugins/cloud/public/types.ts index a7e34c79a8505..dd3dcf27c1a61 100644 --- a/x-pack/plugins/cloud/public/types.ts +++ b/x-pack/plugins/cloud/public/types.ts @@ -97,6 +97,10 @@ export interface CloudSetup { * Cloud ID. Undefined if not running on Cloud. */ cloudId?: string; + /** + * The Elastic Cloud Organization that owns this deployment/project. + */ + organizationId?: string; /** * The deployment's ID. Only available when running on Elastic Cloud. */ @@ -208,5 +212,10 @@ export interface CloudSetup { * Will always be present if `isServerlessEnabled` is `true` */ projectType?: string; + /** + * The serverless orchestrator target. The potential values are `canary` or `non-canary` + * Will always be present if `isServerlessEnabled` is `true` + */ + orchestratorTarget?: string; }; } diff --git a/x-pack/plugins/cloud/server/__snapshots__/plugin.test.ts.snap b/x-pack/plugins/cloud/server/__snapshots__/plugin.test.ts.snap index 41002d0c48e6b..fa873a89a85d7 100644 --- a/x-pack/plugins/cloud/server/__snapshots__/plugin.test.ts.snap +++ b/x-pack/plugins/cloud/server/__snapshots__/plugin.test.ts.snap @@ -20,8 +20,10 @@ Object { "onboarding": Object { "defaultSolution": undefined, }, + "organizationId": undefined, "projectsUrl": "https://cloud.elastic.co/projects/", "serverless": Object { + "orchestratorTarget": undefined, "projectId": undefined, "projectName": undefined, "projectType": undefined, diff --git a/x-pack/plugins/cloud/server/collectors/cloud_usage_collector.test.ts b/x-pack/plugins/cloud/server/collectors/cloud_usage_collector.test.ts index b9442fb74f94f..ec9a81ad0272b 100644 --- a/x-pack/plugins/cloud/server/collectors/cloud_usage_collector.test.ts +++ b/x-pack/plugins/cloud/server/collectors/cloud_usage_collector.test.ts @@ -38,10 +38,12 @@ describe('createCloudUsageCollector', () => { expect(await collector.fetch(collectorFetchContext)).toStrictEqual({ isCloudEnabled: true, isElasticStaffOwned: undefined, + organizationId: undefined, trialEndDate: undefined, deploymentId: undefined, projectId: undefined, projectType: undefined, + orchestratorTarget: undefined, }); }); @@ -54,11 +56,13 @@ describe('createCloudUsageCollector', () => { expect(await collector.fetch(collectorFetchContext)).toStrictEqual({ isCloudEnabled: true, isElasticStaffOwned: undefined, + organizationId: undefined, trialEndDate: '2020-10-01T14:30:16Z', inTrial: false, deploymentId: undefined, projectId: undefined, projectType: undefined, + orchestratorTarget: undefined, }); }); @@ -67,9 +71,11 @@ describe('createCloudUsageCollector', () => { isCloudEnabled: true, trialEndDate: '2020-10-01T14:30:16Z', isElasticStaffOwned: true, + organizationId: '1234', deploymentId: 'a-deployment-id', projectId: 'a-project-id', projectType: 'security', + orchestratorTarget: 'canary', }); expect(await collector.fetch(collectorFetchContext)).toStrictEqual({ @@ -77,9 +83,11 @@ describe('createCloudUsageCollector', () => { trialEndDate: '2020-10-01T14:30:16Z', inTrial: false, isElasticStaffOwned: true, + organizationId: '1234', deploymentId: 'a-deployment-id', projectId: 'a-project-id', projectType: 'security', + orchestratorTarget: 'canary', }); }); }); diff --git a/x-pack/plugins/cloud/server/collectors/cloud_usage_collector.ts b/x-pack/plugins/cloud/server/collectors/cloud_usage_collector.ts index 0b8415f755a76..2d1924817e56e 100644 --- a/x-pack/plugins/cloud/server/collectors/cloud_usage_collector.ts +++ b/x-pack/plugins/cloud/server/collectors/cloud_usage_collector.ts @@ -12,9 +12,11 @@ export interface CloudUsageCollectorConfig { // Using * | undefined instead of ?: to force the calling code to list all the options (even when they can be undefined) trialEndDate: string | undefined; isElasticStaffOwned: boolean | undefined; + organizationId: string | undefined; deploymentId: string | undefined; projectId: string | undefined; projectType: string | undefined; + orchestratorTarget: string | undefined; } interface CloudUsage { @@ -22,9 +24,11 @@ interface CloudUsage { trialEndDate?: string; inTrial?: boolean; isElasticStaffOwned?: boolean; + organizationId?: string; deploymentId?: string; projectId?: string; projectType?: string; + orchestratorTarget?: string; } export function createCloudUsageCollector( @@ -35,19 +39,36 @@ export function createCloudUsageCollector( isCloudEnabled, trialEndDate, isElasticStaffOwned, + organizationId, deploymentId, projectId, projectType, + orchestratorTarget, } = config; const trialEndDateMs = trialEndDate ? new Date(trialEndDate).getTime() : undefined; return usageCollection.makeUsageCollector({ type: 'cloud', isReady: () => true, schema: { - isCloudEnabled: { type: 'boolean' }, - trialEndDate: { type: 'date' }, - inTrial: { type: 'boolean' }, - isElasticStaffOwned: { type: 'boolean' }, + isCloudEnabled: { + type: 'boolean', + _meta: { description: 'Is the deployment running in Elastic Cloud (ESS or Serverless)?' }, + }, + trialEndDate: { type: 'date', _meta: { description: 'End of the trial period' } }, + inTrial: { + type: 'boolean', + _meta: { description: 'Is the organization during the trial period?' }, + }, + isElasticStaffOwned: { + type: 'boolean', + _meta: { description: 'Is the deploymend owned by an Elastician' }, + }, + organizationId: { + type: 'keyword', + _meta: { + description: 'The Elastic Cloud Organization ID that owns the deployment/project', + }, + }, deploymentId: { type: 'keyword', _meta: { description: 'The ESS Deployment ID' }, @@ -60,16 +81,22 @@ export function createCloudUsageCollector( type: 'keyword', _meta: { description: 'The Serverless Project type' }, }, + orchestratorTarget: { + type: 'keyword', + _meta: { description: 'The Orchestrator Target where it is deployed (canary/non-canary)' }, + }, }, fetch: () => { return { isCloudEnabled, isElasticStaffOwned, + organizationId, trialEndDate, ...(trialEndDateMs ? { inTrial: Date.now() <= trialEndDateMs } : {}), deploymentId, projectId, projectType, + orchestratorTarget, }; }, }); diff --git a/x-pack/plugins/cloud/server/config.ts b/x-pack/plugins/cloud/server/config.ts index de4ebd94b6f2b..371f895b92e09 100644 --- a/x-pack/plugins/cloud/server/config.ts +++ b/x-pack/plugins/cloud/server/config.ts @@ -25,6 +25,7 @@ const configSchema = schema.object({ deployments_url: schema.string({ defaultValue: '/deployments' }), deployment_url: schema.maybe(schema.string()), id: schema.maybe(schema.string()), + organization_id: schema.maybe(schema.string()), billing_url: schema.maybe(schema.string()), performance_url: schema.maybe(schema.string()), users_and_roles_url: schema.maybe(schema.string()), @@ -44,6 +45,7 @@ const configSchema = schema.object({ project_id: schema.maybe(schema.string()), project_name: schema.maybe(schema.string()), project_type: schema.maybe(schema.string()), + orchestrator_target: schema.maybe(schema.string()), }, // avoid future chicken-and-egg situation with the component populating the config { unknowns: 'ignore' } @@ -60,6 +62,7 @@ export const config: PluginConfigDescriptor = { deployments_url: true, deployment_url: true, id: true, + organization_id: true, billing_url: true, users_and_roles_url: true, performance_url: true, @@ -72,6 +75,7 @@ export const config: PluginConfigDescriptor = { project_id: true, project_name: true, project_type: true, + orchestrator_target: true, }, onboarding: { default_solution: true, diff --git a/x-pack/plugins/cloud/server/plugin.ts b/x-pack/plugins/cloud/server/plugin.ts index d8d5d397655e3..362a69b4ac0a6 100644 --- a/x-pack/plugins/cloud/server/plugin.ts +++ b/x-pack/plugins/cloud/server/plugin.ts @@ -35,6 +35,10 @@ export interface CloudSetup { * @note The `cloudId` is a concatenation of the deployment name and a hash. Users can update the deployment name, changing the `cloudId`. However, the changed `cloudId` will not be re-injected into `kibana.yml`. If you need the current `cloudId` the best approach is to split the injected `cloudId` on the semi-colon, and replace the first element with the `persistent.cluster.metadata.display_name` value as provided by a call to `GET _cluster/settings`. */ cloudId?: string; + /** + * The Elastic Cloud Organization that owns this deployment/project. + */ + organizationId?: string; /** * The deployment's ID. Only available when running on Elastic Cloud. */ @@ -127,6 +131,11 @@ export interface CloudSetup { * Will always be present if `isServerlessEnabled` is `true` */ projectType?: string; + /** + * The serverless orchestrator target. The potential values are `canary` or `non-canary` + * Will always be present if `isServerlessEnabled` is `true` + */ + orchestratorTarget?: string; }; } @@ -163,19 +172,23 @@ export class CloudPlugin implements Plugin { public setup(core: CoreSetup, { usageCollection }: PluginsSetup): CloudSetup { const isCloudEnabled = getIsCloudEnabled(this.config.id); + const organizationId = this.config.organization_id; const projectId = this.config.serverless?.project_id; const projectType = this.config.serverless?.project_type; + const orchestratorTarget = this.config.serverless?.orchestrator_target; const isServerlessEnabled = !!projectId; const deploymentId = parseDeploymentIdFromDeploymentUrl(this.config.deployment_url); registerCloudDeploymentMetadataAnalyticsContext(core.analytics, this.config); registerCloudUsageCollector(usageCollection, { isCloudEnabled, + organizationId, trialEndDate: this.config.trial_end_date, isElasticStaffOwned: this.config.is_elastic_staff_owned, deploymentId, projectId, projectType, + orchestratorTarget, }); let decodedId: DecodedCloudId | undefined; @@ -186,6 +199,7 @@ export class CloudPlugin implements Plugin { return { ...this.getCloudUrls(), cloudId: this.config.id, + organizationId, instanceSizeMb: readInstanceSizeMb(), deploymentId, elasticsearchUrl: decodedId?.elasticsearchUrl, @@ -207,6 +221,7 @@ export class CloudPlugin implements Plugin { projectId, projectName: this.config.serverless?.project_name, projectType, + orchestratorTarget, }, }; } diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/single_page_layout/hooks/setup_technology.test.ts b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/single_page_layout/hooks/setup_technology.test.ts index 8543c54a9ea2f..b3fadedcc964e 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/single_page_layout/hooks/setup_technology.test.ts +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/single_page_layout/hooks/setup_technology.test.ts @@ -157,7 +157,7 @@ describe('useAgentless', () => { }); describe('useSetupTechnology', () => { - const updateNewAgentPolicyMock = jest.fn(); + const setNewAgentPolicy = jest.fn(); const updateAgentPoliciesMock = jest.fn(); const setSelectedPolicyTabMock = jest.fn(); const newAgentPolicyMock = { @@ -202,7 +202,7 @@ describe('useSetupTechnology', () => { const { result } = renderHook(() => useSetupTechnology({ - updateNewAgentPolicy: updateNewAgentPolicyMock, + setNewAgentPolicy, newAgentPolicy: newAgentPolicyMock, updateAgentPolicies: updateAgentPoliciesMock, setSelectedPolicyTab: setSelectedPolicyTabMock, @@ -217,7 +217,7 @@ describe('useSetupTechnology', () => { it('should fetch agentless policy if agentless feature is enabled and isServerless is true', async () => { const { waitForNextUpdate } = renderHook(() => useSetupTechnology({ - updateNewAgentPolicy: updateNewAgentPolicyMock, + setNewAgentPolicy, newAgentPolicy: newAgentPolicyMock, updateAgentPolicies: updateAgentPoliciesMock, setSelectedPolicyTab: setSelectedPolicyTabMock, @@ -246,7 +246,7 @@ describe('useSetupTechnology', () => { }); const { result, waitForNextUpdate } = renderHook(() => useSetupTechnology({ - updateNewAgentPolicy: updateNewAgentPolicyMock, + setNewAgentPolicy, newAgentPolicy: newAgentPolicyMock, updateAgentPolicies: updateAgentPoliciesMock, setSelectedPolicyTab: setSelectedPolicyTabMock, @@ -262,7 +262,7 @@ describe('useSetupTechnology', () => { waitForNextUpdate(); expect(result.current.selectedSetupTechnology).toBe(SetupTechnology.AGENTLESS); - expect(updateNewAgentPolicyMock).toHaveBeenCalledWith({ + expect(setNewAgentPolicy).toHaveBeenCalledWith({ name: 'Agentless policy for endpoint-1', supports_agentless: true, }); @@ -284,7 +284,7 @@ describe('useSetupTechnology', () => { }); const { result, rerender } = renderHook(() => useSetupTechnology({ - updateNewAgentPolicy: updateNewAgentPolicyMock, + setNewAgentPolicy, newAgentPolicy: newAgentPolicyMock, updateAgentPolicies: updateAgentPoliciesMock, setSelectedPolicyTab: setSelectedPolicyTabMock, @@ -299,13 +299,13 @@ describe('useSetupTechnology', () => { }); expect(result.current.selectedSetupTechnology).toBe(SetupTechnology.AGENTLESS); - expect(updateNewAgentPolicyMock).toHaveBeenCalledWith({ + expect(setNewAgentPolicy).toHaveBeenCalledWith({ name: 'Agentless policy for endpoint-1', supports_agentless: true, }); rerender({ - updateNewAgentPolicy: updateNewAgentPolicyMock, + setNewAgentPolicy, newAgentPolicy: newAgentPolicyMock, updateAgentPolicies: updateAgentPoliciesMock, setSelectedPolicyTab: setSelectedPolicyTabMock, @@ -316,7 +316,7 @@ describe('useSetupTechnology', () => { }); waitFor(() => { - expect(updateNewAgentPolicyMock).toHaveBeenCalledWith({ + expect(setNewAgentPolicy).toHaveBeenCalledWith({ name: 'Agentless policy for endpoint-2', supports_agentless: true, }); @@ -333,7 +333,7 @@ describe('useSetupTechnology', () => { const { result, waitForNextUpdate } = renderHook(() => useSetupTechnology({ - updateNewAgentPolicy: updateNewAgentPolicyMock, + setNewAgentPolicy, newAgentPolicy: newAgentPolicyMock, updateAgentPolicies: updateAgentPoliciesMock, setSelectedPolicyTab: setSelectedPolicyTabMock, @@ -348,7 +348,7 @@ describe('useSetupTechnology', () => { }); waitForNextUpdate(); - expect(updateNewAgentPolicyMock).toHaveBeenCalledTimes(0); + expect(setNewAgentPolicy).toHaveBeenCalledTimes(0); }); it('should not fetch agentless policy if agentless is enabled but serverless is disabled', async () => { @@ -360,7 +360,7 @@ describe('useSetupTechnology', () => { const { result } = renderHook(() => useSetupTechnology({ - updateNewAgentPolicy: updateNewAgentPolicyMock, + setNewAgentPolicy, newAgentPolicy: newAgentPolicyMock, updateAgentPolicies: updateAgentPoliciesMock, setSelectedPolicyTab: setSelectedPolicyTabMock, @@ -375,7 +375,7 @@ describe('useSetupTechnology', () => { it('should update agent policy and selected policy tab when setup technology is agentless', async () => { const { result, waitForNextUpdate } = renderHook(() => useSetupTechnology({ - updateNewAgentPolicy: updateNewAgentPolicyMock, + setNewAgentPolicy, newAgentPolicy: newAgentPolicyMock, updateAgentPolicies: updateAgentPoliciesMock, setSelectedPolicyTab: setSelectedPolicyTabMock, @@ -396,7 +396,7 @@ describe('useSetupTechnology', () => { it('should update new agent policy and selected policy tab when setup technology is agent-based', async () => { const { result, waitForNextUpdate } = renderHook(() => useSetupTechnology({ - updateNewAgentPolicy: updateNewAgentPolicyMock, + setNewAgentPolicy, newAgentPolicy: newAgentPolicyMock, updateAgentPolicies: updateAgentPoliciesMock, setSelectedPolicyTab: setSelectedPolicyTabMock, @@ -420,7 +420,7 @@ describe('useSetupTechnology', () => { expect(result.current.selectedSetupTechnology).toBe(SetupTechnology.AGENT_BASED); - expect(updateNewAgentPolicyMock).toHaveBeenCalledWith(newAgentPolicyMock); + expect(setNewAgentPolicy).toHaveBeenCalledWith(newAgentPolicyMock); expect(setSelectedPolicyTabMock).toHaveBeenCalledWith(SelectedPolicyTab.NEW); }); @@ -431,7 +431,7 @@ describe('useSetupTechnology', () => { const { result } = renderHook(() => useSetupTechnology({ - updateNewAgentPolicy: updateNewAgentPolicyMock, + setNewAgentPolicy, newAgentPolicy: newAgentPolicyMock, updateAgentPolicies: updateAgentPoliciesMock, setSelectedPolicyTab: setSelectedPolicyTabMock, @@ -451,7 +451,7 @@ describe('useSetupTechnology', () => { it('should not update agent policy and selected policy tab when setup technology matches the current one ', async () => { const { result, waitForNextUpdate } = renderHook(() => useSetupTechnology({ - updateNewAgentPolicy: updateNewAgentPolicyMock, + setNewAgentPolicy, newAgentPolicy: newAgentPolicyMock, updateAgentPolicies: updateAgentPoliciesMock, setSelectedPolicyTab: setSelectedPolicyTabMock, @@ -469,14 +469,14 @@ describe('useSetupTechnology', () => { expect(result.current.selectedSetupTechnology).toBe(SetupTechnology.AGENT_BASED); - expect(updateNewAgentPolicyMock).not.toHaveBeenCalled(); + expect(setNewAgentPolicy).not.toHaveBeenCalled(); expect(setSelectedPolicyTabMock).not.toHaveBeenCalled(); }); it('should revert the agent policy name to the original value when switching from agentless back to agent-based', async () => { const { result, waitForNextUpdate } = renderHook(() => useSetupTechnology({ - updateNewAgentPolicy: updateNewAgentPolicyMock, + setNewAgentPolicy, newAgentPolicy: newAgentPolicyMock, updateAgentPolicies: updateAgentPoliciesMock, setSelectedPolicyTab: setSelectedPolicyTabMock, @@ -495,7 +495,7 @@ describe('useSetupTechnology', () => { expect(result.current.selectedSetupTechnology).toBe(SetupTechnology.AGENTLESS); waitFor(() => { - expect(updateNewAgentPolicyMock).toHaveBeenCalledWith({ + expect(setNewAgentPolicy).toHaveBeenCalledWith({ name: 'Agentless policy for endpoint-1', supports_agentless: true, }); @@ -506,6 +506,6 @@ describe('useSetupTechnology', () => { }); expect(result.current.selectedSetupTechnology).toBe(SetupTechnology.AGENT_BASED); - expect(updateNewAgentPolicyMock).toHaveBeenCalledWith(newAgentPolicyMock); + expect(setNewAgentPolicy).toHaveBeenCalledWith(newAgentPolicyMock); }); }); diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/single_page_layout/hooks/setup_technology.ts b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/single_page_layout/hooks/setup_technology.ts index 367fde516ae32..cb72bfd8da245 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/single_page_layout/hooks/setup_technology.ts +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/single_page_layout/hooks/setup_technology.ts @@ -75,7 +75,7 @@ export const useAgentless = () => { }; export function useSetupTechnology({ - updateNewAgentPolicy, + setNewAgentPolicy, newAgentPolicy, updateAgentPolicies, setSelectedPolicyTab, @@ -83,7 +83,7 @@ export function useSetupTechnology({ packagePolicy, isEditPage, }: { - updateNewAgentPolicy: (policy: NewAgentPolicy) => void; + setNewAgentPolicy: (policy: NewAgentPolicy) => void; newAgentPolicy: NewAgentPolicy; updateAgentPolicies: (policies: AgentPolicy[]) => void; setSelectedPolicyTab: (tab: SelectedPolicyTab) => void; @@ -121,7 +121,7 @@ export function useSetupTechnology({ }; if (nextNewAgentlessPolicy.name !== newAgentlessPolicy.name) { setNewAgentlessPolicy(nextNewAgentlessPolicy); - updateNewAgentPolicy(nextNewAgentlessPolicy as NewAgentPolicy); + setNewAgentPolicy(nextNewAgentlessPolicy as NewAgentPolicy); updateAgentPolicies([nextNewAgentlessPolicy] as AgentPolicy[]); } } @@ -132,7 +132,7 @@ export function useSetupTechnology({ packagePolicy.name, selectedSetupTechnology, updateAgentPolicies, - updateNewAgentPolicy, + setNewAgentPolicy, ]); useEffect(() => { @@ -168,23 +168,23 @@ export function useSetupTechnology({ if (setupTechnology === SetupTechnology.AGENTLESS) { if (isAgentlessCloudEnabled) { - updateNewAgentPolicy(newAgentlessPolicy as NewAgentPolicy); + setNewAgentPolicy(newAgentlessPolicy as NewAgentPolicy); setSelectedPolicyTab(SelectedPolicyTab.NEW); updateAgentPolicies([newAgentlessPolicy] as AgentPolicy[]); } // tech debt: remove this when Serverless uses the Agentless API // https://github.com/elastic/security-team/issues/9781 if (isAgentlessServerlessEnabled) { - updateNewAgentPolicy(newAgentlessPolicy as AgentPolicy); + setNewAgentPolicy(newAgentlessPolicy as AgentPolicy); updateAgentPolicies([newAgentlessPolicy] as AgentPolicy[]); setSelectedPolicyTab(SelectedPolicyTab.EXISTING); } } else if (setupTechnology === SetupTechnology.AGENT_BASED) { - updateNewAgentPolicy({ + setNewAgentPolicy({ ...newAgentBasedPolicy.current, supports_agentless: false, is_managed: false, - } as NewAgentPolicy); + }); setSelectedPolicyTab(SelectedPolicyTab.NEW); updateAgentPolicies([newAgentBasedPolicy.current] as AgentPolicy[]); } @@ -195,7 +195,7 @@ export function useSetupTechnology({ selectedSetupTechnology, isAgentlessCloudEnabled, isAgentlessServerlessEnabled, - updateNewAgentPolicy, + setNewAgentPolicy, newAgentlessPolicy, setSelectedPolicyTab, updateAgentPolicies, diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/single_page_layout/index.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/single_page_layout/index.tsx index 67275a3cf4036..7190a90d56198 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/single_page_layout/index.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/single_page_layout/index.tsx @@ -352,7 +352,7 @@ export const CreatePackagePolicySinglePage: CreatePackagePolicyParams = ({ const { isAgentlessEnabled } = useAgentless(); const { handleSetupTechnologyChange, selectedSetupTechnology } = useSetupTechnology({ newAgentPolicy, - updateNewAgentPolicy, + setNewAgentPolicy, updateAgentPolicies, setSelectedPolicyTab, packageInfo, diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/edit_package_policy_page/hooks/use_package_policy_steps.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/edit_package_policy_page/hooks/use_package_policy_steps.tsx index 9ade778c74f31..dd349fca9909e 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/edit_package_policy_page/hooks/use_package_policy_steps.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/edit_package_policy_page/hooks/use_package_policy_steps.tsx @@ -139,7 +139,7 @@ export function usePackagePolicySteps({ const { selectedSetupTechnology } = useSetupTechnology({ newAgentPolicy, - updateNewAgentPolicy, + setNewAgentPolicy, updateAgentPolicies, setSelectedPolicyTab, packageInfo, diff --git a/x-pack/plugins/observability_solution/entity_manager/common/constants_entities.ts b/x-pack/plugins/observability_solution/entity_manager/common/constants_entities.ts index 633dfa2f9fd29..a3194339fac1f 100644 --- a/x-pack/plugins/observability_solution/entity_manager/common/constants_entities.ts +++ b/x-pack/plugins/observability_solution/entity_manager/common/constants_entities.ts @@ -22,6 +22,8 @@ export const ENTITY_HISTORY_BASE_COMPONENT_TEMPLATE_V1 = `${ENTITY_BASE_PREFIX}_${ENTITY_SCHEMA_VERSION_V1}_${ENTITY_HISTORY}_base` as const; export const ENTITY_HISTORY_PREFIX_V1 = `${ENTITY_BASE_PREFIX}-${ENTITY_SCHEMA_VERSION_V1}-${ENTITY_HISTORY}` as const; +export const ENTITY_HISTORY_BACKFILL_PREFIX_V1 = + `${ENTITY_BASE_PREFIX}-${ENTITY_SCHEMA_VERSION_V1}-${ENTITY_HISTORY}-backfill` as const; export const ENTITY_HISTORY_INDEX_PREFIX_V1 = `${ENTITY_INDEX_PREFIX}.${ENTITY_SCHEMA_VERSION_V1}.${ENTITY_HISTORY}` as const; diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/auth/api_key/api_key.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/auth/api_key/api_key.ts index 5d48ff6e36f0d..2535bcc4d64f7 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/auth/api_key/api_key.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/auth/api_key/api_key.ts @@ -8,7 +8,7 @@ import { KibanaRequest } from '@kbn/core-http-server'; import { getFakeKibanaRequest } from '@kbn/security-plugin/server/authentication/api_keys/fake_kibana_request'; import { EntityManagerServerSetup } from '../../../types'; -import { canRunEntityDiscovery, requiredRunTimePrivileges } from '../privileges'; +import { canManageEntityDefinition, entityDefinitionRuntimePrivileges } from '../privileges'; export interface EntityDiscoveryAPIKey { id: string; @@ -45,7 +45,7 @@ export const checkIfEntityDiscoveryAPIKeyIsValid = async ( server.logger.debug('validating API key has runtime privileges for entity discovery'); - return canRunEntityDiscovery(esClient); + return canManageEntityDefinition(esClient); }; export const generateEntityDiscoveryAPIKey = async ( @@ -55,7 +55,7 @@ export const generateEntityDiscoveryAPIKey = async ( const apiKey = await server.security.authc.apiKeys.grantAsInternalUser(req, { name: 'Entity discovery API key', role_descriptors: { - entity_discovery_admin: requiredRunTimePrivileges, + entity_discovery_admin: entityDefinitionRuntimePrivileges, }, metadata: { description: diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/auth/index.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/auth/index.ts index 967a7371ac100..61e31367da706 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/auth/index.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/auth/index.ts @@ -15,7 +15,11 @@ import { saveEntityDiscoveryAPIKey, deleteEntityDiscoveryAPIKey, } from './api_key/saved_object'; -import { canEnableEntityDiscovery, canRunEntityDiscovery } from './privileges'; +import { + canEnableEntityDiscovery, + canManageEntityDefinition, + canDisableEntityDiscovery, +} from './privileges'; export { readEntityDiscoveryAPIKey, @@ -24,6 +28,7 @@ export { checkIfAPIKeysAreEnabled, checkIfEntityDiscoveryAPIKeyIsValid, canEnableEntityDiscovery, - canRunEntityDiscovery, + canManageEntityDefinition, + canDisableEntityDiscovery, generateEntityDiscoveryAPIKey, }; diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/auth/privileges.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/auth/privileges.ts index 3bc88127a5964..9de76ac7b4c5c 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/auth/privileges.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/auth/privileges.ts @@ -7,10 +7,55 @@ import { ElasticsearchClient } from '@kbn/core/server'; import { ENTITY_INDICES_PATTERN } from '../../../common/constants_entities'; +import { SO_ENTITY_DEFINITION_TYPE, SO_ENTITY_DISCOVERY_API_KEY_TYPE } from '../../saved_objects'; import { BUILT_IN_ALLOWED_INDICES } from '../entities/built_in/constants'; -export const requiredRunTimePrivileges = { - // all of +export const canManageEntityDefinition = async (client: ElasticsearchClient) => { + const { has_all_requested: hasAllRequested } = await client.security.hasPrivileges({ + body: entityDefinitionRuntimePrivileges, + }); + + return hasAllRequested; +}; + +const canDeleteEntityDefinition = async (client: ElasticsearchClient) => { + const { has_all_requested: hasAllRequested } = await client.security.hasPrivileges({ + body: entityDefinitionDeletionPrivileges, + }); + + return hasAllRequested; +}; + +const canManageAPIKey = async (client: ElasticsearchClient) => { + const { has_all_requested: hasAllRequested } = await client.security.hasPrivileges({ + body: apiKeyCreationPrivileges, + }); + + return hasAllRequested; +}; + +const canDeleteAPIKey = async (client: ElasticsearchClient) => { + const { has_all_requested: hasAllRequested } = await client.security.hasPrivileges({ + body: apiKeyDeletionPrivileges, + }); + + return hasAllRequested; +}; + +export const canEnableEntityDiscovery = async (client: ElasticsearchClient) => { + return Promise.all([canManageAPIKey(client), canManageEntityDefinition(client)]).then((results) => + results.every(Boolean) + ); +}; + +export const canDisableEntityDiscovery = async (client: ElasticsearchClient) => { + return Promise.all([canDeleteAPIKey(client), canDeleteEntityDefinition(client)]).then((results) => + results.every(Boolean) + ); +}; + +export const entityDefinitionRuntimePrivileges = { + cluster: ['manage_transform', 'manage_ingest_pipelines', 'manage_index_templates'], index: [ { names: [ENTITY_INDICES_PATTERN], @@ -21,50 +66,48 @@ export const requiredRunTimePrivileges = { privileges: ['read', 'view_index_metadata'], }, ], - cluster: [ - 'manage_transform', - 'monitor_transform', - 'manage_ingest_pipelines', - 'monitor', - 'manage_index_templates', - ], application: [ { application: 'kibana-.kibana', - privileges: ['saved_object:entity-definition/*'], + privileges: [`saved_object:${SO_ENTITY_DEFINITION_TYPE}/*`], resources: ['*'], }, ], }; -export const requiredEnablementPrivileges = { - // any one of - cluster: ['manage_security', 'manage_api_key', 'manage_own_api_key'], +export const entityDefinitionDeletionPrivileges = { + cluster: ['manage_transform', 'manage_ingest_pipelines', 'manage_index_templates'], + index: [ + { + names: [ENTITY_INDICES_PATTERN], + privileges: ['delete_index'], + }, + ], + application: [ + { + application: 'kibana-.kibana', + privileges: [`saved_object:${SO_ENTITY_DEFINITION_TYPE}/delete`], + resources: ['*'], + }, + ], }; -export const canRunEntityDiscovery = async (client: ElasticsearchClient) => { - const { has_all_requested: hasAllRequested } = await client.security.hasPrivileges({ - body: { - cluster: requiredRunTimePrivileges.cluster, - index: requiredRunTimePrivileges.index, - application: requiredRunTimePrivileges.application, +export const apiKeyCreationPrivileges = { + application: [ + { + application: 'kibana-.kibana', + privileges: [`saved_object:${SO_ENTITY_DISCOVERY_API_KEY_TYPE}/*`], + resources: ['*'], }, - }); - - return hasAllRequested; + ], }; -export const canEnableEntityDiscovery = async (client: ElasticsearchClient) => { - const [canRun, { cluster: grantedClusterPrivileges }] = await Promise.all([ - canRunEntityDiscovery(client), - client.security.hasPrivileges({ - body: { - cluster: requiredEnablementPrivileges.cluster, - }, - }), - ]); - - return ( - canRun && requiredEnablementPrivileges.cluster.some((k) => grantedClusterPrivileges[k] === true) - ); +const apiKeyDeletionPrivileges = { + application: [ + { + application: 'kibana-.kibana', + privileges: [`saved_object:${SO_ENTITY_DISCOVERY_API_KEY_TYPE}/delete`], + resources: ['*'], + }, + ], }; diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/create_and_install_transform.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/create_and_install_transform.ts index aca51df235fea..99c089ac14600 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/create_and_install_transform.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/create_and_install_transform.ts @@ -27,6 +27,24 @@ export async function createAndInstallHistoryTransform( } } +export async function createAndInstallHistoryBackfillTransform( + esClient: ElasticsearchClient, + definition: EntityDefinition, + logger: Logger +) { + try { + const historyTransform = generateHistoryTransform(definition, true); + await retryTransientEsErrors(() => esClient.transform.putTransform(historyTransform), { + logger, + }); + } catch (e) { + logger.error( + `Cannot create entity history backfill transform for [${definition.id}] entity definition` + ); + throw e; + } +} + export async function createAndInstallLatestTransform( esClient: ElasticsearchClient, definition: EntityDefinition, diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/helpers/fixtures/entity_definition_with_backfill.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/helpers/fixtures/entity_definition_with_backfill.ts new file mode 100644 index 0000000000000..6d4026973ca38 --- /dev/null +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/helpers/fixtures/entity_definition_with_backfill.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { entityDefinitionSchema } from '@kbn/entities-schema'; +export const entityDefinitionWithBackfill = entityDefinitionSchema.parse({ + id: 'admin-console-services', + version: '999.999.999', + name: 'Services for Admin Console', + type: 'service', + indexPatterns: ['kbn-data-forge-fake_stack.*'], + history: { + timestampField: '@timestamp', + interval: '1m', + settings: { + backfillSyncDelay: '15m', + backfillLookbackPeriod: '72h', + backfillFrequency: '5m', + }, + }, + identityFields: ['log.logger', { field: 'event.category', optional: true }], + displayNameTemplate: '{{log.logger}}{{#event.category}}:{{.}}{{/event.category}}', + metadata: ['tags', 'host.name', 'host.os.name', { source: '_index', destination: 'sourceIndex' }], + metrics: [ + { + name: 'logRate', + equation: 'A', + metrics: [ + { + name: 'A', + aggregation: 'doc_count', + filter: 'log.level: *', + }, + ], + }, + { + name: 'errorRate', + equation: 'A', + metrics: [ + { + name: 'A', + aggregation: 'doc_count', + filter: 'log.level: "ERROR"', + }, + ], + }, + ], +}); diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/helpers/generate_component_id.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/helpers/generate_component_id.ts index c41aae4f21347..1be53fc0af8c9 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/helpers/generate_component_id.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/helpers/generate_component_id.ts @@ -7,6 +7,7 @@ import { EntityDefinition } from '@kbn/entities-schema'; import { + ENTITY_HISTORY_BACKFILL_PREFIX_V1, ENTITY_HISTORY_INDEX_PREFIX_V1, ENTITY_HISTORY_PREFIX_V1, ENTITY_LATEST_INDEX_PREFIX_V1, @@ -18,6 +19,11 @@ function generateHistoryId(definition: EntityDefinition) { return `${ENTITY_HISTORY_PREFIX_V1}-${definition.id}`; } +// History Backfill +export function generateHistoryBackfillTransformId(definition: EntityDefinition) { + return `${ENTITY_HISTORY_BACKFILL_PREFIX_V1}-${definition.id}`; +} + export const generateHistoryTransformId = generateHistoryId; export const generateHistoryIngestPipelineId = generateHistoryId; diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/helpers/is_backfill_enabled.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/helpers/is_backfill_enabled.ts new file mode 100644 index 0000000000000..6a97d3c950eec --- /dev/null +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/helpers/is_backfill_enabled.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EntityDefinition } from '@kbn/entities-schema'; + +export function isBackfillEnabled(definition: EntityDefinition) { + return definition.history.settings?.backfillSyncDelay != null; +} diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.ts index 43f18b2b81bf0..36c3f32342477 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.ts @@ -26,7 +26,7 @@ function createMetadataPainlessScript(definition: EntityDefinition) { } return definition.metadata.reduce((acc, def) => { - const destination = def.destination || def.source; + const destination = def.destination; const optionalFieldPath = destination.replaceAll('.', '?.'); const next = ` if (ctx.entity?.metadata?.${optionalFieldPath} != null) { diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/install_entity_definition.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/install_entity_definition.ts index b47f17b6b00fa..875242f73d751 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/install_entity_definition.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/install_entity_definition.ts @@ -18,6 +18,7 @@ import { createAndInstallLatestIngestPipeline, } from './create_and_install_ingest_pipeline'; import { + createAndInstallHistoryBackfillTransform, createAndInstallHistoryTransform, createAndInstallLatestTransform, } from './create_and_install_transform'; @@ -28,10 +29,12 @@ import { findEntityDefinitions } from './find_entity_definition'; import { saveEntityDefinition } from './save_entity_definition'; import { startTransform } from './start_transform'; import { + stopAndDeleteHistoryBackfillTransform, stopAndDeleteHistoryTransform, stopAndDeleteLatestTransform, } from './stop_and_delete_transform'; import { uninstallEntityDefinition } from './uninstall_entity_definition'; +import { isBackfillEnabled } from './helpers/is_backfill_enabled'; import { deleteTemplate, upsertTemplate } from '../manage_index_templates'; import { getEntitiesLatestIndexTemplateConfig } from '../../templates/entities_latest_template'; import { getEntitiesHistoryIndexTemplateConfig } from '../../templates/entities_history_template'; @@ -56,6 +59,7 @@ export async function installEntityDefinition({ }, transforms: { history: false, + backfill: false, latest: false, }, definition: false, @@ -98,6 +102,10 @@ export async function installEntityDefinition({ logger.debug(`Installing transforms for definition ${definition.id}`); await createAndInstallHistoryTransform(esClient, entityDefinition, logger); installState.transforms.history = true; + if (isBackfillEnabled(entityDefinition)) { + await createAndInstallHistoryBackfillTransform(esClient, entityDefinition, logger); + installState.transforms.backfill = true; + } await createAndInstallLatestTransform(esClient, entityDefinition, logger); installState.transforms.latest = true; @@ -120,6 +128,10 @@ export async function installEntityDefinition({ await stopAndDeleteHistoryTransform(esClient, definition, logger); } + if (installState.transforms.backfill) { + await stopAndDeleteHistoryBackfillTransform(esClient, definition, logger); + } + if (installState.transforms.latest) { await stopAndDeleteLatestTransform(esClient, definition, logger); } diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/start_transform.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/start_transform.ts index 7de64394fabee..46bb16ff00ae3 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/start_transform.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/start_transform.ts @@ -8,10 +8,12 @@ import { ElasticsearchClient, Logger } from '@kbn/core/server'; import { EntityDefinition } from '@kbn/entities-schema'; import { + generateHistoryBackfillTransformId, generateHistoryTransformId, generateLatestTransformId, } from './helpers/generate_component_id'; import { retryTransientEsErrors } from './helpers/retry'; +import { isBackfillEnabled } from './helpers/is_backfill_enabled'; export async function startTransform( esClient: ElasticsearchClient, @@ -26,6 +28,17 @@ export async function startTransform( esClient.transform.startTransform({ transform_id: historyTransformId }, { ignore: [409] }), { logger } ); + if (isBackfillEnabled(definition)) { + const historyBackfillTransformId = generateHistoryBackfillTransformId(definition); + await retryTransientEsErrors( + () => + esClient.transform.startTransform( + { transform_id: historyBackfillTransformId }, + { ignore: [409] } + ), + { logger } + ); + } await retryTransientEsErrors( () => esClient.transform.startTransform({ transform_id: latestTransformId }, { ignore: [409] }), diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/stop_and_delete_transform.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/stop_and_delete_transform.ts index 2b978217fdf98..d49165be22106 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/stop_and_delete_transform.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/stop_and_delete_transform.ts @@ -8,6 +8,7 @@ import { ElasticsearchClient, Logger } from '@kbn/core/server'; import { EntityDefinition } from '@kbn/entities-schema'; import { + generateHistoryBackfillTransformId, generateHistoryTransformId, generateLatestTransformId, } from './helpers/generate_component_id'; @@ -42,6 +43,35 @@ export async function stopAndDeleteHistoryTransform( } } +export async function stopAndDeleteHistoryBackfillTransform( + esClient: ElasticsearchClient, + definition: EntityDefinition, + logger: Logger +) { + try { + const historyBackfillTransformId = generateHistoryBackfillTransformId(definition); + await retryTransientEsErrors( + () => + esClient.transform.stopTransform( + { transform_id: historyBackfillTransformId, wait_for_completion: true, force: true }, + { ignore: [409, 404] } + ), + { logger } + ); + await retryTransientEsErrors( + () => + esClient.transform.deleteTransform( + { transform_id: historyBackfillTransformId, force: true }, + { ignore: [404] } + ), + { logger } + ); + } catch (e) { + logger.error(`Cannot stop or delete history backfill transform [${definition.id}]`); + throw e; + } +} + export async function stopAndDeleteLatestTransform( esClient: ElasticsearchClient, definition: EntityDefinition, diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/__snapshots__/generate_history_transform.test.ts.snap b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/__snapshots__/generate_history_transform.test.ts.snap index b76cd81f6ecf9..4ecdd0c3ab024 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/__snapshots__/generate_history_transform.test.ts.snap +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/__snapshots__/generate_history_transform.test.ts.snap @@ -1,6 +1,153 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`generateHistoryTransform(definition) should generate a valid latest transform 1`] = ` +exports[`generateHistoryTransform(definition) should generate a valid history backfill transform 1`] = ` +Object { + "_meta": Object { + "definitionVersion": "999.999.999", + "managed": false, + }, + "defer_validation": true, + "dest": Object { + "index": ".entities.v1.history.noop", + "pipeline": "entities-v1-history-admin-console-services", + }, + "frequency": "5m", + "pivot": Object { + "aggs": Object { + "_errorRate_A": Object { + "filter": Object { + "bool": Object { + "minimum_should_match": 1, + "should": Array [ + Object { + "match_phrase": Object { + "log.level": "ERROR", + }, + }, + ], + }, + }, + }, + "_logRate_A": Object { + "filter": Object { + "bool": Object { + "minimum_should_match": 1, + "should": Array [ + Object { + "exists": Object { + "field": "log.level", + }, + }, + ], + }, + }, + }, + "entity.lastSeenTimestamp": Object { + "max": Object { + "field": "@timestamp", + }, + }, + "entity.metadata.host.name": Object { + "terms": Object { + "field": "host.name", + "size": 1000, + }, + }, + "entity.metadata.host.os.name": Object { + "terms": Object { + "field": "host.os.name", + "size": 1000, + }, + }, + "entity.metadata.sourceIndex": Object { + "terms": Object { + "field": "_index", + "size": 1000, + }, + }, + "entity.metadata.tags": Object { + "terms": Object { + "field": "tags", + "size": 1000, + }, + }, + "entity.metrics.errorRate": Object { + "bucket_script": Object { + "buckets_path": Object { + "A": "_errorRate_A>_count", + }, + "script": Object { + "lang": "painless", + "source": "params.A", + }, + }, + }, + "entity.metrics.logRate": Object { + "bucket_script": Object { + "buckets_path": Object { + "A": "_logRate_A>_count", + }, + "script": Object { + "lang": "painless", + "source": "params.A", + }, + }, + }, + }, + "group_by": Object { + "@timestamp": Object { + "date_histogram": Object { + "field": "@timestamp", + "fixed_interval": "1m", + }, + }, + "entity.identity.event.category": Object { + "terms": Object { + "field": "event.category", + "missing_bucket": true, + }, + }, + "entity.identity.log.logger": Object { + "terms": Object { + "field": "log.logger", + "missing_bucket": false, + }, + }, + }, + }, + "settings": Object { + "deduce_mappings": false, + "unattended": true, + }, + "source": Object { + "index": Array [ + "kbn-data-forge-fake_stack.*", + ], + "query": Object { + "bool": Object { + "filter": Array [ + Object { + "range": Object { + "@timestamp": Object { + "gte": "now-72h", + }, + }, + }, + ], + }, + }, + }, + "sync": Object { + "time": Object { + "delay": "15m", + "field": "@timestamp", + }, + }, + "transform_id": "entities-v1-history-backfill-admin-console-services", +} +`; + +exports[`generateHistoryTransform(definition) should generate a valid history transform 1`] = ` Object { "_meta": Object { "definitionVersion": "999.999.999", diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_history_transform.test.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_history_transform.test.ts index 8bb9f494d5f4e..cde87d670c8c2 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_history_transform.test.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_history_transform.test.ts @@ -6,11 +6,16 @@ */ import { entityDefinition } from '../helpers/fixtures/entity_definition'; +import { entityDefinitionWithBackfill } from '../helpers/fixtures/entity_definition_with_backfill'; import { generateHistoryTransform } from './generate_history_transform'; describe('generateHistoryTransform(definition)', () => { - it('should generate a valid latest transform', () => { + it('should generate a valid history transform', () => { const transform = generateHistoryTransform(entityDefinition); expect(transform).toMatchSnapshot(); }); + it('should generate a valid history backfill transform', () => { + const transform = generateHistoryTransform(entityDefinitionWithBackfill, true); + expect(transform).toMatchSnapshot(); + }); }); diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_history_transform.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_history_transform.ts index 645225eaf688c..05b0e7ee7fd54 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_history_transform.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_history_transform.ts @@ -21,19 +21,50 @@ import { generateHistoryTransformId, generateHistoryIngestPipelineId, generateHistoryIndexName, + generateHistoryBackfillTransformId, } from '../helpers/generate_component_id'; +import { isBackfillEnabled } from '../helpers/is_backfill_enabled'; export function generateHistoryTransform( - definition: EntityDefinition + definition: EntityDefinition, + backfill = false ): TransformPutTransformRequest { + if (backfill && !isBackfillEnabled(definition)) { + throw new Error( + 'This function was called with backfill=true without history.settings.backfillSyncDelay' + ); + } + const filter: QueryDslQueryContainer[] = []; if (definition.filter) { filter.push(getElasticsearchQueryOrThrow(definition.filter)); } + if (backfill && definition.history.settings?.backfillLookbackPeriod) { + filter.push({ + range: { + [definition.history.timestampField]: { + gte: `now-${definition.history.settings?.backfillLookbackPeriod.toJSON()}`, + }, + }, + }); + } + + const syncDelay = backfill + ? definition.history.settings?.backfillSyncDelay + : definition.history.settings?.syncDelay; + + const transformId = backfill + ? generateHistoryBackfillTransformId(definition) + : generateHistoryTransformId(definition); + + const frequency = backfill + ? definition.history.settings?.backfillFrequency + : definition.history.settings?.frequency; + return { - transform_id: generateHistoryTransformId(definition), + transform_id: transformId, _meta: { definitionVersion: definition.version, managed: definition.managed, @@ -53,11 +84,11 @@ export function generateHistoryTransform( index: `${generateHistoryIndexName({ id: 'noop' } as EntityDefinition)}`, pipeline: generateHistoryIngestPipelineId(definition), }, - frequency: definition.history.settings?.frequency ?? ENTITY_DEFAULT_HISTORY_FREQUENCY, + frequency: frequency || ENTITY_DEFAULT_HISTORY_FREQUENCY, sync: { time: { field: definition.history.settings?.syncField ?? definition.history.timestampField, - delay: definition.history.settings?.syncDelay ?? ENTITY_DEFAULT_HISTORY_SYNC_DELAY, + delay: syncDelay || ENTITY_DEFAULT_HISTORY_SYNC_DELAY, }, }, settings: { diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts index 809ed5f2b57b9..31ba3e9add0dc 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts @@ -34,7 +34,7 @@ export function generateLatestMetadataAggregations(definition: EntityDefinition) return definition.metadata.reduce( (aggs, metadata) => ({ ...aggs, - [`entity.metadata.${metadata.destination ?? metadata.source}`]: { + [`entity.metadata.${metadata.destination}`]: { filter: { range: { 'event.ingested': { diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/uninstall_entity_definition.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/uninstall_entity_definition.ts index 9b8685031642a..11f772ce2e938 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/uninstall_entity_definition.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/uninstall_entity_definition.ts @@ -18,9 +18,11 @@ import { deleteIndices } from './delete_index'; import { deleteHistoryIngestPipeline, deleteLatestIngestPipeline } from './delete_ingest_pipeline'; import { findEntityDefinitions } from './find_entity_definition'; import { + stopAndDeleteHistoryBackfillTransform, stopAndDeleteHistoryTransform, stopAndDeleteLatestTransform, } from './stop_and_delete_transform'; +import { isBackfillEnabled } from './helpers/is_backfill_enabled'; import { deleteTemplate } from '../manage_index_templates'; export async function uninstallEntityDefinition({ @@ -37,6 +39,9 @@ export async function uninstallEntityDefinition({ deleteData?: boolean; }) { await stopAndDeleteHistoryTransform(esClient, definition, logger); + if (isBackfillEnabled(definition)) { + await stopAndDeleteHistoryBackfillTransform(esClient, definition, logger); + } await stopAndDeleteLatestTransform(esClient, definition, logger); await deleteHistoryIngestPipeline(esClient, definition, logger); await deleteLatestIngestPipeline(esClient, definition, logger); diff --git a/x-pack/plugins/observability_solution/entity_manager/server/routes/enablement/check.ts b/x-pack/plugins/observability_solution/entity_manager/server/routes/enablement/check.ts index 66fabf2c91327..8ee8de3751ab2 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/routes/enablement/check.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/routes/enablement/check.ts @@ -6,7 +6,6 @@ */ import { RequestHandlerContext } from '@kbn/core/server'; -import { getFakeKibanaRequest } from '@kbn/security-plugin/server/authentication/api_keys/fake_kibana_request'; import { SetupRouteOptions } from '../types'; import { checkIfEntityDiscoveryAPIKeyIsValid, readEntityDiscoveryAPIKey } from '../../lib/auth'; import { @@ -44,9 +43,8 @@ export function checkEntityDiscoveryEnabledRoute { diff --git a/x-pack/plugins/observability_solution/entity_manager/server/routes/enablement/disable.ts b/x-pack/plugins/observability_solution/entity_manager/server/routes/enablement/disable.ts index a81e1a28cd1a1..4a0500e7efbca 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/routes/enablement/disable.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/routes/enablement/disable.ts @@ -6,16 +6,13 @@ */ import { RequestHandlerContext } from '@kbn/core/server'; -import { getFakeKibanaRequest } from '@kbn/security-plugin/server/authentication/api_keys/fake_kibana_request'; import { schema } from '@kbn/config-schema'; import { SetupRouteOptions } from '../types'; -import { - checkIfEntityDiscoveryAPIKeyIsValid, - deleteEntityDiscoveryAPIKey, - readEntityDiscoveryAPIKey, -} from '../../lib/auth'; -import { ERROR_API_KEY_NOT_FOUND, ERROR_API_KEY_NOT_VALID } from '../../../common/errors'; +import { deleteEntityDiscoveryAPIKey, readEntityDiscoveryAPIKey } from '../../lib/auth'; +import { ERROR_USER_NOT_AUTHORIZED } from '../../../common/errors'; import { uninstallBuiltInEntityDefinitions } from '../../lib/entities/uninstall_entity_definition'; +import { canDisableEntityDiscovery } from '../../lib/auth/privileges'; +import { EntityDiscoveryApiKeyType } from '../../saved_objects'; export function disableEntityDiscoveryRoute({ router, @@ -33,23 +30,21 @@ export function disableEntityDiscoveryRoute({ }, async (context, req, res) => { try { - server.logger.debug('reading entity discovery API key from saved object'); - const apiKey = await readEntityDiscoveryAPIKey(server); - - if (apiKey === undefined) { - return res.ok({ body: { success: false, reason: ERROR_API_KEY_NOT_FOUND } }); - } - - server.logger.debug('validating existing entity discovery API key'); - const isValid = await checkIfEntityDiscoveryAPIKeyIsValid(server, apiKey); - - if (!isValid) { - return res.ok({ body: { success: false, reason: ERROR_API_KEY_NOT_VALID } }); + const esClient = (await context.core).elasticsearch.client.asCurrentUser; + const canDisable = await canDisableEntityDiscovery(esClient); + if (!canDisable) { + return res.ok({ + body: { + success: false, + reason: ERROR_USER_NOT_AUTHORIZED, + message: + 'Current Kibana user does not have the required permissions to disable entity discovery', + }, + }); } - - const fakeRequest = getFakeKibanaRequest({ id: apiKey.id, api_key: apiKey.apiKey }); - const soClient = server.core.savedObjects.getScopedClient(fakeRequest); - const esClient = server.core.elasticsearch.client.asScoped(fakeRequest).asCurrentUser; + const soClient = (await context.core).savedObjects.getClient({ + includedHiddenTypes: [EntityDiscoveryApiKeyType.name], + }); await uninstallBuiltInEntityDefinitions({ soClient, @@ -58,10 +53,16 @@ export function disableEntityDiscoveryRoute({ deleteData: req.query.deleteData, }); - await deleteEntityDiscoveryAPIKey((await context.core).savedObjects.client); - await server.security.authc.apiKeys.invalidateAsInternalUser({ - ids: [apiKey.id], - }); + server.logger.debug('reading entity discovery API key from saved object'); + const apiKey = await readEntityDiscoveryAPIKey(server); + // api key could be deleted outside of the apis, it does not affect the + // disablement flow + if (apiKey) { + await deleteEntityDiscoveryAPIKey(soClient); + await server.security.authc.apiKeys.invalidateAsInternalUser({ + ids: [apiKey.id], + }); + } return res.ok({ body: { success: true } }); } catch (err) { diff --git a/x-pack/plugins/observability_solution/entity_manager/server/routes/enablement/enable.ts b/x-pack/plugins/observability_solution/entity_manager/server/routes/enablement/enable.ts index 84977cf785ce2..d9af1105e42b4 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/routes/enablement/enable.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/routes/enablement/enable.ts @@ -6,7 +6,6 @@ */ import { RequestHandlerContext } from '@kbn/core/server'; -import { getFakeKibanaRequest } from '@kbn/security-plugin/server/authentication/api_keys/fake_kibana_request'; import { SetupRouteOptions } from '../types'; import { canEnableEntityDiscovery, @@ -19,8 +18,8 @@ import { } from '../../lib/auth'; import { builtInDefinitions } from '../../lib/entities/built_in'; import { installBuiltInEntityDefinitions } from '../../lib/entities/install_entity_definition'; -import { EntityDiscoveryApiKeyType } from '../../saved_objects'; import { ERROR_API_KEY_SERVICE_DISABLED, ERROR_USER_NOT_AUTHORIZED } from '../../../common/errors'; +import { EntityDiscoveryApiKeyType } from '../../saved_objects'; export function enableEntityDiscoveryRoute({ router, @@ -62,7 +61,6 @@ export function enableEntityDiscoveryRoute({ const soClient = (await context.core).savedObjects.getClient({ includedHiddenTypes: [EntityDiscoveryApiKeyType.name], }); - const existingApiKey = await readEntityDiscoveryAPIKey(server); if (existingApiKey !== undefined) { @@ -79,20 +77,19 @@ export function enableEntityDiscoveryRoute({ const apiKey = await generateEntityDiscoveryAPIKey(server, req); if (apiKey === undefined) { - throw new Error('could not generate entity discovery API key'); + return res.customError({ + statusCode: 500, + body: new Error('could not generate entity discovery API key'), + }); } await saveEntityDiscoveryAPIKey(soClient, apiKey); - const fakeRequest = getFakeKibanaRequest({ id: apiKey.id, api_key: apiKey.apiKey }); - const scopedSoClient = server.core.savedObjects.getScopedClient(fakeRequest); - const scopedEsClient = server.core.elasticsearch.client.asScoped(fakeRequest).asCurrentUser; - await installBuiltInEntityDefinitions({ logger, builtInDefinitions, - esClient: scopedEsClient, - soClient: scopedSoClient, + esClient, + soClient, }); return res.ok({ body: { success: true } }); diff --git a/x-pack/plugins/observability_solution/entity_manager/server/routes/entities/get.ts b/x-pack/plugins/observability_solution/entity_manager/server/routes/entities/get.ts index 3f1ffde5afef4..e9a6a8dbd3167 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/routes/entities/get.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/routes/entities/get.ts @@ -31,7 +31,7 @@ export function getEntityDefinitionRoute({ page: req.query.page ?? 1, perPage: req.query.perPage ?? 10, }); - return res.ok({ body: definitions }); + return res.ok({ body: { definitions } }); } catch (e) { return res.customError({ body: e, statusCode: 500 }); } diff --git a/x-pack/plugins/observability_solution/entity_manager/server/routes/entities/reset.ts b/x-pack/plugins/observability_solution/entity_manager/server/routes/entities/reset.ts index 6f97a5fbe0d51..7755fcf65b3c3 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/routes/entities/reset.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/routes/entities/reset.ts @@ -13,6 +13,7 @@ import { EntitySecurityException } from '../../lib/entities/errors/entity_securi import { InvalidTransformError } from '../../lib/entities/errors/invalid_transform_error'; import { readEntityDefinition } from '../../lib/entities/read_entity_definition'; import { + stopAndDeleteHistoryBackfillTransform, stopAndDeleteHistoryTransform, stopAndDeleteLatestTransform, } from '../../lib/entities/stop_and_delete_transform'; @@ -26,11 +27,13 @@ import { createAndInstallLatestIngestPipeline, } from '../../lib/entities/create_and_install_ingest_pipeline'; import { + createAndInstallHistoryBackfillTransform, createAndInstallHistoryTransform, createAndInstallLatestTransform, } from '../../lib/entities/create_and_install_transform'; import { startTransform } from '../../lib/entities/start_transform'; import { EntityDefinitionNotFound } from '../../lib/entities/errors/entity_not_found'; +import { isBackfillEnabled } from '../../lib/entities/helpers/is_backfill_enabled'; export function resetEntityDefinitionRoute({ router, @@ -52,6 +55,9 @@ export function resetEntityDefinitionRoute({ // Delete the transform and ingest pipeline await stopAndDeleteHistoryTransform(esClient, definition, logger); + if (isBackfillEnabled(definition)) { + await stopAndDeleteHistoryBackfillTransform(esClient, definition, logger); + } await stopAndDeleteLatestTransform(esClient, definition, logger); await deleteHistoryIngestPipeline(esClient, definition, logger); await deleteLatestIngestPipeline(esClient, definition, logger); @@ -61,6 +67,9 @@ export function resetEntityDefinitionRoute({ await createAndInstallHistoryIngestPipeline(esClient, definition, logger); await createAndInstallLatestIngestPipeline(esClient, definition, logger); await createAndInstallHistoryTransform(esClient, definition, logger); + if (isBackfillEnabled(definition)) { + await createAndInstallHistoryBackfillTransform(esClient, definition, logger); + } await createAndInstallLatestTransform(esClient, definition, logger); await startTransform(esClient, definition, logger); diff --git a/x-pack/plugins/observability_solution/entity_manager/server/saved_objects/entity_discovery_api_key.ts b/x-pack/plugins/observability_solution/entity_manager/server/saved_objects/entity_discovery_api_key.ts index bed5a5ca73673..7508f56636da3 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/saved_objects/entity_discovery_api_key.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/saved_objects/entity_discovery_api_key.ts @@ -7,11 +7,11 @@ import { SavedObjectsType } from '@kbn/core/server'; -const SO_ENTITY_DISCOVERY_API_KEY_TYPE = 'entity-discovery-api-key'; +export const SO_ENTITY_DISCOVERY_API_KEY_TYPE = 'entity-discovery-api-key'; export const EntityDiscoveryApiKeyType: SavedObjectsType = { name: SO_ENTITY_DISCOVERY_API_KEY_TYPE, - hidden: false, + hidden: true, namespaceType: 'multiple-isolated', mappings: { dynamic: false, diff --git a/x-pack/plugins/observability_solution/entity_manager/server/saved_objects/index.ts b/x-pack/plugins/observability_solution/entity_manager/server/saved_objects/index.ts index fd88e2e16e4a2..d787672d17d85 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/saved_objects/index.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/saved_objects/index.ts @@ -6,4 +6,7 @@ */ export { entityDefinition, SO_ENTITY_DEFINITION_TYPE } from './entity_definition'; -export { EntityDiscoveryApiKeyType } from './entity_discovery_api_key'; +export { + EntityDiscoveryApiKeyType, + SO_ENTITY_DISCOVERY_API_KEY_TYPE, +} from './entity_discovery_api_key'; diff --git a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json index a42825ed281f6..39024e4592e11 100644 --- a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json +++ b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json @@ -7100,16 +7100,34 @@ "cloud": { "properties": { "isCloudEnabled": { - "type": "boolean" + "type": "boolean", + "_meta": { + "description": "Is the deployment running in Elastic Cloud (ESS or Serverless)?" + } }, "trialEndDate": { - "type": "date" + "type": "date", + "_meta": { + "description": "End of the trial period" + } }, "inTrial": { - "type": "boolean" + "type": "boolean", + "_meta": { + "description": "Is the organization during the trial period?" + } }, "isElasticStaffOwned": { - "type": "boolean" + "type": "boolean", + "_meta": { + "description": "Is the deploymend owned by an Elastician" + } + }, + "organizationId": { + "type": "keyword", + "_meta": { + "description": "The Elastic Cloud Organization ID that owns the deployment/project" + } }, "deploymentId": { "type": "keyword", @@ -7128,6 +7146,12 @@ "_meta": { "description": "The Serverless Project type" } + }, + "orchestratorTarget": { + "type": "keyword", + "_meta": { + "description": "The Orchestrator Target where it is deployed (canary/non-canary)" + } } } }, diff --git a/x-pack/test/api_integration/apis/entity_manager/config.ts b/x-pack/test/api_integration/apis/entity_manager/config.ts new file mode 100644 index 0000000000000..4280c8a0393ef --- /dev/null +++ b/x-pack/test/api_integration/apis/entity_manager/config.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrConfigProviderContext, GenericFtrProviderContext } from '@kbn/test'; +import { FtrProviderContext } from '../../ftr_provider_context'; + +type InheritedServices = FtrProviderContext extends GenericFtrProviderContext + ? TServices + : {}; + +interface EntityManagerConfig { + services: InheritedServices & {}; +} + +export default async function createTestConfig({ + readConfigFile, +}: FtrConfigProviderContext): Promise { + const baseIntegrationTestsConfig = await readConfigFile(require.resolve('../../config.ts')); + const services = baseIntegrationTestsConfig.get('services'); + + return { + ...baseIntegrationTestsConfig.getAll(), + testFiles: [require.resolve('.')], + services: { + ...services, + }, + }; +} + +export type CreateTestConfig = Awaited>; + +export type AssetManagerServices = CreateTestConfig['services']; diff --git a/x-pack/test/api_integration/apis/entity_manager/enablement.ts b/x-pack/test/api_integration/apis/entity_manager/enablement.ts new file mode 100644 index 0000000000000..8ec5f6743a51f --- /dev/null +++ b/x-pack/test/api_integration/apis/entity_manager/enablement.ts @@ -0,0 +1,127 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { ERROR_USER_NOT_AUTHORIZED } from '@kbn/entityManager-plugin/common/errors'; +import { builtInDefinitions } from '@kbn/entityManager-plugin/server/lib/entities/built_in'; +import { EntityDefinitionWithState } from '@kbn/entityManager-plugin/server/lib/entities/types'; +import { FtrProviderContext } from '../../ftr_provider_context'; +import { createAdmin, createRuntimeUser } from './helpers/user'; + +interface Auth { + username: string; + password: string; +} + +export default function ({ getService }: FtrProviderContext) { + const esClient = getService('es'); + const supertest = getService('supertestWithoutAuth'); + + const enablementRequest = + (method: 'get' | 'put' | 'delete') => async (auth: Auth, query?: { [key: string]: any }) => { + const response = await supertest[method]('/internal/entities/managed/enablement') + .auth(auth.username, auth.password) + .query(query) + .set('kbn-xsrf', 'xxx') + .send() + .expect(200); + return response.body; + }; + + const getInstalledDefinitions = async (auth: Auth) => { + const response = await supertest + .get('/internal/entities/definition') + .auth(auth.username, auth.password) + .set('kbn-xsrf', 'xxx') + .send() + .expect(200); + return response.body; + }; + + const entityDiscoveryState = enablementRequest('get'); + const enableEntityDiscovery = enablementRequest('put'); + const disableEntityDiscovery = enablementRequest('delete'); + + describe('Entity discovery enablement', () => { + let authorizedUser: { username: string; password: string }; + let unauthorizedUser: { username: string; password: string }; + + before(async () => { + [authorizedUser, unauthorizedUser] = await Promise.all([ + createAdmin({ esClient }), + createRuntimeUser({ esClient }), + ]); + }); + + describe('with authorized user', () => { + it('should enable and disable entity discovery', async () => { + const enableResponse = await enableEntityDiscovery(authorizedUser); + expect(enableResponse.success).to.eql(true, "authorized user can't enable EEM"); + + let definitionsResponse = await getInstalledDefinitions(authorizedUser); + expect(definitionsResponse.definitions.length).to.eql(builtInDefinitions.length); + expect( + builtInDefinitions.every((builtin) => { + return definitionsResponse.definitions.find( + (installedDefinition: EntityDefinitionWithState) => { + return ( + installedDefinition.id === builtin.id && + installedDefinition.state.installed && + installedDefinition.state.running + ); + } + ); + }) + ).to.eql(true, 'all builtin definitions are not installed/running'); + + let stateResponse = await entityDiscoveryState(authorizedUser); + expect(stateResponse.enabled).to.eql( + true, + `EEM is not enabled; response: ${JSON.stringify(stateResponse)}` + ); + + const disableResponse = await disableEntityDiscovery(authorizedUser, { deleteData: false }); + expect(disableResponse.success).to.eql( + true, + `authorized user failed to disable EEM; response: ${JSON.stringify(disableResponse)}` + ); + + stateResponse = await entityDiscoveryState(authorizedUser); + expect(stateResponse.enabled).to.eql(false, 'EEM is not disabled'); + + definitionsResponse = await getInstalledDefinitions(authorizedUser); + expect(definitionsResponse.definitions).to.eql([]); + }); + }); + + describe('with unauthorized user', () => { + it('should fail to enable entity discovery', async () => { + const enableResponse = await enableEntityDiscovery(unauthorizedUser); + expect(enableResponse.success).to.eql(false, 'unauthorized user can enable EEM'); + expect(enableResponse.reason).to.eql(ERROR_USER_NOT_AUTHORIZED); + + const stateResponse = await entityDiscoveryState(unauthorizedUser); + expect(stateResponse.enabled).to.eql(false, 'EEM is enabled'); + + const definitionsResponse = await getInstalledDefinitions(unauthorizedUser); + expect(definitionsResponse.definitions).to.eql([]); + }); + + it('should fail to disable entity discovery', async () => { + const enableResponse = await enableEntityDiscovery(authorizedUser); + expect(enableResponse.success).to.eql(true, "authorized user can't enable EEM"); + + let disableResponse = await enableEntityDiscovery(unauthorizedUser); + expect(disableResponse.success).to.eql(false, 'unauthorized user can disable EEM'); + expect(disableResponse.reason).to.eql(ERROR_USER_NOT_AUTHORIZED); + + disableResponse = await enableEntityDiscovery(authorizedUser); + expect(disableResponse.success).to.eql(true, "authorized user can't disable EEM"); + }); + }); + }); +} diff --git a/x-pack/test/api_integration/apis/entity_manager/helpers/user.ts b/x-pack/test/api_integration/apis/entity_manager/helpers/user.ts new file mode 100644 index 0000000000000..89181b7e0c155 --- /dev/null +++ b/x-pack/test/api_integration/apis/entity_manager/helpers/user.ts @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { mergeWith, uniq } from 'lodash'; +import { Client } from '@elastic/elasticsearch'; +import { + apiKeyCreationPrivileges, + entityDefinitionDeletionPrivileges, + entityDefinitionRuntimePrivileges, +} from '@kbn/entityManager-plugin/server/lib/auth/privileges'; + +export const createAdmin = async ({ + esClient, + username = 'entities_admin', + password = 'changeme', +}: { + esClient: Client; + username?: string; + password?: string; +}) => { + const privileges = mergeWith( + { application: [], index: [], cluster: [] }, + apiKeyCreationPrivileges, + entityDefinitionRuntimePrivileges, + entityDefinitionDeletionPrivileges, + (src, other) => uniq(src.concat(other)) + ); + const role = 'entities_all'; + + await esClient.security.putRole({ + name: role, + applications: privileges.application, + cluster: privileges.cluster, + indices: privileges.index, + }); + await esClient.security.putUser({ username, password, roles: [role] }); + + return { username, password }; +}; + +export const createRuntimeUser = async ({ + esClient, + username = 'entities_runtime_user', + password = 'changeme', +}: { + esClient: Client; + username?: string; + password?: string; +}) => { + const privileges = entityDefinitionRuntimePrivileges; + const role = 'entities_runtime'; + + await esClient.security.putRole({ + name: role, + applications: privileges.application, + cluster: privileges.cluster, + indices: privileges.index, + }); + await esClient.security.putUser({ username, password, roles: [role] }); + + return { username, password }; +}; diff --git a/x-pack/test/api_integration/apis/entity_manager/index.ts b/x-pack/test/api_integration/apis/entity_manager/index.ts new file mode 100644 index 0000000000000..74a5493401fa3 --- /dev/null +++ b/x-pack/test/api_integration/apis/entity_manager/index.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { FtrProviderContext } from '../../ftr_provider_context'; + +export default function ({ loadTestFile }: FtrProviderContext) { + describe('Entity Manager', function () { + this.tags(['entityManager']); + + loadTestFile(require.resolve('./enablement')); + }); +} diff --git a/x-pack/test/apm_api_integration/common/fixtures/es_archiver/8.0.0/mappings.json b/x-pack/test/apm_api_integration/common/fixtures/es_archiver/8.0.0/mappings.json index 4a40e66195db3..a64e037343bb3 100644 --- a/x-pack/test/apm_api_integration/common/fixtures/es_archiver/8.0.0/mappings.json +++ b/x-pack/test/apm_api_integration/common/fixtures/es_archiver/8.0.0/mappings.json @@ -22988,6 +22988,9 @@ "multi-line": { "type": "keyword" }, + "served_from_cache": { + "type": "keyword" + }, "this-is-a-very-long-tag-name-without-any-spaces": { "type": "keyword" } diff --git a/x-pack/test/functional/apps/lens/group1/multiple_data_views.ts b/x-pack/test/functional/apps/lens/group1/multiple_data_views.ts index 75d3d7ca6caf1..d1ab8555d3bdb 100644 --- a/x-pack/test/functional/apps/lens/group1/multiple_data_views.ts +++ b/x-pack/test/functional/apps/lens/group1/multiple_data_views.ts @@ -45,7 +45,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { ).to.eql(expectedData); } - describe('lens with multiple data views', () => { + // Failing: See https://github.com/elastic/kibana/issues/189056 + describe.skip('lens with multiple data views', () => { const visTitle = 'xyChart with multiple data views'; before(async () => { diff --git a/x-pack/test/tsconfig.json b/x-pack/test/tsconfig.json index 3b525dfc316ec..30d1dfa7a2e90 100644 --- a/x-pack/test/tsconfig.json +++ b/x-pack/test/tsconfig.json @@ -173,6 +173,7 @@ "@kbn/ml-trained-models-utils", "@kbn/openapi-common", "@kbn/securitysolution-lists-common", - "@kbn/securitysolution-exceptions-common" + "@kbn/securitysolution-exceptions-common", + "@kbn/entityManager-plugin" ] }