From e3425ccf733219d046c01f1ce63d2de74ab570ca Mon Sep 17 00:00:00 2001 From: Georgii Gorbachev Date: Mon, 7 Jun 2021 12:37:14 +0300 Subject: [PATCH 1/2] Build a consolidated implementation (Draft) --- .../plugins/rule_registry/common/constants.ts | 1 + .../assets/common_component_templates.ts | 35 ++++ .../common/event_log/assets/common_schema.ts | 14 ++ .../event_log/assets/default_ilm_policy.ts} | 33 ++-- .../event_log/assets/index.ts} | 8 +- .../assets/schema/common_ecs_fields.ts | 39 ++++ .../assets/schema/technical_fields.ts | 40 ++++ .../definition/event_log_definition.ts | 17 ++ .../event_log/definition/event_schema.ts | 27 +++ .../event_log/definition/ilm_policy.ts} | 5 +- .../common/event_log/definition/index.ts | 12 ++ .../event_log/definition}/index_names.ts | 42 ++++- .../event_log/definition/index_template.ts | 77 ++++++++ .../rule_registry/server/event_log/common.ts | 9 + .../server/event_log/elasticsearch/index.ts | 5 +- .../elasticsearch/index_bootstrapper.ts | 157 +++++++++++----- .../elasticsearch/index_management_gateway.ts | 171 +++++++++++++----- .../elasticsearch/resources/index_spec.ts | 49 +++++ .../elasticsearch/resources/index_template.ts | 52 ------ .../resources/index_templates.ts | 95 ++++++++++ .../server/event_log/event_schema/schema.ts | 51 ------ .../event_log/event_schema/schema_types.ts | 20 -- .../rule_registry/server/event_log/index.ts | 1 - .../log/bootstrapper_of_common_resources.ts | 69 +++++++ .../log/bootstrapper_of_log_resources.ts | 43 +++++ .../server/event_log/log/event_log.ts | 4 +- .../event_log/log/event_log_bootstrapper.ts | 51 ------ .../event_log/log/event_log_definition.ts | 37 ---- .../event_log/log/event_log_object_factory.ts | 119 ++++++++++++ .../event_log/log/event_log_provider.ts | 9 +- .../event_log/log/event_log_registry.ts | 23 ++- .../event_log/log/event_log_resolver.ts | 145 +++------------ .../server/event_log/log/event_log_service.ts | 28 ++- .../server/event_log/log/event_logger.ts | 5 +- .../event_log/log/event_logger_template.ts | 4 +- .../server/event_log/log/event_query.ts | 2 +- .../event_log/log/event_query_builder.ts | 5 +- .../server/event_log/log/index.ts | 1 - .../server/event_log/log/internal_api.ts | 20 +- .../server/event_log/log/public_api.ts | 44 ++--- .../log/utils/mapping_from_field_map.ts | 33 ---- x-pack/plugins/rule_registry/server/plugin.ts | 45 ++++- x-pack/plugins/rule_registry/server/types.ts | 13 ++ .../detection_engine/reference_rules/query.ts | 1 - .../routes/rules/find_rules_route.ts | 52 ++++++ .../schemas/execution_event_schema.ts | 7 + .../security_solution/server/plugin.ts | 1 + .../plugins/security_solution/server/types.ts | 2 + 48 files changed, 1151 insertions(+), 572 deletions(-) create mode 100644 x-pack/plugins/rule_registry/common/constants.ts create mode 100644 x-pack/plugins/rule_registry/common/event_log/assets/common_component_templates.ts create mode 100644 x-pack/plugins/rule_registry/common/event_log/assets/common_schema.ts rename x-pack/plugins/rule_registry/{server/event_log/elasticsearch/resources/ilm_policy.ts => common/event_log/assets/default_ilm_policy.ts} (50%) rename x-pack/plugins/rule_registry/{server/event_log/elasticsearch/resources/index_mappings.ts => common/event_log/assets/index.ts} (60%) create mode 100644 x-pack/plugins/rule_registry/common/event_log/assets/schema/common_ecs_fields.ts create mode 100644 x-pack/plugins/rule_registry/common/event_log/assets/schema/technical_fields.ts create mode 100644 x-pack/plugins/rule_registry/common/event_log/definition/event_log_definition.ts create mode 100644 x-pack/plugins/rule_registry/common/event_log/definition/event_schema.ts rename x-pack/plugins/rule_registry/{server/event_log/event_schema/index.ts => common/event_log/definition/ilm_policy.ts} (72%) create mode 100644 x-pack/plugins/rule_registry/common/event_log/definition/index.ts rename x-pack/plugins/rule_registry/{server/event_log/elasticsearch/resources => common/event_log/definition}/index_names.ts (69%) create mode 100644 x-pack/plugins/rule_registry/common/event_log/definition/index_template.ts create mode 100644 x-pack/plugins/rule_registry/server/event_log/common.ts create mode 100644 x-pack/plugins/rule_registry/server/event_log/elasticsearch/resources/index_spec.ts delete mode 100644 x-pack/plugins/rule_registry/server/event_log/elasticsearch/resources/index_template.ts create mode 100644 x-pack/plugins/rule_registry/server/event_log/elasticsearch/resources/index_templates.ts delete mode 100644 x-pack/plugins/rule_registry/server/event_log/event_schema/schema.ts delete mode 100644 x-pack/plugins/rule_registry/server/event_log/event_schema/schema_types.ts create mode 100644 x-pack/plugins/rule_registry/server/event_log/log/bootstrapper_of_common_resources.ts create mode 100644 x-pack/plugins/rule_registry/server/event_log/log/bootstrapper_of_log_resources.ts delete mode 100644 x-pack/plugins/rule_registry/server/event_log/log/event_log_bootstrapper.ts delete mode 100644 x-pack/plugins/rule_registry/server/event_log/log/event_log_definition.ts create mode 100644 x-pack/plugins/rule_registry/server/event_log/log/event_log_object_factory.ts delete mode 100644 x-pack/plugins/rule_registry/server/event_log/log/utils/mapping_from_field_map.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/schemas/execution_event_schema.ts diff --git a/x-pack/plugins/rule_registry/common/constants.ts b/x-pack/plugins/rule_registry/common/constants.ts new file mode 100644 index 0000000000000..0eeb0e47ecb81 --- /dev/null +++ b/x-pack/plugins/rule_registry/common/constants.ts @@ -0,0 +1 @@ +export const APP_ID = 'ruleRegistry'; diff --git a/x-pack/plugins/rule_registry/common/event_log/assets/common_component_templates.ts b/x-pack/plugins/rule_registry/common/event_log/assets/common_component_templates.ts new file mode 100644 index 0000000000000..f400d4147e397 --- /dev/null +++ b/x-pack/plugins/rule_registry/common/event_log/assets/common_component_templates.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { merge } from 'lodash'; +import { ComponentTemplateOptions } from '../definition'; +import { commonEcsMappings } from './schema/common_ecs_fields'; +import { technicalFieldMappings } from './schema/technical_fields'; + +/** + * Based on these options the Event Log mechanism will create and maintain + * `.alerts-settings` component template. + */ +export const commonSettingsTemplate: ComponentTemplateOptions = { + version: 1, + settings: { + number_of_shards: 1, + auto_expand_replicas: '0-1', + 'mapping.total_fields.limit': 10000, + 'sort.field': '@timestamp', + 'sort.order': 'desc', + }, +}; + +/** + * Based on these options the Event Log mechanism will create and maintain + * `.alerts-mappings` component template. + */ +export const commonMappingsTemplate: ComponentTemplateOptions = { + version: 1, + mappings: merge({}, commonEcsMappings, technicalFieldMappings), +}; diff --git a/x-pack/plugins/rule_registry/common/event_log/assets/common_schema.ts b/x-pack/plugins/rule_registry/common/event_log/assets/common_schema.ts new file mode 100644 index 0000000000000..24bdabcc6334f --- /dev/null +++ b/x-pack/plugins/rule_registry/common/event_log/assets/common_schema.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Schema } from '../definition'; +import { commonEcsSchema } from './schema/common_ecs_fields'; +import { technicalFieldSchema } from './schema/technical_fields'; + +export const commonSchema = Schema.combine(commonEcsSchema, technicalFieldSchema); + +export type CommonFields = typeof commonSchema.event; diff --git a/x-pack/plugins/rule_registry/server/event_log/elasticsearch/resources/ilm_policy.ts b/x-pack/plugins/rule_registry/common/event_log/assets/default_ilm_policy.ts similarity index 50% rename from x-pack/plugins/rule_registry/server/event_log/elasticsearch/resources/ilm_policy.ts rename to x-pack/plugins/rule_registry/common/event_log/assets/default_ilm_policy.ts index 7663ea27fb0df..f7ca7b4fc6769 100644 --- a/x-pack/plugins/rule_registry/server/event_log/elasticsearch/resources/ilm_policy.ts +++ b/x-pack/plugins/rule_registry/common/event_log/assets/default_ilm_policy.ts @@ -5,29 +5,26 @@ * 2.0. */ -import { estypes } from '@elastic/elasticsearch'; - -export interface IlmPolicy { - policy: estypes.IlmPolicy; -} +import { IlmPolicy } from '../definition'; export const defaultIlmPolicy: IlmPolicy = { - policy: { - phases: { - hot: { - min_age: '0ms', - actions: { - rollover: { - max_age: '90d', - max_size: '50gb', - }, + phases: { + hot: { + min_age: '0ms', + actions: { + rollover: { + max_age: '30d', + max_primary_shard_size: '50gb', }, - }, - delete: { - actions: { - delete: {}, + set_priority: { + priority: 100, }, }, }, + delete: { + actions: { + delete: {}, + }, + }, }, }; diff --git a/x-pack/plugins/rule_registry/server/event_log/elasticsearch/resources/index_mappings.ts b/x-pack/plugins/rule_registry/common/event_log/assets/index.ts similarity index 60% rename from x-pack/plugins/rule_registry/server/event_log/elasticsearch/resources/index_mappings.ts rename to x-pack/plugins/rule_registry/common/event_log/assets/index.ts index 064bde5001f7b..f4fc23f935ff0 100644 --- a/x-pack/plugins/rule_registry/server/event_log/elasticsearch/resources/index_mappings.ts +++ b/x-pack/plugins/rule_registry/common/event_log/assets/index.ts @@ -5,8 +5,6 @@ * 2.0. */ -export interface IndexMappings { - dynamic: 'strict' | boolean; - properties: Record; - _meta?: Record; -} +export * from './common_component_templates'; +export * from './common_schema'; +export * from './default_ilm_policy'; diff --git a/x-pack/plugins/rule_registry/common/event_log/assets/schema/common_ecs_fields.ts b/x-pack/plugins/rule_registry/common/event_log/assets/schema/common_ecs_fields.ts new file mode 100644 index 0000000000000..44385f0e2dfb5 --- /dev/null +++ b/x-pack/plugins/rule_registry/common/event_log/assets/schema/common_ecs_fields.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { pickWithPatterns } from '../../../../common/pick_with_patterns'; +import { + TIMESTAMP, + TAGS, + EVENT_KIND, + EVENT_ACTION, + RULE_UUID, + RULE_ID, + RULE_NAME, + RULE_CATEGORY, +} from '../../../../common/technical_rule_data_field_names'; +import { ecsFieldMap } from '../../../assets/field_maps/ecs_field_map'; +import { mappingFromFieldMap } from '../../../mapping_from_field_map'; +import { runtimeTypeFromFieldMap } from '../../../field_map/runtime_type_from_fieldmap'; +import { Schema } from '../../definition'; + +export const commonEcsFieldMap = { + ...pickWithPatterns( + ecsFieldMap, + TIMESTAMP, + TAGS, + EVENT_KIND, + EVENT_ACTION, + RULE_UUID, + RULE_ID, + RULE_NAME, + RULE_CATEGORY + ), +} as const; + +export const commonEcsMappings = mappingFromFieldMap(commonEcsFieldMap); +export const commonEcsSchema = Schema.create(runtimeTypeFromFieldMap(commonEcsFieldMap)); diff --git a/x-pack/plugins/rule_registry/common/event_log/assets/schema/technical_fields.ts b/x-pack/plugins/rule_registry/common/event_log/assets/schema/technical_fields.ts new file mode 100644 index 0000000000000..db0389a49f5c8 --- /dev/null +++ b/x-pack/plugins/rule_registry/common/event_log/assets/schema/technical_fields.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + ALERT_DURATION, + ALERT_END, + ALERT_EVALUATION_THRESHOLD, + ALERT_EVALUATION_VALUE, + ALERT_ID, + ALERT_SEVERITY_LEVEL, + ALERT_SEVERITY_VALUE, + ALERT_START, + ALERT_STATUS, + ALERT_UUID, + PRODUCER, +} from '../../../../common/technical_rule_data_field_names'; +import { mappingFromFieldMap } from '../../../mapping_from_field_map'; +import { runtimeTypeFromFieldMap } from '../../../field_map/runtime_type_from_fieldmap'; +import { Schema } from '../../definition'; + +export const technicalFieldMap = { + [PRODUCER]: { type: 'keyword' }, + [ALERT_UUID]: { type: 'keyword' }, + [ALERT_ID]: { type: 'keyword' }, + [ALERT_START]: { type: 'date' }, + [ALERT_END]: { type: 'date' }, + [ALERT_DURATION]: { type: 'long' }, + [ALERT_SEVERITY_LEVEL]: { type: 'keyword' }, + [ALERT_SEVERITY_VALUE]: { type: 'long' }, + [ALERT_STATUS]: { type: 'keyword' }, + [ALERT_EVALUATION_THRESHOLD]: { type: 'scaled_float', scaling_factor: 100 }, + [ALERT_EVALUATION_VALUE]: { type: 'scaled_float', scaling_factor: 100 }, +} as const; + +export const technicalFieldMappings = mappingFromFieldMap(technicalFieldMap); +export const technicalFieldSchema = Schema.create(runtimeTypeFromFieldMap(technicalFieldMap)); diff --git a/x-pack/plugins/rule_registry/common/event_log/definition/event_log_definition.ts b/x-pack/plugins/rule_registry/common/event_log/definition/event_log_definition.ts new file mode 100644 index 0000000000000..af5ffb72bd830 --- /dev/null +++ b/x-pack/plugins/rule_registry/common/event_log/definition/event_log_definition.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EventSchema } from './event_schema'; +import { IlmPolicy } from './ilm_policy'; +import { Templates } from './index_template'; + +export interface EventLogDefinition { + logName: string; + schema: EventSchema; + templates: Templates; + ilmPolicy?: IlmPolicy; +} diff --git a/x-pack/plugins/rule_registry/common/event_log/definition/event_schema.ts b/x-pack/plugins/rule_registry/common/event_log/definition/event_schema.ts new file mode 100644 index 0000000000000..dd23dd03ace27 --- /dev/null +++ b/x-pack/plugins/rule_registry/common/event_log/definition/event_schema.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import * as t from 'io-ts'; + +export interface EventSchema { + eventType: t.Type; + event: TEvent; +} + +export abstract class Schema { + public static create(eventType: t.Type): EventSchema { + return { + eventType, + event: {} as t.TypeOf, + }; + } + + public static combine(s1: EventSchema, s2: EventSchema): EventSchema { + const combinedType = t.intersection([s1.eventType, s2.eventType]); + return this.create(combinedType); + } +} diff --git a/x-pack/plugins/rule_registry/server/event_log/event_schema/index.ts b/x-pack/plugins/rule_registry/common/event_log/definition/ilm_policy.ts similarity index 72% rename from x-pack/plugins/rule_registry/server/event_log/event_schema/index.ts rename to x-pack/plugins/rule_registry/common/event_log/definition/ilm_policy.ts index 77c041a4059b5..f45629f9e40cc 100644 --- a/x-pack/plugins/rule_registry/server/event_log/event_schema/index.ts +++ b/x-pack/plugins/rule_registry/common/event_log/definition/ilm_policy.ts @@ -5,5 +5,6 @@ * 2.0. */ -export * from './schema_types'; -export * from './schema'; +import { estypes } from '@elastic/elasticsearch'; + +export type IlmPolicy = estypes.IlmPolicy; diff --git a/x-pack/plugins/rule_registry/common/event_log/definition/index.ts b/x-pack/plugins/rule_registry/common/event_log/definition/index.ts new file mode 100644 index 0000000000000..6ddd76a37afd8 --- /dev/null +++ b/x-pack/plugins/rule_registry/common/event_log/definition/index.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './event_log_definition'; +export * from './event_schema'; +export * from './ilm_policy'; +export * from './index_names'; +export * from './index_template'; diff --git a/x-pack/plugins/rule_registry/server/event_log/elasticsearch/resources/index_names.ts b/x-pack/plugins/rule_registry/common/event_log/definition/index_names.ts similarity index 69% rename from x-pack/plugins/rule_registry/server/event_log/elasticsearch/resources/index_names.ts rename to x-pack/plugins/rule_registry/common/event_log/definition/index_names.ts index 1082c62b95e70..5ae1d586329c7 100644 --- a/x-pack/plugins/rule_registry/server/event_log/elasticsearch/resources/index_names.ts +++ b/x-pack/plugins/rule_registry/common/event_log/definition/index_names.ts @@ -29,14 +29,31 @@ export interface IndexNames extends IndexParams { /** @example '.alerts-security.alerts-default-*' */ indexAliasPattern: string; + /** @example '.alerts-security.alerts-default-000001' */ + indexInitialName: string; + /** @example '.alerts-security.alerts-default-policy' */ indexIlmPolicyName: string; /** @example '.alerts-security.alerts-default-template' */ indexTemplateName: string; - /** @example '.alerts-security.alerts-default-000001' */ - indexInitialName: string; + componentTemplates: { + /** @example '.alerts-mappings' */ + commonMappingsTemplateName: string; + + /** @example '.alerts-settings' */ + commonSettingsTemplateName: string; + + /** @example '.alerts-security.alerts-app' */ + applicationDefinedTemplateName: string; + + /** @example '.alerts-security.alerts-user' */ + userDefinedTemplateName: string; + + /** @example '.alerts-security.alerts-user-default' */ + userDefinedSpaceAwareTemplateName: string; + }; } export abstract class IndexNames { @@ -49,9 +66,19 @@ export abstract class IndexNames { const indexBasePattern = joinWithDash(indexPrefix, logName, '*'); const indexAliasName = joinWithDash(indexPrefix, logName, kibanaSpaceId); const indexAliasPattern = joinWithDash(indexPrefix, logName, kibanaSpaceId, '*'); + const indexInitialName = joinWithDash(indexPrefix, logName, kibanaSpaceId, '000001'); const indexIlmPolicyName = joinWithDash(indexPrefix, logName, kibanaSpaceId, 'policy'); const indexTemplateName = joinWithDash(indexPrefix, logName, kibanaSpaceId, 'template'); - const indexInitialName = joinWithDash(indexPrefix, logName, kibanaSpaceId, '000001'); + const commonMappingsTemplateName = joinWithDash(indexPrefix, 'mappings'); + const commonSettingsTemplateName = joinWithDash(indexPrefix, 'settings'); + const applicationDefinedTemplateName = joinWithDash(indexPrefix, logName, 'app'); + const userDefinedTemplateName = joinWithDash(indexPrefix, logName, 'user'); + const userDefinedSpaceAwareTemplateName = joinWithDash( + indexPrefix, + logName, + 'user', + kibanaSpaceId + ); return { indexPrefix, @@ -61,9 +88,16 @@ export abstract class IndexNames { indexBasePattern, indexAliasName, indexAliasPattern, + indexInitialName, indexIlmPolicyName, indexTemplateName, - indexInitialName, + componentTemplates: { + commonMappingsTemplateName, + commonSettingsTemplateName, + applicationDefinedTemplateName, + userDefinedTemplateName, + userDefinedSpaceAwareTemplateName, + }, }; } diff --git a/x-pack/plugins/rule_registry/common/event_log/definition/index_template.ts b/x-pack/plugins/rule_registry/common/event_log/definition/index_template.ts new file mode 100644 index 0000000000000..eabcc527b30bf --- /dev/null +++ b/x-pack/plugins/rule_registry/common/event_log/definition/index_template.ts @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { estypes } from '@elastic/elasticsearch'; + +type TemplateRequest = NonNullable; + +export type Settings = NonNullable; +export type Mappings = NonNullable; +export type Aliases = NonNullable; +export type Version = NonNullable; +export type Meta = Record; + +export interface TemplateOptions { + settings?: Settings; + mappings?: Mappings; + aliases?: Aliases; + version?: Version; + meta?: Meta; +} + +export type ComponentTemplateOptions = TemplateOptions; + +export interface IndexTemplateOptions extends TemplateOptions { + priority?: number; +} + +/** + * During index bootstrapping a number of templates will be created by the + * Event Log mechanism. They will have a certain order of precedence and + * the "next" template will override properties from all the "previous" ones. + * Here's the list of templates from "start" (most generic, least precedence) + * to "finish" (most specific, most precedence): + * + * 1. Mechanism-level `.alerts-mappings` component template. Specified + * internally by the Event Log mechanism. Contains index mappings common + * to all logs (observability alerts, security execution events, etc). + * 2. Mechanism-level `.alerts-settings` component template. Specified + * internally by the Event Log mechanism. Contains index settings which + * make sense to all logs by default. + * 3. Log-level `.alerts-{log.name}-app` application-defined component template. + * Specified and versioned externally by the application (plugin) which + * defines the log. Contains index mappings and/or settings specific to + * this particular log. This is the place where you as application developer + * can override or extend the default framework mappings and settings. + * 4. Log-level `.alerts-{log.name}-user` user-defined component template. + * Specified internally by the Event Log mechanism, is empty, not versioned. + * By updating it, the user can override default mappings and settings. + * 5. Log-level `.alerts-{log.name}-user-{spaceId}` user-defined space-aware + * component template. Specified internally by the Event Log mechanism, + * is empty, not versioned. By updating it, the user can override default + * mappings and settings of the log in a certain Kibana space. + * 6. Log-level `.alerts-{log.name}-{spaceId}-template` index template. + * Its version and most of its options can be specified externally by the + * application (plugin) which defines the log. This is the place where you + * as application developer can override user settings. However, mind that + * the Event Log mechanism has the last word and injects some hard defaults + * into the final index template to make sure it works as it should. + * + * Template #6 overrides #5, which overrides #4, which overrides #3, etc. + * More on composing multiple templates in the docs: + * https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-put-template.html#multiple-component-templates + * + * As a developer instantiating Templates, you are able to specify templates + * #3 (applicationDefinedComponentTemplate) and optionally #6 (indexTemplate). + * Start with setting application-defined component template options, it should + * be enough in most cases. Specify index template options ONLY if you intend + * to override user settings or mappings for whatever reason. + */ +export interface Templates { + applicationDefinedComponentTemplate: ComponentTemplateOptions; + indexTemplate?: IndexTemplateOptions; +} diff --git a/x-pack/plugins/rule_registry/server/event_log/common.ts b/x-pack/plugins/rule_registry/server/event_log/common.ts new file mode 100644 index 0000000000000..67205659e5664 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/event_log/common.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from '../../common/event_log/assets'; +export * from '../../common/event_log/definition'; diff --git a/x-pack/plugins/rule_registry/server/event_log/elasticsearch/index.ts b/x-pack/plugins/rule_registry/server/event_log/elasticsearch/index.ts index 1941208ed07cd..fd86270b72560 100644 --- a/x-pack/plugins/rule_registry/server/event_log/elasticsearch/index.ts +++ b/x-pack/plugins/rule_registry/server/event_log/elasticsearch/index.ts @@ -9,6 +9,5 @@ export * from './index_bootstrapper'; export * from './index_management_gateway'; export * from './index_reader'; export * from './index_writer'; -export * from './resources/ilm_policy'; -export * from './resources/index_mappings'; -export * from './resources/index_names'; +export * from './resources/index_spec'; +export * from './resources/index_templates'; diff --git a/x-pack/plugins/rule_registry/server/event_log/elasticsearch/index_bootstrapper.ts b/x-pack/plugins/rule_registry/server/event_log/elasticsearch/index_bootstrapper.ts index b0c3927cd7dfe..044672b2fb9b1 100644 --- a/x-pack/plugins/rule_registry/server/event_log/elasticsearch/index_bootstrapper.ts +++ b/x-pack/plugins/rule_registry/server/event_log/elasticsearch/index_bootstrapper.ts @@ -8,10 +8,9 @@ import type { PublicMethodsOf } from '@kbn/utility-types'; import { Logger } from 'src/core/server'; -import { IndexNames } from './resources/index_names'; -import { IndexMappings } from './resources/index_mappings'; -import { createIndexTemplate } from './resources/index_template'; -import { IlmPolicy, defaultIlmPolicy } from './resources/ilm_policy'; +import { IndexNames, commonMappingsTemplate, commonSettingsTemplate } from '../common'; +import { IndexSpec } from './resources/index_spec'; +import { ComponentTemplate, createComponentTemplate } from './resources/index_templates'; import { IIndexManagementGateway } from './index_management_gateway'; interface ConstructorParams { @@ -19,12 +18,6 @@ interface ConstructorParams { logger: Logger; } -export interface IndexSpecification { - indexNames: IndexNames; - indexMappings: IndexMappings; - ilmPolicy?: IlmPolicy; -} - export type IIndexBootstrapper = PublicMethodsOf; // TODO: Converge with the logic of .siem-signals index bootstrapping @@ -46,55 +39,135 @@ export class IndexBootstrapper { this.logger = params.logger.get('IndexBootstrapper'); } - public async run(indexSpec: IndexSpecification): Promise { - this.logger.debug('bootstrapping elasticsearch resources starting'); + public async bootstrapCommonResources(indexNames: IndexNames): Promise { + const { indexPrefix, componentTemplates } = indexNames; + + this.logger.debug(`Bootstrapping common resources for "${indexPrefix}" indices`); try { - const { indexNames, indexMappings, ilmPolicy } = indexSpec; - await this.createIlmPolicyIfNotExists(indexNames, ilmPolicy); - await this.createIndexTemplateIfNotExists(indexNames, indexMappings); - await this.createInitialIndexIfNotExists(indexNames); - } catch (err) { - this.logger.error(`error bootstrapping elasticsearch resources: ${err.message}`); - return false; + await Promise.all([ + this.createOrUpdateComponentTemplate( + componentTemplates.commonMappingsTemplateName, + createComponentTemplate(commonMappingsTemplate) + ), + this.createOrUpdateComponentTemplate( + componentTemplates.commonSettingsTemplateName, + createComponentTemplate(commonSettingsTemplate) + ), + ]); + + this.logger.debug(`Finished bootstrapping common resources for "${indexPrefix}" indices`); + } catch (e) { + this.logger.error( + `Error bootstrapping common resources for "${indexPrefix}" indices: ${e.message}` + ); + throw e; } + } + + public async bootstrapLogLevelResources(indexSpec: IndexSpec): Promise { + const { indexAliasName } = indexSpec.indexNames; + + this.logger.debug(`Bootstrapping index "${indexAliasName}"`); + + try { + await this.createIlmPolicyIfNotExists(indexSpec); + + const componentTemplatesUpdated = await this.createOrUpdateComponentTemplates(indexSpec); + const indexTemplateUpdated = await this.createOrUpdateIndexTemplate(indexSpec); + const anyTemplatesUpdated = componentTemplatesUpdated || indexTemplateUpdated; - this.logger.debug('bootstrapping elasticsearch resources complete'); - return true; + await this.createOrRolloverIndex(indexSpec, anyTemplatesUpdated); + + this.logger.debug(`Finished bootstrapping index "${indexAliasName}"`); + } catch (e) { + this.logger.error(`Error bootstrapping index "${indexAliasName}": ${e.message}`); + throw e; + } } - private async createIlmPolicyIfNotExists(names: IndexNames, policy?: IlmPolicy): Promise { - const { indexIlmPolicyName } = names; + private async createIlmPolicyIfNotExists(indexSpec: IndexSpec): Promise { + const { indexNames, ilmPolicy } = indexSpec; + const { indexIlmPolicyName } = indexNames; const exists = await this.gateway.doesIlmPolicyExist(indexIlmPolicyName); if (!exists) { - const ilmPolicy = policy ?? defaultIlmPolicy; - await this.gateway.createIlmPolicy(indexIlmPolicyName, ilmPolicy); + await this.gateway.setIlmPolicy(indexIlmPolicyName, ilmPolicy); } } - private async createIndexTemplateIfNotExists( - names: IndexNames, - mappings: IndexMappings - ): Promise { - const { indexTemplateName } = names; + private async createOrUpdateComponentTemplates(indexSpec: IndexSpec): Promise { + const names = indexSpec.indexNames.componentTemplates; + + const results = await Promise.all([ + this.createOrUpdateComponentTemplate( + names.applicationDefinedTemplateName, + indexSpec.applicationDefinedTemplate + ), + this.createOrUpdateComponentTemplate( + names.userDefinedTemplateName, + indexSpec.userDefinedTemplate + ), + this.createOrUpdateComponentTemplate( + names.userDefinedSpaceAwareTemplateName, + indexSpec.userDefinedSpaceAwareTemplate + ), + ]); + + return results.some(Boolean); + } - const templateVersion = 1; // TODO: get from EventSchema definition - const template = createIndexTemplate(names, mappings, templateVersion); + private async createOrUpdateComponentTemplate( + componentTemplateName: string, + componentTemplate: ComponentTemplate + ): Promise { + const result = await this.gateway.getComponentTemplateVersion(componentTemplateName); - const exists = await this.gateway.doesIndexTemplateExist(indexTemplateName); - if (!exists) { - await this.gateway.createIndexTemplate(indexTemplateName, template); - } else { - await this.gateway.updateIndexTemplate(indexTemplateName, template); + const currentVersion = result.templateVersion; + const targetVersion = componentTemplate.version; + + const templateNeedsUpdate = + !result.templateExists || + (currentVersion == null && targetVersion != null) || + (currentVersion != null && targetVersion != null && currentVersion < targetVersion); + + if (templateNeedsUpdate) { + await this.gateway.setComponentTemplate(componentTemplateName, componentTemplate); } + + return templateNeedsUpdate; } - private async createInitialIndexIfNotExists(names: IndexNames): Promise { - const { indexAliasName, indexInitialName } = names; + private async createOrUpdateIndexTemplate(indexSpec: IndexSpec): Promise { + const { indexNames, indexTemplate } = indexSpec; + const { indexTemplateName } = indexNames; - const exists = await this.gateway.doesAliasExist(indexAliasName); - if (!exists) { + const result = await this.gateway.getIndexTemplateVersion(indexTemplateName); + + const currentVersion = result.templateVersion; + const targetVersion = indexTemplate.version; + + const templateNeedsUpdate = + !result.templateExists || + (currentVersion == null && targetVersion != null) || + (currentVersion != null && targetVersion != null && currentVersion < targetVersion); + + if (templateNeedsUpdate) { + await this.gateway.setIndexTemplate(indexTemplateName, indexTemplate); + } + + return templateNeedsUpdate; + } + + private async createOrRolloverIndex( + indexSpec: IndexSpec, + anyTemplatesUpdated: boolean + ): Promise { + const { indexNames } = indexSpec; + const { indexAliasName, indexInitialName } = indexNames; + + const indexAliasExists = await this.gateway.doesAliasExist(indexAliasName); + if (!indexAliasExists) { await this.gateway.createIndex(indexInitialName, { aliases: { [indexAliasName]: { @@ -102,6 +175,8 @@ export class IndexBootstrapper { }, }, }); + } else if (anyTemplatesUpdated) { + await this.gateway.rolloverAlias(indexAliasName); } } } diff --git a/x-pack/plugins/rule_registry/server/event_log/elasticsearch/index_management_gateway.ts b/x-pack/plugins/rule_registry/server/event_log/elasticsearch/index_management_gateway.ts index cb04a442d0b34..5ce706385f139 100644 --- a/x-pack/plugins/rule_registry/server/event_log/elasticsearch/index_management_gateway.ts +++ b/x-pack/plugins/rule_registry/server/event_log/elasticsearch/index_management_gateway.ts @@ -7,8 +7,8 @@ import type { PublicMethodsOf } from '@kbn/utility-types'; import { ElasticsearchClient, Logger } from 'src/core/server'; -import { IlmPolicy } from './resources/ilm_policy'; -import { IndexTemplate } from './resources/index_template'; +import { IlmPolicy } from '../common'; +import { ComponentTemplate, IndexTemplate } from './resources/index_templates'; interface ConstructorParams { elasticsearch: Promise; @@ -27,101 +27,167 @@ export class IndexManagementGateway { } public async doesIlmPolicyExist(policyName: string): Promise { - this.logger.debug(`Checking if ILM policy exists; name="${policyName}"`); + this.logger.debug(`Checking existence of ILM policy "${policyName}"`); try { const es = await this.elasticsearch; - await es.transport.request({ - method: 'GET', - path: `/_ilm/policy/${policyName}`, - }); + await es.ilm.getLifecycle({ policy: policyName }); + return true; } catch (e) { - if (e.statusCode === 404) return false; - throw new Error(`Error checking existence of ILM policy: ${e.message}`); + if (e.statusCode === 404) { + return false; + } + + this.logger.error(e); + throw new Error(`Error checking existence of ILM policy "${policyName}": ${e.message}`); } - return true; } - public async createIlmPolicy(policyName: string, policy: IlmPolicy): Promise { - this.logger.debug(`Creating ILM policy; name="${policyName}"`); + public async setIlmPolicy(policyName: string, policy: IlmPolicy): Promise { + this.logger.debug(`Setting ILM policy "${policyName}"`); try { const es = await this.elasticsearch; - await es.transport.request({ - method: 'PUT', - path: `/_ilm/policy/${policyName}`, - body: policy, + await es.ilm.putLifecycle({ + policy: policyName, + body: { policy }, }); } catch (e) { - throw new Error(`Error creating ILM policy: ${e.message}`); + this.logger.error(e); + throw new Error(`Error setting ILM policy "${policyName}": ${e.message}`); } } - public async doesIndexTemplateExist(templateName: string): Promise { - this.logger.debug(`Checking if index template exists; name="${templateName}"`); + public async getComponentTemplateVersion(templateName: string): Promise { + this.logger.debug(`Getting component template version "${templateName}"`); try { const es = await this.elasticsearch; - const { body } = await es.indices.existsTemplate({ name: templateName }); - return body as boolean; + const response = await es.cluster.getComponentTemplate({ + name: templateName, + }); + + // const response = await es.transport.request({ + // method: 'GET', + // path: `/_component_template/${templateName}`, + // }); + + const template = response.body.component_templates.find(({ name }) => name === templateName); + + if (template) { + return { + templateName, + templateExists: true, + templateVersion: template.component_template.version ?? undefined, + }; + } else { + return { + templateName, + templateExists: false, + templateVersion: undefined, + }; + } } catch (e) { - throw new Error(`Error checking existence of index template: ${e.message}`); + this.logger.error(e); + return { + templateName, + templateExists: false, + templateVersion: undefined, + }; } } - public async createIndexTemplate(templateName: string, template: IndexTemplate): Promise { - this.logger.debug(`Creating index template; name="${templateName}"`); + public async getIndexTemplateVersion(templateName: string): Promise { + this.logger.debug(`Getting index template version "${templateName}"`); try { const es = await this.elasticsearch; - await es.indices.putTemplate({ create: true, name: templateName, body: template }); - } catch (e) { - // The error message doesn't have a type attribute we can look to guarantee it's due - // to the template already existing (only long message) so we'll check ourselves to see - // if the template now exists. This scenario would happen if you startup multiple Kibana - // instances at the same time. - const existsNow = await this.doesIndexTemplateExist(templateName); - if (!existsNow) { - const error = new Error(`Error creating index template: ${e.message}`); - Object.assign(error, { wrapped: e }); - throw error; + const response = await es.indices.getIndexTemplate({ name: templateName }); + + const template = response.body.index_templates.find(({ name }) => name === templateName); + if (template) { + return { + templateName, + templateExists: true, + templateVersion: template.index_template.version ?? undefined, + }; + } else { + return { + templateName, + templateExists: false, + templateVersion: undefined, + }; } + } catch (e) { + this.logger.error(e); + return { + templateName, + templateExists: false, + templateVersion: undefined, + }; } } - public async updateIndexTemplate(templateName: string, template: IndexTemplate): Promise { - this.logger.debug(`Updating index template; name="${templateName}"`); + public async setComponentTemplate( + templateName: string, + template: ComponentTemplate + ): Promise { + this.logger.debug(`Setting component template "${templateName}"`); try { - // @ts-expect-error settings is optional - const { settings, ...templateWithoutSettings } = template; + const es = await this.elasticsearch; + await es.cluster.putComponentTemplate({ + name: templateName, + body: template, + }); + } catch (e) { + this.logger.error(e); + throw new Error(`Error setting component template "${templateName}": ${e.message}`); + } + } + public async setIndexTemplate(templateName: string, template: IndexTemplate): Promise { + this.logger.debug(`Setting index template "${templateName}"`); + + try { const es = await this.elasticsearch; - await es.indices.putTemplate({ - create: false, + await es.indices.putIndexTemplate({ name: templateName, - body: templateWithoutSettings, + body: template, }); } catch (e) { - throw new Error(`Error updating index template: ${e.message}`); + this.logger.error(e); + throw new Error(`Error setting index template "${templateName}": ${e.message}`); } } public async doesAliasExist(aliasName: string): Promise { - this.logger.debug(`Checking if index alias exists; name="${aliasName}"`); + this.logger.debug(`Checking existence of index alias "${aliasName}"`); try { const es = await this.elasticsearch; const { body } = await es.indices.existsAlias({ name: aliasName }); - return body as boolean; + return body; } catch (e) { - throw new Error(`Error checking existence of initial index: ${e.message}`); + this.logger.error(e); + throw new Error(`Error checking existence of index alias "${aliasName}": ${e.message}`); + } + } + + public async rolloverAlias(aliasName: string): Promise { + this.logger.debug(`Starting rollover of index alias "${aliasName}"`); + + try { + const es = await this.elasticsearch; + await es.indices.rollover({ alias: aliasName }); + } catch (e) { + this.logger.error(e); + throw new Error(`Error in rollover of index alias "${aliasName}": ${e.message}`); } } public async createIndex(indexName: string, body: Record = {}): Promise { - this.logger.debug(`Creating index; name="${indexName}"`); - this.logger.debug(JSON.stringify(body, null, 2)); + this.logger.debug(`Creating index "${indexName}"`); try { const es = await this.elasticsearch; @@ -132,9 +198,14 @@ export class IndexManagementGateway { } catch (e) { if (e.body?.error?.type !== 'resource_already_exists_exception') { this.logger.error(e); - this.logger.error(JSON.stringify(e.meta, null, 2)); - throw new Error(`Error creating initial index: ${e.message}`); + throw new Error(`Error creating index "${indexName}": ${e.message}`); } } } } + +export interface TemplateVersion { + templateName: string; + templateExists: boolean; + templateVersion?: number; +} diff --git a/x-pack/plugins/rule_registry/server/event_log/elasticsearch/resources/index_spec.ts b/x-pack/plugins/rule_registry/server/event_log/elasticsearch/resources/index_spec.ts new file mode 100644 index 0000000000000..a8e48861ba0c1 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/event_log/elasticsearch/resources/index_spec.ts @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EventLogDefinition, IndexNames, IlmPolicy, defaultIlmPolicy } from '../../common'; +import { + ComponentTemplate, + IndexTemplate, + createComponentTemplate, + createIndexTemplate, +} from './index_templates'; + +export interface IndexSpec { + indexNames: IndexNames; + + applicationDefinedTemplate: ComponentTemplate; + userDefinedTemplate: ComponentTemplate; + userDefinedSpaceAwareTemplate: ComponentTemplate; + indexTemplate: IndexTemplate; + + ilmPolicy: IlmPolicy; +} + +export const createIndexSpec = ( + logDefinition: EventLogDefinition, + indexPrefix: string, + kibanaSpaceId: string +): IndexSpec => { + const { logName, templates, ilmPolicy } = logDefinition; + const { applicationDefinedComponentTemplate, indexTemplate } = templates; + + const indexNames = IndexNames.create({ + indexPrefix, + logName, + kibanaSpaceId, + }); + + return { + indexNames, + applicationDefinedTemplate: createComponentTemplate(applicationDefinedComponentTemplate), + userDefinedTemplate: createComponentTemplate({}), + userDefinedSpaceAwareTemplate: createComponentTemplate({}), + indexTemplate: createIndexTemplate(indexNames, indexTemplate), + ilmPolicy: ilmPolicy ?? defaultIlmPolicy, + }; +}; diff --git a/x-pack/plugins/rule_registry/server/event_log/elasticsearch/resources/index_template.ts b/x-pack/plugins/rule_registry/server/event_log/elasticsearch/resources/index_template.ts deleted file mode 100644 index 0045bce3ed873..0000000000000 --- a/x-pack/plugins/rule_registry/server/event_log/elasticsearch/resources/index_template.ts +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { estypes } from '@elastic/elasticsearch'; -import { IndexNames } from './index_names'; -import { IndexMappings } from './index_mappings'; - -export type IndexTemplate = estypes.IndicesPutTemplateRequest['body']; - -export const createIndexTemplate = ( - names: IndexNames, - mappings: IndexMappings, - version: number -): IndexTemplate => { - const { indexAliasName, indexAliasPattern, indexIlmPolicyName } = names; - - return { - index_patterns: [indexAliasPattern], - settings: { - number_of_shards: 1, // TODO: do we need to set this? - auto_expand_replicas: '0-1', // TODO: do we need to set? - index: { - lifecycle: { - name: indexIlmPolicyName, - rollover_alias: indexAliasName, - }, - }, - mapping: { - total_fields: { - limit: 10000, - }, - }, - sort: { - field: '@timestamp', - order: 'desc', - }, - }, - // @ts-expect-error IndexMappings is not assignale to estypes.MappingTypeMapping - mappings: { - ...mappings, - _meta: { - ...mappings._meta, - version, - }, - }, - version, - }; -}; diff --git a/x-pack/plugins/rule_registry/server/event_log/elasticsearch/resources/index_templates.ts b/x-pack/plugins/rule_registry/server/event_log/elasticsearch/resources/index_templates.ts new file mode 100644 index 0000000000000..18633f430eea1 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/event_log/elasticsearch/resources/index_templates.ts @@ -0,0 +1,95 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + ComponentTemplateOptions, + IndexTemplateOptions, + IndexNames, + Settings, + Mappings, + Aliases, + Version, + Meta, +} from '../../common'; + +export interface ComponentTemplate { + template: { + settings: Settings; + mappings?: Mappings; + aliases?: Aliases; + }; + version?: Version; + _meta?: Meta; +} + +export const createComponentTemplate = (options: ComponentTemplateOptions): ComponentTemplate => { + const { settings, mappings, aliases, version, meta } = options; + + return { + template: { + settings: settings ?? {}, + mappings: mappings ?? {}, + aliases: aliases ?? {}, + }, + version, + _meta: meta, + }; +}; + +export interface IndexTemplate { + index_patterns: string[]; + composed_of?: string[]; + template: { + settings?: Settings; + mappings?: Mappings; + aliases?: Aliases; + }; + priority?: number; + version?: Version; + _meta?: Meta; +} + +export const createIndexTemplate = ( + indexNames: IndexNames, + options?: IndexTemplateOptions +): IndexTemplate => { + const { indexAliasName, indexAliasPattern, indexIlmPolicyName, componentTemplates } = indexNames; + const { settings, mappings, aliases, version, meta, priority } = options ?? {}; + + return { + index_patterns: [indexAliasPattern], + composed_of: [ + componentTemplates.commonMappingsTemplateName, + componentTemplates.commonSettingsTemplateName, + componentTemplates.applicationDefinedTemplateName, + componentTemplates.userDefinedTemplateName, + componentTemplates.userDefinedSpaceAwareTemplateName, + ], + template: { + settings: { + // External settings provided by an application (plugin) + ...settings, + + // Hard defaults - application or user cannot override them. + // These are necessary to make the event log mechanism work as expected. + 'index.lifecycle.name': indexIlmPolicyName, + 'index.lifecycle.rollover_alias': indexAliasName, + }, + mappings: { + ...mappings, + _meta: { + ...mappings?._meta, + version, + }, + }, + aliases, + }, + priority, + version, + _meta: meta, + }; +}; diff --git a/x-pack/plugins/rule_registry/server/event_log/event_schema/schema.ts b/x-pack/plugins/rule_registry/server/event_log/event_schema/schema.ts deleted file mode 100644 index 9b5d94918a83f..0000000000000 --- a/x-pack/plugins/rule_registry/server/event_log/event_schema/schema.ts +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { EventSchema, Event } from './schema_types'; -import { FieldMap, runtimeTypeFromFieldMap, mergeFieldMaps } from '../../../common/field_map'; -import { - TechnicalRuleFieldMaps, - technicalRuleFieldMap, -} from '../../../common/assets/field_maps/technical_rule_field_map'; - -const baseSchema = createSchema(technicalRuleFieldMap); - -export abstract class Schema { - public static create(fields: TMap): EventSchema { - return createSchema(fields); - } - - public static combine( - s1: EventSchema, - s2: EventSchema - ): EventSchema { - const combinedFields = mergeFieldMaps(s1.objectFields, s2.objectFields); - return createSchema(combinedFields); - } - - public static getBase(): EventSchema { - return baseSchema; - } - - public static extendBase( - fields: TMap - ): EventSchema { - const extensionSchema = createSchema(fields); - return this.combine(baseSchema, extensionSchema); - } -} - -function createSchema(fields: TMap): EventSchema { - const objectType: Event = ({} as unknown) as Event; - const runtimeType = runtimeTypeFromFieldMap(fields); - - return { - objectFields: fields, - objectType, - runtimeType, - }; -} diff --git a/x-pack/plugins/rule_registry/server/event_log/event_schema/schema_types.ts b/x-pack/plugins/rule_registry/server/event_log/event_schema/schema_types.ts deleted file mode 100644 index e5c665652fe97..0000000000000 --- a/x-pack/plugins/rule_registry/server/event_log/event_schema/schema_types.ts +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { FieldMap, FieldMapType, TypeOfFieldMap } from '../../../common/field_map'; - -export interface EventSchema { - objectFields: TMap; - objectType: Event; - runtimeType: EventRuntimeType; -} - -export type Event = TypeOfFieldMap; - -export type EventRuntimeType = FieldMapType; - -export { FieldMap }; diff --git a/x-pack/plugins/rule_registry/server/event_log/index.ts b/x-pack/plugins/rule_registry/server/event_log/index.ts index cf7467588c22f..2fe3a7dfb217e 100644 --- a/x-pack/plugins/rule_registry/server/event_log/index.ts +++ b/x-pack/plugins/rule_registry/server/event_log/index.ts @@ -6,5 +6,4 @@ */ export * from './elasticsearch'; -export * from './event_schema'; export * from './log'; diff --git a/x-pack/plugins/rule_registry/server/event_log/log/bootstrapper_of_common_resources.ts b/x-pack/plugins/rule_registry/server/event_log/log/bootstrapper_of_common_resources.ts new file mode 100644 index 0000000000000..c73de94f5887b --- /dev/null +++ b/x-pack/plugins/rule_registry/server/event_log/log/bootstrapper_of_common_resources.ts @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { once } from 'lodash'; +import { Logger } from 'kibana/server'; +import { IndexNames } from '../common'; +import { IIndexBootstrapper } from '../elasticsearch'; +import { ReadySignal, createReadySignal } from '../utils/ready_signal'; + +interface ConstructorParams { + indexNames: IndexNames; + indexBootstrapper: IIndexBootstrapper; + isWriteEnabled: boolean; + logger: Logger; +} + +export type BootstrappingResult = 'success' | Error; + +export class BootstrapperOfCommonResources { + private readonly indexNames: IndexNames; + private readonly indexBootstrapper: IIndexBootstrapper; + private readonly isWriteEnabled: boolean; + private readonly logger: Logger; + private readonly result: ReadySignal; + + constructor(params: ConstructorParams) { + this.indexNames = params.indexNames; + this.indexBootstrapper = params.indexBootstrapper; + this.isWriteEnabled = params.isWriteEnabled; + this.logger = params.logger.get('BootstrapperOfCommonResources'); + this.result = createReadySignal(); + } + + private startOnce = once((): void => { + const { indexNames, indexBootstrapper, isWriteEnabled, logger, result } = this; + const { indexPrefix } = indexNames; + + if (!isWriteEnabled) { + return; + } + + Promise.resolve() + .then(async () => { + logger.debug(`Bootstrapping common resources for "${indexPrefix}"`); + await indexBootstrapper.bootstrapCommonResources(indexNames); + logger.debug(`Bootstrapping done for "${indexPrefix}"`); + + result.signal('success'); + }) + .catch((e: Error) => { + logger.error(e); + logger.debug(`Bootstrapping failed for "${indexPrefix}": ${e.message}`); + + result.signal(e); + }); + }); + + public start(): void { + this.startOnce(); + } + + public waitUntilFinished(): Promise { + return this.result.wait(); + } +} diff --git a/x-pack/plugins/rule_registry/server/event_log/log/bootstrapper_of_log_resources.ts b/x-pack/plugins/rule_registry/server/event_log/log/bootstrapper_of_log_resources.ts new file mode 100644 index 0000000000000..c77a88a3a371f --- /dev/null +++ b/x-pack/plugins/rule_registry/server/event_log/log/bootstrapper_of_log_resources.ts @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Logger } from 'kibana/server'; +import { IIndexBootstrapper, IndexSpec } from '../elasticsearch'; +import { BootstrappingResult } from './bootstrapper_of_common_resources'; + +interface ConstructorParams { + indexSpec: IndexSpec; + indexBootstrapper: IIndexBootstrapper; + isWriteEnabled: boolean; + isMechanismReady: Promise; + logger: Logger; +} + +export class BootstrapperOfLogResources { + private isIndexBootstrapped: boolean; + + constructor(private readonly params: ConstructorParams) { + this.isIndexBootstrapped = false; + } + + public async run(): Promise { + const { indexSpec, indexBootstrapper, isWriteEnabled, isMechanismReady } = this.params; + + if (this.isIndexBootstrapped || !isWriteEnabled) { + return; + } + + const mechanismBootstrappingResult = await isMechanismReady; + if (mechanismBootstrappingResult !== 'success') { + throw mechanismBootstrappingResult; + } + + await indexBootstrapper.bootstrapLogLevelResources(indexSpec); + + this.isIndexBootstrapped = true; + } +} diff --git a/x-pack/plugins/rule_registry/server/event_log/log/event_log.ts b/x-pack/plugins/rule_registry/server/event_log/log/event_log.ts index 2b1ecde48d2db..e48e3fddf40e7 100644 --- a/x-pack/plugins/rule_registry/server/event_log/log/event_log.ts +++ b/x-pack/plugins/rule_registry/server/event_log/log/event_log.ts @@ -7,13 +7,13 @@ import { estypes } from '@elastic/elasticsearch'; import { DeepPartial } from '../utils/utility_types'; -import { IndexNames } from '../elasticsearch'; +import { CommonFields, IndexNames } from '../common'; import { IEventLog, IEventLogger, IEventLoggerTemplate, IEventQueryBuilder } from './public_api'; import { EventLogParams } from './internal_api'; import { EventLoggerTemplate } from './event_logger_template'; import { EventQueryBuilder } from './event_query_builder'; -export class EventLog implements IEventLog { +export class EventLog implements IEventLog { private readonly params: EventLogParams; private readonly initialTemplate: IEventLoggerTemplate; diff --git a/x-pack/plugins/rule_registry/server/event_log/log/event_log_bootstrapper.ts b/x-pack/plugins/rule_registry/server/event_log/log/event_log_bootstrapper.ts deleted file mode 100644 index 0498a7cd97b2f..0000000000000 --- a/x-pack/plugins/rule_registry/server/event_log/log/event_log_bootstrapper.ts +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { Logger } from 'kibana/server'; -import { IIndexBootstrapper, IndexSpecification } from '../elasticsearch'; - -interface ConstructorParams { - indexSpec: IndexSpecification; - indexBootstrapper: IIndexBootstrapper; - isWriteEnabled: boolean; - logger: Logger; -} - -export class EventLogBootstrapper { - private readonly indexSpec: IndexSpecification; - private readonly indexBootstrapper: IIndexBootstrapper; - private readonly logger: Logger; - private readonly isWriteEnabled: boolean; - private isIndexBootstrapped: boolean; - - constructor(params: ConstructorParams) { - this.indexSpec = params.indexSpec; - this.indexBootstrapper = params.indexBootstrapper; - this.logger = params.logger.get('EventLogBootstrapper'); - this.isWriteEnabled = params.isWriteEnabled; - this.isIndexBootstrapped = false; - } - - public async run(): Promise { - if (this.isIndexBootstrapped || !this.isWriteEnabled) { - return; - } - - const { logName, indexAliasName } = this.indexSpec.indexNames; - const logInfo = `log="${logName}" index="${indexAliasName}"`; - - this.logger.debug(`Bootstrapping started, ${logInfo}`); - this.isIndexBootstrapped = await this.indexBootstrapper.run(this.indexSpec); - this.logger.debug( - `Bootstrapping ${this.isIndexBootstrapped ? 'succeeded' : 'failed'}, ${logInfo}` - ); - - if (!this.isIndexBootstrapped) { - throw new Error(`Event log bootstrapping failed, ${logInfo}`); - } - } -} diff --git a/x-pack/plugins/rule_registry/server/event_log/log/event_log_definition.ts b/x-pack/plugins/rule_registry/server/event_log/log/event_log_definition.ts deleted file mode 100644 index 124664d5578b0..0000000000000 --- a/x-pack/plugins/rule_registry/server/event_log/log/event_log_definition.ts +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { IlmPolicy, defaultIlmPolicy, IndexNames } from '../elasticsearch'; -import { EventSchema, FieldMap, Schema } from '../event_schema'; -import { EventLogOptions, IEventLogDefinition } from './public_api'; - -export class EventLogDefinition implements IEventLogDefinition { - public readonly eventLogName: string; - public readonly eventSchema: EventSchema; - public readonly ilmPolicy: IlmPolicy; - - constructor(options: EventLogOptions) { - // TODO: validate options; options.name should not contain "-" and "." - this.eventLogName = options.name; - this.eventSchema = options.schema; - this.ilmPolicy = options.ilmPolicy ?? defaultIlmPolicy; - } - - public defineChild( - options: EventLogOptions - ): IEventLogDefinition { - const childName = IndexNames.createChildLogName(this.eventLogName, options.name); - const childSchema = Schema.combine(this.eventSchema, options.schema); - const childPolicy = options.ilmPolicy ?? this.ilmPolicy; - - return new EventLogDefinition({ - name: childName, - schema: childSchema, - ilmPolicy: childPolicy, - }); - } -} diff --git a/x-pack/plugins/rule_registry/server/event_log/log/event_log_object_factory.ts b/x-pack/plugins/rule_registry/server/event_log/log/event_log_object_factory.ts new file mode 100644 index 0000000000000..552dd32bee729 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/event_log/log/event_log_object_factory.ts @@ -0,0 +1,119 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { CommonFields, EventLogDefinition, IndexNames } from '../common'; +import { + IndexBootstrapper, + IndexManagementGateway, + IndexReader, + IndexWriter, + IndexSpec, + createIndexSpec, +} from '../elasticsearch'; + +import { + BootstrapperOfCommonResources, + BootstrappingResult, +} from './bootstrapper_of_common_resources'; +import { BootstrapperOfLogResources } from './bootstrapper_of_log_resources'; +import { EventLog } from './event_log'; +import { IEventLog, EventLogServiceConfig, EventLogServiceDependencies } from './public_api'; + +export class EventLogObjectFactory { + constructor( + private readonly config: EventLogServiceConfig, + private readonly deps: EventLogServiceDependencies + ) {} + + public createIndexSpec(logDefinition: EventLogDefinition, kibanaSpaceId: string): IndexSpec { + const { indexPrefix } = this.config; + return createIndexSpec(logDefinition, indexPrefix, kibanaSpaceId); + } + + public createIndexReader(indexSpec: IndexSpec): IndexReader { + const { clusterClient, logger } = this.deps; + const { indexNames } = indexSpec; + + return new IndexReader({ + indexName: indexNames.indexAliasPattern, + elasticsearch: clusterClient.then((c) => c.asInternalUser), + logger, + }); + } + + public createIndexWriter(indexSpec: IndexSpec): IndexWriter { + const { clusterClient, logger } = this.deps; + const { isWriteEnabled } = this.config; + const { indexNames } = indexSpec; + + return new IndexWriter({ + indexName: indexNames.indexAliasName, + elasticsearch: clusterClient.then((c) => c.asInternalUser), + isWriteEnabled, + logger, + }); + } + + public createIndexBootstrapper(): IndexBootstrapper { + const { clusterClient, logger } = this.deps; + + return new IndexBootstrapper({ + gateway: new IndexManagementGateway({ + elasticsearch: clusterClient.then((c) => c.asInternalUser), + logger, + }), + logger, + }); + } + + public createBootstrapperOfCommonResources(): BootstrapperOfCommonResources { + const { logger } = this.deps; + const { indexPrefix, isWriteEnabled } = this.config; + + return new BootstrapperOfCommonResources({ + indexNames: IndexNames.create({ + indexPrefix, + logName: 'stub', + kibanaSpaceId: 'stub', + }), + indexBootstrapper: this.createIndexBootstrapper(), + isWriteEnabled, + logger, + }); + } + + public createBootstrapperOfLogResources( + indexSpec: IndexSpec, + isMechanismReady: Promise + ): BootstrapperOfLogResources { + const { logger } = this.deps; + const { isWriteEnabled } = this.config; + + return new BootstrapperOfLogResources({ + indexSpec, + indexBootstrapper: this.createIndexBootstrapper(), + isWriteEnabled, + isMechanismReady, + logger, + }); + } + + public createEventLog( + indexSpec: IndexSpec, + indexReader: IndexReader, + indexWriter: IndexWriter + ): IEventLog { + const { logger } = this.deps; + + return new EventLog({ + indexNames: indexSpec.indexNames, + indexReader, + indexWriter, + logger, + }); + } +} diff --git a/x-pack/plugins/rule_registry/server/event_log/log/event_log_provider.ts b/x-pack/plugins/rule_registry/server/event_log/log/event_log_provider.ts index d1ecd6a977a08..d7ac3dd91eb0a 100644 --- a/x-pack/plugins/rule_registry/server/event_log/log/event_log_provider.ts +++ b/x-pack/plugins/rule_registry/server/event_log/log/event_log_provider.ts @@ -5,18 +5,19 @@ * 2.0. */ +import { CommonFields } from '../common'; import { IIndexWriter } from '../elasticsearch'; import { IEventLog } from './public_api'; import { IEventLogProvider } from './internal_api'; -import { EventLogBootstrapper } from './event_log_bootstrapper'; +import { BootstrapperOfLogResources } from './bootstrapper_of_log_resources'; -interface ConstructorParams { +interface ConstructorParams { log: IEventLog; - logBootstrapper: EventLogBootstrapper; + logBootstrapper: BootstrapperOfLogResources; indexWriter: IIndexWriter; } -export class EventLogProvider implements IEventLogProvider { +export class EventLogProvider implements IEventLogProvider { constructor(private readonly params: ConstructorParams) {} public getLog(): IEventLog { diff --git a/x-pack/plugins/rule_registry/server/event_log/log/event_log_registry.ts b/x-pack/plugins/rule_registry/server/event_log/log/event_log_registry.ts index 52f6c6bd918d4..2e7d8618b85eb 100644 --- a/x-pack/plugins/rule_registry/server/event_log/log/event_log_registry.ts +++ b/x-pack/plugins/rule_registry/server/event_log/log/event_log_registry.ts @@ -5,15 +5,14 @@ * 2.0. */ -import { Event, FieldMap } from '../event_schema'; -import { IEventLogDefinition } from './public_api'; +import { CommonFields, EventLogDefinition } from '../common'; import { IEventLogRegistry, IEventLogProvider } from './internal_api'; -const getRegistryKey = (definition: IEventLogDefinition, spaceId: string) => - `${definition.eventLogName}-${spaceId}`; +const getRegistryKey = (definition: EventLogDefinition, spaceId: string) => + `${definition.logName}-${spaceId}`; interface RegistryEntry { - definition: IEventLogDefinition; + definition: EventLogDefinition; spaceId: string; provider: IEventLogProvider; } @@ -21,19 +20,19 @@ interface RegistryEntry { export class EventLogRegistry implements IEventLogRegistry { private readonly map = new Map(); - public get( - definition: IEventLogDefinition, + public get( + definition: EventLogDefinition, spaceId: string - ): IEventLogProvider> | null { + ): IEventLogProvider | null { const key = getRegistryKey(definition, spaceId); const entry = this.map.get(key); - return entry != null ? (entry.provider as IEventLogProvider>) : null; + return entry != null ? (entry.provider as IEventLogProvider) : null; } - public add( - definition: IEventLogDefinition, + public add( + definition: EventLogDefinition, spaceId: string, - provider: IEventLogProvider> + provider: IEventLogProvider ): void { const key = getRegistryKey(definition, spaceId); diff --git a/x-pack/plugins/rule_registry/server/event_log/log/event_log_resolver.ts b/x-pack/plugins/rule_registry/server/event_log/log/event_log_resolver.ts index 8440f55432304..6d7c698c48a17 100644 --- a/x-pack/plugins/rule_registry/server/event_log/log/event_log_resolver.ts +++ b/x-pack/plugins/rule_registry/server/event_log/log/event_log_resolver.ts @@ -5,47 +5,27 @@ * 2.0. */ -import { - IndexBootstrapper, - IndexManagementGateway, - IndexNames, - IndexReader, - IndexSpecification, - IndexWriter, -} from '../elasticsearch'; +import { CommonFields, EventLogDefinition } from '../common'; -import { Event, FieldMap } from '../event_schema'; +import { IEventLog, IEventLogResolver } from './public_api'; import { IEventLogRegistry, IEventLogProvider } from './internal_api'; -import { - EventLogServiceConfig, - EventLogServiceDependencies, - IEventLog, - IEventLogDefinition, - IEventLogResolver, -} from './public_api'; - -import { EventLog } from './event_log'; -import { EventLogBootstrapper } from './event_log_bootstrapper'; +import { EventLogObjectFactory } from './event_log_object_factory'; import { EventLogProvider } from './event_log_provider'; -import { mappingFromFieldMap } from './utils/mapping_from_field_map'; +import { BootstrappingResult } from './bootstrapper_of_common_resources'; export class EventLogResolver implements IEventLogResolver { - private readonly indexBootstrapper: IndexBootstrapper; - constructor( - private readonly config: EventLogServiceConfig, - private readonly deps: EventLogServiceDependencies, + private readonly factory: EventLogObjectFactory, private readonly registry: IEventLogRegistry, + private readonly isMechanismReady: Promise, private readonly bootstrapLog: boolean - ) { - this.indexBootstrapper = this.createIndexBootstrapper(); - } + ) {} - public async resolve( - definition: IEventLogDefinition, + public async resolve( + logDefinition: EventLogDefinition, kibanaSpaceId: string - ): Promise>> { - const provider = this.resolveLogProvider(definition, kibanaSpaceId); + ): Promise> { + const provider = this.resolveLogProvider(logDefinition, kibanaSpaceId); if (this.bootstrapLog) { await provider.bootstrapLog(); @@ -54,109 +34,30 @@ export class EventLogResolver implements IEventLogResolver { return provider.getLog(); } - private resolveLogProvider( - definition: IEventLogDefinition, + private resolveLogProvider( + logDefinition: EventLogDefinition, kibanaSpaceId: string - ): IEventLogProvider> { - const existingProvider = this.registry.get(definition, kibanaSpaceId); + ): IEventLogProvider { + const { factory, registry, isMechanismReady } = this; + + const existingProvider = registry.get(logDefinition, kibanaSpaceId); if (existingProvider) { return existingProvider; } - const indexSpec = this.createIndexSpec(definition, kibanaSpaceId); - const indexReader = this.createIndexReader(indexSpec); - const indexWriter = this.createIndexWriter(indexSpec); - const logBootstrapper = this.createEventLogBootstrapper(indexSpec); - const log = this.createEventLog(indexSpec, indexReader, indexWriter); + const indexSpec = factory.createIndexSpec(logDefinition, kibanaSpaceId); + const indexReader = factory.createIndexReader(indexSpec); + const indexWriter = factory.createIndexWriter(indexSpec); + const logBootstrapper = factory.createBootstrapperOfLogResources(indexSpec, isMechanismReady); + const log = factory.createEventLog(indexSpec, indexReader, indexWriter); const logProvider = new EventLogProvider({ log, logBootstrapper, indexWriter, }); - this.registry.add(definition, kibanaSpaceId, logProvider); + registry.add(logDefinition, kibanaSpaceId, logProvider); return logProvider; } - - private createIndexSpec( - definition: IEventLogDefinition, - kibanaSpaceId: string - ): IndexSpecification { - const { indexPrefix } = this.config; - const { eventLogName, eventSchema, ilmPolicy } = definition; - - const indexNames = IndexNames.create({ - indexPrefix, - logName: eventLogName, - kibanaSpaceId, - }); - - const indexMappings = mappingFromFieldMap(eventSchema.objectFields); - - return { indexNames, indexMappings, ilmPolicy }; - } - - private createIndexBootstrapper(): IndexBootstrapper { - const { clusterClient, logger } = this.deps; - - return new IndexBootstrapper({ - gateway: new IndexManagementGateway({ - elasticsearch: clusterClient.then((c) => c.asInternalUser), - logger, - }), - logger, - }); - } - - private createIndexReader(indexSpec: IndexSpecification): IndexReader { - const { clusterClient, logger } = this.deps; - const { indexNames } = indexSpec; - - return new IndexReader({ - indexName: indexNames.indexAliasPattern, - elasticsearch: clusterClient.then((c) => c.asInternalUser), // TODO: internal or current? - logger, - }); - } - - private createIndexWriter(indexSpec: IndexSpecification): IndexWriter { - const { clusterClient, logger } = this.deps; - const { isWriteEnabled } = this.config; - const { indexNames } = indexSpec; - - return new IndexWriter({ - indexName: indexNames.indexAliasName, - elasticsearch: clusterClient.then((c) => c.asInternalUser), // TODO: internal or current? - isWriteEnabled, - logger, - }); - } - - private createEventLogBootstrapper(indexSpec: IndexSpecification): EventLogBootstrapper { - const { logger } = this.deps; - const { isWriteEnabled } = this.config; - - return new EventLogBootstrapper({ - indexSpec, - indexBootstrapper: this.indexBootstrapper, - isWriteEnabled, - logger, - }); - } - - private createEventLog( - indexSpec: IndexSpecification, - indexReader: IndexReader, - indexWriter: IndexWriter - ): IEventLog> { - const { logger } = this.deps; - - return new EventLog>({ - indexNames: indexSpec.indexNames, - indexReader, - indexWriter, - logger, - }); - } } diff --git a/x-pack/plugins/rule_registry/server/event_log/log/event_log_service.ts b/x-pack/plugins/rule_registry/server/event_log/log/event_log_service.ts index b5b1d23f2e215..59fe1e85b3c7b 100644 --- a/x-pack/plugins/rule_registry/server/event_log/log/event_log_service.ts +++ b/x-pack/plugins/rule_registry/server/event_log/log/event_log_service.ts @@ -7,19 +7,20 @@ import { KibanaRequest } from 'kibana/server'; -import { Event, FieldMap } from '../event_schema'; +import { CommonFields, EventLogDefinition } from '../common'; import { EventLogServiceConfig, EventLogServiceDependencies, IEventLog, - IEventLogDefinition, IEventLogResolver, IEventLogService, IScopedEventLogResolver, } from './public_api'; +import { EventLogObjectFactory } from './event_log_object_factory'; import { EventLogRegistry } from './event_log_registry'; import { EventLogResolver } from './event_log_resolver'; +import { BootstrapperOfCommonResources } from './bootstrapper_of_common_resources'; const BOOTSTRAP_BY_DEFAULT = true; @@ -29,17 +30,19 @@ interface ConstructorParams { } export class EventLogService implements IEventLogService { + private readonly factory: EventLogObjectFactory; private readonly registry: EventLogRegistry; + private readonly bootstrapper: BootstrapperOfCommonResources; constructor(private readonly params: ConstructorParams) { + this.factory = new EventLogObjectFactory(params.config, params.dependencies); this.registry = new EventLogRegistry(); + this.bootstrapper = this.factory.createBootstrapperOfCommonResources(); } public getResolver(bootstrapLog = BOOTSTRAP_BY_DEFAULT): IEventLogResolver { - const { params, registry } = this; - const { config, dependencies } = params; - - return new EventLogResolver(config, dependencies, registry, bootstrapLog); + const isMechanismReady = this.bootstrapper.waitUntilFinished(); + return new EventLogResolver(this.factory, this.registry, isMechanismReady, bootstrapLog); } public getScopedResolver( @@ -49,19 +52,24 @@ export class EventLogService implements IEventLogService { const resolver = this.getResolver(bootstrapLog); return { - resolve: async ( - definition: IEventLogDefinition - ): Promise>> => { + resolve: async ( + logDefinition: EventLogDefinition + ): Promise> => { const spaces = await this.params.dependencies.spacesService; const spaceId = spaces.getSpaceId(request); - const log = await resolver.resolve(definition, spaceId); + const log = await resolver.resolve(logDefinition, spaceId); return log; }, }; } + public start(): void { + this.bootstrapper.start(); + } + public async stop(): Promise { + await this.bootstrapper.waitUntilFinished(); await this.registry.shutdown(); } } diff --git a/x-pack/plugins/rule_registry/server/event_log/log/event_logger.ts b/x-pack/plugins/rule_registry/server/event_log/log/event_logger.ts index c6f88f49835d7..733b1dae98f0d 100644 --- a/x-pack/plugins/rule_registry/server/event_log/log/event_logger.ts +++ b/x-pack/plugins/rule_registry/server/event_log/log/event_logger.ts @@ -7,10 +7,11 @@ import { DeepPartial } from '../utils/utility_types'; import { mergeFields } from '../utils/fields'; -import { EventLoggerParams } from './internal_api'; +import { CommonFields } from '../common'; import { IEventLogger, IEventLoggerTemplate } from './public_api'; +import { EventLoggerParams } from './internal_api'; -export class EventLogger implements IEventLogger { +export class EventLogger implements IEventLogger { private readonly params: EventLoggerParams; private readonly ownTemplate: IEventLoggerTemplate; diff --git a/x-pack/plugins/rule_registry/server/event_log/log/event_logger_template.ts b/x-pack/plugins/rule_registry/server/event_log/log/event_logger_template.ts index 3872a5c744269..564576ae9c73e 100644 --- a/x-pack/plugins/rule_registry/server/event_log/log/event_logger_template.ts +++ b/x-pack/plugins/rule_registry/server/event_log/log/event_logger_template.ts @@ -7,11 +7,13 @@ import { DeepPartial } from '../utils/utility_types'; import { mergeFields } from '../utils/fields'; +import { CommonFields } from '../common'; import { IEventLogger, IEventLoggerTemplate } from './public_api'; import { EventLoggerParams } from './internal_api'; import { EventLogger } from './event_logger'; -export class EventLoggerTemplate implements IEventLoggerTemplate { +export class EventLoggerTemplate + implements IEventLoggerTemplate { private readonly params: EventLoggerParams; constructor(params: EventLoggerParams) { diff --git a/x-pack/plugins/rule_registry/server/event_log/log/event_query.ts b/x-pack/plugins/rule_registry/server/event_log/log/event_query.ts index 0eabe4be64837..78ca513159c98 100644 --- a/x-pack/plugins/rule_registry/server/event_log/log/event_query.ts +++ b/x-pack/plugins/rule_registry/server/event_log/log/event_query.ts @@ -6,8 +6,8 @@ */ import { estypes } from '@elastic/elasticsearch'; -import { IIndexReader } from '../elasticsearch'; import { truthy } from '../utils/predicates'; +import { IIndexReader } from '../elasticsearch'; import { IEventQuery } from './public_api'; export interface EventQueryParams { diff --git a/x-pack/plugins/rule_registry/server/event_log/log/event_query_builder.ts b/x-pack/plugins/rule_registry/server/event_log/log/event_query_builder.ts index bf9aca74f7800..24fd59fe693e9 100644 --- a/x-pack/plugins/rule_registry/server/event_log/log/event_query_builder.ts +++ b/x-pack/plugins/rule_registry/server/event_log/log/event_query_builder.ts @@ -11,11 +11,12 @@ import { esKuery } from '../../../../../../src/plugins/data/server'; import { DeepPartial } from '../utils/utility_types'; import { mergeFields } from '../utils/fields'; -import { EventLogParams } from './internal_api'; +import { CommonFields } from '../common'; import { IEventQueryBuilder, IEventQuery, SortingParams, PaginationParams } from './public_api'; +import { EventLogParams } from './internal_api'; import { EventQuery } from './event_query'; -export class EventQueryBuilder implements IEventQueryBuilder { +export class EventQueryBuilder implements IEventQueryBuilder { private readonly params: EventLogParams; private loggerName: string; private fields: DeepPartial | null; diff --git a/x-pack/plugins/rule_registry/server/event_log/log/index.ts b/x-pack/plugins/rule_registry/server/event_log/log/index.ts index e5593390733e4..06cfc090bb734 100644 --- a/x-pack/plugins/rule_registry/server/event_log/log/index.ts +++ b/x-pack/plugins/rule_registry/server/event_log/log/index.ts @@ -5,6 +5,5 @@ * 2.0. */ -export * from './event_log_definition'; export * from './event_log_service'; export * from './public_api'; diff --git a/x-pack/plugins/rule_registry/server/event_log/log/internal_api.ts b/x-pack/plugins/rule_registry/server/event_log/log/internal_api.ts index 8db931b35912d..583746eee1985 100644 --- a/x-pack/plugins/rule_registry/server/event_log/log/internal_api.ts +++ b/x-pack/plugins/rule_registry/server/event_log/log/internal_api.ts @@ -7,27 +7,27 @@ import { Logger } from 'kibana/server'; -import { IIndexReader, IIndexWriter, IndexNames } from '../elasticsearch'; -import { Event, FieldMap } from '../event_schema'; +import { CommonFields, EventLogDefinition, IndexNames } from '../common'; +import { IIndexReader, IIndexWriter } from '../elasticsearch'; import { DeepPartial } from '../utils/utility_types'; -import { IEventLogDefinition, IEventLog } from './public_api'; +import { IEventLog } from './public_api'; export interface IEventLogRegistry { - get( - definition: IEventLogDefinition, + get( + definition: EventLogDefinition, spaceId: string - ): IEventLogProvider> | null; + ): IEventLogProvider | null; - add( - definition: IEventLogDefinition, + add( + definition: EventLogDefinition, spaceId: string, - provider: IEventLogProvider> + provider: IEventLogProvider ): void; shutdown(): Promise; } -export interface IEventLogProvider { +export interface IEventLogProvider { getLog(): IEventLog; bootstrapLog(): Promise; shutdownLog(): Promise; diff --git a/x-pack/plugins/rule_registry/server/event_log/log/public_api.ts b/x-pack/plugins/rule_registry/server/event_log/log/public_api.ts index 7dcaee9d382b1..ea23bc286c2a8 100644 --- a/x-pack/plugins/rule_registry/server/event_log/log/public_api.ts +++ b/x-pack/plugins/rule_registry/server/event_log/log/public_api.ts @@ -9,31 +9,9 @@ import { estypes } from '@elastic/elasticsearch'; import { IClusterClient, KibanaRequest, Logger } from 'kibana/server'; import { SpacesServiceStart } from '../../../../spaces/server'; -import { IlmPolicy, IndexNames, IndexSpecification } from '../elasticsearch'; -import { FieldMap, Event, EventSchema } from '../event_schema'; +import { CommonFields, EventLogDefinition, IndexNames } from '../common'; import { DeepPartial } from '../utils/utility_types'; -export { IlmPolicy, IndexSpecification }; - -// ------------------------------------------------------------------------------------------------- -// Definition API (defining log hierarchies as simple objects) - -export interface EventLogOptions { - name: string; - schema: EventSchema; - ilmPolicy?: IlmPolicy; -} - -export interface IEventLogDefinition { - eventLogName: string; - eventSchema: EventSchema; - ilmPolicy: IlmPolicy; - - defineChild( - options: EventLogOptions - ): IEventLogDefinition; -} - // ------------------------------------------------------------------------------------------------- // Resolving and bootstrapping API (creating runtime objects representing logs, bootstrapping indices) @@ -54,19 +32,19 @@ export interface IEventLogService { } export interface IEventLogResolver { - resolve( - definition: IEventLogDefinition, + resolve( + logDefinition: EventLogDefinition, spaceId: string - ): Promise>>; + ): Promise>; } export interface IScopedEventLogResolver { - resolve( - definition: IEventLogDefinition - ): Promise>>; + resolve( + logDefinition: EventLogDefinition + ): Promise>; } -export interface IEventLog extends IEventLoggerTemplate { +export interface IEventLog extends IEventLoggerTemplate { getNames(): IndexNames; getQueryBuilder(): IEventQueryBuilder; @@ -79,19 +57,19 @@ export interface IEventLog extends IEventLoggerTemplate { // ------------------------------------------------------------------------------------------------- // Write API (logging events) -export interface IEventLoggerTemplate { +export interface IEventLoggerTemplate { getLoggerTemplate(fields: DeepPartial): IEventLoggerTemplate; getLogger(name: string, fields?: DeepPartial): IEventLogger; } -export interface IEventLogger extends IEventLoggerTemplate { +export interface IEventLogger extends IEventLoggerTemplate { logEvent(fields: DeepPartial): void; } // ------------------------------------------------------------------------------------------------- // Read API (searching, filtering, sorting, pagination, aggregation over events) -export interface IEventQueryBuilder { +export interface IEventQueryBuilder { filterByLogger(loggerName: string): IEventQueryBuilder; filterByFields(fields: DeepPartial): IEventQueryBuilder; filterByKql(kql: string): IEventQueryBuilder; diff --git a/x-pack/plugins/rule_registry/server/event_log/log/utils/mapping_from_field_map.ts b/x-pack/plugins/rule_registry/server/event_log/log/utils/mapping_from_field_map.ts deleted file mode 100644 index fd5dc3ae02288..0000000000000 --- a/x-pack/plugins/rule_registry/server/event_log/log/utils/mapping_from_field_map.ts +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { set } from '@elastic/safer-lodash-set'; -import { FieldMap } from '../../../../common/field_map'; -import { IndexMappings } from '../../elasticsearch'; - -export function mappingFromFieldMap(fieldMap: FieldMap): IndexMappings { - const mappings = { - dynamic: 'strict' as const, - properties: {}, - }; - - const fields = Object.keys(fieldMap).map((key) => { - const field = fieldMap[key]; - return { - name: key, - ...field, - }; - }); - - fields.forEach((field) => { - const { name, required, array, ...rest } = field; - - set(mappings.properties, field.name.split('.').join('.properties.'), rest); - }); - - return mappings; -} diff --git a/x-pack/plugins/rule_registry/server/plugin.ts b/x-pack/plugins/rule_registry/server/plugin.ts index 043b07f9d67c1..ef3c10482c991 100644 --- a/x-pack/plugins/rule_registry/server/plugin.ts +++ b/x-pack/plugins/rule_registry/server/plugin.ts @@ -5,12 +5,20 @@ * 2.0. */ -import { PluginInitializerContext, Plugin, CoreSetup, Logger } from 'src/core/server'; +import { + PluginInitializerContext, + Plugin, + CoreSetup, + Logger, + IContextProvider, +} from 'src/core/server'; import { SpacesPluginStart } from '../../spaces/server'; - +import { APP_ID } from '../common/constants'; import { RuleRegistryPluginConfig } from './config'; import { RuleDataPluginService } from './rule_data_plugin_service'; import { EventLogService, IEventLogService } from './event_log'; +import { RuleRegistryApiRequestHandlerContext, RuleRegistryRequestHandlerContext } from './types'; +import { CommonFields, EventLogDefinition } from './event_log/common'; // eslint-disable-next-line @typescript-eslint/no-empty-interface interface RuleRegistryPluginSetupDependencies {} @@ -86,10 +94,41 @@ export class RuleRegistryPlugin }); this.eventLogService = eventLogService; + + core.http.registerRouteHandlerContext( + APP_ID, + this.createRouteHandlerContext() + ); + return { ruleDataService, eventLogService }; } - public start(): RuleRegistryPluginStartContract {} + public start(): RuleRegistryPluginStartContract { + const { eventLogService } = this; + + if (eventLogService) { + eventLogService.start(); + } + } + + private createRouteHandlerContext = (): IContextProvider< + RuleRegistryRequestHandlerContext, + typeof APP_ID + > => { + const { eventLogService } = this; + return async (context, request): Promise => { + return { + getEventLogClient: async ( + logDefinition: EventLogDefinition + ) => { + const eventLogClient = await eventLogService! + .getScopedResolver(request) + .resolve(logDefinition); + return eventLogClient; + }, + }; + }; + }; public stop() { const { eventLogService, logger } = this; diff --git a/x-pack/plugins/rule_registry/server/types.ts b/x-pack/plugins/rule_registry/server/types.ts index 959c05fd1334e..8bdb27f100a67 100644 --- a/x-pack/plugins/rule_registry/server/types.ts +++ b/x-pack/plugins/rule_registry/server/types.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { RequestHandlerContext } from '../../../../src/core/server'; import { AlertInstanceContext, AlertInstanceState, @@ -12,6 +13,8 @@ import { AlertTypeState, } from '../../alerting/common'; import { AlertType } from '../../alerting/server'; +import { IEventLog } from './event_log'; +import { CommonFields, EventLogDefinition } from './event_log/common'; type SimpleAlertType< TParams extends AlertTypeParams = {}, @@ -38,3 +41,13 @@ export type AlertTypeWithExecutor< > & { executor: AlertTypeExecutor; }; + +export interface RuleRegistryApiRequestHandlerContext { + getEventLogClient: ( + logDefinition: EventLogDefinition + ) => Promise>; +} + +export interface RuleRegistryRequestHandlerContext extends RequestHandlerContext { + ruleRegistry: RuleRegistryApiRequestHandlerContext; +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/reference_rules/query.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/reference_rules/query.ts index 4ca9448f5e3c7..1934a17d3d204 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/reference_rules/query.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/reference_rules/query.ts @@ -73,7 +73,6 @@ export const createQueryAlertType = (ruleDataClient: RuleDataClient, logger: Log }; const alerts = await findAlerts(query); - // console.log('alerts', alerts); alertWithPersistence(alerts).forEach((alert) => { alert.scheduleActions('default', { server: 'server-test' }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_route.ts index 428978fe1d820..42bb3efe921af 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_route.ts @@ -20,6 +20,7 @@ import { ruleStatusSavedObjectsClientFactory } from '../../signals/rule_status_s import { buildRouteValidation } from '../../../../utils/build_validation/route_validation'; import { transformFindAlerts } from './utils'; import { getBulkRuleActionsSavedObject } from '../../rule_actions/get_bulk_rule_actions_saved_object'; +// import { commonSchema } from '../../../../../../rule_registry/server/event_log/common'; export const findRulesRoute = ( router: SecuritySolutionPluginRouter, @@ -53,6 +54,57 @@ export const findRulesRoute = ( return siemResponse.error({ statusCode: 404 }); } + // const eventLogClient = await context.ruleRegistry.getEventLogClient({ + // logName: 'execution.log', + // schema: commonSchema, + // templates: { + // applicationDefinedComponentTemplate: { + // mappings: { + // dynamic: 'strict', + // properties: { + // kibana: { + // properties: { + // rac: { + // properties: { + // event_log: { + // properties: { + // log_name: { + // type: 'keyword', + // }, + // logger_name: { + // type: 'keyword', + // }, + // }, + // }, + // }, + // }, + // }, + // }, + // }, + // }, + // }, + // }, + // }); + + // const ruleLogger = eventLogClient.getLogger('ruleExecutionLogger'); + + // ruleLogger.logEvent({ + // '@timestamp': [`${new Date().getTime()}`], + // tags: ['test'], + // 'kibana.rac.alert.id': 'testId', + // 'kibana.rac.alert.status': 'warning', + // }); + + // const searchQuery = eventLogClient + // .getQueryBuilder() + // .filterByLogger('ruleExecutionLogger') + // .sortBy([{ '@timestamp': { order: 'desc' } }]) + // .buildQuery(); + + // const events = await searchQuery.execute(); + + // console.log({ events }); + const ruleStatusClient = ruleStatusSavedObjectsClientFactory(savedObjectsClient); const rules = await findRules({ alertsClient, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/schemas/execution_event_schema.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/schemas/execution_event_schema.ts new file mode 100644 index 0000000000000..539ca7c1d98e0 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/schemas/execution_event_schema.ts @@ -0,0 +1,7 @@ +import * as t from 'io-ts'; + +import { commonSchema, Schema } from '../../../../../rule_registry/server/event_log/common'; + +export const executionEventFieldsSchema = Schema.create(t.type({})); + +export const executionEventSchema = Schema.combine(commonSchema, executionEventFieldsSchema); diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index cd923a4b0619f..5f1f8f5f59e76 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -248,6 +248,7 @@ export class Plugin implements IPlugin; From fa04b8d66b5ce53c600d291eb3144af129d006f6 Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Tue, 15 Jun 2021 00:47:47 -0700 Subject: [PATCH 2/2] Add IndexWriter indexManyNow method --- .../server/event_log/elasticsearch/index_writer.ts | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/rule_registry/server/event_log/elasticsearch/index_writer.ts b/x-pack/plugins/rule_registry/server/event_log/elasticsearch/index_writer.ts index 7f83421ec80d8..04b7129d4c7c7 100644 --- a/x-pack/plugins/rule_registry/server/event_log/elasticsearch/index_writer.ts +++ b/x-pack/plugins/rule_registry/server/event_log/elasticsearch/index_writer.ts @@ -6,6 +6,7 @@ */ import type { PublicMethodsOf } from '@kbn/utility-types'; +import { estypes } from '@elastic/elasticsearch'; import util from 'util'; import { Logger, ElasticsearchClient } from 'src/core/server'; import { BufferedStream } from './utils/buffered_stream'; @@ -40,7 +41,9 @@ export class IndexWriter { this.logger = params.logger.get('IndexWriter'); this.buffer = new BufferedStream({ - flush: (items) => this.bulkIndex(items), + flush: async (items) => { + this.bulkIndex(items); + }, }); } @@ -60,11 +63,16 @@ export class IndexWriter { } } + public async indexManyNow(docs: Document[]): Promise { + const bufferItems = docs.map((doc) => ({ index: this.indexName, doc })); + return this.bulkIndex(bufferItems); + } + public async close(): Promise { await this.buffer.closeAndWaitUntilFlushed(); } - private async bulkIndex(items: BufferItem[]): Promise { + private async bulkIndex(items: BufferItem[]): Promise { this.logger.debug(`Indexing ${items.length} documents`); const bulkBody: Array> = []; @@ -85,10 +93,12 @@ export class IndexWriter { error.stack += '\n' + util.inspect(response.body.items, { depth: null }); this.logger.error(error); } + return response.body; } catch (e) { this.logger.error( `error writing bulk events: "${e.message}"; docs: ${JSON.stringify(bulkBody)}` ); + return undefined; } } }