From 1b85455d27fb0c94099b174da0b7e4304bd1961c Mon Sep 17 00:00:00 2001 From: Milton Hultgren Date: Thu, 8 Aug 2024 11:08:13 +0200 Subject: [PATCH] [EEM] Add entities aliases (#190055) ## Summary When an entity definition is installed and the transforms write the first documents to the `.entities-*` indices, the index templates applied also set up an alias like `entities-{type}-latest` to make it easier to query data by entity type. ## How to test Ingest some data using data forge, install a definition of a given type, try to query for the data via the new alias. ## Open question Do we need to do anything related to users/roles/privileges for the entities data, to make it easier for admins to create their users with the right access? The built in `viewer` role has read access to all indices and it seems trivial to create a new role that limits that down to `entities-*`. --- .../create_assets_es_clients.ts | 8 +++++--- .../lib/entities/install_entity_definition.ts | 4 ++-- .../entities_history_template.test.ts.snap | 3 +++ .../entities_latest_template.test.ts.snap | 3 +++ .../templates/entities_history_template.test.ts | 2 +- .../templates/entities_history_template.ts | 16 +++++++++++----- .../templates/entities_latest_template.test.ts | 2 +- .../templates/entities_latest_template.ts | 16 +++++++++++----- 8 files changed, 37 insertions(+), 17 deletions(-) diff --git a/x-pack/plugins/observability_solution/apm/server/lib/helpers/create_es_client/create_assets_es_client/create_assets_es_clients.ts b/x-pack/plugins/observability_solution/apm/server/lib/helpers/create_es_client/create_assets_es_client/create_assets_es_clients.ts index 17193a6d508b6..4ddd64fd0a3bc 100644 --- a/x-pack/plugins/observability_solution/apm/server/lib/helpers/create_es_client/create_assets_es_client/create_assets_es_clients.ts +++ b/x-pack/plugins/observability_solution/apm/server/lib/helpers/create_es_client/create_assets_es_client/create_assets_es_clients.ts @@ -13,9 +13,10 @@ import { MsearchMultisearchHeader, } from '@elastic/elasticsearch/lib/api/types'; import { withApmSpan } from '../../../../utils/with_apm_span'; +import { EntityType } from '../../../../routes/entities/types'; -const ENTITIES_LATEST_INDEX_NAME = '.entities.v1.latest.builtin_services*'; -const ENTITIES_HISTORY_INDEX_NAME = '.entities.v1.history.builtin_services*'; +const ENTITIES_LATEST_INDEX_NAME = `entities-${EntityType.SERVICE}-latest`; +const ENTITIES_HISTORY_INDEX_NAME = `entities-${EntityType.SERVICE}-history`; export function cancelEsRequestOnAbort>( promise: T, @@ -60,7 +61,7 @@ export async function createEntitiesESClient({ const promise = withApmSpan(operationName, () => { return cancelEsRequestOnAbort( esClient.search( - { ...searchRequest, index: [indexName] }, + { ...searchRequest, index: [indexName], ignore_unavailable: true }, { signal: controller.signal, meta: true, @@ -99,6 +100,7 @@ export async function createEntitiesESClient({ const searchParams: [MsearchMultisearchHeader, MsearchMultisearchBody] = [ { index: [ENTITIES_LATEST_INDEX_NAME], + ignore_unavailable: true, }, { ...params.body, diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/install_entity_definition.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/install_entity_definition.ts index a58019bf236ae..5c3c954b45333 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/install_entity_definition.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/install_entity_definition.ts @@ -81,13 +81,13 @@ export async function installEntityDefinition({ await upsertTemplate({ esClient, logger, - template: getEntitiesHistoryIndexTemplateConfig(definition.id), + template: getEntitiesHistoryIndexTemplateConfig(definition), }); installState.indexTemplates.history = true; await upsertTemplate({ esClient, logger, - template: getEntitiesLatestIndexTemplateConfig(definition.id), + template: getEntitiesLatestIndexTemplateConfig(definition), }); installState.indexTemplates.latest = true; diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/templates/__snapshots__/entities_history_template.test.ts.snap b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/templates/__snapshots__/entities_history_template.test.ts.snap index 94af9f3307f04..464293cc1bca8 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/templates/__snapshots__/entities_history_template.test.ts.snap +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/templates/__snapshots__/entities_history_template.test.ts.snap @@ -29,6 +29,9 @@ Object { "name": "entities_v1_history_admin-console-services_index_template", "priority": 200, "template": Object { + "aliases": Object { + "entities-service-history": Object {}, + }, "mappings": Object { "_meta": Object { "version": "1.6.0", diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/templates/__snapshots__/entities_latest_template.test.ts.snap b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/templates/__snapshots__/entities_latest_template.test.ts.snap index b4247098d9498..448286b16d84a 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/templates/__snapshots__/entities_latest_template.test.ts.snap +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/templates/__snapshots__/entities_latest_template.test.ts.snap @@ -29,6 +29,9 @@ Object { "name": "entities_v1_latest_admin-console-services_index_template", "priority": 200, "template": Object { + "aliases": Object { + "entities-service-latest": Object {}, + }, "mappings": Object { "_meta": Object { "version": "1.6.0", diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/templates/entities_history_template.test.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/templates/entities_history_template.test.ts index 11aad78741020..055ebe75cd608 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/templates/entities_history_template.test.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/templates/entities_history_template.test.ts @@ -10,7 +10,7 @@ import { getEntitiesHistoryIndexTemplateConfig } from './entities_history_templa describe('getEntitiesHistoryIndexTemplateConfig(definitionId)', () => { it('should generate a valid index template', () => { - const template = getEntitiesHistoryIndexTemplateConfig(entityDefinition.id); + const template = getEntitiesHistoryIndexTemplateConfig(entityDefinition); expect(template).toMatchSnapshot(); }); }); diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/templates/entities_history_template.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/templates/entities_history_template.ts index a0fb4b032a6e1..03f3a3510f627 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/templates/entities_history_template.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/templates/entities_history_template.ts @@ -6,19 +6,22 @@ */ import { IndicesPutIndexTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; +import { EntityDefinition } from '@kbn/entities-schema'; import { getEntityHistoryIndexTemplateV1 } from '../../../../common/helpers'; import { + ENTITY_BASE_PREFIX, ENTITY_ENTITY_COMPONENT_TEMPLATE_V1, ENTITY_EVENT_COMPONENT_TEMPLATE_V1, + ENTITY_HISTORY, ENTITY_HISTORY_BASE_COMPONENT_TEMPLATE_V1, ENTITY_HISTORY_INDEX_PREFIX_V1, } from '../../../../common/constants_entities'; import { getCustomHistoryTemplateComponents } from '../../../templates/components/helpers'; export const getEntitiesHistoryIndexTemplateConfig = ( - definitionId: string + definition: EntityDefinition ): IndicesPutIndexTemplateRequest => ({ - name: getEntityHistoryIndexTemplateV1(definitionId), + name: getEntityHistoryIndexTemplateV1(definition.id), _meta: { description: "Index template for indices managed by the Elastic Entity Model's entity discovery framework for the history dataset", @@ -26,16 +29,19 @@ export const getEntitiesHistoryIndexTemplateConfig = ( managed: true, managed_by: 'elastic_entity_model', }, - ignore_missing_component_templates: getCustomHistoryTemplateComponents(definitionId), + ignore_missing_component_templates: getCustomHistoryTemplateComponents(definition.id), composed_of: [ ENTITY_HISTORY_BASE_COMPONENT_TEMPLATE_V1, ENTITY_ENTITY_COMPONENT_TEMPLATE_V1, ENTITY_EVENT_COMPONENT_TEMPLATE_V1, - ...getCustomHistoryTemplateComponents(definitionId), + ...getCustomHistoryTemplateComponents(definition.id), ], - index_patterns: [`${ENTITY_HISTORY_INDEX_PREFIX_V1}.${definitionId}.*`], + index_patterns: [`${ENTITY_HISTORY_INDEX_PREFIX_V1}.${definition.id}.*`], priority: 200, template: { + aliases: { + [`${ENTITY_BASE_PREFIX}-${definition.type}-${ENTITY_HISTORY}`]: {}, + }, mappings: { _meta: { version: '1.6.0', diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/templates/entities_latest_template.test.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/templates/entities_latest_template.test.ts index 72583d941492c..f1012d0bcb7f7 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/templates/entities_latest_template.test.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/templates/entities_latest_template.test.ts @@ -10,7 +10,7 @@ import { getEntitiesLatestIndexTemplateConfig } from './entities_latest_template describe('getEntitiesLatestIndexTemplateConfig(definitionId)', () => { it('should generate a valid index template', () => { - const template = getEntitiesLatestIndexTemplateConfig(entityDefinition.id); + const template = getEntitiesLatestIndexTemplateConfig(entityDefinition); expect(template).toMatchSnapshot(); }); }); diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/templates/entities_latest_template.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/templates/entities_latest_template.ts index 466346f86b44d..7b0093bb4b83f 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/templates/entities_latest_template.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/templates/entities_latest_template.ts @@ -6,19 +6,22 @@ */ import { IndicesPutIndexTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; +import { EntityDefinition } from '@kbn/entities-schema'; import { getEntityLatestIndexTemplateV1 } from '../../../../common/helpers'; import { + ENTITY_BASE_PREFIX, ENTITY_ENTITY_COMPONENT_TEMPLATE_V1, ENTITY_EVENT_COMPONENT_TEMPLATE_V1, + ENTITY_LATEST, ENTITY_LATEST_BASE_COMPONENT_TEMPLATE_V1, ENTITY_LATEST_INDEX_PREFIX_V1, } from '../../../../common/constants_entities'; import { getCustomLatestTemplateComponents } from '../../../templates/components/helpers'; export const getEntitiesLatestIndexTemplateConfig = ( - definitionId: string + definition: EntityDefinition ): IndicesPutIndexTemplateRequest => ({ - name: getEntityLatestIndexTemplateV1(definitionId), + name: getEntityLatestIndexTemplateV1(definition.id), _meta: { description: "Index template for indices managed by the Elastic Entity Model's entity discovery framework for the latest dataset", @@ -26,16 +29,19 @@ export const getEntitiesLatestIndexTemplateConfig = ( managed: true, managed_by: 'elastic_entity_model', }, - ignore_missing_component_templates: getCustomLatestTemplateComponents(definitionId), + ignore_missing_component_templates: getCustomLatestTemplateComponents(definition.id), composed_of: [ ENTITY_LATEST_BASE_COMPONENT_TEMPLATE_V1, ENTITY_ENTITY_COMPONENT_TEMPLATE_V1, ENTITY_EVENT_COMPONENT_TEMPLATE_V1, - ...getCustomLatestTemplateComponents(definitionId), + ...getCustomLatestTemplateComponents(definition.id), ], - index_patterns: [`${ENTITY_LATEST_INDEX_PREFIX_V1}.${definitionId}`], + index_patterns: [`${ENTITY_LATEST_INDEX_PREFIX_V1}.${definition.id}`], priority: 200, template: { + aliases: { + [`${ENTITY_BASE_PREFIX}-${definition.type}-${ENTITY_LATEST}`]: {}, + }, mappings: { _meta: { version: '1.6.0',