From 0b3f4fbd3cd60663289fc13f8f01e3f4c9131479 Mon Sep 17 00:00:00 2001 From: Kevin Lacabane Date: Fri, 22 Nov 2024 16:12:04 +0100 Subject: [PATCH] [eem] _search endpoint / initial entity manager UI (#199609) ## Summary - create `_search` endpoint to discover entities with esql queries. It currently reads sources of the provided `type` from `kibana_entity_definitions` index. Run this query to insert a definition: ``` POST kibana_entity_definitions/_doc { "entity_type": "service", "index_patterns": ["remote_cluster:logs-*"], "metadata_fields": [], "identity_fields": ["service.name"], "filters": [], "timestamp_field": "@timestamp" } ``` By default `_search` will look at data in the last 5m. The lookup period can be overriden by providing `start`/`end` parameters in ISO format. It also accepts a `limit` to specify the number of entities returned which defaults to 10 ``` POST kbn:/internal/entities/v2/_search { "type": "service", "start": "2024-11-19T20:40:00.000Z", "end": "2024-11-19T20:50:00.000Z", "limit": 20 } ``` - create `_search/preview` endpoint to preview output of entity sources without persisting them - create UI to preview results of an entity definition at `/app/entity_manager`. The application is living in its own plugin at `observability_solution/entity_manager_app` ![Screenshot 2024-11-11 at 11 37 18](https://github.com/user-attachments/assets/f284342d-21a3-4ba1-be94-38cff311266c) --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Milton Hultgren --- .github/CODEOWNERS | 1 + docs/developer/plugin-list.asciidoc | 4 + package.json | 1 + packages/kbn-optimizer/limits.yml | 1 + .../collectors/application_usage/schema.ts | 1 + src/plugins/telemetry/schema/oss_plugins.json | 131 +++++++ tsconfig.base.json | 2 + .../kbn-entities-schema/src/schema/entity.ts | 7 + x-pack/plugins/entity_manager/kibana.jsonc | 6 +- x-pack/plugins/entity_manager/public/index.ts | 2 + .../plugins/entity_manager/public/plugin.ts | 6 +- x-pack/plugins/entity_manager/public/types.ts | 1 - .../errors/unknown_entity_type.ts} | 9 +- .../server/lib/entity_client.ts | 117 +++++- .../server/lib/queries/index.test.ts | 38 ++ .../server/lib/queries/index.ts | 91 +++++ .../server/lib/queries/utils.test.ts | 135 +++++++ .../server/lib/queries/utils.ts | 90 +++++ .../server/routes/entities/index.ts | 3 + .../entity_manager/server/routes/v2/search.ts | 96 +++++ x-pack/plugins/entity_manager/tsconfig.json | 1 + .../entity_manager_app/README.md | 3 + .../entity_manager_app/jest.config.js | 20 + .../entity_manager_app/kibana.jsonc | 29 ++ .../entity_manager_app/public/application.tsx | 103 ++++++ .../public/context/plugin_context.ts | 21 ++ .../public/hooks/use_kibana.ts | 22 ++ .../public/hooks/use_plugin_context.ts | 19 + .../entity_manager_app/public/index.ts | 13 + .../public/pages/overview/index.tsx | 347 ++++++++++++++++++ .../entity_manager_app/public/plugin.ts | 80 ++++ .../entity_manager_app/public/routes.tsx | 27 ++ .../entity_manager_app/public/types.ts | 32 ++ .../entity_manager_app/tsconfig.json | 33 ++ yarn.lock | 4 + 35 files changed, 1482 insertions(+), 14 deletions(-) rename x-pack/plugins/entity_manager/server/lib/{errors.ts => entities/errors/unknown_entity_type.ts} (51%) create mode 100644 x-pack/plugins/entity_manager/server/lib/queries/index.test.ts create mode 100644 x-pack/plugins/entity_manager/server/lib/queries/index.ts create mode 100644 x-pack/plugins/entity_manager/server/lib/queries/utils.test.ts create mode 100644 x-pack/plugins/entity_manager/server/lib/queries/utils.ts create mode 100644 x-pack/plugins/entity_manager/server/routes/v2/search.ts create mode 100644 x-pack/plugins/observability_solution/entity_manager_app/README.md create mode 100644 x-pack/plugins/observability_solution/entity_manager_app/jest.config.js create mode 100644 x-pack/plugins/observability_solution/entity_manager_app/kibana.jsonc create mode 100644 x-pack/plugins/observability_solution/entity_manager_app/public/application.tsx create mode 100644 x-pack/plugins/observability_solution/entity_manager_app/public/context/plugin_context.ts create mode 100644 x-pack/plugins/observability_solution/entity_manager_app/public/hooks/use_kibana.ts create mode 100644 x-pack/plugins/observability_solution/entity_manager_app/public/hooks/use_plugin_context.ts create mode 100644 x-pack/plugins/observability_solution/entity_manager_app/public/index.ts create mode 100644 x-pack/plugins/observability_solution/entity_manager_app/public/pages/overview/index.tsx create mode 100644 x-pack/plugins/observability_solution/entity_manager_app/public/plugin.ts create mode 100644 x-pack/plugins/observability_solution/entity_manager_app/public/routes.tsx create mode 100644 x-pack/plugins/observability_solution/entity_manager_app/public/types.ts create mode 100644 x-pack/plugins/observability_solution/entity_manager_app/tsconfig.json diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index c0adb7f980d4..46f771371a3c 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -920,6 +920,7 @@ x-pack/plugins/observability_solution/apm_data_access @elastic/obs-knowledge-tea x-pack/plugins/observability_solution/apm/ftr_e2e @elastic/obs-ux-infra_services-team x-pack/plugins/observability_solution/dataset_quality @elastic/obs-ux-logs-team x-pack/plugins/observability_solution/entities_data_access @elastic/obs-entities +x-pack/plugins/observability_solution/entity_manager_app @elastic/obs-entities x-pack/plugins/observability_solution/exploratory_view @elastic/obs-ux-management-team x-pack/plugins/observability_solution/infra @elastic/obs-ux-logs-team @elastic/obs-ux-infra_services-team x-pack/plugins/observability_solution/inventory @elastic/obs-ux-infra_services-team diff --git a/docs/developer/plugin-list.asciidoc b/docs/developer/plugin-list.asciidoc index ea84b25dcda4..efe65e4a9954 100644 --- a/docs/developer/plugin-list.asciidoc +++ b/docs/developer/plugin-list.asciidoc @@ -579,6 +579,10 @@ security and spaces filtering. |This plugin provides access to observed entity data, such as information about hosts, pods, containers, services, and more. +|{kib-repo}blob/{branch}/x-pack/plugins/observability_solution/entity_manager_app/README.md[entityManagerApp] +|This plugin provides a user interface to interact with the Entity Manager. + + |{kib-repo}blob/{branch}/x-pack/plugins/event_log/README.md[eventLog] |The event log plugin provides a persistent history of alerting and action activities. diff --git a/package.json b/package.json index 2d667eb6c8dc..981a947a9a91 100644 --- a/package.json +++ b/package.json @@ -479,6 +479,7 @@ "@kbn/entities-data-access-plugin": "link:x-pack/plugins/observability_solution/entities_data_access", "@kbn/entities-schema": "link:x-pack/packages/kbn-entities-schema", "@kbn/entity-manager-fixture-plugin": "link:x-pack/test/api_integration/apis/entity_manager/fixture_plugin", + "@kbn/entityManager-app-plugin": "link:x-pack/plugins/observability_solution/entity_manager_app", "@kbn/entityManager-plugin": "link:x-pack/plugins/entity_manager", "@kbn/error-boundary-example-plugin": "link:examples/error_boundary", "@kbn/es-errors": "link:packages/kbn-es-errors", diff --git a/packages/kbn-optimizer/limits.yml b/packages/kbn-optimizer/limits.yml index c7189dc1598d..34355f36541c 100644 --- a/packages/kbn-optimizer/limits.yml +++ b/packages/kbn-optimizer/limits.yml @@ -42,6 +42,7 @@ pageLoadAssetSize: embeddableEnhanced: 22107 enterpriseSearch: 66810 entityManager: 17175 + entityManagerApp: 20378 esql: 37000 esqlDataGrid: 24582 esUiShared: 326654 diff --git a/src/plugins/kibana_usage_collection/server/collectors/application_usage/schema.ts b/src/plugins/kibana_usage_collection/server/collectors/application_usage/schema.ts index 88d60b1a86b2..0cc56676137e 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/application_usage/schema.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/application_usage/schema.ts @@ -143,6 +143,7 @@ export const applicationUsageSchema = { enterpriseSearchSemanticSearch: commonSchema, enterpriseSearchVectorSearch: commonSchema, enterpriseSearchElasticsearch: commonSchema, + entity_manager: commonSchema, appSearch: commonSchema, workplaceSearch: commonSchema, searchExperiences: commonSchema, diff --git a/src/plugins/telemetry/schema/oss_plugins.json b/src/plugins/telemetry/schema/oss_plugins.json index 511ab4cf89bf..44fcda4f2858 100644 --- a/src/plugins/telemetry/schema/oss_plugins.json +++ b/src/plugins/telemetry/schema/oss_plugins.json @@ -3015,6 +3015,137 @@ } } }, + "entity_manager": { + "properties": { + "appId": { + "type": "keyword", + "_meta": { + "description": "The application being tracked" + } + }, + "viewId": { + "type": "keyword", + "_meta": { + "description": "Always `main`" + } + }, + "clicks_total": { + "type": "long", + "_meta": { + "description": "General number of clicks in the application since we started counting them" + } + }, + "clicks_7_days": { + "type": "long", + "_meta": { + "description": "General number of clicks in the application over the last 7 days" + } + }, + "clicks_30_days": { + "type": "long", + "_meta": { + "description": "General number of clicks in the application over the last 30 days" + } + }, + "clicks_90_days": { + "type": "long", + "_meta": { + "description": "General number of clicks in the application over the last 90 days" + } + }, + "minutes_on_screen_total": { + "type": "float", + "_meta": { + "description": "Minutes the application is active and on-screen since we started counting them." + } + }, + "minutes_on_screen_7_days": { + "type": "float", + "_meta": { + "description": "Minutes the application is active and on-screen over the last 7 days" + } + }, + "minutes_on_screen_30_days": { + "type": "float", + "_meta": { + "description": "Minutes the application is active and on-screen over the last 30 days" + } + }, + "minutes_on_screen_90_days": { + "type": "float", + "_meta": { + "description": "Minutes the application is active and on-screen over the last 90 days" + } + }, + "views": { + "type": "array", + "items": { + "properties": { + "appId": { + "type": "keyword", + "_meta": { + "description": "The application being tracked" + } + }, + "viewId": { + "type": "keyword", + "_meta": { + "description": "The application view being tracked" + } + }, + "clicks_total": { + "type": "long", + "_meta": { + "description": "General number of clicks in the application sub view since we started counting them" + } + }, + "clicks_7_days": { + "type": "long", + "_meta": { + "description": "General number of clicks in the active application sub view over the last 7 days" + } + }, + "clicks_30_days": { + "type": "long", + "_meta": { + "description": "General number of clicks in the active application sub view over the last 30 days" + } + }, + "clicks_90_days": { + "type": "long", + "_meta": { + "description": "General number of clicks in the active application sub view over the last 90 days" + } + }, + "minutes_on_screen_total": { + "type": "float", + "_meta": { + "description": "Minutes the application sub view is active and on-screen since we started counting them." + } + }, + "minutes_on_screen_7_days": { + "type": "float", + "_meta": { + "description": "Minutes the application is active and on-screen active application sub view over the last 7 days" + } + }, + "minutes_on_screen_30_days": { + "type": "float", + "_meta": { + "description": "Minutes the application is active and on-screen active application sub view over the last 30 days" + } + }, + "minutes_on_screen_90_days": { + "type": "float", + "_meta": { + "description": "Minutes the application is active and on-screen active application sub view over the last 90 days" + } + } + } + } + } + } + }, "appSearch": { "properties": { "appId": { diff --git a/tsconfig.base.json b/tsconfig.base.json index d69eca86ea54..23be79df3418 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -822,6 +822,8 @@ "@kbn/entities-schema/*": ["x-pack/packages/kbn-entities-schema/*"], "@kbn/entity-manager-fixture-plugin": ["x-pack/test/api_integration/apis/entity_manager/fixture_plugin"], "@kbn/entity-manager-fixture-plugin/*": ["x-pack/test/api_integration/apis/entity_manager/fixture_plugin/*"], + "@kbn/entityManager-app-plugin": ["x-pack/plugins/observability_solution/entity_manager_app"], + "@kbn/entityManager-app-plugin/*": ["x-pack/plugins/observability_solution/entity_manager_app/*"], "@kbn/entityManager-plugin": ["x-pack/plugins/entity_manager"], "@kbn/entityManager-plugin/*": ["x-pack/plugins/entity_manager/*"], "@kbn/error-boundary-example-plugin": ["examples/error_boundary"], diff --git a/x-pack/packages/kbn-entities-schema/src/schema/entity.ts b/x-pack/packages/kbn-entities-schema/src/schema/entity.ts index 7bfe505face1..5df10e11bb7e 100644 --- a/x-pack/packages/kbn-entities-schema/src/schema/entity.ts +++ b/x-pack/packages/kbn-entities-schema/src/schema/entity.ts @@ -23,6 +23,13 @@ export interface MetadataRecord { [key: string]: string[] | MetadataRecord | string; } +export interface EntityV2 { + 'entity.id': string; + 'entity.last_seen_timestamp': string; + 'entity.type': string; + [metadata: string]: any; +} + const literalSchema = z.union([z.string(), z.number(), z.boolean(), z.null()]); type Literal = z.infer; diff --git a/x-pack/plugins/entity_manager/kibana.jsonc b/x-pack/plugins/entity_manager/kibana.jsonc index d5dadcf8fd2b..c18822d48ac0 100644 --- a/x-pack/plugins/entity_manager/kibana.jsonc +++ b/x-pack/plugins/entity_manager/kibana.jsonc @@ -8,9 +8,13 @@ "plugin": { "id": "entityManager", "configPath": ["xpack", "entityManager"], - "requiredPlugins": ["security", "encryptedSavedObjects", "licensing"], "browser": true, "server": true, + "requiredPlugins": [ + "security", + "encryptedSavedObjects", + "licensing" + ], "requiredBundles": [] } } diff --git a/x-pack/plugins/entity_manager/public/index.ts b/x-pack/plugins/entity_manager/public/index.ts index 85a9285b1692..73d23ad45e9c 100644 --- a/x-pack/plugins/entity_manager/public/index.ts +++ b/x-pack/plugins/entity_manager/public/index.ts @@ -16,6 +16,8 @@ export const plugin: PluginInitializer< return new Plugin(context); }; +export { EntityClient } from './lib/entity_client'; + export type { EntityManagerPublicPluginSetup, EntityManagerPublicPluginStart }; export type EntityManagerAppId = 'entityManager'; diff --git a/x-pack/plugins/entity_manager/public/plugin.ts b/x-pack/plugins/entity_manager/public/plugin.ts index 6d6d56a95b75..7ff6354c997e 100644 --- a/x-pack/plugins/entity_manager/public/plugin.ts +++ b/x-pack/plugins/entity_manager/public/plugin.ts @@ -22,16 +22,14 @@ export class Plugin implements EntityManagerPluginClass { } setup(core: CoreSetup) { - const entityClient = new EntityClient(core); return { - entityClient, + entityClient: new EntityClient(core), }; } start(core: CoreStart) { - const entityClient = new EntityClient(core); return { - entityClient, + entityClient: new EntityClient(core), }; } diff --git a/x-pack/plugins/entity_manager/public/types.ts b/x-pack/plugins/entity_manager/public/types.ts index 66499479299d..90d9026e8b9b 100644 --- a/x-pack/plugins/entity_manager/public/types.ts +++ b/x-pack/plugins/entity_manager/public/types.ts @@ -10,7 +10,6 @@ import type { EntityClient } from './lib/entity_client'; export interface EntityManagerPublicPluginSetup { entityClient: EntityClient; } - export interface EntityManagerPublicPluginStart { entityClient: EntityClient; } diff --git a/x-pack/plugins/entity_manager/server/lib/errors.ts b/x-pack/plugins/entity_manager/server/lib/entities/errors/unknown_entity_type.ts similarity index 51% rename from x-pack/plugins/entity_manager/server/lib/errors.ts rename to x-pack/plugins/entity_manager/server/lib/entities/errors/unknown_entity_type.ts index e0d341f87a9f..5d29e24a1cca 100644 --- a/x-pack/plugins/entity_manager/server/lib/errors.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/errors/unknown_entity_type.ts @@ -5,10 +5,9 @@ * 2.0. */ -export class AssetNotFoundError extends Error { - constructor(ean: string) { - super(`Asset with ean (${ean}) not found in the provided time range`); - Object.setPrototypeOf(this, new.target.prototype); - this.name = 'AssetNotFoundError'; +export class UnknownEntityType extends Error { + constructor(message: string) { + super(message); + this.name = 'UnknownEntityType'; } } diff --git a/x-pack/plugins/entity_manager/server/lib/entity_client.ts b/x-pack/plugins/entity_manager/server/lib/entity_client.ts index 8bb51941092f..7045bee1fc53 100644 --- a/x-pack/plugins/entity_manager/server/lib/entity_client.ts +++ b/x-pack/plugins/entity_manager/server/lib/entity_client.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { EntityDefinition, EntityDefinitionUpdate } from '@kbn/entities-schema'; +import { EntityV2, EntityDefinition, EntityDefinitionUpdate } from '@kbn/entities-schema'; import { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server'; import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; import { Logger } from '@kbn/logging'; @@ -23,6 +23,9 @@ import { stopTransforms } from './entities/stop_transforms'; import { deleteIndices } from './entities/delete_index'; import { EntityDefinitionWithState } from './entities/types'; import { EntityDefinitionUpdateConflict } from './entities/errors/entity_definition_update_conflict'; +import { EntitySource, getEntityInstancesQuery } from './queries'; +import { mergeEntitiesList, runESQLQuery } from './queries/utils'; +import { UnknownEntityType } from './entities/errors/unknown_entity_type'; export class EntityClient { constructor( @@ -126,8 +129,6 @@ export class EntityClient { }); if (deleteData) { - // delete data with current user as system user does not have - // .entities privileges await deleteIndices(this.options.esClient, definition, this.options.logger); } } @@ -170,4 +171,114 @@ export class EntityClient { this.options.logger.info(`Stopping transforms for definition [${definition.id}]`); return stopTransforms(this.options.esClient, definition, this.options.logger); } + + async getEntitySources({ type }: { type: string }) { + const result = await this.options.esClient.search({ + index: 'kibana_entity_definitions', + query: { + bool: { + must: { + term: { entity_type: type }, + }, + }, + }, + }); + + return result.hits.hits.map((hit) => hit._source) as EntitySource[]; + } + + async searchEntities({ + type, + start, + end, + metadataFields = [], + filters = [], + limit = 10, + }: { + type: string; + start: string; + end: string; + metadataFields?: string[]; + filters?: string[]; + limit?: number; + }) { + const sources = await this.getEntitySources({ type }); + if (sources.length === 0) { + throw new UnknownEntityType(`No sources found for entity type [${type}]`); + } + + return this.searchEntitiesBySources({ + sources, + start, + end, + metadataFields, + filters, + limit, + }); + } + + async searchEntitiesBySources({ + sources, + start, + end, + metadataFields = [], + filters = [], + limit = 10, + }: { + sources: EntitySource[]; + start: string; + end: string; + metadataFields?: string[]; + filters?: string[]; + limit?: number; + }) { + const entities = await Promise.all( + sources.map(async (source) => { + const mandatoryFields = [source.timestamp_field, ...source.identity_fields]; + const metaFields = [...metadataFields, ...source.metadata_fields]; + const { fields } = await this.options.esClient.fieldCaps({ + index: source.index_patterns, + fields: [...mandatoryFields, ...metaFields], + }); + + const sourceHasMandatoryFields = mandatoryFields.every((field) => !!fields[field]); + if (!sourceHasMandatoryFields) { + // we can't build entities without id fields so we ignore the source. + // filters should likely behave similarly. + this.options.logger.info( + `Ignoring source for type [${source.type}] with index_patterns [${source.index_patterns}] because some mandatory fields [${mandatoryFields}] are not mapped` + ); + return []; + } + + // but metadata field not being available is fine + const availableMetadataFields = metaFields.filter((field) => fields[field]); + + const query = getEntityInstancesQuery({ + source: { + ...source, + metadata_fields: availableMetadataFields, + filters: [...source.filters, ...filters], + }, + start, + end, + limit, + }); + this.options.logger.debug(`Entity query: ${query}`); + + const rawEntities = await runESQLQuery({ + query, + esClient: this.options.esClient, + }); + + return rawEntities.map((entity) => { + entity['entity.id'] = source.identity_fields.map((field) => entity[field]).join(':'); + entity['entity.type'] = source.type; + return entity; + }); + }) + ).then((results) => results.flat()); + + return mergeEntitiesList(entities).slice(0, limit); + } } diff --git a/x-pack/plugins/entity_manager/server/lib/queries/index.test.ts b/x-pack/plugins/entity_manager/server/lib/queries/index.test.ts new file mode 100644 index 000000000000..539d20c46479 --- /dev/null +++ b/x-pack/plugins/entity_manager/server/lib/queries/index.test.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getEntityInstancesQuery } from '.'; + +describe('getEntityInstancesQuery', () => { + describe('getEntityInstancesQuery', () => { + it('generates a valid esql query', () => { + const query = getEntityInstancesQuery({ + source: { + type: 'service', + index_patterns: ['logs-*', 'metrics-*'], + identity_fields: ['service.name'], + metadata_fields: ['host.name'], + filters: [], + timestamp_field: 'custom_timestamp_field', + }, + limit: 5, + start: '2024-11-20T19:00:00.000Z', + end: '2024-11-20T20:00:00.000Z', + }); + + expect(query).toEqual( + 'FROM logs-*,metrics-*|' + + 'WHERE custom_timestamp_field >= "2024-11-20T19:00:00.000Z"|' + + 'WHERE custom_timestamp_field <= "2024-11-20T20:00:00.000Z"|' + + 'WHERE service.name IS NOT NULL|' + + 'STATS entity.last_seen_timestamp=MAX(custom_timestamp_field),metadata.host.name=VALUES(host.name) BY service.name|' + + 'SORT entity.last_seen_timestamp DESC|' + + 'LIMIT 5' + ); + }); + }); +}); diff --git a/x-pack/plugins/entity_manager/server/lib/queries/index.ts b/x-pack/plugins/entity_manager/server/lib/queries/index.ts new file mode 100644 index 000000000000..9fc7ae00c9aa --- /dev/null +++ b/x-pack/plugins/entity_manager/server/lib/queries/index.ts @@ -0,0 +1,91 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { z } from '@kbn/zod'; + +export const entitySourceSchema = z.object({ + type: z.string(), + timestamp_field: z.optional(z.string()).default('@timestamp'), + index_patterns: z.array(z.string()), + identity_fields: z.array(z.string()), + metadata_fields: z.array(z.string()), + filters: z.array(z.string()), +}); + +export type EntitySource = z.infer; + +const sourceCommand = ({ source }: { source: EntitySource }) => { + let query = `FROM ${source.index_patterns}`; + + const esMetadataFields = source.metadata_fields.filter((field) => + ['_index', '_id'].includes(field) + ); + if (esMetadataFields.length) { + query += ` METADATA ${esMetadataFields.join(',')}`; + } + + return query; +}; + +const filterCommands = ({ + source, + start, + end, +}: { + source: EntitySource; + start: string; + end: string; +}) => { + const commands = [ + `WHERE ${source.timestamp_field} >= "${start}"`, + `WHERE ${source.timestamp_field} <= "${end}"`, + ]; + + source.identity_fields.forEach((field) => { + commands.push(`WHERE ${field} IS NOT NULL`); + }); + + source.filters.forEach((filter) => { + commands.push(`WHERE ${filter}`); + }); + + return commands; +}; + +const statsCommand = ({ source }: { source: EntitySource }) => { + const aggs = [ + // default 'last_seen' attribute + `entity.last_seen_timestamp=MAX(${source.timestamp_field})`, + ...source.metadata_fields + .filter((field) => !source.identity_fields.some((idField) => idField === field)) + .map((field) => `metadata.${field}=VALUES(${field})`), + ]; + + return `STATS ${aggs.join(',')} BY ${source.identity_fields.join(',')}`; +}; + +export function getEntityInstancesQuery({ + source, + limit, + start, + end, +}: { + source: EntitySource; + limit: number; + start: string; + end: string; +}): string { + const commands = [ + sourceCommand({ source }), + ...filterCommands({ source, start, end }), + statsCommand({ source }), + `SORT entity.last_seen_timestamp DESC`, + `LIMIT ${limit}`, + ]; + + return commands.join('|'); +} diff --git a/x-pack/plugins/entity_manager/server/lib/queries/utils.test.ts b/x-pack/plugins/entity_manager/server/lib/queries/utils.test.ts new file mode 100644 index 000000000000..5d5702567172 --- /dev/null +++ b/x-pack/plugins/entity_manager/server/lib/queries/utils.test.ts @@ -0,0 +1,135 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { mergeEntitiesList } from './utils'; + +describe('mergeEntitiesList', () => { + describe('mergeEntitiesList', () => { + it('merges entities on entity.id', () => { + const entities = [ + { + 'entity.id': 'foo', + 'entity.last_seen_timestamp': '2024-11-20T18:00:00.000Z', + 'entity.type': 'service', + }, + { + 'entity.id': 'foo', + 'entity.last_seen_timestamp': '2024-11-20T18:00:00.000Z', + 'entity.type': 'service', + }, + ]; + + const mergedEntities = mergeEntitiesList(entities); + expect(mergedEntities.length).toEqual(1); + expect(mergedEntities[0]).toEqual({ + 'entity.id': 'foo', + 'entity.last_seen_timestamp': '2024-11-20T18:00:00.000Z', + 'entity.type': 'service', + }); + }); + + it('merges metadata fields', () => { + const entities = [ + { + 'entity.id': 'foo', + 'entity.last_seen_timestamp': '2024-11-20T18:00:00.000Z', + 'entity.type': 'service', + 'metadata.host.name': 'host-1', + 'metadata.agent.name': 'agent-1', + 'metadata.service.environment': ['dev', 'staging'], + 'metadata.only_in_record_1': 'foo', + }, + { + 'entity.id': 'foo', + 'entity.last_seen_timestamp': '2024-11-20T18:00:00.000Z', + 'entity.type': 'service', + 'metadata.host.name': ['host-2', 'host-3'], + 'metadata.agent.name': 'agent-2', + 'metadata.service.environment': 'prod', + 'metadata.only_in_record_2': 'bar', + }, + ]; + + const mergedEntities = mergeEntitiesList(entities); + expect(mergedEntities.length).toEqual(1); + expect(mergedEntities[0]).toEqual({ + 'entity.id': 'foo', + 'entity.last_seen_timestamp': '2024-11-20T18:00:00.000Z', + 'entity.type': 'service', + 'metadata.host.name': ['host-1', 'host-2', 'host-3'], + 'metadata.agent.name': ['agent-1', 'agent-2'], + 'metadata.service.environment': ['dev', 'staging', 'prod'], + 'metadata.only_in_record_1': 'foo', + 'metadata.only_in_record_2': 'bar', + }); + }); + + it('picks most recent timestamp when merging', () => { + const entities = [ + { + 'entity.id': 'foo', + 'entity.last_seen_timestamp': '2024-11-20T18:00:00.000Z', + 'entity.type': 'service', + 'metadata.host.name': 'host-1', + }, + { + 'entity.id': 'foo', + 'entity.last_seen_timestamp': '2024-11-20T20:00:00.000Z', + 'entity.type': 'service', + 'metadata.host.name': 'host-2', + }, + { + 'entity.id': 'foo', + 'entity.last_seen_timestamp': '2024-11-20T16:00:00.000Z', + 'entity.type': 'service', + 'metadata.host.name': 'host-3', + }, + ]; + + const mergedEntities = mergeEntitiesList(entities); + expect(mergedEntities.length).toEqual(1); + expect(mergedEntities[0]).toEqual({ + 'entity.id': 'foo', + 'entity.last_seen_timestamp': '2024-11-20T20:00:00.000Z', + 'entity.type': 'service', + 'metadata.host.name': ['host-1', 'host-2', 'host-3'], + }); + }); + + it('deduplicates metadata values', () => { + const entities = [ + { + 'entity.id': 'foo', + 'entity.last_seen_timestamp': '2024-11-20T18:00:00.000Z', + 'entity.type': 'service', + 'metadata.host.name': 'host-1', + }, + { + 'entity.id': 'foo', + 'entity.last_seen_timestamp': '2024-11-20T20:00:00.000Z', + 'entity.type': 'service', + 'metadata.host.name': 'host-2', + }, + { + 'entity.id': 'foo', + 'entity.last_seen_timestamp': '2024-11-20T16:00:00.000Z', + 'entity.type': 'service', + 'metadata.host.name': ['host-1', 'host-2'], + }, + ]; + + const mergedEntities = mergeEntitiesList(entities); + expect(mergedEntities.length).toEqual(1); + expect(mergedEntities[0]).toEqual({ + 'entity.id': 'foo', + 'entity.last_seen_timestamp': '2024-11-20T20:00:00.000Z', + 'entity.type': 'service', + 'metadata.host.name': ['host-1', 'host-2'], + }); + }); + }); +}); diff --git a/x-pack/plugins/entity_manager/server/lib/queries/utils.ts b/x-pack/plugins/entity_manager/server/lib/queries/utils.ts new file mode 100644 index 000000000000..68f5b0f11aff --- /dev/null +++ b/x-pack/plugins/entity_manager/server/lib/queries/utils.ts @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import { EntityV2 } from '@kbn/entities-schema'; +import { ESQLSearchResponse } from '@kbn/es-types'; +import { uniq } from 'lodash'; + +function mergeEntities(entity1: EntityV2, entity2: EntityV2): EntityV2 { + const merged: EntityV2 = { + ...entity1, + 'entity.last_seen_timestamp': new Date( + Math.max( + Date.parse(entity1['entity.last_seen_timestamp']), + Date.parse(entity2['entity.last_seen_timestamp']) + ) + ).toISOString(), + }; + + for (const [key, value] of Object.entries(entity2).filter(([_key]) => + _key.startsWith('metadata.') + )) { + if (merged[key]) { + merged[key] = uniq([ + ...(Array.isArray(merged[key]) ? merged[key] : [merged[key]]), + ...(Array.isArray(value) ? value : [value]), + ]); + } else { + merged[key] = value; + } + } + return merged; +} + +export function mergeEntitiesList(entities: EntityV2[]): EntityV2[] { + const instances: { [key: string]: EntityV2 } = {}; + + for (let i = 0; i < entities.length; i++) { + const entity = entities[i]; + const id = entity['entity.id']; + + if (instances[id]) { + instances[id] = mergeEntities(instances[id], entity); + } else { + instances[id] = entity; + } + } + + return Object.values(instances); +} + +export async function runESQLQuery({ + esClient, + query, +}: { + esClient: ElasticsearchClient; + query: string; +}): Promise { + const esqlResponse = (await esClient.esql.query( + { + query, + format: 'json', + }, + { querystring: { drop_null_columns: true } } + )) as unknown as ESQLSearchResponse; + + const documents = esqlResponse.values.map((row) => + row.reduce>((acc, value, index) => { + const column = esqlResponse.columns[index]; + + if (!column) { + return acc; + } + + // Removes the type suffix from the column name + const name = column.name.replace(/\.(text|keyword)$/, ''); + if (!acc[name]) { + acc[name] = value; + } + + return acc; + }, {}) + ) as T[]; + + return documents; +} diff --git a/x-pack/plugins/entity_manager/server/routes/entities/index.ts b/x-pack/plugins/entity_manager/server/routes/entities/index.ts index 539423c6a5e1..52300ab2601b 100644 --- a/x-pack/plugins/entity_manager/server/routes/entities/index.ts +++ b/x-pack/plugins/entity_manager/server/routes/entities/index.ts @@ -10,6 +10,7 @@ import { deleteEntityDefinitionRoute } from './delete'; import { getEntityDefinitionRoute } from './get'; import { resetEntityDefinitionRoute } from './reset'; import { updateEntityDefinitionRoute } from './update'; +import { searchEntitiesRoute, searchEntitiesPreviewRoute } from '../v2/search'; export const entitiesRoutes = { ...createEntityDefinitionRoute, @@ -17,4 +18,6 @@ export const entitiesRoutes = { ...getEntityDefinitionRoute, ...resetEntityDefinitionRoute, ...updateEntityDefinitionRoute, + ...searchEntitiesRoute, + ...searchEntitiesPreviewRoute, }; diff --git a/x-pack/plugins/entity_manager/server/routes/v2/search.ts b/x-pack/plugins/entity_manager/server/routes/v2/search.ts new file mode 100644 index 000000000000..0b975da748a8 --- /dev/null +++ b/x-pack/plugins/entity_manager/server/routes/v2/search.ts @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import moment from 'moment'; +import { z } from '@kbn/zod'; +import { createEntityManagerServerRoute } from '../create_entity_manager_server_route'; +import { entitySourceSchema } from '../../lib/queries'; +import { UnknownEntityType } from '../../lib/entities/errors/unknown_entity_type'; + +export const searchEntitiesRoute = createEntityManagerServerRoute({ + endpoint: 'POST /internal/entities/v2/_search', + params: z.object({ + body: z.object({ + type: z.string(), + metadata_fields: z.optional(z.array(z.string())).default([]), + filters: z.optional(z.array(z.string())).default([]), + start: z + .optional(z.string()) + .default(() => moment().subtract(5, 'minutes').toISOString()) + .refine((val) => moment(val).isValid(), { + message: 'start should be a date in ISO format', + }), + end: z + .optional(z.string()) + .default(() => moment().toISOString()) + .refine((val) => moment(val).isValid(), { + message: 'start should be a date in ISO format', + }), + limit: z.optional(z.number()).default(10), + }), + }), + handler: async ({ request, response, params, logger, getScopedClient }) => { + try { + const { type, start, end, limit, filters, metadata_fields: metadataFields } = params.body; + + const client = await getScopedClient({ request }); + const entities = await client.searchEntities({ + type, + filters, + metadataFields, + start, + end, + limit, + }); + + return response.ok({ body: { entities } }); + } catch (e) { + logger.error(e); + + if (e instanceof UnknownEntityType) { + return response.notFound({ body: e }); + } + + return response.customError({ body: e, statusCode: 500 }); + } + }, +}); + +export const searchEntitiesPreviewRoute = createEntityManagerServerRoute({ + endpoint: 'POST /internal/entities/v2/_search/preview', + params: z.object({ + body: z.object({ + sources: z.array(entitySourceSchema), + start: z + .optional(z.string()) + .default(() => moment().subtract(5, 'minutes').toISOString()) + .refine((val) => moment(val).isValid(), { + message: 'start should be a date in ISO format', + }), + end: z + .optional(z.string()) + .default(() => moment().toISOString()) + .refine((val) => moment(val).isValid(), { + message: 'start should be a date in ISO format', + }), + limit: z.optional(z.number()).default(10), + }), + }), + handler: async ({ request, response, params, logger, getScopedClient }) => { + const { sources, start, end, limit } = params.body; + + const client = await getScopedClient({ request }); + const entities = await client.searchEntitiesBySources({ + sources, + start, + end, + limit, + }); + + return response.ok({ body: { entities } }); + }, +}); diff --git a/x-pack/plugins/entity_manager/tsconfig.json b/x-pack/plugins/entity_manager/tsconfig.json index 34c57a27dd82..2ef8551f373f 100644 --- a/x-pack/plugins/entity_manager/tsconfig.json +++ b/x-pack/plugins/entity_manager/tsconfig.json @@ -35,5 +35,6 @@ "@kbn/encrypted-saved-objects-plugin", "@kbn/licensing-plugin", "@kbn/core-saved-objects-server", + "@kbn/es-types", ] } diff --git a/x-pack/plugins/observability_solution/entity_manager_app/README.md b/x-pack/plugins/observability_solution/entity_manager_app/README.md new file mode 100644 index 000000000000..1fd230a046c5 --- /dev/null +++ b/x-pack/plugins/observability_solution/entity_manager_app/README.md @@ -0,0 +1,3 @@ +# Entity Manager App Plugin + +This plugin provides a user interface to interact with the Entity Manager. \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/entity_manager_app/jest.config.js b/x-pack/plugins/observability_solution/entity_manager_app/jest.config.js new file mode 100644 index 000000000000..d8217a43063a --- /dev/null +++ b/x-pack/plugins/observability_solution/entity_manager_app/jest.config.js @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +const path = require('path'); + +module.exports = { + preset: '@kbn/test', + rootDir: path.resolve(__dirname, '../../../..'), + roots: ['/x-pack/plugins/observability_solution/entity_manager_app'], + coverageDirectory: + '/target/kibana-coverage/jest/x-pack/plugins/observability_solution/entity_manager_app', + coverageReporters: ['text', 'html'], + collectCoverageFrom: [ + '/x-pack/plugins/observability_solution/entity_manager_app/{common,public,server}/**/*.{js,ts,tsx}', + ], +}; diff --git a/x-pack/plugins/observability_solution/entity_manager_app/kibana.jsonc b/x-pack/plugins/observability_solution/entity_manager_app/kibana.jsonc new file mode 100644 index 000000000000..93e6687f9b4c --- /dev/null +++ b/x-pack/plugins/observability_solution/entity_manager_app/kibana.jsonc @@ -0,0 +1,29 @@ +{ + "type": "plugin", + "id": "@kbn/entityManager-app-plugin", + "owner": "@elastic/obs-entities", + "group": "observability", + "visibility": "private", + "description": "Entity manager plugin for entity assets (inventory, topology, etc)", + "plugin": { + "id": "entityManagerApp", + "configPath": ["xpack", "entityManagerApp"], + "browser": true, + "server": false, + "requiredPlugins": [ + "entityManager", + "observabilityShared", + "presentationUtil", + "usageCollection", + "licensing" + ], + "optionalPlugins": [ + "cloud", + "serverless" + ], + "requiredBundles": [ + "kibanaReact", + "kibanaUtils" + ] + } +} diff --git a/x-pack/plugins/observability_solution/entity_manager_app/public/application.tsx b/x-pack/plugins/observability_solution/entity_manager_app/public/application.tsx new file mode 100644 index 000000000000..8f2e9e2213ba --- /dev/null +++ b/x-pack/plugins/observability_solution/entity_manager_app/public/application.tsx @@ -0,0 +1,103 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { AppMountParameters, APP_WRAPPER_CLASS, CoreStart } from '@kbn/core/public'; +import { PerformanceContextProvider } from '@kbn/ebt-tools'; +import { EuiThemeProvider } from '@kbn/kibana-react-plugin/common'; +import { KibanaContextProvider } from '@kbn/kibana-react-plugin/public'; +import { Storage } from '@kbn/kibana-utils-plugin/public'; +import type { LazyObservabilityPageTemplateProps } from '@kbn/observability-shared-plugin/public'; +import { KibanaRenderContextProvider } from '@kbn/react-kibana-context-render'; +import { KibanaThemeProvider } from '@kbn/react-kibana-context-theme'; +import { RedirectAppLinks } from '@kbn/shared-ux-link-redirect-app'; +import { EntityClient } from '@kbn/entityManager-plugin/public'; +import React from 'react'; +import ReactDOM from 'react-dom'; +import { UsageCollectionSetup } from '@kbn/usage-collection-plugin/public'; +import { Router } from '@kbn/shared-ux-router'; +import { PluginContext } from './context/plugin_context'; +import { EntityManagerPluginStart } from './types'; +import { EntityManagerOverviewPage } from './pages/overview'; + +export function renderApp({ + core, + plugins, + appMountParameters, + ObservabilityPageTemplate, + usageCollection, + isDev, + kibanaVersion, + isServerless, + entityClient, +}: { + core: CoreStart; + plugins: EntityManagerPluginStart; + appMountParameters: AppMountParameters; + ObservabilityPageTemplate: React.ComponentType; + usageCollection: UsageCollectionSetup; + isDev?: boolean; + kibanaVersion: string; + isServerless?: boolean; + entityClient: EntityClient; +}) { + const { element, history, theme$ } = appMountParameters; + const isDarkMode = core.theme.getTheme().darkMode; + + // ensure all divs are .kbnAppWrappers + element.classList.add(APP_WRAPPER_CLASS); + + const ApplicationUsageTrackingProvider = + usageCollection?.components.ApplicationUsageTrackingProvider ?? React.Fragment; + + const CloudProvider = plugins.cloud?.CloudContextProvider ?? React.Fragment; + + ReactDOM.render( + + + + + + + + + + + + + + + + + + + + + , + element + ); + + return () => { + ReactDOM.unmountComponentAtNode(element); + }; +} diff --git a/x-pack/plugins/observability_solution/entity_manager_app/public/context/plugin_context.ts b/x-pack/plugins/observability_solution/entity_manager_app/public/context/plugin_context.ts new file mode 100644 index 000000000000..7da2833be439 --- /dev/null +++ b/x-pack/plugins/observability_solution/entity_manager_app/public/context/plugin_context.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createContext } from 'react'; +import type { AppMountParameters } from '@kbn/core/public'; +import type { LazyObservabilityPageTemplateProps } from '@kbn/observability-shared-plugin/public'; +import { EntityClient } from '@kbn/entityManager-plugin/public'; + +export interface PluginContextValue { + isDev?: boolean; + isServerless?: boolean; + appMountParameters?: AppMountParameters; + ObservabilityPageTemplate: React.ComponentType; + entityClient: EntityClient; +} + +export const PluginContext = createContext(null); diff --git a/x-pack/plugins/observability_solution/entity_manager_app/public/hooks/use_kibana.ts b/x-pack/plugins/observability_solution/entity_manager_app/public/hooks/use_kibana.ts new file mode 100644 index 000000000000..a515b9b80b01 --- /dev/null +++ b/x-pack/plugins/observability_solution/entity_manager_app/public/hooks/use_kibana.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { CoreStart } from '@kbn/core/public'; +import { useKibana } from '@kbn/kibana-react-plugin/public'; +import { Storage } from '@kbn/kibana-utils-plugin/public'; +import { EntityClient } from '@kbn/entityManager-plugin/public'; + +export type StartServices = CoreStart & + AdditionalServices & { + storage: Storage; + kibanaVersion: string; + entityClient: EntityClient; + }; +const useTypedKibana = () => + useKibana>(); + +export { useTypedKibana as useKibana }; diff --git a/x-pack/plugins/observability_solution/entity_manager_app/public/hooks/use_plugin_context.ts b/x-pack/plugins/observability_solution/entity_manager_app/public/hooks/use_plugin_context.ts new file mode 100644 index 000000000000..d0640deb575b --- /dev/null +++ b/x-pack/plugins/observability_solution/entity_manager_app/public/hooks/use_plugin_context.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useContext } from 'react'; +import { PluginContext } from '../context/plugin_context'; +import type { PluginContextValue } from '../context/plugin_context'; + +export function usePluginContext(): PluginContextValue { + const context = useContext(PluginContext); + if (!context) { + throw new Error('Plugin context value is missing!'); + } + + return context; +} diff --git a/x-pack/plugins/observability_solution/entity_manager_app/public/index.ts b/x-pack/plugins/observability_solution/entity_manager_app/public/index.ts new file mode 100644 index 000000000000..5b83ea1d297d --- /dev/null +++ b/x-pack/plugins/observability_solution/entity_manager_app/public/index.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { PluginInitializer, PluginInitializerContext } from '@kbn/core/public'; +import { Plugin } from './plugin'; + +export const plugin: PluginInitializer<{}, {}> = (context: PluginInitializerContext) => { + return new Plugin(context); +}; diff --git a/x-pack/plugins/observability_solution/entity_manager_app/public/pages/overview/index.tsx b/x-pack/plugins/observability_solution/entity_manager_app/public/pages/overview/index.tsx new file mode 100644 index 000000000000..8c978db6f675 --- /dev/null +++ b/x-pack/plugins/observability_solution/entity_manager_app/public/pages/overview/index.tsx @@ -0,0 +1,347 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { useState } from 'react'; +import { v4 as uuid } from 'uuid'; +import { + EuiBasicTable, + EuiButton, + EuiButtonIcon, + EuiCallOut, + EuiFieldText, + EuiFlexGroup, + EuiFlexItem, + EuiForm, + EuiFormRow, + EuiHorizontalRule, + EuiSpacer, + EuiTitle, +} from '@elastic/eui'; +import { EntityV2 } from '@kbn/entities-schema'; +import { usePluginContext } from '../../hooks/use_plugin_context'; + +function EntitySourceForm({ + source, + index, + onFieldChange, +}: { + source: any; + index: number; + onFieldChange: Function; +}) { + const onArrayFieldChange = + (field: Exclude) => (e: React.ChangeEvent) => { + const value = e.target.value.trim(); + if (!value) { + onFieldChange(index, field, []); + } else { + onFieldChange(index, field, e.target.value.trim().split(',')); + } + }; + + return ( + + + + + + + + + + + + + + + + + + + + + + + + + + + + onFieldChange(index, 'timestamp_field', e.target.value)} + /> + + + + ); +} + +interface EntitySource { + id: string; + index_patterns?: string[]; + identity_fields?: string[]; + metadata_fields?: string[]; + filters?: string[]; + timestamp_field?: string; +} + +const newEntitySource = ({ + indexPatterns = [], + identityFields = [], + metadataFields = [], + filters = [], + timestampField = '@timestamp', +}: { + indexPatterns?: string[]; + identityFields?: string[]; + metadataFields?: string[]; + filters?: string[]; + timestampField?: string; +}) => ({ + id: uuid(), + index_patterns: indexPatterns, + identity_fields: identityFields, + metadata_fields: metadataFields, + timestamp_field: timestampField, + filters, +}); + +export function EntityManagerOverviewPage() { + const { ObservabilityPageTemplate, entityClient } = usePluginContext(); + const [previewEntities, setPreviewEntities] = useState([]); + const [isSearchingEntities, setIsSearchingEntities] = useState(false); + const [previewError, setPreviewError] = useState(null); + const [formErrors, setFormErrors] = useState([]); + const [entityType, setEntityType] = useState('service'); + const [entitySources, setEntitySources] = useState([ + newEntitySource({ + indexPatterns: ['remote_cluster:logs-*'], + identityFields: ['service.name'], + }), + ]); + + const searchEntities = async () => { + if ( + !entitySources.some( + (source) => source.identity_fields.length > 0 && source.index_patterns.length > 0 + ) + ) { + setFormErrors(['No valid source found']); + return; + } + + setIsSearchingEntities(true); + setFormErrors([]); + setPreviewError(null); + + try { + const { entities } = await entityClient.repositoryClient( + 'POST /internal/entities/v2/_search/preview', + { + params: { + body: { + sources: entitySources + .filter( + (source) => source.index_patterns.length > 0 && source.identity_fields.length > 0 + ) + .map((source) => ({ ...source, type: entityType })), + }, + }, + } + ); + + setPreviewEntities(entities); + } catch (err) { + setPreviewError(err.body?.message); + } finally { + setIsSearchingEntities(false); + } + }; + + return ( + + 0} error={formErrors}> + + + +

Entity type

+ + + + + + { + setEntityType(e.target.value.trim()); + }} + /> + + + + + + + + + +

Entity sources

+ + + + + setEntitySources([...entitySources, newEntitySource({})])} + /> + + + + + + {entitySources.map((source, i) => ( +
+ + + +

Source {i + 1}

+ + + {entitySources.length > 1 ? ( + + { + entitySources.splice(i, 1); + setEntitySources(entitySources.map((_source) => ({ ..._source }))); + }} + /> + + ) : null} + + + + + , + value: any + ) => { + entitySources[index][field] = value; + setEntitySources([...entitySources]); + }} + /> + {i === entitySources.length - 1 ? ( + + ) : ( + + )} +
+ ))} + + + + + + Preview + + + + + Create + + + + + + + + + {previewError ? ( + +

{previewError}

+ + ) : null} + + source.identity_fields))).map( + (field) => ({ + field, + name: field, + }) + ), + ...Array.from(new Set(entitySources.flatMap((source) => source.metadata_fields))).map( + (field) => ({ + field: `metadata.${field}`, + name: `metadata.${field}`, + }) + ), + ]} + /> + + ); +} diff --git a/x-pack/plugins/observability_solution/entity_manager_app/public/plugin.ts b/x-pack/plugins/observability_solution/entity_manager_app/public/plugin.ts new file mode 100644 index 000000000000..0db381522b02 --- /dev/null +++ b/x-pack/plugins/observability_solution/entity_manager_app/public/plugin.ts @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { BehaviorSubject } from 'rxjs'; +import { + App, + AppMountParameters, + AppStatus, + AppUpdater, + CoreSetup, + DEFAULT_APP_CATEGORIES, + PluginInitializerContext, +} from '@kbn/core/public'; +import { Logger } from '@kbn/logging'; +import { EntityClient } from '@kbn/entityManager-plugin/public'; + +import { + EntityManagerAppPluginClass, + EntityManagerPluginStart, + EntityManagerPluginSetup, +} from './types'; + +export class Plugin implements EntityManagerAppPluginClass { + public logger: Logger; + private readonly appUpdater$ = new BehaviorSubject(() => ({})); + + constructor(private readonly context: PluginInitializerContext<{}>) { + this.logger = context.logger.get(); + } + + setup(core: CoreSetup, pluginSetup: EntityManagerPluginSetup) { + const kibanaVersion = this.context.env.packageInfo.version; + + const mount = async (params: AppMountParameters) => { + const { renderApp } = await import('./application'); + const [coreStart, pluginsStart] = await core.getStartServices(); + + return renderApp({ + appMountParameters: params, + core: coreStart, + isDev: this.context.env.mode.dev, + kibanaVersion, + usageCollection: pluginSetup.usageCollection, + ObservabilityPageTemplate: pluginsStart.observabilityShared.navigation.PageTemplate, + plugins: pluginsStart, + isServerless: !!pluginsStart.serverless, + entityClient: new EntityClient(core), + }); + }; + + const appUpdater$ = this.appUpdater$; + const app: App = { + id: 'entity_manager', + title: 'Entity Manager', + order: 8002, + updater$: appUpdater$, + euiIconType: 'logoObservability', + appRoute: '/app/entity_manager', + category: DEFAULT_APP_CATEGORIES.observability, + mount, + visibleIn: [], + keywords: ['observability', 'monitor', 'entities'], + status: AppStatus.inaccessible, + }; + + core.application.register(app); + + return {}; + } + + start() { + return {}; + } + + stop() {} +} diff --git a/x-pack/plugins/observability_solution/entity_manager_app/public/routes.tsx b/x-pack/plugins/observability_solution/entity_manager_app/public/routes.tsx new file mode 100644 index 000000000000..80baa45422bf --- /dev/null +++ b/x-pack/plugins/observability_solution/entity_manager_app/public/routes.tsx @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { EntityManagerOverviewPage } from './pages/overview'; + +interface RouteDef { + [key: string]: { + handler: () => React.ReactElement; + params: Record; + exact: boolean; + }; +} + +export function getRoutes(): RouteDef { + return { + '/app/entity_manager': { + handler: () => , + params: {}, + exact: true, + }, + }; +} diff --git a/x-pack/plugins/observability_solution/entity_manager_app/public/types.ts b/x-pack/plugins/observability_solution/entity_manager_app/public/types.ts new file mode 100644 index 000000000000..b735771d79f8 --- /dev/null +++ b/x-pack/plugins/observability_solution/entity_manager_app/public/types.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { Plugin as PluginClass } from '@kbn/core/public'; +import { PresentationUtilPluginStart } from '@kbn/presentation-util-plugin/public'; +import { CloudStart } from '@kbn/cloud-plugin/public'; +import { ServerlessPluginSetup, ServerlessPluginStart } from '@kbn/serverless/public'; +import { + ObservabilitySharedPluginSetup, + ObservabilitySharedPluginStart, +} from '@kbn/observability-shared-plugin/public'; +import { UsageCollectionSetup } from '@kbn/usage-collection-plugin/public'; +import { EntityManagerPublicPluginSetup } from '@kbn/entityManager-plugin/public/types'; + +export interface EntityManagerPluginSetup { + observabilityShared: ObservabilitySharedPluginSetup; + serverless?: ServerlessPluginSetup; + usageCollection: UsageCollectionSetup; + entityManager: EntityManagerPublicPluginSetup; +} + +export interface EntityManagerPluginStart { + presentationUtil: PresentationUtilPluginStart; + cloud?: CloudStart; + serverless?: ServerlessPluginStart; + observabilityShared: ObservabilitySharedPluginStart; +} + +export type EntityManagerAppPluginClass = PluginClass<{}, {}>; diff --git a/x-pack/plugins/observability_solution/entity_manager_app/tsconfig.json b/x-pack/plugins/observability_solution/entity_manager_app/tsconfig.json new file mode 100644 index 000000000000..64c0a293a4e2 --- /dev/null +++ b/x-pack/plugins/observability_solution/entity_manager_app/tsconfig.json @@ -0,0 +1,33 @@ +{ + "extends": "../../../../tsconfig.base.json", + "compilerOptions": { + "outDir": "target/types" + }, + "include": [ + "../../../../typings/**/*", + "common/**/*", + "public/**/*", + "types/**/*" + ], + "exclude": [ + "target/**/*" + ], + "kbn_references": [ + "@kbn/core", + "@kbn/logging", + "@kbn/ebt-tools", + "@kbn/kibana-react-plugin", + "@kbn/kibana-utils-plugin", + "@kbn/observability-shared-plugin", + "@kbn/react-kibana-context-render", + "@kbn/react-kibana-context-theme", + "@kbn/shared-ux-link-redirect-app", + "@kbn/usage-collection-plugin", + "@kbn/shared-ux-router", + "@kbn/presentation-util-plugin", + "@kbn/cloud-plugin", + "@kbn/serverless", + "@kbn/entityManager-plugin", + "@kbn/entities-schema", + ] +} diff --git a/yarn.lock b/yarn.lock index b8abc0d2e5f2..07783adc7944 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5430,6 +5430,10 @@ version "0.0.0" uid "" +"@kbn/entityManager-app-plugin@link:x-pack/plugins/observability_solution/entity_manager_app": + version "0.0.0" + uid "" + "@kbn/entityManager-plugin@link:x-pack/plugins/entity_manager": version "0.0.0" uid ""