[eem] _search endpoint / initial entity manager UI

.github/CODEOWNERS

@@ -919,6 +919,7 @@ x-pack/plugins/observability_solution/apm_data_access @elastic/obs-knowledge-tea
 x-pack/plugins/observability_solution/apm/ftr_e2e @elastic/obs-ux-infra_services-team
 x-pack/plugins/observability_solution/dataset_quality @elastic/obs-ux-logs-team
 x-pack/plugins/observability_solution/entities_data_access @elastic/obs-entities
+x-pack/plugins/observability_solution/entity_manager_app @elastic/obs-entities
 x-pack/plugins/observability_solution/exploratory_view @elastic/obs-ux-management-team
 x-pack/plugins/observability_solution/infra @elastic/obs-ux-logs-team @elastic/obs-ux-infra_services-team
 x-pack/plugins/observability_solution/inventory @elastic/obs-ux-infra_services-team

docs/developer/plugin-list.asciidoc

@@ -579,6 +579,10 @@ security and spaces filtering.
 |This plugin provides access to observed entity data, such as information about hosts, pods, containers, services, and more.
 
 
+|{kib-repo}blob/{branch}/x-pack/plugins/observability_solution/entity_manager_app/README.md[entityManagerApp]
+|This plugin provides a user interface to interact with the Entity Manager.
+
+
 |{kib-repo}blob/{branch}/x-pack/plugins/event_log/README.md[eventLog]
 |The event log plugin provides a persistent history of alerting and action
 activities.

package.json

@@ -478,6 +478,7 @@
     "@kbn/entities-data-access-plugin": "link:x-pack/plugins/observability_solution/entities_data_access",
     "@kbn/entities-schema": "link:x-pack/packages/kbn-entities-schema",
     "@kbn/entity-manager-fixture-plugin": "link:x-pack/test/api_integration/apis/entity_manager/fixture_plugin",
+    "@kbn/entityManager-app-plugin": "link:x-pack/plugins/observability_solution/entity_manager_app",
     "@kbn/entityManager-plugin": "link:x-pack/plugins/entity_manager",
     "@kbn/error-boundary-example-plugin": "link:examples/error_boundary",
     "@kbn/es-errors": "link:packages/kbn-es-errors",

packages/kbn-optimizer/limits.yml

@@ -42,6 +42,7 @@ pageLoadAssetSize:
   embeddableEnhanced: 22107
   enterpriseSearch: 66810
   entityManager: 17175
+  entityManagerApp: 20378
   esql: 37000
   esqlDataGrid: 24582
   esUiShared: 326654

src/plugins/kibana_usage_collection/server/collectors/application_usage/schema.ts

@@ -143,6 +143,7 @@ export const applicationUsageSchema = {
   enterpriseSearchSemanticSearch: commonSchema,
   enterpriseSearchVectorSearch: commonSchema,
   enterpriseSearchElasticsearch: commonSchema,
+  entity_manager: commonSchema,
   appSearch: commonSchema,
   workplaceSearch: commonSchema,
   searchExperiences: commonSchema,

src/plugins/telemetry/schema/oss_plugins.json

@@ -3015,6 +3015,137 @@
             }
           }
         },
+        "entity_manager": {
+          "properties": {
+            "appId": {
+              "type": "keyword",
+              "_meta": {
+                "description": "The application being tracked"
+              }
+            },
+            "viewId": {
+              "type": "keyword",
+              "_meta": {
+                "description": "Always `main`"
+              }
+            },
+            "clicks_total": {
+              "type": "long",
+              "_meta": {
+                "description": "General number of clicks in the application since we started counting them"
+              }
+            },
+            "clicks_7_days": {
+              "type": "long",
+              "_meta": {
+                "description": "General number of clicks in the application over the last 7 days"
+              }
+            },
+            "clicks_30_days": {
+              "type": "long",
+              "_meta": {
+                "description": "General number of clicks in the application over the last 30 days"
+              }
+            },
+            "clicks_90_days": {
+              "type": "long",
+              "_meta": {
+                "description": "General number of clicks in the application over the last 90 days"
+              }
+            },
+            "minutes_on_screen_total": {
+              "type": "float",
+              "_meta": {
+                "description": "Minutes the application is active and on-screen since we started counting them."
+              }
+            },
+            "minutes_on_screen_7_days": {
+              "type": "float",
+              "_meta": {
+                "description": "Minutes the application is active and on-screen over the last 7 days"
+              }
+            },
+            "minutes_on_screen_30_days": {
+              "type": "float",
+              "_meta": {
+                "description": "Minutes the application is active and on-screen over the last 30 days"
+              }
+            },
+            "minutes_on_screen_90_days": {
+              "type": "float",
+              "_meta": {
+                "description": "Minutes the application is active and on-screen over the last 90 days"
+              }
+            },
+            "views": {
+              "type": "array",
+              "items": {
+                "properties": {
+                  "appId": {
+                    "type": "keyword",
+                    "_meta": {
+                      "description": "The application being tracked"
+                    }
+                  },
+                  "viewId": {
+                    "type": "keyword",
+                    "_meta": {
+                      "description": "The application view being tracked"
+                    }
+                  },
+                  "clicks_total": {
+                    "type": "long",
+                    "_meta": {
+                      "description": "General number of clicks in the application sub view since we started counting them"
+                    }
+                  },
+                  "clicks_7_days": {
+                    "type": "long",
+                    "_meta": {
+                      "description": "General number of clicks in the active application sub view over the last 7 days"
+                    }
+                  },
+                  "clicks_30_days": {
+                    "type": "long",
+                    "_meta": {
+                      "description": "General number of clicks in the active application sub view over the last 30 days"
+                    }
+                  },
+                  "clicks_90_days": {
+                    "type": "long",
+                    "_meta": {
+                      "description": "General number of clicks in the active application sub view over the last 90 days"
+                    }
+                  },
+                  "minutes_on_screen_total": {
+                    "type": "float",
+                    "_meta": {
+                      "description": "Minutes the application sub view is active and on-screen since we started counting them."
+                    }
+                  },
+                  "minutes_on_screen_7_days": {
+                    "type": "float",
+                    "_meta": {
+                      "description": "Minutes the application is active and on-screen active application sub view over the last 7 days"
+                    }
+                  },
+                  "minutes_on_screen_30_days": {
+                    "type": "float",
+                    "_meta": {
+                      "description": "Minutes the application is active and on-screen active application sub view over the last 30 days"
+                    }
+                  },
+                  "minutes_on_screen_90_days": {
+                    "type": "float",
+                    "_meta": {
+                      "description": "Minutes the application is active and on-screen active application sub view over the last 90 days"
+                    }
+                  }
+                }
+              }
+            }
+          }
+        },
         "appSearch": {
           "properties": {
             "appId": {

tsconfig.base.json

@@ -820,6 +820,8 @@
       "@kbn/entities-schema/*": ["x-pack/packages/kbn-entities-schema/*"],
       "@kbn/entity-manager-fixture-plugin": ["x-pack/test/api_integration/apis/entity_manager/fixture_plugin"],
       "@kbn/entity-manager-fixture-plugin/*": ["x-pack/test/api_integration/apis/entity_manager/fixture_plugin/*"],
+      "@kbn/entityManager-app-plugin": ["x-pack/plugins/observability_solution/entity_manager_app"],
+      "@kbn/entityManager-app-plugin/*": ["x-pack/plugins/observability_solution/entity_manager_app/*"],
       "@kbn/entityManager-plugin": ["x-pack/plugins/entity_manager"],
       "@kbn/entityManager-plugin/*": ["x-pack/plugins/entity_manager/*"],
       "@kbn/error-boundary-example-plugin": ["examples/error_boundary"],

x-pack/packages/kbn-entities-schema/src/schema/entity.ts

@@ -23,6 +23,13 @@ export interface MetadataRecord {
   [key: string]: string[] | MetadataRecord | string;
 }
 
+export interface Entity {
+  'entity.id': string;
+  'entity.last_seen_timestamp': string;
+  'entity.type': string;
+  [metadata: string]: any;
+}
+
 const literalSchema = z.union([z.string(), z.number(), z.boolean(), z.null()]);
 
 type Literal = z.infer<typeof literalSchema>;

x-pack/plugins/entity_manager/kibana.jsonc

@@ -8,9 +8,13 @@
   "plugin": {
     "id": "entityManager",
     "configPath": ["xpack", "entityManager"],
-    "requiredPlugins": ["security", "encryptedSavedObjects", "licensing"],
     "browser": true,
     "server": true,
+    "requiredPlugins": [
+      "security",
+      "encryptedSavedObjects",
+      "licensing"
+    ],
     "requiredBundles": []
   }
 }

x-pack/plugins/entity_manager/public/index.ts

@@ -16,6 +16,8 @@ export const plugin: PluginInitializer<
   return new Plugin(context);
 };
 
+export { EntityClient } from './lib/entity_client';
+
 export type { EntityManagerPublicPluginSetup, EntityManagerPublicPluginStart };
 export type EntityManagerAppId = 'entityManager';
 

x-pack/plugins/entity_manager/public/plugin.ts

@@ -22,16 +22,14 @@ export class Plugin implements EntityManagerPluginClass {
   }
 
   setup(core: CoreSetup) {
-    const entityClient = new EntityClient(core);
     return {
-      entityClient,
+      entityClient: new EntityClient(core),
     };
   }
 
   start(core: CoreStart) {
-    const entityClient = new EntityClient(core);
     return {
-      entityClient,
+      entityClient: new EntityClient(core),
     };
   }
 

x-pack/plugins/entity_manager/public/types.ts

@@ -10,7 +10,6 @@ import type { EntityClient } from './lib/entity_client';
 export interface EntityManagerPublicPluginSetup {
   entityClient: EntityClient;
 }
-
 export interface EntityManagerPublicPluginStart {
   entityClient: EntityClient;
 }

x-pack/plugins/entity_manager/server/lib/entity_client.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { EntityDefinition, EntityDefinitionUpdate } from '@kbn/entities-schema';
+import { Entity, EntityDefinition, EntityDefinitionUpdate } from '@kbn/entities-schema';
 import { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server';
 import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
 import { Logger } from '@kbn/logging';
@@ -23,6 +23,8 @@ import { stopTransforms } from './entities/stop_transforms';
 import { deleteIndices } from './entities/delete_index';
 import { EntityDefinitionWithState } from './entities/types';
 import { EntityDefinitionUpdateConflict } from './entities/errors/entity_definition_update_conflict';
+import { EntitySource, getEntityInstancesQuery } from './queries';
+import { mergeEntitiesList, runESQLQuery } from './queries/utils';
 
 export class EntityClient {
   constructor(
@@ -126,8 +128,6 @@ export class EntityClient {
     });
 
     if (deleteData) {
-      // delete data with current user as system user does not have
-      // .entities privileges
       await deleteIndices(this.options.esClient, definition, this.options.logger);
     }
   }
@@ -170,4 +170,72 @@ export class EntityClient {
     this.options.logger.info(`Stopping transforms for definition [${definition.id}]`);
     return stopTransforms(this.options.esClient, definition, this.options.logger);
   }
+
+  async getEntitySources({ type }: { type: string }) {
+    const result = await this.options.esClient.search<EntitySource>({
+      index: 'kibana_entity_definitions',
+      query: {
+        bool: {
+          must: {
+            term: { entity_type: type },
+          },
+        },
+      },
+    });
+
+    return result.hits.hits.map((hit) => hit._source) as EntitySource[];
+  }
+
+  async searchEntities({
+    sources,
+    start,
+    end,
+    limit = 10,
+  }: {
+    sources: EntitySource[];
+    start: string;
+    end: string;
+    limit?: number;
+  }) {
+    const entities = await Promise.all(
+      sources.map(async (source) => {
+        const mandatoryFields = [source.timestamp_field, ...source.identity_fields];
+        const { fields } = await this.options.esClient.fieldCaps({
+          index: source.index_patterns,
+          fields: [...mandatoryFields, ...source.metadata_fields],
+        });
+
+        const sourceHasMandatoryFields = mandatoryFields.every((field) => !!fields[field]);
+        if (!sourceHasMandatoryFields) {
+          // we can't build entities without id fields so we ignore the source.
+          // filters should likely behave similarly.
+          this.options.logger.info(
+            `Ignoring source for type [${source.type}] with index_patterns [${source.index_patterns}] because some mandatory fields [${mandatoryFields}] are not mapped`
+          );
+          return [];
+        }
+
+        // but metadata field not being available is fine
+        const availableMetadataFields = source.metadata_fields.filter((field) => fields[field]);
+
+        const query = getEntityInstancesQuery({
+          source: { ...source, metadata_fields: availableMetadataFields },
+          start,
+          end,
+          limit,
+        });
+        this.options.logger.debug(`Entity query: ${query}`);
+
+        const rawEntities = await runESQLQuery<Entity>({ query, esClient: this.options.esClient });
+
+        return rawEntities.map((entity) => {
+          entity['entity.id'] = source.identity_fields.map((field) => entity[field]).join(':');
+          entity['entity.type'] = source.type;
+          return entity;
+        });
+      })
+    ).then((results) => results.flat());
+
+    return mergeEntitiesList(entities).slice(0, limit);
+  }
 }

x-pack/plugins/entity_manager/server/lib/queries/index.test.ts

@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { getEntityInstancesQuery } from '.';
+
+describe(__filename, () => {
+  describe('getEntityInstancesQuery', () => {
+    it('generates a valid esql query', () => {
+      const query = getEntityInstancesQuery({
+        source: {
+          type: 'service',
+          index_patterns: ['logs-*', 'metrics-*'],
+          identity_fields: ['service.name'],
+          metadata_fields: ['host.name'],
+          filters: [],
+          timestamp_field: 'custom_timestamp_field',
+        },
+        limit: 5,
+        start: '2024-11-20T19:00:00.000Z',
+        end: '2024-11-20T20:00:00.000Z',
+      });
+
+      expect(query).toEqual(
+        'FROM logs-*,metrics-*|' +
+          'WHERE custom_timestamp_field >= "2024-11-20T19:00:00.000Z"|' +
+          'WHERE custom_timestamp_field <= "2024-11-20T20:00:00.000Z"|' +
+          'WHERE service.name IS NOT NULL|' +
+          'STATS entity.last_seen_timestamp=MAX(custom_timestamp_field),metadata.host.name=VALUES(host.name) BY service.name|' +
+          'SORT entity.last_seen_timestamp DESC|' +
+          'LIMIT 5'
+      );
+    });
+  });
+});