[eem] _search endpoint / initial entity manager UI (elastic#199609)

## Summary - create `_search` endpoint to discover entities with esql queries. It currently reads sources of the provided `type` from `kibana_entity_definitions` index. Run this query to insert a definition: ``` POST kibana_entity_definitions/_doc { "entity_type": "service", "index_patterns": ["remote_cluster:logs-*"], "metadata_fields": [], "identity_fields": ["service.name"], "filters": [], "timestamp_field": "@timestamp" } ``` By default `_search` will look at data in the last 5m. The lookup period can be overriden by providing `start`/`end` parameters in ISO format. It also accepts a `limit` to specify the number of entities returned which defaults to 10 ``` POST kbn:/internal/entities/v2/_search { "type": "service", "start": "2024-11-19T20:40:00.000Z", "end": "2024-11-19T20:50:00.000Z", "limit": 20 } ``` - create `_search/preview` endpoint to preview output of entity sources without persisting them - create UI to preview results of an entity definition at `/app/entity_manager`. The application is living in its own plugin at `observability_solution/entity_manager_app` ![Screenshot 2024-11-11 at 11 37 18](https://github.com/user-attachments/assets/f284342d-21a3-4ba1-be94-38cff311266c) --------- Co-authored-by: kibanamachine <[email protected]> Co-authored-by: Milton Hultgren <[email protected]>
klacabane · Nov 22, 2024 · 0b3f4fb · 0b3f4fb
1 parent c3c872c
commit 0b3f4fb
Show file tree

Hide file tree

Showing 35 changed files with 1,482 additions and 14 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -920,6 +920,7 @@ x-pack/plugins/observability_solution/apm_data_access @elastic/obs-knowledge-tea
 x-pack/plugins/observability_solution/apm/ftr_e2e @elastic/obs-ux-infra_services-team
 x-pack/plugins/observability_solution/dataset_quality @elastic/obs-ux-logs-team
 x-pack/plugins/observability_solution/entities_data_access @elastic/obs-entities
+x-pack/plugins/observability_solution/entity_manager_app @elastic/obs-entities
 x-pack/plugins/observability_solution/exploratory_view @elastic/obs-ux-management-team
 x-pack/plugins/observability_solution/infra @elastic/obs-ux-logs-team @elastic/obs-ux-infra_services-team
 x-pack/plugins/observability_solution/inventory @elastic/obs-ux-infra_services-team

diff --git a/docs/developer/plugin-list.asciidoc b/docs/developer/plugin-list.asciidoc
@@ -579,6 +579,10 @@ security and spaces filtering.
 |This plugin provides access to observed entity data, such as information about hosts, pods, containers, services, and more.
 
 
+|{kib-repo}blob/{branch}/x-pack/plugins/observability_solution/entity_manager_app/README.md[entityManagerApp]
+|This plugin provides a user interface to interact with the Entity Manager.
+
+
 |{kib-repo}blob/{branch}/x-pack/plugins/event_log/README.md[eventLog]
 |The event log plugin provides a persistent history of alerting and action
 activities.

diff --git a/package.json b/package.json
@@ -479,6 +479,7 @@
     "@kbn/entities-data-access-plugin": "link:x-pack/plugins/observability_solution/entities_data_access",
     "@kbn/entities-schema": "link:x-pack/packages/kbn-entities-schema",
     "@kbn/entity-manager-fixture-plugin": "link:x-pack/test/api_integration/apis/entity_manager/fixture_plugin",
+    "@kbn/entityManager-app-plugin": "link:x-pack/plugins/observability_solution/entity_manager_app",
     "@kbn/entityManager-plugin": "link:x-pack/plugins/entity_manager",
     "@kbn/error-boundary-example-plugin": "link:examples/error_boundary",
     "@kbn/es-errors": "link:packages/kbn-es-errors",

diff --git a/packages/kbn-optimizer/limits.yml b/packages/kbn-optimizer/limits.yml
@@ -42,6 +42,7 @@ pageLoadAssetSize:
   embeddableEnhanced: 22107
   enterpriseSearch: 66810
   entityManager: 17175
+  entityManagerApp: 20378
   esql: 37000
   esqlDataGrid: 24582
   esUiShared: 326654

diff --git a/src/plugins/kibana_usage_collection/server/collectors/application_usage/schema.ts b/src/plugins/kibana_usage_collection/server/collectors/application_usage/schema.ts
@@ -143,6 +143,7 @@ export const applicationUsageSchema = {
   enterpriseSearchSemanticSearch: commonSchema,
   enterpriseSearchVectorSearch: commonSchema,
   enterpriseSearchElasticsearch: commonSchema,
+  entity_manager: commonSchema,
   appSearch: commonSchema,
   workplaceSearch: commonSchema,
   searchExperiences: commonSchema,

diff --git a/src/plugins/telemetry/schema/oss_plugins.json b/src/plugins/telemetry/schema/oss_plugins.json
@@ -3015,6 +3015,137 @@
             }
           }
         },
+        "entity_manager": {
+          "properties": {
+            "appId": {
+              "type": "keyword",
+              "_meta": {
+                "description": "The application being tracked"
+              }
+            },
+            "viewId": {
+              "type": "keyword",
+              "_meta": {
+                "description": "Always `main`"
+              }
+            },
+            "clicks_total": {
+              "type": "long",
+              "_meta": {
+                "description": "General number of clicks in the application since we started counting them"
+              }
+            },
+            "clicks_7_days": {
+              "type": "long",
+              "_meta": {
+                "description": "General number of clicks in the application over the last 7 days"
+              }
+            },
+            "clicks_30_days": {
+              "type": "long",
+              "_meta": {
+                "description": "General number of clicks in the application over the last 30 days"
+              }
+            },
+            "clicks_90_days": {
+              "type": "long",
+              "_meta": {
+                "description": "General number of clicks in the application over the last 90 days"
+              }
+            },
+            "minutes_on_screen_total": {
+              "type": "float",
+              "_meta": {
+                "description": "Minutes the application is active and on-screen since we started counting them."
+              }
+            },
+            "minutes_on_screen_7_days": {
+              "type": "float",
+              "_meta": {
+                "description": "Minutes the application is active and on-screen over the last 7 days"
+              }
+            },
+            "minutes_on_screen_30_days": {
+              "type": "float",
+              "_meta": {
+                "description": "Minutes the application is active and on-screen over the last 30 days"
+              }
+            },
+            "minutes_on_screen_90_days": {
+              "type": "float",
+              "_meta": {
+                "description": "Minutes the application is active and on-screen over the last 90 days"
+              }
+            },
+            "views": {
+              "type": "array",
+              "items": {
+                "properties": {
+                  "appId": {
+                    "type": "keyword",
+                    "_meta": {
+                      "description": "The application being tracked"
+                    }
+                  },
+                  "viewId": {
+                    "type": "keyword",
+                    "_meta": {
+                      "description": "The application view being tracked"
+                    }
+                  },
+                  "clicks_total": {
+                    "type": "long",
+                    "_meta": {
+                      "description": "General number of clicks in the application sub view since we started counting them"
+                    }
+                  },
+                  "clicks_7_days": {
+                    "type": "long",
+                    "_meta": {
+                      "description": "General number of clicks in the active application sub view over the last 7 days"
+                    }
+                  },
+                  "clicks_30_days": {
+                    "type": "long",
+                    "_meta": {
+                      "description": "General number of clicks in the active application sub view over the last 30 days"
+                    }
+                  },
+                  "clicks_90_days": {
+                    "type": "long",
+                    "_meta": {
+                      "description": "General number of clicks in the active application sub view over the last 90 days"
+                    }
+                  },
+                  "minutes_on_screen_total": {
+                    "type": "float",
+                    "_meta": {
+                      "description": "Minutes the application sub view is active and on-screen since we started counting them."
+                    }
+                  },
+                  "minutes_on_screen_7_days": {
+                    "type": "float",
+                    "_meta": {
+                      "description": "Minutes the application is active and on-screen active application sub view over the last 7 days"
+                    }
+                  },
+                  "minutes_on_screen_30_days": {
+                    "type": "float",
+                    "_meta": {
+                      "description": "Minutes the application is active and on-screen active application sub view over the last 30 days"
+                    }
+                  },
+                  "minutes_on_screen_90_days": {
+                    "type": "float",
+                    "_meta": {
+                      "description": "Minutes the application is active and on-screen active application sub view over the last 90 days"
+                    }
+                  }
+                }
+              }
+            }
+          }
+        },
         "appSearch": {
           "properties": {
             "appId": {

diff --git a/tsconfig.base.json b/tsconfig.base.json
@@ -822,6 +822,8 @@
       "@kbn/entities-schema/*": ["x-pack/packages/kbn-entities-schema/*"],
       "@kbn/entity-manager-fixture-plugin": ["x-pack/test/api_integration/apis/entity_manager/fixture_plugin"],
       "@kbn/entity-manager-fixture-plugin/*": ["x-pack/test/api_integration/apis/entity_manager/fixture_plugin/*"],
+      "@kbn/entityManager-app-plugin": ["x-pack/plugins/observability_solution/entity_manager_app"],
+      "@kbn/entityManager-app-plugin/*": ["x-pack/plugins/observability_solution/entity_manager_app/*"],
       "@kbn/entityManager-plugin": ["x-pack/plugins/entity_manager"],
       "@kbn/entityManager-plugin/*": ["x-pack/plugins/entity_manager/*"],
       "@kbn/error-boundary-example-plugin": ["examples/error_boundary"],

diff --git a/x-pack/packages/kbn-entities-schema/src/schema/entity.ts b/x-pack/packages/kbn-entities-schema/src/schema/entity.ts
@@ -23,6 +23,13 @@ export interface MetadataRecord {
   [key: string]: string[] | MetadataRecord | string;
 }
 
+export interface EntityV2 {
+  'entity.id': string;
+  'entity.last_seen_timestamp': string;
+  'entity.type': string;
+  [metadata: string]: any;
+}
+
 const literalSchema = z.union([z.string(), z.number(), z.boolean(), z.null()]);
 
 type Literal = z.infer<typeof literalSchema>;

diff --git a/x-pack/plugins/entity_manager/kibana.jsonc b/x-pack/plugins/entity_manager/kibana.jsonc
@@ -8,9 +8,13 @@
   "plugin": {
     "id": "entityManager",
     "configPath": ["xpack", "entityManager"],
-    "requiredPlugins": ["security", "encryptedSavedObjects", "licensing"],
     "browser": true,
     "server": true,
+    "requiredPlugins": [
+      "security",
+      "encryptedSavedObjects",
+      "licensing"
+    ],
     "requiredBundles": []
   }
 }
diff --git a/x-pack/plugins/entity_manager/public/index.ts b/x-pack/plugins/entity_manager/public/index.ts
@@ -16,6 +16,8 @@ export const plugin: PluginInitializer<
   return new Plugin(context);
 };
 
+export { EntityClient } from './lib/entity_client';
+
 export type { EntityManagerPublicPluginSetup, EntityManagerPublicPluginStart };
 export type EntityManagerAppId = 'entityManager';
 

diff --git a/x-pack/plugins/entity_manager/public/plugin.ts b/x-pack/plugins/entity_manager/public/plugin.ts
@@ -22,16 +22,14 @@ export class Plugin implements EntityManagerPluginClass {
   }
 
   setup(core: CoreSetup) {
-    const entityClient = new EntityClient(core);
     return {
-      entityClient,
+      entityClient: new EntityClient(core),
     };
   }
 
   start(core: CoreStart) {
-    const entityClient = new EntityClient(core);
     return {
-      entityClient,
+      entityClient: new EntityClient(core),
     };
   }
 

diff --git a/x-pack/plugins/entity_manager/public/types.ts b/x-pack/plugins/entity_manager/public/types.ts
@@ -10,7 +10,6 @@ import type { EntityClient } from './lib/entity_client';
 export interface EntityManagerPublicPluginSetup {
   entityClient: EntityClient;
 }
-
 export interface EntityManagerPublicPluginStart {
   entityClient: EntityClient;
 }

diff --git a/...ugins/entity_manager/server/lib/errors.ts → ...ib/entities/errors/unknown_entity_type.ts b/...ugins/entity_manager/server/lib/errors.ts → ...ib/entities/errors/unknown_entity_type.ts
@@ -5,10 +5,9 @@
  * 2.0.
  */
 
-export class AssetNotFoundError extends Error {
-  constructor(ean: string) {
-    super(`Asset with ean (${ean}) not found in the provided time range`);
-    Object.setPrototypeOf(this, new.target.prototype);
-    this.name = 'AssetNotFoundError';
+export class UnknownEntityType extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = 'UnknownEntityType';
   }
 }