[8.x] [Synthtrace] Adding Entities support (#196258) (#196645)

# Backport This will backport the following commits from `main` to `8.x`: - [[Synthtrace] Adding Entities support (#196258)](#196258)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport)
elastic · Oct 17, 2024 · 466c71b · 466c71b
1 parent 5453ac0
commit 466c71b
Show file tree

Hide file tree

Showing 26 changed files with 491 additions and 485 deletions.
diff --git a/packages/kbn-apm-synthtrace-client/index.ts b/packages/kbn-apm-synthtrace-client/index.ts
@@ -35,5 +35,5 @@ export { generateLongId, generateShortId } from './src/lib/utils/generate_id';
 export { appendHash, hashKeysOf } from './src/lib/utils/hash';
 export type { ESDocumentWithOperation, SynthtraceESAction, SynthtraceGenerator } from './src/types';
 export { log, type LogDocument, LONG_FIELD_NAME } from './src/lib/logs';
-export { type AssetDocument } from './src/lib/assets';
 export { syntheticsMonitor, type SyntheticsMonitorDocument } from './src/lib/synthetics';
+export { type EntityFields, entities } from './src/lib/entities';
diff --git a/packages/kbn-apm-synthtrace-client/src/lib/assets/asset.ts b/packages/kbn-apm-synthtrace-client/src/lib/assets/asset.ts
diff --git a/packages/kbn-apm-synthtrace-client/src/lib/assets/index.ts b/packages/kbn-apm-synthtrace-client/src/lib/assets/index.ts
diff --git a/packages/kbn-apm-synthtrace-client/src/lib/assets/service_assets.ts b/packages/kbn-apm-synthtrace-client/src/lib/assets/service_assets.ts
diff --git a/packages/kbn-apm-synthtrace-client/src/lib/entities/container_entity.ts b/packages/kbn-apm-synthtrace-client/src/lib/entities/container_entity.ts
@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { EntityDataStreamType, EntityFields } from '.';
+import { Serializable } from '../serializable';
+
+class ContainerEntity extends Serializable<EntityFields> {
+  constructor(fields: EntityFields) {
+    super({
+      ...fields,
+      'entity.type': 'container',
+      'entity.definitionId': 'latest',
+    });
+  }
+}
+
+export function containerEntity({
+  agentName,
+  dataStreamType,
+  dataStreamDataset,
+  containerId,
+  entityId,
+}: {
+  agentName: string[];
+  dataStreamType: EntityDataStreamType[];
+  dataStreamDataset: string;
+  containerId: string;
+  entityId: string;
+}) {
+  return new ContainerEntity({
+    'source_data_stream.type': dataStreamType,
+    'source_data_stream.dataset': dataStreamDataset,
+    'agent.name': agentName,
+    'container.id': containerId,
+    'entity.id': entityId,
+  });
+}
diff --git a/packages/kbn-apm-synthtrace-client/src/lib/entities/host_entity.ts b/packages/kbn-apm-synthtrace-client/src/lib/entities/host_entity.ts
@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { EntityDataStreamType, EntityFields } from '.';
+import { Serializable } from '../serializable';
+
+class HostEntity extends Serializable<EntityFields> {
+  constructor(fields: EntityFields) {
+    super({
+      ...fields,
+      'entity.type': 'host',
+      'entity.definitionId': 'latest',
+    });
+  }
+}
+
+export function hostEntity({
+  agentName,
+  dataStreamType,
+  dataStreamDataset,
+  hostName,
+  entityId,
+}: {
+  agentName: string[];
+  dataStreamType: EntityDataStreamType[];
+  dataStreamDataset: string;
+  hostName: string;
+  entityId: string;
+}) {
+  return new HostEntity({
+    'source_data_stream.type': dataStreamType,
+    'source_data_stream.dataset': dataStreamDataset,
+    'agent.name': agentName,
+    'host.name': hostName,
+    'entity.id': entityId,
+  });
+}
diff --git a/packages/kbn-apm-synthtrace-client/src/lib/entities/index.ts b/packages/kbn-apm-synthtrace-client/src/lib/entities/index.ts
@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { Fields } from '../entity';
+import { serviceEntity } from './service_entity';
+import { hostEntity } from './host_entity';
+import { containerEntity } from './container_entity';
+
+export type EntityDataStreamType = 'metrics' | 'logs' | 'traces';
+
+export type EntityFields = Fields &
+  Partial<{
+    'agent.name': string[];
+    'source_data_stream.type': string | string[];
+    'source_data_stream.dataset': string | string[];
+    'event.ingested': string;
+    sourceIndex: string;
+    'entity.lastSeenTimestamp': string;
+    'entity.schemaVersion': string;
+    'entity.definitionVersion': string;
+    'entity.displayName': string;
+    'entity.identityFields': string | string[];
+    'entity.id': string;
+    'entity.type': string;
+    'entity.definitionId': string;
+    [key: string]: any;
+  }>;
+
+export const entities = { serviceEntity, hostEntity, containerEntity };
diff --git a/packages/kbn-apm-synthtrace-client/src/lib/entities/service_entity.ts b/packages/kbn-apm-synthtrace-client/src/lib/entities/service_entity.ts
@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { EntityDataStreamType, EntityFields } from '.';
+import { Serializable } from '../serializable';
+
+class ServiceEntity extends Serializable<EntityFields> {
+  constructor(fields: EntityFields) {
+    super({
+      ...fields,
+      'entity.type': 'service',
+      'entity.definitionId': 'latest',
+    });
+  }
+}
+
+export function serviceEntity({
+  agentName,
+  dataStreamType,
+  serviceName,
+  environment,
+  entityId,
+}: {
+  agentName: string[];
+  serviceName: string;
+  dataStreamType: EntityDataStreamType[];
+  environment?: string;
+  entityId: string;
+}) {
+  return new ServiceEntity({
+    'service.name': serviceName,
+    'service.environment': environment,
+    'source_data_stream.type': dataStreamType,
+    'agent.name': agentName,
+    'entity.id': entityId,
+  });
+}
diff --git a/packages/kbn-apm-synthtrace/index.ts b/packages/kbn-apm-synthtrace/index.ts
@@ -15,7 +15,7 @@ export { InfraSynthtraceEsClient } from './src/lib/infra/infra_synthtrace_es_cli
 export { InfraSynthtraceKibanaClient } from './src/lib/infra/infra_synthtrace_kibana_client';
 export { MonitoringSynthtraceEsClient } from './src/lib/monitoring/monitoring_synthtrace_es_client';
 export { LogsSynthtraceEsClient } from './src/lib/logs/logs_synthtrace_es_client';
-export { AssetsSynthtraceEsClient } from './src/lib/assets/assets_synthtrace_es_client';
+export { EntitiesSynthtraceEsClient } from './src/lib/entities/entities_synthtrace_es_client';
 export { SyntheticsSynthtraceEsClient } from './src/lib/synthetics/synthetics_synthtrace_es_client';
 export {
   addObserverVersionTransform,

diff --git a/packages/kbn-apm-synthtrace/src/cli/scenario.ts b/packages/kbn-apm-synthtrace/src/cli/scenario.ts
@@ -13,18 +13,23 @@ import {
   InfraSynthtraceEsClient,
   LogsSynthtraceEsClient,
   SyntheticsSynthtraceEsClient,
+  EntitiesSynthtraceEsClient,
 } from '../..';
-import { AssetsSynthtraceEsClient } from '../lib/assets/assets_synthtrace_es_client';
 import { Logger } from '../lib/utils/create_logger';
 import { ScenarioReturnType } from '../lib/utils/with_client';
 import { RunOptions } from './utils/parse_run_cli_flags';
+import { EntitiesSynthtraceKibanaClient } from '../lib/apm/client/entities_synthtrace_kibana_client';
 
 interface EsClients {
   apmEsClient: ApmSynthtraceEsClient;
   logsEsClient: LogsSynthtraceEsClient;
   infraEsClient: InfraSynthtraceEsClient;
-  assetsEsClient: AssetsSynthtraceEsClient;
   syntheticsEsClient: SyntheticsSynthtraceEsClient;
+  entitiesEsClient: EntitiesSynthtraceEsClient;
+}
+
+interface KibanaClients {
+  entitiesKibanaClient: EntitiesSynthtraceKibanaClient;
 }
 
 type Generate<TFields> = (options: {
@@ -33,6 +38,6 @@ type Generate<TFields> = (options: {
 }) => ScenarioReturnType<TFields> | Array<ScenarioReturnType<TFields>>;
 
 export type Scenario<TFields> = (options: RunOptions & { logger: Logger }) => Promise<{
-  bootstrap?: (options: EsClients) => Promise<void>;
+  bootstrap?: (options: EsClients & KibanaClients) => Promise<void>;
   generate: Generate<TFields>;
 }>;
diff --git a/packages/kbn-apm-synthtrace/src/cli/utils/bootstrap.ts b/packages/kbn-apm-synthtrace/src/cli/utils/bootstrap.ts
@@ -14,8 +14,9 @@ import { getInfraEsClient } from './get_infra_es_client';
 import { getKibanaClient } from './get_kibana_client';
 import { getServiceUrls } from './get_service_urls';
 import { RunOptions } from './parse_run_cli_flags';
-import { getAssetsEsClient } from './get_assets_es_client';
 import { getSyntheticsEsClient } from './get_synthetics_es_client';
+import { getEntitiesEsClient } from './get_entities_es_client';
+import { getEntitiesKibanaClient } from './get_entites_kibana_client';
 
 export async function bootstrap(runOptions: RunOptions) {
   const logger = createLogger(runOptions.logLevel);
@@ -57,12 +58,17 @@ export async function bootstrap(runOptions: RunOptions) {
     concurrency: runOptions.concurrency,
   });
 
-  const assetsEsClient = getAssetsEsClient({
+  const entitiesEsClient = getEntitiesEsClient({
     target: esUrl,
     logger,
     concurrency: runOptions.concurrency,
   });
 
+  const entitiesKibanaClient = getEntitiesKibanaClient({
+    target: kibanaUrl,
+    logger,
+  });
+
   const syntheticsEsClient = getSyntheticsEsClient({
     target: esUrl,
     logger,
@@ -73,7 +79,7 @@ export async function bootstrap(runOptions: RunOptions) {
     await apmEsClient.clean();
     await logsEsClient.clean();
     await infraEsClient.clean();
-    await assetsEsClient.clean();
+    await entitiesEsClient.clean();
     await syntheticsEsClient.clean();
   }
 
@@ -82,10 +88,11 @@ export async function bootstrap(runOptions: RunOptions) {
     apmEsClient,
     logsEsClient,
     infraEsClient,
-    assetsEsClient,
+    entitiesEsClient,
     syntheticsEsClient,
     version,
     kibanaUrl,
     esUrl,
+    entitiesKibanaClient,
   };
 }
diff --git a/...regators/create_logs_assets_aggregator.ts → ...rc/cli/utils/get_entites_kibana_client.ts b/...regators/create_logs_assets_aggregator.ts → ...rc/cli/utils/get_entites_kibana_client.ts
@@ -7,7 +7,14 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import { LogDocument } from '@kbn/apm-synthtrace-client';
-import { createAssetsAggregatorFactory } from '../../utils/create_assets_aggregator_factory';
+import { EntitiesSynthtraceKibanaClient } from '../../lib/apm/client/entities_synthtrace_kibana_client';
+import { Logger } from '../../lib/utils/create_logger';
 
-export const createLogsAssetsAggregator = createAssetsAggregatorFactory<LogDocument>();
+export function getEntitiesKibanaClient({ target, logger }: { target: string; logger: Logger }) {
+  const kibanaClient = new EntitiesSynthtraceKibanaClient({
+    logger,
+    target,
+  });
+
+  return kibanaClient;
+}
diff --git a/...ace/src/cli/utils/get_assets_es_client.ts → ...e/src/cli/utils/get_entities_es_client.ts b/...ace/src/cli/utils/get_assets_es_client.ts → ...e/src/cli/utils/get_entities_es_client.ts
@@ -8,12 +8,12 @@
  */
 
 import { Client } from '@elastic/elasticsearch';
-import { AssetsSynthtraceEsClient } from '../../lib/assets/assets_synthtrace_es_client';
+import { EntitiesSynthtraceEsClient } from '../../lib/entities/entities_synthtrace_es_client';
 import { Logger } from '../../lib/utils/create_logger';
 import { RunOptions } from './parse_run_cli_flags';
 import { getEsClientTlsSettings } from './ssl';
 
-export function getAssetsEsClient({
+export function getEntitiesEsClient({
   target,
   logger,
   concurrency,
@@ -26,7 +26,7 @@ export function getAssetsEsClient({
     tls: getEsClientTlsSettings(target),
   });
 
-  return new AssetsSynthtraceEsClient({
+  return new EntitiesSynthtraceEsClient({
     client,
     logger,
     concurrency,

diff --git a/packages/kbn-apm-synthtrace/src/cli/utils/start_historical_data_upload.ts b/packages/kbn-apm-synthtrace/src/cli/utils/start_historical_data_upload.ts
@@ -26,7 +26,7 @@ export async function startHistoricalDataUpload({
   from: Date;
   to: Date;
 }) {
-  const { logger, esUrl, version } = await bootstrap(runOptions);
+  const { logger, esUrl, version, kibanaUrl } = await bootstrap(runOptions);
 
   const cores = cpus().length;
 
@@ -93,6 +93,7 @@ export async function startHistoricalDataUpload({
         workerId: workerIndex.toString(),
         esUrl,
         version,
+        kibanaUrl,
       };
       const worker = new Worker(Path.join(__dirname, './worker.js'), {
         workerData,