[8.x] [EEM] Add built in definitions for core Kubernetes entities (el…

…astic#196916) (elastic#201660) # Backport This will backport the following commits from `main` to `8.x`: - [[EEM] Add built in definitions for core Kubernetes entities (elastic#196916)](elastic#196916)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport)  Co-authored-by: Milton Hultgren <[email protected]>
kibanamachine · Nov 26, 2024 · 456316b · 456316b
1 parent c5b964e
commit 456316b
Show file tree

Hide file tree

Showing 30 changed files with 802 additions and 4 deletions.
diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/index.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/index.ts
@@ -10,10 +10,13 @@ import { builtInServicesFromEcsEntityDefinition } from './services_from_ecs_data
 import { builtInHostsFromEcsEntityDefinition } from './hosts_from_ecs_data';
 import { builtInContainersFromEcsEntityDefinition } from './containers_from_ecs_data';
 
+import * as kubernetes from './kubernetes';
+
 export { BUILT_IN_ID_PREFIX } from './constants';
 
 export const builtInDefinitions: EntityDefinition[] = [
   builtInServicesFromEcsEntityDefinition,
   builtInHostsFromEcsEntityDefinition,
   builtInContainersFromEcsEntityDefinition,
+  ...Object.values(kubernetes),
 ];
diff --git a/...ugins/entity_manager/server/lib/entities/built_in/kubernetes/common/ecs_index_patterns.ts b/...ugins/entity_manager/server/lib/entities/built_in/kubernetes/common/ecs_index_patterns.ts
@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const commonEcsIndexPatterns = ['metrics-kubernetes*', 'logs-*'];
diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/common/ecs_metadata.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/common/ecs_metadata.ts
@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { MetadataField } from '@kbn/entities-schema';
+import { globalMetadata } from './global_metadata';
+
+export const commonEcsMetadata: MetadataField[] = [
+  ...globalMetadata,
+  {
+    source: 'orchestrator.namespace',
+    destination: 'orchestrator.namespace',
+    aggregation: { type: 'terms', limit: 10 },
+  },
+  {
+    source: 'orchestrator.cluster_ip',
+    destination: 'orchestrator.cluster_id',
+    aggregation: { type: 'top_value', sort: { '@timestamp': 'desc' } },
+  },
+  {
+    source: 'orchestrator.cluster_name',
+    destination: 'orchestrator.cluster_name',
+    aggregation: { type: 'top_value', sort: { '@timestamp': 'desc' } },
+  },
+];
diff --git a/.../plugins/entity_manager/server/lib/entities/built_in/kubernetes/common/global_metadata.ts b/.../plugins/entity_manager/server/lib/entities/built_in/kubernetes/common/global_metadata.ts
@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { MetadataField } from '@kbn/entities-schema';
+
+export const globalMetadata: MetadataField[] = [
+  {
+    source: '_index',
+    destination: 'source_index',
+    aggregation: { type: 'top_value', sort: { '@timestamp': 'desc' } },
+  },
+  {
+    source: 'data_stream.type',
+    destination: 'source_data_stream.type',
+    aggregation: { type: 'top_value', sort: { '@timestamp': 'desc' } },
+  },
+  {
+    source: 'data_stream.dataset',
+    destination: 'source_data_stream.dataset',
+    aggregation: { type: 'top_value', sort: { '@timestamp': 'desc' } },
+  },
+];
diff --git a/...gins/entity_manager/server/lib/entities/built_in/kubernetes/common/otel_index_patterns.ts b/...gins/entity_manager/server/lib/entities/built_in/kubernetes/common/otel_index_patterns.ts
@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const commonOtelIndexPatterns = ['metrics-*otel*', 'logs-*'];
diff --git a/...ck/plugins/entity_manager/server/lib/entities/built_in/kubernetes/common/otel_metadata.ts b/...ck/plugins/entity_manager/server/lib/entities/built_in/kubernetes/common/otel_metadata.ts
@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { MetadataField } from '@kbn/entities-schema';
+import { globalMetadata } from './global_metadata';
+
+export const commonOtelMetadata: MetadataField[] = [
+  ...globalMetadata,
+  {
+    source: 'k8s.namespace.name',
+    destination: 'k8s.namespace.name',
+    aggregation: { type: 'terms', limit: 10 },
+  },
+  {
+    source: 'k8s.cluster.name',
+    destination: 'k8s.cluster.name',
+    aggregation: { type: 'top_value', sort: { '@timestamp': 'desc' } },
+  },
+];
diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/cluster.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/cluster.ts
@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { EntityDefinition, entityDefinitionSchema } from '@kbn/entities-schema';
+import { BUILT_IN_ID_PREFIX } from '../../constants';
+import { commonEcsIndexPatterns } from '../common/ecs_index_patterns';
+import { globalMetadata } from '../common/global_metadata';
+
+export const builtInKubernetesClusterEcsEntityDefinition: EntityDefinition =
+  entityDefinitionSchema.parse({
+    id: `${BUILT_IN_ID_PREFIX}kubernetes_cluster_ecs`,
+    filter: 'orchestrator.cluster.name: *',
+    managed: true,
+    version: '0.1.0',
+    name: 'Kubernetes Clusters from ECS data',
+    description:
+      'This definition extracts Kubernetes cluster entities from the Kubernetes integration data streams',
+    type: 'k8s.cluster.ecs',
+    indexPatterns: commonEcsIndexPatterns,
+    identityFields: ['orchestrator.cluster.name'],
+    displayNameTemplate: '{{orchestrator.cluster.name}}',
+    latest: {
+      timestampField: '@timestamp',
+      lookbackPeriod: '10m',
+      settings: {
+        frequency: '5m',
+      },
+    },
+    metadata: [
+      ...globalMetadata,
+      {
+        source: 'orchestrator.namespace',
+        destination: 'orchestrator.namespace',
+        aggregation: { type: 'terms', limit: 10 },
+      },
+      {
+        source: 'orchestrator.cluster_ip',
+        destination: 'orchestrator.cluster_id',
+        aggregation: { type: 'top_value', sort: { '@timestamp': 'desc' } },
+      },
+    ],
+  });
diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/cron_job.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/cron_job.ts
@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { EntityDefinition, entityDefinitionSchema } from '@kbn/entities-schema';
+import { BUILT_IN_ID_PREFIX } from '../../constants';
+import { commonEcsIndexPatterns } from '../common/ecs_index_patterns';
+import { commonEcsMetadata } from '../common/ecs_metadata';
+
+export const builtInKubernetesCronJobEcsEntityDefinition: EntityDefinition =
+  entityDefinitionSchema.parse({
+    id: `${BUILT_IN_ID_PREFIX}kubernetes_cron_job_ecs`,
+    filter: 'kubernetes.cronjob.uid : *',
+    managed: true,
+    version: '0.1.0',
+    name: 'Kubernetes CronJob from ECS data',
+    description:
+      'This definition extracts Kubernetes cron job entities from the Kubernetes integration data streams',
+    type: 'k8s.cronjob.ecs',
+    indexPatterns: commonEcsIndexPatterns,
+    identityFields: ['kubernetes.cronjob.uid'],
+    displayNameTemplate: '{{kubernetes.cronjob.name}}',
+    latest: {
+      timestampField: '@timestamp',
+      lookbackPeriod: '10m',
+      settings: {
+        frequency: '5m',
+      },
+    },
+    metadata: commonEcsMetadata,
+  });
diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/daemon_set.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/daemon_set.ts
@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { EntityDefinition, entityDefinitionSchema } from '@kbn/entities-schema';
+import { BUILT_IN_ID_PREFIX } from '../../constants';
+import { commonEcsIndexPatterns } from '../common/ecs_index_patterns';
+import { commonEcsMetadata } from '../common/ecs_metadata';
+
+export const builtInKubernetesDaemonSetEcsEntityDefinition: EntityDefinition =
+  entityDefinitionSchema.parse({
+    id: `${BUILT_IN_ID_PREFIX}kubernetes_daemon_set_ecs`,
+    filter: 'kubernetes.daemonset.uid : *',
+    managed: true,
+    version: '0.1.0',
+    name: 'Kubernetes DaemonSet from ECS data',
+    description:
+      'This definition extracts Kubernetes daemon set entities from the Kubernetes integration data streams',
+    type: 'k8s.daemonset.ecs',
+    indexPatterns: commonEcsIndexPatterns,
+    identityFields: ['kubernetes.daemonset.name'],
+    displayNameTemplate: '{{kubernetes.daemonset.name}}',
+    latest: {
+      timestampField: '@timestamp',
+      lookbackPeriod: '10m',
+      settings: {
+        frequency: '5m',
+      },
+    },
+    metadata: commonEcsMetadata,
+  });
diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/deployment.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/deployment.ts
@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { EntityDefinition, entityDefinitionSchema } from '@kbn/entities-schema';
+import { BUILT_IN_ID_PREFIX } from '../../constants';
+import { commonEcsMetadata } from '../common/ecs_metadata';
+import { commonEcsIndexPatterns } from '../common/ecs_index_patterns';
+
+export const builtInKubernetesDeploymentEcsEntityDefinition: EntityDefinition =
+  entityDefinitionSchema.parse({
+    id: `${BUILT_IN_ID_PREFIX}kubernetes_deployment_ecs`,
+    filter: 'kubernetes.deployment.uid : *',
+    managed: true,
+    version: '0.1.0',
+    name: 'Kubernetes Deployment from ECS data',
+    description:
+      'This definition extracts Kubernetes deployment entities from the Kubernetes integration data streams',
+    type: 'k8s.deployment.ecs',
+    indexPatterns: commonEcsIndexPatterns,
+    identityFields: ['kubernetes.deployment.uid'],
+    displayNameTemplate: '{{kubernetes.deployment.name}}',
+    latest: {
+      timestampField: '@timestamp',
+      lookbackPeriod: '10m',
+      settings: {
+        frequency: '5m',
+      },
+    },
+    metadata: commonEcsMetadata,
+  });
diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/index.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/index.ts
@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export { builtInKubernetesClusterEcsEntityDefinition } from './cluster';
+export { builtInKubernetesNodeEcsEntityDefinition } from './node';
+export { builtInKubernetesPodEcsEntityDefinition } from './pod';
+export { builtInKubernetesReplicaSetEcsEntityDefinition } from './replica_set';
+export { builtInKubernetesDeploymentEcsEntityDefinition } from './deployment';
+export { builtInKubernetesStatefulSetEcsEntityDefinition } from './stateful_set';
+export { builtInKubernetesDaemonSetEcsEntityDefinition } from './daemon_set';
+export { builtInKubernetesJobEcsEntityDefinition } from './job';
+export { builtInKubernetesCronJobEcsEntityDefinition } from './cron_job';
+export { builtInKubernetesServiceEcsEntityDefinition } from './service';
diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/job.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/job.ts
@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { EntityDefinition, entityDefinitionSchema } from '@kbn/entities-schema';
+import { BUILT_IN_ID_PREFIX } from '../../constants';
+import { commonEcsIndexPatterns } from '../common/ecs_index_patterns';
+import { commonEcsMetadata } from '../common/ecs_metadata';
+
+export const builtInKubernetesJobEcsEntityDefinition: EntityDefinition =
+  entityDefinitionSchema.parse({
+    id: `${BUILT_IN_ID_PREFIX}kubernetes_job_ecs`,
+    filter: 'kubernetes.job.uid : *',
+    managed: true,
+    version: '0.1.0',
+    name: 'Kubernetes Job from ECS data',
+    description:
+      'This definition extracts Kubernetes job entities from the Kubernetes integration data streams',
+    type: 'k8s.job.ecs',
+    indexPatterns: commonEcsIndexPatterns,
+    identityFields: ['kubernetes.job.uid'],
+    displayNameTemplate: '{{kubernetes.job.name}}',
+    latest: {
+      timestampField: '@timestamp',
+      lookbackPeriod: '10m',
+      settings: {
+        frequency: '5m',
+      },
+    },
+    metadata: commonEcsMetadata,
+  });
diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/node.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/node.ts
@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { EntityDefinition, entityDefinitionSchema } from '@kbn/entities-schema';
+import { BUILT_IN_ID_PREFIX } from '../../constants';
+import { commonEcsIndexPatterns } from '../common/ecs_index_patterns';
+import { commonEcsMetadata } from '../common/ecs_metadata';
+
+export const builtInKubernetesNodeEcsEntityDefinition: EntityDefinition =
+  entityDefinitionSchema.parse({
+    id: `${BUILT_IN_ID_PREFIX}kubernetes_node_ecs`,
+    filer: 'kubernetes.node.uid : *',
+    managed: true,
+    version: '0.1.0',
+    name: 'Kubernetes Node from ECS data',
+    description:
+      'This definition extracts Kubernetes node entities from the Kubernetes integration data streams',
+    type: 'k8s.node.ecs',
+    indexPatterns: commonEcsIndexPatterns,
+    identityFields: ['kubernetes.node.uid'],
+    displayNameTemplate: '{{kubernetes.node.name}}',
+    latest: {
+      timestampField: '@timestamp',
+      lookbackPeriod: '10m',
+      settings: {
+        frequency: '5m',
+      },
+    },
+    metadata: commonEcsMetadata,
+  });
diff --git a/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/pod.ts b/x-pack/plugins/entity_manager/server/lib/entities/built_in/kubernetes/ecs/pod.ts
@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { EntityDefinition, entityDefinitionSchema } from '@kbn/entities-schema';
+import { BUILT_IN_ID_PREFIX } from '../../constants';
+import { commonEcsMetadata } from '../common/ecs_metadata';
+import { commonEcsIndexPatterns } from '../common/ecs_index_patterns';
+
+export const builtInKubernetesPodEcsEntityDefinition: EntityDefinition =
+  entityDefinitionSchema.parse({
+    id: `${BUILT_IN_ID_PREFIX}kubernetes_pod_ecs`,
+    filter: 'kubernetes.pod.uid: *',
+    managed: true,
+    version: '0.1.0',
+    name: 'Kubernetes Pod from ECS data',
+    description:
+      'This definition extracts Kubernetes pod entities from the Kubernetes integration data streams',
+    type: 'k8s.pod.ecs',
+    indexPatterns: commonEcsIndexPatterns,
+    identityFields: ['kubernetes.pod.name'],
+    displayNameTemplate: '{{kubernetes.pod.name}}',
+    latest: {
+      timestampField: '@timestamp',
+      lookbackPeriod: '10m',
+      settings: {
+        frequency: '5m',
+      },
+    },
+    metadata: commonEcsMetadata,
+  });