generating routing_path from dimension fields

elastic · Nov 15, 2022 · 980b1b3 · 980b1b3
1 parent fc64c1b
commit 980b1b3
Show file tree

Hide file tree

Showing 2 changed files with 124 additions and 11 deletions.
diff --git a/...k/plugins/fleet/server/services/package_policies/experimental_datastream_features.test.ts b/...k/plugins/fleet/server/services/package_policies/experimental_datastream_features.test.ts
@@ -10,7 +10,10 @@ import { savedObjectsClientMock } from '@kbn/core-saved-objects-api-server-mocks
 
 import type { NewPackagePolicy, PackagePolicy } from '../../types';
 
-import { handleExperimentalDatastreamFeatureOptIn } from './experimental_datastream_features';
+import {
+  builRoutingPath,
+  handleExperimentalDatastreamFeatureOptIn,
+} from './experimental_datastream_features';
 
 function getNewTestPackagePolicy({
   isSyntheticSourceEnabled,
@@ -102,6 +105,12 @@ describe('experimental_datastream_features', () => {
                   // @ts-expect-error
                   mode: 'stored',
                 },
+                properties: {
+                  test_dimension: {
+                    type: 'keyword',
+                    time_series_dimension: true,
+                  },
+                },
               },
             },
           },
@@ -178,7 +187,9 @@ describe('experimental_datastream_features', () => {
         expect.objectContaining({
           body: expect.objectContaining({
             template: expect.objectContaining({
-              settings: expect.objectContaining({ index: { mode: 'time_series' } }),
+              settings: expect.objectContaining({
+                index: { mode: 'time_series', routing_path: ['test_dimension'] },
+              }),
             }),
           }),
         })
@@ -280,12 +291,74 @@ describe('experimental_datastream_features', () => {
           expect.objectContaining({
             body: expect.objectContaining({
               template: expect.objectContaining({
-                settings: expect.objectContaining({ index: { mode: 'time_series' } }),
+                settings: expect.objectContaining({
+                  index: { mode: 'time_series', routing_path: ['test_dimension'] },
+                }),
               }),
             }),
           })
         );
       });
     });
   });
+  it('should build routing path', () => {
+    const mappingProperties = {
+      cloud: {
+        properties: {
+          availability_zone: {
+            ignore_above: 1024,
+            type: 'keyword',
+          },
+          image: {
+            properties: {
+              id: {
+                ignore_above: 1024,
+                type: 'keyword',
+              },
+            },
+          },
+        },
+      },
+      test_dimension: {
+        time_series_dimension: true,
+        type: 'keyword',
+      },
+      '@timestamp': {
+        type: 'date',
+      },
+    };
+    const routingPath = builRoutingPath(mappingProperties as any);
+    expect(routingPath).toEqual(['test_dimension']);
+  });
+
+  it('should build routing path from nested properties', () => {
+    const mappingProperties = {
+      cloud: {
+        properties: {
+          availability_zone: {
+            ignore_above: 1024,
+            type: 'keyword',
+          },
+          image: {
+            properties: {
+              id: {
+                ignore_above: 1024,
+                type: 'keyword',
+                time_series_dimension: true,
+              },
+            },
+          },
+        },
+      },
+      test_dimension: {
+        time_series_dimension: true,
+        type: 'keyword',
+      },
+      '@timestamp': {
+        type: 'date',
+      },
+    };
+    const routingPath = builRoutingPath(mappingProperties as any);
+    expect(routingPath).toEqual(['cloud.image.id', 'test_dimension']);
+  });
 });
diff --git a/x-pack/plugins/fleet/server/services/package_policies/experimental_datastream_features.ts b/x-pack/plugins/fleet/server/services/package_policies/experimental_datastream_features.ts
@@ -5,13 +5,42 @@
  * 2.0.
  */
 
+import type {
+  MappingProperty,
+  PropertyName,
+} from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
 import type { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server';
 
 import type { NewPackagePolicy, PackagePolicy } from '../../types';
 import { getInstallation } from '../epm/packages';
 import { updateDatastreamExperimentalFeatures } from '../epm/packages/update';
 
+function mapFields(mappingProperties: Record<PropertyName, MappingProperty>) {
+  const mappings = Object.keys(mappingProperties).reduce((acc, curr) => {
+    const property = mappingProperties[curr] as any;
+    if (property.properties) {
+      const childMappings = mapFields(property.properties);
+      Object.keys(childMappings).forEach((key) => {
+        acc[curr + '.' + key] = childMappings[key];
+      });
+    } else {
+      acc[curr] = property;
+    }
+    return acc;
+  }, {} as any);
+  return mappings;
+}
+
+export function builRoutingPath(properties: Record<PropertyName, MappingProperty>) {
+  const mappingsProperties = mapFields(properties);
+  return Object.keys(mappingsProperties).filter(
+    (mapping) =>
+      mappingsProperties[mapping].type === 'keyword' &&
+      mappingsProperties[mapping].time_series_dimension
+  );
+}
+
 export async function handleExperimentalDatastreamFeatureOptIn({
   soClient,
   esClient,
@@ -45,14 +74,18 @@ export async function handleExperimentalDatastreamFeatureOptIn({
     const isSyntheticSourceOptInChanged =
       existingOptIn?.features.synthetic_source !== featureMapEntry.features.synthetic_source;
 
-    if (isSyntheticSourceOptInChanged) {
-      const componentTemplateName = `${featureMapEntry.data_stream}@package`;
-      const componentTemplateRes = await esClient.cluster.getComponentTemplate({
-        name: componentTemplateName,
-      });
+    const isTSDBOptInChanged = existingOptIn?.features.tsdb !== featureMapEntry.features.tsdb;
+
+    if (!isSyntheticSourceOptInChanged && !isTSDBOptInChanged) continue;
 
-      const componentTemplate = componentTemplateRes.component_templates[0].component_template;
+    const componentTemplateName = `${featureMapEntry.data_stream}@package`;
+    const componentTemplateRes = await esClient.cluster.getComponentTemplate({
+      name: componentTemplateName,
+    });
 
+    const componentTemplate = componentTemplateRes.component_templates[0].component_template;
+
+    if (isSyntheticSourceOptInChanged) {
       const body = {
         template: {
           ...componentTemplate.template,
@@ -72,9 +105,15 @@ export async function handleExperimentalDatastreamFeatureOptIn({
       });
     }
 
-    const isTSDBOptInChanged = existingOptIn?.features.tsdb !== featureMapEntry.features.tsdb;
-
     if (isTSDBOptInChanged && featureMapEntry.features.tsdb) {
+      const mappingsProperties = componentTemplate.template?.mappings?.properties ?? {};
+
+      // All mapped fields of type keyword and time_series_dimension enabled will be included in the generated routing path
+      // Temporarily generating routing_path here until fixed in elasticsearch https://github.com/elastic/elasticsearch/issues/91592
+      const routingPath = builRoutingPath(mappingsProperties);
+
+      if (routingPath.length === 0) continue;
+
       const indexTemplateRes = await esClient.indices.getIndexTemplate({
         name: featureMapEntry.data_stream,
       });
@@ -88,6 +127,7 @@ export async function handleExperimentalDatastreamFeatureOptIn({
             ...(indexTemplate.template?.settings ?? {}),
             index: {
               mode: 'time_series',
+              routing_path: routingPath,
             },
           },
         },