[Synthetics] Process processors in kibana as yaml

src/core/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts

@@ -106,7 +106,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "ingest-agent-policies": "f11cc19275f4c3e4ee7c5cd6423b6706b21b989d",
         "ingest-download-sources": "d7edc5e588d9afa61c4b831604582891c54ef1c7",
         "ingest-outputs": "bffa0fd93dfdde904d7f5aff77df72d1c35938d9",
-        "ingest-package-policies": "55816507db0134b8efbe0509e311a91ce7e1c6cc",
+        "ingest-package-policies": "134dc7dbe337d4e4b289073be2e1ef80b17818fc",
         "ingest_manager_settings": "418311b03c8eda53f5d2ea6f54c1356afaa65511",
         "inventory-view": "b8683c8e352a286b4aca1ab21003115a4800af83",
         "kql-telemetry": "93c1d16c1a0dfca9c8842062cf5ef8f62ae401ad",

x-pack/plugins/fleet/server/saved_objects/index.ts

@@ -25,6 +25,8 @@ import {
   UNINSTALL_TOKENS_SAVED_OBJECT_TYPE,
 } from '../constants';
 
+import { migrateSyntheticsPackagePolicyToV8100 } from './migrations/synthetics/to_v8_10_0';
+
 import { migratePackagePolicyEvictionsFromV8100 } from './migrations/security_solution/to_v8_10_0';
 
 import {
@@ -281,6 +283,10 @@ const getSavedObjectTypes = (): { [key: string]: SavedObjectsType } => ({
             type: 'data_backfill',
             backfillFn: migratePackagePolicyToV8100,
           },
+          {
+            type: 'data_backfill',
+            backfillFn: migrateSyntheticsPackagePolicyToV8100,
+          },
         ],
         schemas: {
           forwardCompatibility: migratePackagePolicyEvictionsFromV8100,

x-pack/plugins/fleet/server/saved_objects/migrations/synthetics/fixtures/8.7.0.ts

@@ -826,7 +826,12 @@ export const icmpPolicy = {
   typeMigrationVersion: '8.7.0',
 } as unknown as SavedObjectUnsanitizedDoc<PackagePolicy>;
 
-export const getBrowserPolicy = (throttling = '5d/3u/20l') =>
+export const getBrowserPolicy = (
+  throttling = '5d/3u/20l',
+  project: string | null = null,
+  testRunId?: string,
+  runOnce = false
+) =>
   ({
     type: 'ingest-package-policies',
     id: '420754e9-40f2-486c-bc2e-265bafd735c5-fe200580-dee2-11ed-933e-0f85f8c5dd40-default',
@@ -948,7 +953,7 @@ export const getBrowserPolicy = (throttling = '5d/3u/20l') =>
                 location_name: { value: 'Fleet managed', type: 'text' },
                 id: { type: 'text' },
                 config_id: { type: 'text' },
-                run_once: { value: false, type: 'bool' },
+                run_once: { value: runOnce, type: 'bool' },
                 origin: { type: 'text' },
                 'monitor.project.id': { type: 'text' },
                 'monitor.project.name': { type: 'text' },
@@ -1012,10 +1017,11 @@ export const getBrowserPolicy = (throttling = '5d/3u/20l') =>
                 location_name: { value: 'A private location', type: 'text' },
                 id: { value: '420754e9-40f2-486c-bc2e-265bafd735c5', type: 'text' },
                 config_id: { value: '420754e9-40f2-486c-bc2e-265bafd735c5', type: 'text' },
-                run_once: { value: false, type: 'bool' },
+                run_once: { value: runOnce, type: 'bool' },
                 origin: { value: 'ui', type: 'text' },
-                'monitor.project.id': { value: null, type: 'text' },
-                'monitor.project.name': { value: null, type: 'text' },
+                'monitor.project.id': { value: project, type: 'text' },
+                'monitor.project.name': { value: project, type: 'text' },
+                ...(testRunId ? { test_run_id: { value: testRunId, type: 'text' } } : {}),
               },
               id: 'synthetics/browser-browser-420754e9-40f2-486c-bc2e-265bafd735c5-fe200580-dee2-11ed-933e-0f85f8c5dd40-default',
               compiled_stream: {

x-pack/plugins/fleet/server/saved_objects/migrations/synthetics/to_v8_10_0.test.ts

@@ -0,0 +1,107 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { SavedObjectModelTransformationContext } from '@kbn/core-saved-objects-server';
+
+import { getBrowserPolicy } from './fixtures/8.7.0';
+
+import { migrateSyntheticsPackagePolicyToV8100 as migration } from './to_v8_10_0';
+
+describe('8.10.0 Synthetics Package Policy migration', () => {
+  describe('processors migration', () => {
+    it('handles processors field for empty values', () => {
+      const actual = migration(
+        getBrowserPolicy('false'),
+        {} as SavedObjectModelTransformationContext
+      );
+      expect(actual.attributes?.inputs?.[3]?.streams[0]?.vars?.processors?.value).toEqual(
+        '[{"add_fields":{"fields":{"monitor.fleet_managed":true,"config_id":"420754e9-40f2-486c-bc2e-265bafd735c5"},"target":""}}]'
+      );
+      expect(actual.attributes?.inputs?.[3]?.streams[0]?.compiled_stream?.processors).toEqual(
+        '[{"add_fields":{"fields":{"monitor.fleet_managed":true,"config_id":"420754e9-40f2-486c-bc2e-265bafd735c5"},"target":""}}]'
+      );
+    });
+
+    it('handles processors field for project monitor', () => {
+      const actual = migration(
+        getBrowserPolicy('', 'test-project'),
+        {} as SavedObjectModelTransformationContext
+      );
+      expect(actual.attributes?.inputs?.[3]?.streams[0]?.vars?.processors?.value).toEqual(
+        JSON.stringify([
+          {
+            add_fields: {
+              fields: {
+                'monitor.fleet_managed': true,
+                config_id: '420754e9-40f2-486c-bc2e-265bafd735c5',
+                'monitor.project.name': 'test-project',
+                'monitor.project.id': 'test-project',
+              },
+              target: '',
+            },
+          },
+        ])
+      );
+      expect(actual.attributes?.inputs?.[3]?.streams[0]?.compiled_stream.processors).toEqual(
+        JSON.stringify([
+          {
+            add_fields: {
+              fields: {
+                'monitor.fleet_managed': true,
+                config_id: '420754e9-40f2-486c-bc2e-265bafd735c5',
+                'monitor.project.name': 'test-project',
+                'monitor.project.id': 'test-project',
+              },
+              target: '',
+            },
+          },
+        ])
+      );
+    });
+
+    it('handles processors field for test now fields', () => {
+      const actual = migration(
+        getBrowserPolicy('', 'test-project', 'test-run-id', true),
+        {} as SavedObjectModelTransformationContext
+      );
+      expect(actual.attributes?.inputs?.[3]?.streams[0]?.vars?.processors?.value).toEqual(
+        JSON.stringify([
+          {
+            add_fields: {
+              fields: {
+                'monitor.fleet_managed': true,
+                test_run_id: 'test-run-id',
+                run_once: true,
+                config_id: '420754e9-40f2-486c-bc2e-265bafd735c5',
+                'monitor.project.name': 'test-project',
+                'monitor.project.id': 'test-project',
+              },
+              target: '',
+            },
+          },
+        ])
+      );
+      expect(actual.attributes?.inputs?.[3]?.streams[0]?.compiled_stream.processors).toEqual(
+        JSON.stringify([
+          {
+            add_fields: {
+              fields: {
+                'monitor.fleet_managed': true,
+                test_run_id: 'test-run-id',
+                run_once: true,
+                config_id: '420754e9-40f2-486c-bc2e-265bafd735c5',
+                'monitor.project.name': 'test-project',
+                'monitor.project.id': 'test-project',
+              },
+              target: '',
+            },
+          },
+        ])
+      );
+    });
+  });
+});

x-pack/plugins/fleet/server/saved_objects/migrations/synthetics/to_v8_10_0.ts

@@ -0,0 +1,89 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { SavedObjectModelDataBackfillFn } from '@kbn/core-saved-objects-server';
+
+import type { PackagePolicy, PackagePolicyConfigRecord } from '../../../../common';
+
+export const migrateSyntheticsPackagePolicyToV8100: SavedObjectModelDataBackfillFn<
+  PackagePolicy,
+  PackagePolicy
+> = (packagePolicyDoc) => {
+  if (
+    packagePolicyDoc.attributes.package?.name !== 'synthetics' ||
+    !packagePolicyDoc.attributes.is_managed
+  ) {
+    return packagePolicyDoc;
+  }
+  const updatedAttributes = packagePolicyDoc.attributes;
+
+  const enabledInput = updatedAttributes.inputs.find((input) => input.enabled === true);
+  const enabledStream = enabledInput?.streams.find((stream) => {
+    return ['browser', 'http', 'icmp', 'tcp'].includes(stream.data_stream.dataset);
+  });
+  if (!enabledStream) {
+    return {
+      attributes: updatedAttributes,
+    };
+  }
+
+  if (enabledStream.vars) {
+    const processors = processorsFormatter(enabledStream.vars);
+    enabledStream.vars.processors = { value: processors, type: 'yaml' };
+    enabledStream.compiled_stream.processors = processors;
+  }
+
+  return {
+    attributes: updatedAttributes,
+  };
+};
+
+type Fields = Record<string, string | boolean>;
+
+interface FieldProcessor {
+  add_fields: {
+    target: string;
+    fields: Fields;
+  };
+}
+
+export const processorsFormatter = (vars: PackagePolicyConfigRecord) => {
+  const fields: Fields = {
+    'monitor.fleet_managed': true,
+  };
+  if (vars.test_run_id?.value) {
+    fields.test_run_id = vars.test_run_id.value;
+  }
+  if (vars.run_once?.value) {
+    fields.run_once = vars.run_once.value;
+  }
+  if (vars.config_id?.value) {
+    fields.config_id = vars.config_id.value;
+  }
+  const projName = vars['monitor.project.name']?.value;
+  if (projName) {
+    fields['monitor.project.name'] = projName;
+  }
+  const projId = vars['monitor.project.id']?.value;
+  if (projId) {
+    fields['monitor.project.id'] = projId;
+  }
+  const monId = vars['monitor.id']?.value;
+  if (monId) {
+    fields['monitor.id'] = monId;
+  }
+  const processors: FieldProcessor[] = [
+    {
+      add_fields: {
+        fields,
+        target: '',
+      },
+    },
+  ];
+
+  return JSON.stringify(processors);
+};

x-pack/plugins/synthetics/server/synthetics_service/formatters/private_formatters/format_synthetics_policy.ts

@@ -7,27 +7,28 @@
 
 import { NewPackagePolicy } from '@kbn/fleet-plugin/common';
 import { cloneDeep } from 'lodash';
+import { processorsFormatter } from './processors_formatter';
 import { LegacyConfigKey } from '../../../../common/constants/monitor_management';
 import { ConfigKey, DataStream, MonitorFields } from '../../../../common/runtime_types';
 import { throttlingFormatter } from './browser_formatters';
 import { replaceStringWithParams } from '../formatting_utils';
 import { syntheticsPolicyFormatters } from './formatters';
 import { PARAMS_KEYS_TO_SKIP } from '../common';
 
+export interface ProcessorFields {
+  location_name: string;
+  location_id: string;
+  'monitor.project.name': string;
+  'monitor.project.id': string;
+  'monitor.id': string;
+  test_run_id: string;
+  run_once: boolean;
+}
+
 export const formatSyntheticsPolicy = (
   newPolicy: NewPackagePolicy,
   monitorType: DataStream,
-  config: Partial<
-    MonitorFields & {
-      location_name: string;
-      location_id: string;
-      'monitor.project.name': string;
-      'monitor.project.id': string;
-      'monitor.id': string;
-      test_run_id: string;
-      run_once: boolean;
-    }
-  >,
+  config: Partial<MonitorFields & ProcessorFields>,
   params: Record<string, string>,
   isLegacy?: boolean
 ) => {
@@ -67,6 +68,11 @@ export const formatSyntheticsPolicy = (
     }
   });
 
+  const processorItem = dataStream?.vars?.processors;
+  if (processorItem) {
+    processorItem.value = processorsFormatter(config);
+  }
+
   // TODO: remove this once we remove legacy support
   const throttling = dataStream?.vars?.[LegacyConfigKey.THROTTLING_CONFIG];
   if (throttling) {

x-pack/plugins/synthetics/server/synthetics_service/formatters/private_formatters/processors_formatter.ts

@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ProcessorFields } from './format_synthetics_policy';
+import { MonitorFields } from '../../../../common/runtime_types';
+
+type Fields = Record<string, string | boolean>;
+
+interface FieldProcessor {
+  add_fields: {
+    target: string;
+    fields: Fields;
+  };
+}
+
+export const processorsFormatter = (config: Partial<MonitorFields & ProcessorFields>) => {
+  const fields: Fields = {
+    'monitor.fleet_managed': true,
+  };
+  if (config.test_run_id) {
+    fields.test_run_id = config.test_run_id;
+  }
+  if (config.run_once) {
+    fields.run_once = config.run_once;
+  }
+  if (config.config_id) {
+    fields.config_id = config.config_id;
+  }
+  if (config['monitor.project.name']) {
+    fields['monitor.project.name'] = config['monitor.project.name'];
+  }
+  if (config['monitor.project.id']) {
+    fields['monitor.project.id'] = config['monitor.project.id'];
+  }
+  if (config['monitor.id']) {
+    fields['monitor.id'] = config['monitor.id'];
+  }
+  const processors: FieldProcessor[] = [
+    {
+      add_fields: {
+        fields,
+        target: '',
+      },
+    },
+  ];
+
+  return JSON.stringify(processors);
+};

x-pack/plugins/synthetics/server/synthetics_service/private_location/synthetics_private_location.ts

@@ -102,12 +102,10 @@ export class SyntheticsPrivateLocation {
           config_id: config.fields?.config_id,
           location_name: stringifyString(privateLocation.label),
           location_id: privateLocation.id,
-          'monitor.project.id': stringifyString(
-            config.fields?.['monitor.project.id'] ?? config[ConfigKey.PROJECT_ID]
-          ),
-          'monitor.project.name': stringifyString(
-            config.fields?.['monitor.project.name'] ?? config[ConfigKey.PROJECT_ID]
-          ),
+          'monitor.project.id':
+            config.fields?.['monitor.project.id'] ?? config[ConfigKey.PROJECT_ID],
+          'monitor.project.name':
+            config.fields?.['monitor.project.name'] ?? config[ConfigKey.PROJECT_ID],
           ...(testRunId
             ? {
                 test_run_id: testRunId,