From 618538482d0c4940526b236a9003c5e278915ee5 Mon Sep 17 00:00:00 2001
From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
Date: Tue, 26 Nov 2024 01:49:33 +1100
Subject: [PATCH] [8.x] [Automatic Import] Remove fields with @ from the script
 processor (#201548) (#201589)

# Backport

This will backport the following commits from `main` to `8.x`:
- [[Automatic Import] Remove fields with @ from the script processor
(#201548)](https://github.com/elastic/kibana/pull/201548)

<!--- Backport version: 9.4.3 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Bharat
Pasupula","email":"123897612+bhapas@users.noreply.github.com"},"sourceCommit":{"committedDate":"2024-11-25T13:00:14Z","message":"[Automatic
Import] Remove fields with @ from the script processor (#201548)\n\n##
Summary\r\n\r\nThis PR filters the fields containing `@` in date type
from `script`\r\nprocessor.\r\n\r\n## Before this
PR\r\n\r\n\r\n![image](https://github.com/user-attachments/assets/a733d81f-aaaf-4787-b974-1e5d35ff4b8f)\r\n\r\n```json\r\n
{\r\n \"script\": {\r\n \"tag\": \"script_convert_array_to_string\",\r\n
\"description\": \"Ensures the date processor does not receive an array
value.\",\r\n \"lang\": \"painless\",\r\n \"source\": \"if
(ctx.varonis?.varonis_alerts?.@timestamp != null &&\\n
ctx.varonis.varonis_alerts.@timestamp instanceof ArrayList){\\n
ctx.varonis.varonis_alerts.@timestamp =
ctx.varonis.varonis_alerts.@timestamp[0];\\n}\\n\"\r\n }\r\n },\r\n
{\r\n \"date\": {\r\n \"if\": \"ctx.varonis?.varonis_alerts?.@timestamp
!= null\",\r\n \"tag\":
\"date_processor_varonis.varonis_alerts.@timestamp\",\r\n \"field\":
\"varonis.varonis_alerts.@timestamp\",\r\n \"target_field\":
\"event.start\",\r\n \"formats\": [\r\n
\"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\",\r\n \"ISO8601\"\r\n ]\r\n }\r\n
},\r\n```\r\n\r\n## After this PR\r\n\r\n```json\r\n \"date\": {\r\n
\"if\": \"ctx.varonis?.varonis_alerts?.@timestamp != null\",\r\n
\"tag\": \"date_processor_varonis.varonis_alerts.@timestamp\",\r\n
\"field\": \"varonis.varonis_alerts.@timestamp\",\r\n \"target_field\":
\"event.start\",\r\n \"formats\": [\r\n
\"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\",\r\n \"ISO8601\"\r\n ]\r\n }\r\n
},\r\n```\r\n\r\n### Checklist\r\n\r\nCheck the PR satisfies following
conditions. \r\n\r\nReviewers should verify this PR satisfies this list
as well.\r\n\r\n- [x] [Unit or
functional\r\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\r\nwere
updated or added to match the most common
scenarios","sha":"8964dc92c774d9ac5c82a411022ece3fb91e3cfd","branchLabelMapping":{"^v9.0.0$":"main","^v8.18.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["bug","release_note:fix","v9.0.0","backport:prev-major","Team:Security-Scalability"],"title":"[Automatic
Import] Remove fields with @ from the script
processor","number":201548,"url":"https://github.com/elastic/kibana/pull/201548","mergeCommit":{"message":"[Automatic
Import] Remove fields with @ from the script processor (#201548)\n\n##
Summary\r\n\r\nThis PR filters the fields containing `@` in date type
from `script`\r\nprocessor.\r\n\r\n## Before this
PR\r\n\r\n\r\n![image](https://github.com/user-attachments/assets/a733d81f-aaaf-4787-b974-1e5d35ff4b8f)\r\n\r\n```json\r\n
{\r\n \"script\": {\r\n \"tag\": \"script_convert_array_to_string\",\r\n
\"description\": \"Ensures the date processor does not receive an array
value.\",\r\n \"lang\": \"painless\",\r\n \"source\": \"if
(ctx.varonis?.varonis_alerts?.@timestamp != null &&\\n
ctx.varonis.varonis_alerts.@timestamp instanceof ArrayList){\\n
ctx.varonis.varonis_alerts.@timestamp =
ctx.varonis.varonis_alerts.@timestamp[0];\\n}\\n\"\r\n }\r\n },\r\n
{\r\n \"date\": {\r\n \"if\": \"ctx.varonis?.varonis_alerts?.@timestamp
!= null\",\r\n \"tag\":
\"date_processor_varonis.varonis_alerts.@timestamp\",\r\n \"field\":
\"varonis.varonis_alerts.@timestamp\",\r\n \"target_field\":
\"event.start\",\r\n \"formats\": [\r\n
\"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\",\r\n \"ISO8601\"\r\n ]\r\n }\r\n
},\r\n```\r\n\r\n## After this PR\r\n\r\n```json\r\n \"date\": {\r\n
\"if\": \"ctx.varonis?.varonis_alerts?.@timestamp != null\",\r\n
\"tag\": \"date_processor_varonis.varonis_alerts.@timestamp\",\r\n
\"field\": \"varonis.varonis_alerts.@timestamp\",\r\n \"target_field\":
\"event.start\",\r\n \"formats\": [\r\n
\"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\",\r\n \"ISO8601\"\r\n ]\r\n }\r\n
},\r\n```\r\n\r\n### Checklist\r\n\r\nCheck the PR satisfies following
conditions. \r\n\r\nReviewers should verify this PR satisfies this list
as well.\r\n\r\n- [x] [Unit or
functional\r\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\r\nwere
updated or added to match the most common
scenarios","sha":"8964dc92c774d9ac5c82a411022ece3fb91e3cfd"}},"sourceBranch":"main","suggestedTargetBranches":[],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","branchLabelMappingKey":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/201548","number":201548,"mergeCommit":{"message":"[Automatic
Import] Remove fields with @ from the script processor (#201548)\n\n##
Summary\r\n\r\nThis PR filters the fields containing `@` in date type
from `script`\r\nprocessor.\r\n\r\n## Before this
PR\r\n\r\n\r\n![image](https://github.com/user-attachments/assets/a733d81f-aaaf-4787-b974-1e5d35ff4b8f)\r\n\r\n```json\r\n
{\r\n \"script\": {\r\n \"tag\": \"script_convert_array_to_string\",\r\n
\"description\": \"Ensures the date processor does not receive an array
value.\",\r\n \"lang\": \"painless\",\r\n \"source\": \"if
(ctx.varonis?.varonis_alerts?.@timestamp != null &&\\n
ctx.varonis.varonis_alerts.@timestamp instanceof ArrayList){\\n
ctx.varonis.varonis_alerts.@timestamp =
ctx.varonis.varonis_alerts.@timestamp[0];\\n}\\n\"\r\n }\r\n },\r\n
{\r\n \"date\": {\r\n \"if\": \"ctx.varonis?.varonis_alerts?.@timestamp
!= null\",\r\n \"tag\":
\"date_processor_varonis.varonis_alerts.@timestamp\",\r\n \"field\":
\"varonis.varonis_alerts.@timestamp\",\r\n \"target_field\":
\"event.start\",\r\n \"formats\": [\r\n
\"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\",\r\n \"ISO8601\"\r\n ]\r\n }\r\n
},\r\n```\r\n\r\n## After this PR\r\n\r\n```json\r\n \"date\": {\r\n
\"if\": \"ctx.varonis?.varonis_alerts?.@timestamp != null\",\r\n
\"tag\": \"date_processor_varonis.varonis_alerts.@timestamp\",\r\n
\"field\": \"varonis.varonis_alerts.@timestamp\",\r\n \"target_field\":
\"event.start\",\r\n \"formats\": [\r\n
\"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\",\r\n \"ISO8601\"\r\n ]\r\n }\r\n
},\r\n```\r\n\r\n### Checklist\r\n\r\nCheck the PR satisfies following
conditions. \r\n\r\nReviewers should verify this PR satisfies this list
as well.\r\n\r\n- [x] [Unit or
functional\r\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\r\nwere
updated or added to match the most common
scenarios","sha":"8964dc92c774d9ac5c82a411022ece3fb91e3cfd"}}]}]
BACKPORT-->

Co-authored-by: Bharat Pasupula <123897612+bhapas@users.noreply.github.com>
---
 .../__jest__/fixtures/ecs_mapping.ts          | 381 ++++++++++++++++++
 .../server/graphs/ecs/pipeline.test.ts        | 306 ++++++++++++++
 .../server/graphs/ecs/pipeline.ts             |   3 +
 .../server/templates/pipeline.yml.njk         |   3 +-
 4 files changed, 692 insertions(+), 1 deletion(-)
 create mode 100644 x-pack/plugins/integration_assistant/server/graphs/ecs/pipeline.test.ts

diff --git a/x-pack/plugins/integration_assistant/__jest__/fixtures/ecs_mapping.ts b/x-pack/plugins/integration_assistant/__jest__/fixtures/ecs_mapping.ts
index 67e6c7d8f6a5e..8b905ccd64f96 100644
--- a/x-pack/plugins/integration_assistant/__jest__/fixtures/ecs_mapping.ts
+++ b/x-pack/plugins/integration_assistant/__jest__/fixtures/ecs_mapping.ts
@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { EcsMappingState } from '../../server/types';
 import { SamplesFormatName } from '../../common';
 
 export const ecsMappingExpectedResults = {
@@ -480,3 +481,383 @@ export const ecsTestState = {
   combinedSamples: '{"test1": "test1"}',
   additionalProcessors: [],
 };
+
+export const ecsPipelineState: EcsMappingState = {
+  lastExecutedChain: 'validateMappings',
+  rawSamples: [],
+  additionalProcessors: [],
+  prefixedSamples: [
+    '{"xdfsfs":{"ds":{"ei":0,"event":"cert.create","uid":"efd326fc-dd13-4df8-erre-3102c2d717d3","code":"TC000I","time":"2024-02-24T06:56:50.648137154Z","cluster_name":"teleport.ericbeahan.com","cert_type":"user","identity":{"user":"teleport-admin","roles":["access","editor"],"logins":["root","ubuntu","ec2-user","-teleport-internal-join"],"expires":"2024-02-24T06:56:50.648137154Z","route_to_cluster":"teleport.ericbeahan.com","traits":{"aws_role_arns":null,"azure_identities":null,"db_names":null,"db_roles":null,"db_users":null,"gcp_service_accounts":null,"host_user_gid":[""],"host_user_uid":[""],"kubernetes_groups":null,"kubernetes_users":null,"logins":["root","ubuntu","ec2-user"],"windows_logins":null},"teleport_cluster":"teleport.ericbeahan.com","client_ip":"1.2.3.4","prev_identity_expires":"0001-01-01T00:00:00Z","private_key_policy":"none"}}}}',
+    '{"xdfsfs":{"ds":{"ei":0,"event":"session.start","uid":"fff30583-13be-49e8-b159-32952c6ea34f","code":"T2000I","time":"2024-02-23T18:56:57.648137154Z","cluster_name":"teleport.ericbeahan.com","user":"teleport-admin","login":"ec2-user","user_kind":1,"sid":"293fda2d-2266-4d4d-b9d1-bd5ea9dd9fc3","private_key_policy":"none","namespace":"default","server_id":"face0091-2bf1-54er-a16a-f1514b4119f4","server_hostname":"ip-172-31-8-163.us-east-2.compute.internal","server_labels":{"hostname":"ip-172-31-8-163.us-east-2.compute.internal","teleport.internal/resource-id":"dccb2999-9fb8-4169-aded-ec7a1c0a26de"},"addr.remote":"1.2.3.4:50339","proto":"ssh","size":"80:25","initial_command":[""],"session_recording":"node"}}}',
+  ],
+  combinedSamples:
+    '{\n  "xdfsfs": {\n    "ds": {\n      "identity": {\n        "client_ip": "1.2.3.4",\n        "prev_identity_expires": "0001-01-01T00:00:00Z",\n        "private_key_policy": "none"\n      },\n      "user": "teleport-admin",\n      "login": "ec2-user",\n      "user_kind": 1,\n      "sid": "293fda2d-2266-4d4d-b9d1-bd5ea9dd9fc3",\n      "private_key_policy": "none",\n      "namespace": "default",\n      "server_id": "face0091-2bf1-43fd-a16a-f1514b4119f4",\n      "server_hostname": "ip-172-31-8-163.us-east-2.compute.internal",\n      "server_labels": {\n        "hostname": "ip-172-31-8-163.us-east-2.compute.internal",\n        "teleport.internal/resource-id": "dccb2999-9fb8-4169-aded-ec7a1c0a26de"\n      },\n      "addr.remote": "1.2.3.4:50339",\n      "proto": "ssh",\n      "size": "80:25",\n      "initial_command": [\n        ""\n      ],\n      "session_recording": "node"\n    }\n  }\n}',
+  sampleChunks: [],
+  exAnswer:
+    '{\n  "crowdstrike": {\n    "falcon": {\n      "metadata": {\n        "customerIDString": null,\n        "offset": null,\n        "eventType": {\n          "target": "event.code",\n          "confidence": 0.94,\n          "type": "string",\n          "date_formats": []\n        },\n        "eventCreationTime": {\n          "target": "event.created",\n          "confidence": 0.85,\n          "type": "date",\n          "date_formats": [\n            "UNIX"\n          ]\n        },\n        "version": null,\n        "event": {\n          "DeviceId": null,\n          "CustomerId": null,\n          "Ipv": {\n            "target": "network.type",\n            "confidence": 0.99,\n            "type": "string",\n            "date_formats": []\n          }\n        }\n      }\n    }\n  }\n}',
+  packageName: 'xdfsfs',
+  dataStreamName: 'ds',
+  finalized: false,
+  currentMapping: {
+    xdfsfs: {
+      ds: {
+        identity: {
+          client_ip: {
+            target: 'client.ip',
+            confidence: 0.95,
+            type: 'string',
+            date_formats: [],
+          },
+          prev_identity_expires: {
+            target: 'event.end',
+            confidence: 0.7,
+            type: 'date',
+            date_formats: ["yyyy-MM-dd'T'HH:mm:ss'Z'"],
+          },
+          private_key_policy: null,
+        },
+        user: {
+          target: 'user.name',
+          confidence: 0.9,
+          type: 'string',
+          date_formats: [],
+        },
+        login: {
+          target: 'user.id',
+          confidence: 0.8,
+          type: 'string',
+          date_formats: [],
+        },
+        user_kind: null,
+        sid: {
+          target: 'event.id',
+          confidence: 0.85,
+          type: 'string',
+          date_formats: [],
+        },
+        private_key_policy: null,
+        namespace: null,
+        server_id: {
+          target: 'host.id',
+          confidence: 0.9,
+          type: 'string',
+          date_formats: [],
+        },
+        server_hostname: {
+          target: 'host.hostname',
+          confidence: 0.95,
+          type: 'string',
+          date_formats: [],
+        },
+        server_labels: {
+          hostname: null,
+          'teleport.internal/resource-id': null,
+        },
+        'addr.remote': {
+          target: 'source.address',
+          confidence: 0.9,
+          type: 'string',
+          date_formats: [],
+        },
+        proto: {
+          target: 'network.protocol',
+          confidence: 0.95,
+          type: 'string',
+          date_formats: [],
+        },
+        size: null,
+        initial_command: null,
+        session_recording: null,
+      },
+    },
+  },
+  chunkMapping: {
+    xdfsfs: {
+      ds: {
+        ei: null,
+        event: {
+          target: 'event.action',
+          confidence: 0.9,
+          type: 'string',
+          date_formats: [],
+        },
+        uid: {
+          target: 'event.id',
+          confidence: 0.95,
+          type: 'string',
+          date_formats: [],
+        },
+        code: {
+          target: 'event.code',
+          confidence: 0.9,
+          type: 'string',
+          date_formats: [],
+        },
+        time: {
+          target: 'event.created',
+          confidence: 0.95,
+          type: 'date',
+          date_formats: ["yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSS'Z'"],
+        },
+        cluster_name: {
+          target: 'cloud.account.name',
+          confidence: 0.8,
+          type: 'string',
+          date_formats: [],
+        },
+        cert_type: null,
+        identity: {
+          user: {
+            target: 'user.name',
+            confidence: 0.95,
+            type: 'string',
+            date_formats: [],
+          },
+          roles: {
+            target: 'user.roles',
+            confidence: 0.9,
+            type: 'string',
+            date_formats: [],
+          },
+          logins: null,
+          expires: {
+            target: 'user.changes.name',
+            confidence: 0.7,
+            type: 'date',
+            date_formats: ["yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSS'Z'"],
+          },
+          route_to_cluster: null,
+          traits: {
+            aws_role_arns: null,
+            azure_identities: null,
+            db_names: null,
+            db_roles: null,
+            db_users: null,
+            gcp_service_accounts: null,
+            host_user_gid: null,
+            host_user_uid: null,
+            kubernetes_groups: null,
+            kubernetes_users: null,
+            logins: null,
+            windows_logins: null,
+          },
+          teleport_cluster: null,
+          client_ip: {
+            target: 'client.ip',
+            confidence: 0.95,
+            type: 'string',
+            date_formats: [],
+          },
+          prev_identity_expires: {
+            target: 'event.end',
+            confidence: 0.7,
+            type: 'date',
+            date_formats: ["yyyy-MM-dd'T'HH:mm:ss'Z'"],
+          },
+          private_key_policy: null,
+        },
+        user: {
+          target: 'user.name',
+          confidence: 0.9,
+          type: 'string',
+          date_formats: [],
+        },
+        login: {
+          target: 'user.id',
+          confidence: 0.8,
+          type: 'string',
+          date_formats: [],
+        },
+        user_kind: null,
+        sid: {
+          target: 'event.id',
+          confidence: 0.85,
+          type: 'string',
+          date_formats: [],
+        },
+        private_key_policy: null,
+        namespace: null,
+        server_id: {
+          target: 'host.id',
+          confidence: 0.9,
+          type: 'string',
+          date_formats: [],
+        },
+        server_hostname: {
+          target: 'host.hostname',
+          confidence: 0.95,
+          type: 'string',
+          date_formats: [],
+        },
+        server_labels: {
+          hostname: null,
+          'teleport.internal/resource-id': null,
+        },
+        'addr.remote': {
+          target: 'source.address',
+          confidence: 0.9,
+          type: 'string',
+          date_formats: [],
+        },
+        proto: {
+          target: 'network.protocol',
+          confidence: 0.95,
+          type: 'string',
+          date_formats: [],
+        },
+        size: null,
+        initial_command: null,
+        session_recording: null,
+      },
+    },
+  },
+  finalMapping: {
+    xdfsfs: {
+      ds: {
+        ei: null,
+        event: {
+          target: 'event.action',
+          confidence: 0.9,
+          type: 'string',
+          date_formats: [],
+        },
+        uid: {
+          target: 'event.id',
+          confidence: 0.95,
+          type: 'string',
+          date_formats: [],
+        },
+        code: {
+          target: 'event.code',
+          confidence: 0.9,
+          type: 'string',
+          date_formats: [],
+        },
+        '@timestamp': {
+          target: '@timestamp',
+          confidence: 0.95,
+          type: 'date',
+          date_formats: ["yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSS'Z'"],
+        },
+        cluster_name: {
+          target: 'cloud.account.name',
+          confidence: 0.8,
+          type: 'string',
+          date_formats: [],
+        },
+        cert_type: null,
+        identity: {
+          user: {
+            target: 'user.name',
+            confidence: 0.95,
+            type: 'string',
+            date_formats: [],
+          },
+          roles: {
+            target: 'user.roles',
+            confidence: 0.9,
+            type: 'string',
+            date_formats: [],
+          },
+          logins: null,
+          expires: {
+            target: 'user.changes.name',
+            confidence: 0.7,
+            type: 'date',
+            date_formats: ["yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSS'Z'"],
+          },
+          route_to_cluster: null,
+          traits: {
+            aws_role_arns: null,
+            azure_identities: null,
+            db_names: null,
+            db_roles: null,
+            db_users: null,
+            gcp_service_accounts: null,
+            host_user_gid: null,
+            host_user_uid: null,
+            kubernetes_groups: null,
+            kubernetes_users: null,
+            logins: null,
+            windows_logins: null,
+          },
+          teleport_cluster: null,
+          client_ip: {
+            target: 'client.ip',
+            confidence: 0.95,
+            type: 'string',
+            date_formats: [],
+          },
+          prev_identity_expires: {
+            target: 'event.end',
+            confidence: 0.7,
+            type: 'date',
+            date_formats: ["yyyy-MM-dd'T'HH:mm:ss'Z'"],
+          },
+          private_key_policy: null,
+        },
+        user: {
+          target: 'user.name',
+          confidence: 0.9,
+          type: 'string',
+          date_formats: [],
+        },
+        login: {
+          target: 'user.id',
+          confidence: 0.8,
+          type: 'string',
+          date_formats: [],
+        },
+        user_kind: null,
+        sid: null,
+        private_key_policy: null,
+        namespace: null,
+        server_id: {
+          target: 'host.id',
+          confidence: 0.9,
+          type: 'string',
+          date_formats: [],
+        },
+        server_hostname: {
+          target: 'host.hostname',
+          confidence: 0.95,
+          type: 'string',
+          date_formats: [],
+        },
+        server_labels: {
+          hostname: null,
+          'teleport.internal/resource-id': null,
+        },
+        'addr.remote': {
+          target: 'source.address',
+          confidence: 0.9,
+          type: 'string',
+          date_formats: [],
+        },
+        proto: {
+          target: 'network.protocol',
+          confidence: 0.95,
+          type: 'string',
+          date_formats: [],
+        },
+        size: null,
+        initial_command: null,
+        session_recording: null,
+      },
+    },
+  },
+  useFinalMapping: true,
+  hasTriedOnce: true,
+  currentPipeline: {},
+  duplicateFields: [],
+  missingKeys: [],
+  invalidEcsFields: [],
+  results: {},
+  samplesFormat: {
+    name: 'json',
+    json_path: [],
+  },
+  ecsVersion: '8.11.0',
+  ecs: '',
+  chunkSize: 0,
+};
diff --git a/x-pack/plugins/integration_assistant/server/graphs/ecs/pipeline.test.ts b/x-pack/plugins/integration_assistant/server/graphs/ecs/pipeline.test.ts
new file mode 100644
index 0000000000000..3dda9208c3094
--- /dev/null
+++ b/x-pack/plugins/integration_assistant/server/graphs/ecs/pipeline.test.ts
@@ -0,0 +1,306 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ecsPipelineState } from '../../../__jest__/fixtures/ecs_mapping';
+import type { EcsMappingState } from '../../types';
+import { createPipeline } from './pipeline';
+
+const state: EcsMappingState = ecsPipelineState;
+
+describe('Testing pipeline templates', () => {
+  it('handle pipeline creation', async () => {
+    const pipeline = createPipeline(state);
+    expect(pipeline.processors).toEqual([
+      {
+        set: { field: 'ecs.version', tag: 'set_ecs_version', value: '8.11.0' },
+      },
+      {
+        set: {
+          field: 'originalMessage',
+          copy_from: 'message',
+          tag: 'copy_original_message',
+        },
+      },
+      {
+        rename: {
+          field: 'originalMessage',
+          target_field: 'event.original',
+          tag: 'rename_message',
+          ignore_missing: true,
+          if: 'ctx.event?.original == null',
+        },
+      },
+      {
+        remove: {
+          field: 'originalMessage',
+          ignore_missing: true,
+          tag: 'remove_copied_message',
+          if: 'ctx.event?.original != null',
+        },
+      },
+      {
+        remove: { field: 'message', ignore_missing: true, tag: 'remove_message' },
+      },
+      {
+        json: {
+          field: 'event.original',
+          tag: 'json_original',
+          target_field: 'xdfsfs.ds',
+        },
+      },
+      {
+        rename: {
+          field: 'xdfsfs.ds.event',
+          target_field: 'event.action',
+          ignore_missing: true,
+        },
+      },
+      {
+        rename: {
+          field: 'xdfsfs.ds.uid',
+          target_field: 'event.id',
+          ignore_missing: true,
+        },
+      },
+      {
+        rename: {
+          field: 'xdfsfs.ds.code',
+          target_field: 'event.code',
+          ignore_missing: true,
+        },
+      },
+      {
+        date: {
+          field: 'xdfsfs.ds.@timestamp',
+          target_field: '@timestamp',
+          formats: ["yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSS'Z'", 'ISO8601'],
+          tag: 'date_processor_xdfsfs.ds.@timestamp',
+          if: 'ctx.xdfsfs?.ds?.@timestamp != null',
+        },
+      },
+      {
+        rename: {
+          field: 'xdfsfs.ds.cluster_name',
+          target_field: 'cloud.account.name',
+          ignore_missing: true,
+        },
+      },
+      {
+        rename: {
+          field: 'xdfsfs.ds.identity.user',
+          target_field: 'user.name',
+          ignore_missing: true,
+        },
+      },
+      {
+        rename: {
+          field: 'xdfsfs.ds.identity.roles',
+          target_field: 'user.roles',
+          ignore_missing: true,
+        },
+      },
+      {
+        script: {
+          description: 'Ensures the date processor does not receive an array value.',
+          tag: 'script_convert_array_to_string',
+          lang: 'painless',
+          source:
+            'if (ctx.xdfsfs?.ds?.identity?.expires != null &&\n' +
+            '    ctx.xdfsfs.ds.identity.expires instanceof ArrayList){\n' +
+            '    ctx.xdfsfs.ds.identity.expires = ctx.xdfsfs.ds.identity.expires[0];\n' +
+            '}\n',
+        },
+      },
+      {
+        date: {
+          field: 'xdfsfs.ds.identity.expires',
+          target_field: 'user.changes.name',
+          formats: ["yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSS'Z'", 'ISO8601'],
+          tag: 'date_processor_xdfsfs.ds.identity.expires',
+          if: 'ctx.xdfsfs?.ds?.identity?.expires != null',
+        },
+      },
+      {
+        convert: {
+          field: 'xdfsfs.ds.identity.client_ip',
+          target_field: 'client.ip',
+          ignore_missing: true,
+          ignore_failure: true,
+          type: 'ip',
+        },
+      },
+      {
+        script: {
+          description: 'Ensures the date processor does not receive an array value.',
+          tag: 'script_convert_array_to_string',
+          lang: 'painless',
+          source:
+            'if (ctx.xdfsfs?.ds?.identity?.prev_identity_expires != null &&\n' +
+            '    ctx.xdfsfs.ds.identity.prev_identity_expires instanceof ArrayList){\n' +
+            '    ctx.xdfsfs.ds.identity.prev_identity_expires = ctx.xdfsfs.ds.identity.prev_identity_expires[0];\n' +
+            '}\n',
+        },
+      },
+      {
+        date: {
+          field: 'xdfsfs.ds.identity.prev_identity_expires',
+          target_field: 'event.end',
+          formats: ["yyyy-MM-dd'T'HH:mm:ss'Z'", 'ISO8601'],
+          tag: 'date_processor_xdfsfs.ds.identity.prev_identity_expires',
+          if: 'ctx.xdfsfs?.ds?.identity?.prev_identity_expires != null',
+        },
+      },
+      {
+        rename: {
+          field: 'xdfsfs.ds.user',
+          target_field: 'user.name',
+          ignore_missing: true,
+        },
+      },
+      {
+        rename: {
+          field: 'xdfsfs.ds.login',
+          target_field: 'user.id',
+          ignore_missing: true,
+        },
+      },
+      {
+        rename: {
+          field: 'xdfsfs.ds.server_id',
+          target_field: 'host.id',
+          ignore_missing: true,
+        },
+      },
+      {
+        rename: {
+          field: 'xdfsfs.ds.server_hostname',
+          target_field: 'host.hostname',
+          ignore_missing: true,
+        },
+      },
+      {
+        rename: {
+          field: 'xdfsfs.ds.addr.remote',
+          target_field: 'source.address',
+          ignore_missing: true,
+        },
+      },
+      {
+        rename: {
+          field: 'xdfsfs.ds.proto',
+          target_field: 'network.protocol',
+          ignore_missing: true,
+        },
+      },
+      {
+        script: {
+          description: 'Drops null/empty values recursively.',
+          tag: 'script_drop_null_empty_values',
+          lang: 'painless',
+          source:
+            'boolean dropEmptyFields(Object object) {\n' +
+            '  if (object == null || object == "") {\n' +
+            '    return true;\n' +
+            '  } else if (object instanceof Map) {\n' +
+            '    ((Map) object).values().removeIf(value -> dropEmptyFields(value));\n' +
+            '    return (((Map) object).size() == 0);\n' +
+            '  } else if (object instanceof List) {\n' +
+            '    ((List) object).removeIf(value -> dropEmptyFields(value));\n' +
+            '    return (((List) object).length == 0);\n' +
+            '  }\n' +
+            '  return false;\n' +
+            '}\n' +
+            'dropEmptyFields(ctx);\n',
+        },
+      },
+      {
+        geoip: {
+          field: 'source.ip',
+          tag: 'geoip_source_ip',
+          target_field: 'source.geo',
+          ignore_missing: true,
+        },
+      },
+      {
+        geoip: {
+          ignore_missing: true,
+          database_file: 'GeoLite2-ASN.mmdb',
+          field: 'source.ip',
+          tag: 'geoip_source_asn',
+          target_field: 'source.as',
+          properties: ['asn', 'organization_name'],
+        },
+      },
+      {
+        rename: {
+          field: 'source.as.asn',
+          tag: 'rename_source_as_asn',
+          target_field: 'source.as.number',
+          ignore_missing: true,
+        },
+      },
+      {
+        rename: {
+          field: 'source.as.organization_name',
+          tag: 'rename_source_as_organization_name',
+          target_field: 'source.as.organization.name',
+          ignore_missing: true,
+        },
+      },
+      {
+        geoip: {
+          field: 'destination.ip',
+          tag: 'geoip_destination_ip',
+          target_field: 'destination.geo',
+          ignore_missing: true,
+        },
+      },
+      {
+        geoip: {
+          database_file: 'GeoLite2-ASN.mmdb',
+          field: 'destination.ip',
+          tag: 'geoip_destination_asn',
+          target_field: 'destination.as',
+          properties: ['asn', 'organization_name'],
+          ignore_missing: true,
+        },
+      },
+      {
+        rename: {
+          field: 'destination.as.asn',
+          tag: 'rename_destination_as_asn',
+          target_field: 'destination.as.number',
+          ignore_missing: true,
+        },
+      },
+      {
+        rename: {
+          field: 'destination.as.organization_name',
+          tag: 'rename_destination_as_organization_name',
+          target_field: 'destination.as.organization.name',
+          ignore_missing: true,
+        },
+      },
+      {
+        remove: {
+          field: ['xdfsfs.ds.identity.client_ip'],
+          ignore_missing: true,
+          tag: 'remove_fields',
+        },
+      },
+      {
+        remove: {
+          field: 'event.original',
+          tag: 'remove_original_event',
+          if: 'ctx?.tags == null || !(ctx.tags.contains("preserve_original_event"))',
+          ignore_failure: true,
+          ignore_missing: true,
+        },
+      },
+    ]);
+  });
+});
diff --git a/x-pack/plugins/integration_assistant/server/graphs/ecs/pipeline.ts b/x-pack/plugins/integration_assistant/server/graphs/ecs/pipeline.ts
index e2d91c094d520..81f7fbcd4507e 100644
--- a/x-pack/plugins/integration_assistant/server/graphs/ecs/pipeline.ts
+++ b/x-pack/plugins/integration_assistant/server/graphs/ecs/pipeline.ts
@@ -186,6 +186,9 @@ export function createPipeline(state: EcsMappingState): IngestPipeline {
   const env = new Environment(new FileSystemLoader(templatesPath), {
     autoescape: false,
   });
+  env.addFilter('includes', function (str, substr) {
+    return str.includes(substr);
+  });
   env.addFilter('startswith', function (str, prefix) {
     return str.startsWith(prefix);
   });
diff --git a/x-pack/plugins/integration_assistant/server/templates/pipeline.yml.njk b/x-pack/plugins/integration_assistant/server/templates/pipeline.yml.njk
index ba846dc50fba9..116d5cc66719f 100644
--- a/x-pack/plugins/integration_assistant/server/templates/pipeline.yml.njk
+++ b/x-pack/plugins/integration_assistant/server/templates/pipeline.yml.njk
@@ -35,6 +35,7 @@ processors:
       target_field: {% if value.target_field | startswith('@') %}"{{ value.target_field }}"{% else %}{{ value.target_field }}{% endif %}
       ignore_missing: true{% endif %}
   {% if key == 'date' %}
+  {% if not value.field | includes('.@') %} {# Leaving fields of type 'test.log.@timestamp' #}
   - script:
       description: Ensures the date processor does not receive an array value.
       tag: script_convert_array_to_string
@@ -43,7 +44,7 @@ processors:
         if (ctx.{% endraw %}{{ value.field.replaceAll('.', '?.') }}{% raw %} != null &&
             ctx.{% endraw %}{{ value.field }}{% raw %} instanceof ArrayList){
             ctx.{% endraw %}{{ value.field }}{% raw %} = ctx.{% endraw %}{{ value.field }}{% raw %}[0];
-        }{% endraw %}
+        }{% endraw %}{% endif %}
   - {{ key }}:
       field: {{ value.field }}
       target_field: {% if value.target_field | startswith('@') %}"{{ value.target_field }}"{% else %}{{ value.target_field }}{% endif %}