Skip to content

Commit

Permalink
[8.x] [Automatic Import] Remove fields with @ from the script process…
Browse files Browse the repository at this point in the history 
…or (#201548) (#201589)

# Backport

This will backport the following commits from `main` to `8.x`:
- [[Automatic Import] Remove fields with @ from the script processor
(#201548)](#201548)

<!--- Backport version: 9.4.3 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Bharat
Pasupula","email":"[email protected]"},"sourceCommit":{"committedDate":"2024-11-25T13:00:14Z","message":"[Automatic
Import] Remove fields with @ from the script processor (#201548)\n\n##
Summary\r\n\r\nThis PR filters the fields containing `@` in date type
from `script`\r\nprocessor.\r\n\r\n## Before this
PR\r\n\r\n\r\n![image](https://github.com/user-attachments/assets/a733d81f-aaaf-4787-b974-1e5d35ff4b8f)\r\n\r\n```json\r\n
{\r\n \"script\": {\r\n \"tag\": \"script_convert_array_to_string\",\r\n
\"description\": \"Ensures the date processor does not receive an array
value.\",\r\n \"lang\": \"painless\",\r\n \"source\": \"if
(ctx.varonis?.varonis_alerts?.@timestamp != null &&\\n
ctx.varonis.varonis_alerts.@timestamp instanceof ArrayList){\\n
ctx.varonis.varonis_alerts.@timestamp =
ctx.varonis.varonis_alerts.@timestamp[0];\\n}\\n\"\r\n }\r\n },\r\n
{\r\n \"date\": {\r\n \"if\": \"ctx.varonis?.varonis_alerts?.@timestamp
!= null\",\r\n \"tag\":
\"date_processor_varonis.varonis_alerts.@timestamp\",\r\n \"field\":
\"varonis.varonis_alerts.@timestamp\",\r\n \"target_field\":
\"event.start\",\r\n \"formats\": [\r\n
\"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\",\r\n \"ISO8601\"\r\n ]\r\n }\r\n
},\r\n```\r\n\r\n## After this PR\r\n\r\n```json\r\n \"date\": {\r\n
\"if\": \"ctx.varonis?.varonis_alerts?.@timestamp != null\",\r\n
\"tag\": \"date_processor_varonis.varonis_alerts.@timestamp\",\r\n
\"field\": \"varonis.varonis_alerts.@timestamp\",\r\n \"target_field\":
\"event.start\",\r\n \"formats\": [\r\n
\"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\",\r\n \"ISO8601\"\r\n ]\r\n }\r\n
},\r\n```\r\n\r\n### Checklist\r\n\r\nCheck the PR satisfies following
conditions. \r\n\r\nReviewers should verify this PR satisfies this list
as well.\r\n\r\n- [x] [Unit or
functional\r\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\r\nwere
updated or added to match the most common
scenarios","sha":"8964dc92c774d9ac5c82a411022ece3fb91e3cfd","branchLabelMapping":{"^v9.0.0$":"main","^v8.18.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["bug","release_note:fix","v9.0.0","backport:prev-major","Team:Security-Scalability"],"title":"[Automatic
Import] Remove fields with @ from the script
processor","number":201548,"url":"https://github.com/elastic/kibana/pull/201548","mergeCommit":{"message":"[Automatic
Import] Remove fields with @ from the script processor (#201548)\n\n##
Summary\r\n\r\nThis PR filters the fields containing `@` in date type
from `script`\r\nprocessor.\r\n\r\n## Before this
PR\r\n\r\n\r\n![image](https://github.com/user-attachments/assets/a733d81f-aaaf-4787-b974-1e5d35ff4b8f)\r\n\r\n```json\r\n
{\r\n \"script\": {\r\n \"tag\": \"script_convert_array_to_string\",\r\n
\"description\": \"Ensures the date processor does not receive an array
value.\",\r\n \"lang\": \"painless\",\r\n \"source\": \"if
(ctx.varonis?.varonis_alerts?.@timestamp != null &&\\n
ctx.varonis.varonis_alerts.@timestamp instanceof ArrayList){\\n
ctx.varonis.varonis_alerts.@timestamp =
ctx.varonis.varonis_alerts.@timestamp[0];\\n}\\n\"\r\n }\r\n },\r\n
{\r\n \"date\": {\r\n \"if\": \"ctx.varonis?.varonis_alerts?.@timestamp
!= null\",\r\n \"tag\":
\"date_processor_varonis.varonis_alerts.@timestamp\",\r\n \"field\":
\"varonis.varonis_alerts.@timestamp\",\r\n \"target_field\":
\"event.start\",\r\n \"formats\": [\r\n
\"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\",\r\n \"ISO8601\"\r\n ]\r\n }\r\n
},\r\n```\r\n\r\n## After this PR\r\n\r\n```json\r\n \"date\": {\r\n
\"if\": \"ctx.varonis?.varonis_alerts?.@timestamp != null\",\r\n
\"tag\": \"date_processor_varonis.varonis_alerts.@timestamp\",\r\n
\"field\": \"varonis.varonis_alerts.@timestamp\",\r\n \"target_field\":
\"event.start\",\r\n \"formats\": [\r\n
\"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\",\r\n \"ISO8601\"\r\n ]\r\n }\r\n
},\r\n```\r\n\r\n### Checklist\r\n\r\nCheck the PR satisfies following
conditions. \r\n\r\nReviewers should verify this PR satisfies this list
as well.\r\n\r\n- [x] [Unit or
functional\r\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\r\nwere
updated or added to match the most common
scenarios","sha":"8964dc92c774d9ac5c82a411022ece3fb91e3cfd"}},"sourceBranch":"main","suggestedTargetBranches":[],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","branchLabelMappingKey":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/201548","number":201548,"mergeCommit":{"message":"[Automatic
Import] Remove fields with @ from the script processor (#201548)\n\n##
Summary\r\n\r\nThis PR filters the fields containing `@` in date type
from `script`\r\nprocessor.\r\n\r\n## Before this
PR\r\n\r\n\r\n![image](https://github.com/user-attachments/assets/a733d81f-aaaf-4787-b974-1e5d35ff4b8f)\r\n\r\n```json\r\n
{\r\n \"script\": {\r\n \"tag\": \"script_convert_array_to_string\",\r\n
\"description\": \"Ensures the date processor does not receive an array
value.\",\r\n \"lang\": \"painless\",\r\n \"source\": \"if
(ctx.varonis?.varonis_alerts?.@timestamp != null &&\\n
ctx.varonis.varonis_alerts.@timestamp instanceof ArrayList){\\n
ctx.varonis.varonis_alerts.@timestamp =
ctx.varonis.varonis_alerts.@timestamp[0];\\n}\\n\"\r\n }\r\n },\r\n
{\r\n \"date\": {\r\n \"if\": \"ctx.varonis?.varonis_alerts?.@timestamp
!= null\",\r\n \"tag\":
\"date_processor_varonis.varonis_alerts.@timestamp\",\r\n \"field\":
\"varonis.varonis_alerts.@timestamp\",\r\n \"target_field\":
\"event.start\",\r\n \"formats\": [\r\n
\"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\",\r\n \"ISO8601\"\r\n ]\r\n }\r\n
},\r\n```\r\n\r\n## After this PR\r\n\r\n```json\r\n \"date\": {\r\n
\"if\": \"ctx.varonis?.varonis_alerts?.@timestamp != null\",\r\n
\"tag\": \"date_processor_varonis.varonis_alerts.@timestamp\",\r\n
\"field\": \"varonis.varonis_alerts.@timestamp\",\r\n \"target_field\":
\"event.start\",\r\n \"formats\": [\r\n
\"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\",\r\n \"ISO8601\"\r\n ]\r\n }\r\n
},\r\n```\r\n\r\n### Checklist\r\n\r\nCheck the PR satisfies following
conditions. \r\n\r\nReviewers should verify this PR satisfies this list
as well.\r\n\r\n- [x] [Unit or
functional\r\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\r\nwere
updated or added to match the most common
scenarios","sha":"8964dc92c774d9ac5c82a411022ece3fb91e3cfd"}}]}]
BACKPORT-->

Co-authored-by: Bharat Pasupula <[email protected]>
  • Loading branch information
@kibanamachine @bhapas
kibanamachine and bhapas authored Nov 25, 2024
1 parent 8fa4fdc commit 6185384
Show file tree
Hide file tree
Showing 4 changed files with 692 additions and 1 deletion.
381 changes: 381 additions & 0 deletions x-pack/plugins/integration_assistant/__jest__/fixtures/ecs_mapping.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
* 2.0.
*/

import { EcsMappingState } from '../../server/types';
import { SamplesFormatName } from '../../common';

export const ecsMappingExpectedResults = {
Expand Down Expand Up @@ -480,3 +481,383 @@ export const ecsTestState = {
combinedSamples: '{"test1": "test1"}',
additionalProcessors: [],
};

export const ecsPipelineState: EcsMappingState = {
lastExecutedChain: 'validateMappings',
rawSamples: [],
additionalProcessors: [],
prefixedSamples: [
'{"xdfsfs":{"ds":{"ei":0,"event":"cert.create","uid":"efd326fc-dd13-4df8-erre-3102c2d717d3","code":"TC000I","time":"2024-02-24T06:56:50.648137154Z","cluster_name":"teleport.ericbeahan.com","cert_type":"user","identity":{"user":"teleport-admin","roles":["access","editor"],"logins":["root","ubuntu","ec2-user","-teleport-internal-join"],"expires":"2024-02-24T06:56:50.648137154Z","route_to_cluster":"teleport.ericbeahan.com","traits":{"aws_role_arns":null,"azure_identities":null,"db_names":null,"db_roles":null,"db_users":null,"gcp_service_accounts":null,"host_user_gid":[""],"host_user_uid":[""],"kubernetes_groups":null,"kubernetes_users":null,"logins":["root","ubuntu","ec2-user"],"windows_logins":null},"teleport_cluster":"teleport.ericbeahan.com","client_ip":"1.2.3.4","prev_identity_expires":"0001-01-01T00:00:00Z","private_key_policy":"none"}}}}',
'{"xdfsfs":{"ds":{"ei":0,"event":"session.start","uid":"fff30583-13be-49e8-b159-32952c6ea34f","code":"T2000I","time":"2024-02-23T18:56:57.648137154Z","cluster_name":"teleport.ericbeahan.com","user":"teleport-admin","login":"ec2-user","user_kind":1,"sid":"293fda2d-2266-4d4d-b9d1-bd5ea9dd9fc3","private_key_policy":"none","namespace":"default","server_id":"face0091-2bf1-54er-a16a-f1514b4119f4","server_hostname":"ip-172-31-8-163.us-east-2.compute.internal","server_labels":{"hostname":"ip-172-31-8-163.us-east-2.compute.internal","teleport.internal/resource-id":"dccb2999-9fb8-4169-aded-ec7a1c0a26de"},"addr.remote":"1.2.3.4:50339","proto":"ssh","size":"80:25","initial_command":[""],"session_recording":"node"}}}',
],
combinedSamples:
'{\n "xdfsfs": {\n "ds": {\n "identity": {\n "client_ip": "1.2.3.4",\n "prev_identity_expires": "0001-01-01T00:00:00Z",\n "private_key_policy": "none"\n },\n "user": "teleport-admin",\n "login": "ec2-user",\n "user_kind": 1,\n "sid": "293fda2d-2266-4d4d-b9d1-bd5ea9dd9fc3",\n "private_key_policy": "none",\n "namespace": "default",\n "server_id": "face0091-2bf1-43fd-a16a-f1514b4119f4",\n "server_hostname": "ip-172-31-8-163.us-east-2.compute.internal",\n "server_labels": {\n "hostname": "ip-172-31-8-163.us-east-2.compute.internal",\n "teleport.internal/resource-id": "dccb2999-9fb8-4169-aded-ec7a1c0a26de"\n },\n "addr.remote": "1.2.3.4:50339",\n "proto": "ssh",\n "size": "80:25",\n "initial_command": [\n ""\n ],\n "session_recording": "node"\n }\n }\n}',
sampleChunks: [],
exAnswer:
'{\n "crowdstrike": {\n "falcon": {\n "metadata": {\n "customerIDString": null,\n "offset": null,\n "eventType": {\n "target": "event.code",\n "confidence": 0.94,\n "type": "string",\n "date_formats": []\n },\n "eventCreationTime": {\n "target": "event.created",\n "confidence": 0.85,\n "type": "date",\n "date_formats": [\n "UNIX"\n ]\n },\n "version": null,\n "event": {\n "DeviceId": null,\n "CustomerId": null,\n "Ipv": {\n "target": "network.type",\n "confidence": 0.99,\n "type": "string",\n "date_formats": []\n }\n }\n }\n }\n }\n}',
packageName: 'xdfsfs',
dataStreamName: 'ds',
finalized: false,
currentMapping: {
xdfsfs: {
ds: {
identity: {
client_ip: {
target: 'client.ip',
confidence: 0.95,
type: 'string',
date_formats: [],
},
prev_identity_expires: {
target: 'event.end',
confidence: 0.7,
type: 'date',
date_formats: ["yyyy-MM-dd'T'HH:mm:ss'Z'"],
},
private_key_policy: null,
},
user: {
target: 'user.name',
confidence: 0.9,
type: 'string',
date_formats: [],
},
login: {
target: 'user.id',
confidence: 0.8,
type: 'string',
date_formats: [],
},
user_kind: null,
sid: {
target: 'event.id',
confidence: 0.85,
type: 'string',
date_formats: [],
},
private_key_policy: null,
namespace: null,
server_id: {
target: 'host.id',
confidence: 0.9,
type: 'string',
date_formats: [],
},
server_hostname: {
target: 'host.hostname',
confidence: 0.95,
type: 'string',
date_formats: [],
},
server_labels: {
hostname: null,
'teleport.internal/resource-id': null,
},
'addr.remote': {
target: 'source.address',
confidence: 0.9,
type: 'string',
date_formats: [],
},
proto: {
target: 'network.protocol',
confidence: 0.95,
type: 'string',
date_formats: [],
},
size: null,
initial_command: null,
session_recording: null,
},
},
},
chunkMapping: {
xdfsfs: {
ds: {
ei: null,
event: {
target: 'event.action',
confidence: 0.9,
type: 'string',
date_formats: [],
},
uid: {
target: 'event.id',
confidence: 0.95,
type: 'string',
date_formats: [],
},
code: {
target: 'event.code',
confidence: 0.9,
type: 'string',
date_formats: [],
},
time: {
target: 'event.created',
confidence: 0.95,
type: 'date',
date_formats: ["yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSS'Z'"],
},
cluster_name: {
target: 'cloud.account.name',
confidence: 0.8,
type: 'string',
date_formats: [],
},
cert_type: null,
identity: {
user: {
target: 'user.name',
confidence: 0.95,
type: 'string',
date_formats: [],
},
roles: {
target: 'user.roles',
confidence: 0.9,
type: 'string',
date_formats: [],
},
logins: null,
expires: {
target: 'user.changes.name',
confidence: 0.7,
type: 'date',
date_formats: ["yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSS'Z'"],
},
route_to_cluster: null,
traits: {
aws_role_arns: null,
azure_identities: null,
db_names: null,
db_roles: null,
db_users: null,
gcp_service_accounts: null,
host_user_gid: null,
host_user_uid: null,
kubernetes_groups: null,
kubernetes_users: null,
logins: null,
windows_logins: null,
},
teleport_cluster: null,
client_ip: {
target: 'client.ip',
confidence: 0.95,
type: 'string',
date_formats: [],
},
prev_identity_expires: {
target: 'event.end',
confidence: 0.7,
type: 'date',
date_formats: ["yyyy-MM-dd'T'HH:mm:ss'Z'"],
},
private_key_policy: null,
},
user: {
target: 'user.name',
confidence: 0.9,
type: 'string',
date_formats: [],
},
login: {
target: 'user.id',
confidence: 0.8,
type: 'string',
date_formats: [],
},
user_kind: null,
sid: {
target: 'event.id',
confidence: 0.85,
type: 'string',
date_formats: [],
},
private_key_policy: null,
namespace: null,
server_id: {
target: 'host.id',
confidence: 0.9,
type: 'string',
date_formats: [],
},
server_hostname: {
target: 'host.hostname',
confidence: 0.95,
type: 'string',
date_formats: [],
},
server_labels: {
hostname: null,
'teleport.internal/resource-id': null,
},
'addr.remote': {
target: 'source.address',
confidence: 0.9,
type: 'string',
date_formats: [],
},
proto: {
target: 'network.protocol',
confidence: 0.95,
type: 'string',
date_formats: [],
},
size: null,
initial_command: null,
session_recording: null,
},
},
},
finalMapping: {
xdfsfs: {
ds: {
ei: null,
event: {
target: 'event.action',
confidence: 0.9,
type: 'string',
date_formats: [],
},
uid: {
target: 'event.id',
confidence: 0.95,
type: 'string',
date_formats: [],
},
code: {
target: 'event.code',
confidence: 0.9,
type: 'string',
date_formats: [],
},
'@timestamp': {
target: '@timestamp',
confidence: 0.95,
type: 'date',
date_formats: ["yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSS'Z'"],
},
cluster_name: {
target: 'cloud.account.name',
confidence: 0.8,
type: 'string',
date_formats: [],
},
cert_type: null,
identity: {
user: {
target: 'user.name',
confidence: 0.95,
type: 'string',
date_formats: [],
},
roles: {
target: 'user.roles',
confidence: 0.9,
type: 'string',
date_formats: [],
},
logins: null,
expires: {
target: 'user.changes.name',
confidence: 0.7,
type: 'date',
date_formats: ["yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSS'Z'"],
},
route_to_cluster: null,
traits: {
aws_role_arns: null,
azure_identities: null,
db_names: null,
db_roles: null,
db_users: null,
gcp_service_accounts: null,
host_user_gid: null,
host_user_uid: null,
kubernetes_groups: null,
kubernetes_users: null,
logins: null,
windows_logins: null,
},
teleport_cluster: null,
client_ip: {
target: 'client.ip',
confidence: 0.95,
type: 'string',
date_formats: [],
},
prev_identity_expires: {
target: 'event.end',
confidence: 0.7,
type: 'date',
date_formats: ["yyyy-MM-dd'T'HH:mm:ss'Z'"],
},
private_key_policy: null,
},
user: {
target: 'user.name',
confidence: 0.9,
type: 'string',
date_formats: [],
},
login: {
target: 'user.id',
confidence: 0.8,
type: 'string',
date_formats: [],
},
user_kind: null,
sid: null,
private_key_policy: null,
namespace: null,
server_id: {
target: 'host.id',
confidence: 0.9,
type: 'string',
date_formats: [],
},
server_hostname: {
target: 'host.hostname',
confidence: 0.95,
type: 'string',
date_formats: [],
},
server_labels: {
hostname: null,
'teleport.internal/resource-id': null,
},
'addr.remote': {
target: 'source.address',
confidence: 0.9,
type: 'string',
date_formats: [],
},
proto: {
target: 'network.protocol',
confidence: 0.95,
type: 'string',
date_formats: [],
},
size: null,
initial_command: null,
session_recording: null,
},
},
},
useFinalMapping: true,
hasTriedOnce: true,
currentPipeline: {},
duplicateFields: [],
missingKeys: [],
invalidEcsFields: [],
results: {},
samplesFormat: {
name: 'json',
json_path: [],
},
ecsVersion: '8.11.0',
ecs: '',
chunkSize: 0,
};
Loading

0 comments on commit 6185384

Please sign in to comment.