From ae7c839a8543ed336e8e94ddd26c54ab5e888484 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Tue, 19 Mar 2019 16:10:22 +0100 Subject: [PATCH 1/5] wong assumption? --- .../actions/decode_json_fields_test.go | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/libbeat/processors/actions/decode_json_fields_test.go b/libbeat/processors/actions/decode_json_fields_test.go index 15afb944988..2eae376b229 100644 --- a/libbeat/processors/actions/decode_json_fields_test.go +++ b/libbeat/processors/actions/decode_json_fields_test.go @@ -199,6 +199,53 @@ func TestTargetRootOption(t *testing.T) { assert.Equal(t, expected.String(), actual.String()) } +func TestArrayWithArraysDisabled(t *testing.T) { + input := common.MapStr{ + "msg": `{ + "arrayOfMap": "[{\"a\":\"b\"}]" + }`, + } + + testConfig, _ = common.NewConfigFrom(map[string]interface{}{ + "fields": fields, + "max_depth": 10, + "process_array": false, + }) + + actual := getActualValue(t, testConfig, input) + + expected := common.MapStr{ + "msg": common.MapStr{ + "arrayOfMap": "[{\"a\":\"b\"}]", + }, + } + + assert.Equal(t, expected.String(), actual.String()) +} +func TestArrayWithArraysEnabled(t *testing.T) { + input := common.MapStr{ + "msg": `{ + "arrayOfMap": "[{\"a\":\"b\"}]" + }`, + } + + testConfig, _ = common.NewConfigFrom(map[string]interface{}{ + "fields": fields, + "max_depth": 10, + "process_array": true, + }) + + actual := getActualValue(t, testConfig, input) + + expected := common.MapStr{ + "msg": common.MapStr{ + "arrayOfMap": []common.MapStr{common.MapStr{"a": "b"}}, + }, + } + + assert.Equal(t, expected.String(), actual.String()) +} + func getActualValue(t *testing.T, config *common.Config, input common.MapStr) common.MapStr { logp.TestingSetup() From bb6bef50b3640402cea21442c0a690612d48b254 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Tue, 19 Mar 2019 17:43:35 +0100 Subject: [PATCH 2/5] indicate skip for arrays --- libbeat/processors/actions/decode_json_fields.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/libbeat/processors/actions/decode_json_fields.go b/libbeat/processors/actions/decode_json_fields.go index d8832858dac..46f6fddd508 100644 --- a/libbeat/processors/actions/decode_json_fields.go +++ b/libbeat/processors/actions/decode_json_fields.go @@ -52,6 +52,7 @@ var ( MaxDepth: 1, ProcessArray: false, } + errProcessingSkipped = errors.New("processing skipped") ) var debug = logp.MakeDebug("filters") @@ -149,6 +150,9 @@ func unmarshal(maxDepth int, text string, fields *interface{}, processArray bool var tmp interface{} err := unmarshal(maxDepth, str, &tmp, processArray) if err != nil { + if err == errProcessingSkipped { + return v, true + } return v, false } @@ -166,7 +170,7 @@ func unmarshal(maxDepth int, text string, fields *interface{}, processArray bool // We want to process arrays here case []interface{}: if !processArray { - break + return errProcessingSkipped } for i, v := range O { From 30530aae80c951367816aca61fb3a254108c73c2 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Tue, 19 Mar 2019 17:45:35 +0100 Subject: [PATCH 3/5] changelog --- CHANGELOG.next.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index edd32bcca81..c77a5b737bb 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -34,7 +34,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - On Google Cloud Engine (GCE) the add_cloud_metadata will now trim the project info from the cloud.machine.type and cloud.availability_zone. {issue}10968[10968] - Rename `migration.enabled` config to `migration.6_to_7.enabled`. {pull}11284[11284] - +- decode_json_field: do not process arrays when flag not set. {pull}11318[11318] *Auditbeat* - Rename `process.exe` to `process.executable` in auditd module to align with ECS. {pull}9949[9949] @@ -170,7 +170,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix a bug when converting NetFlow fields to snake_case. {pull}10950[10950] - Add on_failure handler for Zeek ingest pipelines. Fix one field name error for notice and add an additional test case. {issue}11004[11004] {pull}11105[11105] - Fix issue preventing docker container events to be stored if the container has a network interface without ip address. {issue}11225[11225] {pull}11247[11247] -- Add on_failure handler for Zeek ingest pipelines. Fix one field name error for notice and add an additional test +- Add on_failure handler for Zeek ingest pipelines. Fix one field name error for notice and add an additional test case. {issue}11004[11004] {pull}11105[11105] - Change URLPATH grok pattern to support brackets. {issue}11135[11135] {pull}11252[11252] - Add support for iis log with different address format. {issue}11255[11255] {pull}11256[11256] From 193fb05581182455aa63ddadf8b998bfda8fb6c5 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Tue, 19 Mar 2019 17:46:38 +0100 Subject: [PATCH 4/5] make one liner --- libbeat/processors/actions/decode_json_fields.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/libbeat/processors/actions/decode_json_fields.go b/libbeat/processors/actions/decode_json_fields.go index 46f6fddd508..82b2f2d3ba7 100644 --- a/libbeat/processors/actions/decode_json_fields.go +++ b/libbeat/processors/actions/decode_json_fields.go @@ -150,10 +150,7 @@ func unmarshal(maxDepth int, text string, fields *interface{}, processArray bool var tmp interface{} err := unmarshal(maxDepth, str, &tmp, processArray) if err != nil { - if err == errProcessingSkipped { - return v, true - } - return v, false + return v, err == errProcessingSkipped } return tmp, true From dd8c447a2ba6fae0e31558a3d4b2324d37872714 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Fri, 22 Mar 2019 08:16:12 +0100 Subject: [PATCH 5/5] not marking as breaking change --- CHANGELOG.next.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 43147e1b0bf..7260dfa09b1 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -35,7 +35,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d info from the cloud.machine.type and cloud.availability_zone. {issue}10968[10968] - Empty `meta.json` file will be treated as a missing meta file. {issue}8558[8558] - Rename `migration.enabled` config to `migration.6_to_7.enabled`. {pull}11284[11284] -- decode_json_field: do not process arrays when flag not set. {pull}11318[11318] - Beats Xpack now checks for Basic license on connect. {pull}11296[11296] *Auditbeat* @@ -147,6 +146,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Include ip and boolean type when generating index pattern. {pull}10995[10995] - Cancelling enrollment of a beat will not enroll the beat. {issue}10150[10150] - Add missing fields and test cases for libbeat add_kubernetes_metadata processor. {issue}11133[11133], {pull}11134[11134] +- decode_json_field: do not process arrays when flag not set. {pull}11318[11318] *Auditbeat*