From e114aeb7aaa3b93e875e3daad9f8d8643dbf20fc Mon Sep 17 00:00:00 2001
From: Chao <oleooo@126.com>
Date: Sun, 5 Apr 2020 01:03:54 +0800
Subject: [PATCH 1/7] add urldecode processor

---
 CHANGELOG.next.asciidoc                       |   3 +-
 filebeat/tests/system/test_processors.py      |  26 +++
 libbeat/docs/processors-list.asciidoc         |   6 +
 .../actions/docs/urldecode.asciidoc           |  38 ++++
 libbeat/processors/actions/urldecode.go       | 127 +++++++++++
 libbeat/processors/actions/urldecode_test.go  | 214 ++++++++++++++++++
 6 files changed, 413 insertions(+), 1 deletion(-)
 create mode 100644 libbeat/processors/actions/docs/urldecode.asciidoc
 create mode 100644 libbeat/processors/actions/urldecode.go
 create mode 100644 libbeat/processors/actions/urldecode_test.go

diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
index 91798059ffb..758ec3d8bd3 100644
--- a/CHANGELOG.next.asciidoc
+++ b/CHANGELOG.next.asciidoc
@@ -137,7 +137,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Use max in k8s overview dashboard aggregations. {pull}17015[17015]
 - Fix Disk Used and Disk Usage visualizations in the Metricbeat System dashboards. {issue}12435[12435] {pull}17272[17272]
 - Fix missing Accept header for Prometheus and OpenMetrics module. {issue}16870[16870] {pull}17291[17291]
-- Further revise check for bad data in docker/memory. {pull}17400[17400] 
+- Further revise check for bad data in docker/memory. {pull}17400[17400]
 - Fix issue in Jolokia module when mbean contains multiple quoted properties. {issue}17375[17375] {pull}17374[17374]
 - Combine cloudwatch aggregated metrics into single event. {pull}17345[17345]
 - check if cpuOptions field is nil in DescribeInstances output in ec2 metricset. {pull}17418[17418]
@@ -175,6 +175,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Update supported versions of `redis` output. {pull}17198[17198]
 - Update documentation for system.process.memory fields to include clarification on Windows os's. {pull}17268[17268]
 - Add optional regex based cid extractor to `add_kubernetes_metadata` processor. {pull}17360[17360]
+- Add `urldecode` processor to for decoding URL encoded fields.
 
 *Auditbeat*
 
diff --git a/filebeat/tests/system/test_processors.py b/filebeat/tests/system/test_processors.py
index dae4698a0c9..1dc0b975138 100644
--- a/filebeat/tests/system/test_processors.py
+++ b/filebeat/tests/system/test_processors.py
@@ -302,6 +302,32 @@ def test_decode_csv_fields_all_options(self):
             ["42", "hello world", "string\twith tabs and \"broken\" quotes"],
         ])
 
+    def test_urldecode_defaults(self):
+        """
+        Check URL-decoding using defaults
+        """
+        self.render_config_template(
+            path=os.path.abspath(self.working_dir) + "/test.log",
+            processors=[{
+                "urldecode": {
+                    "fields": [{
+                        "from": "message",
+                        "to": "decoded"
+                    }]
+                },
+            }]
+        )
+
+        self._init_and_read_test_input([
+            "correct data\n",
+            "correct%20data\n",
+        ])
+
+        self._assert_expected_lines([
+            "correct data",
+            "correct data",
+        ], field="decoded")
+
     def test_javascript_processor_add_host_metadata(self):
         """
         Check JS processor with add_host_metadata
diff --git a/libbeat/docs/processors-list.asciidoc b/libbeat/docs/processors-list.asciidoc
index bebffb9d30c..5f78fc0dbea 100644
--- a/libbeat/docs/processors-list.asciidoc
+++ b/libbeat/docs/processors-list.asciidoc
@@ -101,6 +101,9 @@ endif::[]
 ifndef::no_translate_sid_processor[]
 * <<processor-translate-sid, `translate_sid`>>
 endif::[]
+ifndef::no_urldecode_processor[]
+* <<urldecode, `urldecode`>>
+endif::[]
 //# end::processors-list[]
 
 //# tag::processors-include[]
@@ -204,5 +207,8 @@ endif::[]
 ifndef::no_translate_sid_processor[]
 include::{libbeat-processors-dir}/translate_sid/docs/translate_sid.asciidoc[]
 endif::[]
+ifndef::no_urldecode_processor[]
+include::{libbeat-processors-dir}/actions/docs/urldecode.asciidoc[]
+endif::[]
 
 //# end::processors-include[]
diff --git a/libbeat/processors/actions/docs/urldecode.asciidoc b/libbeat/processors/actions/docs/urldecode.asciidoc
new file mode 100644
index 00000000000..ba675e922e7
--- /dev/null
+++ b/libbeat/processors/actions/docs/urldecode.asciidoc
@@ -0,0 +1,38 @@
+[[urldecode]]
+=== URL Decode
+
+++++
+<titleabbrev>urldecode</titleabbrev>
+++++
+
+The `urldecode` processor specifies a list of fields to decode from URL encoded format. Under the `fields`
+key, each entry contains a `from: old-key` and a `to: new-key` pair, where:
+
+* `from` is the original field name
+* `to` is the target field name
+
+[source,yaml]
+-------
+processors:
+- urldecode:
+    fields:
+    - from: "field1"
+      to: "field2"
+    ignore_missing: false
+    fail_on_error: true
+-------
+
+In the example above:
+
+- field1 is decoded in field2
+
+The `urldecode` processor has the following configuration settings:
+
+`ignore_missing`:: (Optional) If set to true, no error is logged in case a key
+which should be URL-decoded is missing. Default is `false`.
+
+`fail_on_error`:: (Optional) If set to true, in case of an error the URL-decoding
+of fields is stopped and the original event is returned. If set to false, decoding
+continues also if an error happened during decoding. Default is `true`.
+
+See <<conditions>> for a list of supported conditions.
diff --git a/libbeat/processors/actions/urldecode.go b/libbeat/processors/actions/urldecode.go
new file mode 100644
index 00000000000..5e1f04a1d25
--- /dev/null
+++ b/libbeat/processors/actions/urldecode.go
@@ -0,0 +1,127 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package actions
+
+import (
+	"fmt"
+	"net/url"
+
+	"github.com/pkg/errors"
+
+	"github.com/elastic/beats/v7/libbeat/beat"
+	"github.com/elastic/beats/v7/libbeat/common"
+	"github.com/elastic/beats/v7/libbeat/logp"
+	"github.com/elastic/beats/v7/libbeat/processors"
+	"github.com/elastic/beats/v7/libbeat/processors/checks"
+	jsprocessor "github.com/elastic/beats/v7/libbeat/processors/script/javascript/module/processor"
+)
+
+type urlDecode struct {
+	config urlDecodeConfig
+	log    *logp.Logger
+}
+
+type urlDecodeConfig struct {
+	Fields        []fromTo `config:"fields"`
+	IgnoreMissing bool     `config:"ignore_missing"`
+	FailOnError   bool     `config:"fail_on_error"`
+}
+
+func init() {
+	processors.RegisterPlugin("urldecode",
+		checks.ConfigChecked(NewURLDecode,
+			checks.RequireFields("fields"),
+			checks.AllowedFields("fields", "ignore_missing", "fail_on_error")))
+	jsprocessor.RegisterPlugin("URLDecode", NewURLDecode)
+}
+
+func NewURLDecode(c *common.Config) (processors.Processor, error) {
+	config := urlDecodeConfig{
+		IgnoreMissing: false,
+		FailOnError:   true,
+	}
+
+	err := c.Unpack(&config)
+	if err != nil {
+		return nil, fmt.Errorf("failed to unpack the configuration of urldecode processor: %s", err)
+	}
+
+	return &urlDecode{
+		config: config,
+		log:    logp.NewLogger("urldecode"),
+	}, nil
+
+}
+
+func (p *urlDecode) Run(event *beat.Event) (*beat.Event, error) {
+	var backup common.MapStr
+	if p.config.FailOnError {
+		backup = event.Fields.Clone()
+	}
+
+	for _, field := range p.config.Fields {
+		err := p.decodeField(field.From, field.To, event)
+		if err != nil {
+			errMsg := fmt.Errorf("failed to decode fields in urldecode processor: %v", err)
+			p.log.Debug(errMsg.Error())
+			if p.config.FailOnError {
+				event.Fields = backup
+				event.PutValue("error.message", errMsg.Error())
+				return event, err
+			}
+		}
+	}
+
+	return event, nil
+}
+
+func (p *urlDecode) decodeField(from string, to string, event *beat.Event) error {
+	value, err := event.GetValue(from)
+	if err != nil {
+		if p.config.IgnoreMissing && errors.Cause(err) == common.ErrKeyNotFound {
+			return nil
+		}
+		return fmt.Errorf("could not fetch value for key: %s, Error: %v", from, err)
+	}
+
+	encodedString, ok := value.(string)
+	if !ok {
+		return fmt.Errorf("invalid type for `from`, expecting a string received %T", value)
+	}
+
+	decodedData, err := url.QueryUnescape(encodedString)
+	if err != nil {
+		return fmt.Errorf("error trying to URL-decode %s: %v", encodedString, err)
+	}
+
+	target := to
+	// If to is empty
+	if to == "" || from == to {
+		target = from
+	}
+
+	if _, err := event.PutValue(target, decodedData); err != nil {
+		return fmt.Errorf("could not put value: %s: %v, %v", decodedData, target, err)
+	}
+
+	return nil
+}
+
+func (p *urlDecode) String() string {
+	return "urldecode=" + fmt.Sprintf("%+v", p.config.Fields)
+}
diff --git a/libbeat/processors/actions/urldecode_test.go b/libbeat/processors/actions/urldecode_test.go
new file mode 100644
index 00000000000..36fb84337ff
--- /dev/null
+++ b/libbeat/processors/actions/urldecode_test.go
@@ -0,0 +1,214 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package actions
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/elastic/beats/v7/libbeat/beat"
+	"github.com/elastic/beats/v7/libbeat/common"
+	"github.com/elastic/beats/v7/libbeat/logp"
+)
+
+func TestURLDecode(t *testing.T) {
+	var testCases = []struct {
+		description string
+		config      urlDecodeConfig
+		Input       common.MapStr
+		Output      common.MapStr
+		error       bool
+	}{
+		{
+			description: "simple field urldecode",
+			config: urlDecodeConfig{
+				Fields: []fromTo{{
+					From: "field1", To: "field2",
+				}},
+				IgnoreMissing: false,
+				FailOnError:   true,
+			},
+			Input: common.MapStr{
+				"field1": "correct%20data",
+			},
+			Output: common.MapStr{
+				"field1": "correct%20data",
+				"field2": "correct data",
+			},
+			error: false,
+		},
+		{
+			description: "simple multiple fields urldecode",
+			config: urlDecodeConfig{
+				Fields: []fromTo{
+					{From: "field1", To: "field2"},
+					{From: "field3", To: "field4"},
+				},
+				IgnoreMissing: false,
+				FailOnError:   true,
+			},
+			Input: common.MapStr{
+				"field1": "correct%20field1",
+				"field3": "correct%20field3",
+			},
+			Output: common.MapStr{
+				"field1": "correct%20field1",
+				"field2": "correct field1",
+				"field3": "correct%20field3",
+				"field4": "correct field3",
+			},
+			error: false,
+		},
+		{
+			description: "simple field urldecode To empty",
+			config: urlDecodeConfig{
+				Fields: []fromTo{{
+					From: "field1", To: "",
+				}},
+				IgnoreMissing: false,
+				FailOnError:   true,
+			},
+			Input: common.MapStr{
+				"field1": "correct%20data",
+			},
+			Output: common.MapStr{
+				"field1": "correct data",
+			},
+			error: false,
+		},
+		{
+			description: "simple field urldecode from and to equals",
+			config: urlDecodeConfig{
+				Fields: []fromTo{{
+					From: "field1", To: "field1",
+				}},
+				IgnoreMissing: false,
+				FailOnError:   true,
+			},
+			Input: common.MapStr{
+				"field1": "correct%20data",
+			},
+			Output: common.MapStr{
+				"field1": "correct data",
+			},
+			error: false,
+		},
+		{
+			description: "simple field bad data - fail on error",
+			config: urlDecodeConfig{
+				Fields: []fromTo{{
+					From: "field1", To: "field1",
+				}},
+				IgnoreMissing: false,
+				FailOnError:   true,
+			},
+			Input: common.MapStr{
+				"field1": "Hello G%ünter",
+			},
+			Output: common.MapStr{
+				"field1": "Hello G%ünter",
+				"error": common.MapStr{
+					"message": "failed to decode fields in urldecode processor: error trying to URL-decode Hello G%ünter: invalid URL escape \"%ü\"",
+				},
+			},
+			error: true,
+		},
+		{
+			description: "simple field bad data fail on error false",
+			config: urlDecodeConfig{
+				Fields: []fromTo{{
+					From: "field1", To: "field1",
+				}},
+				IgnoreMissing: false,
+				FailOnError:   false,
+			},
+			Input: common.MapStr{
+				"field1": "Hello G%ünter",
+			},
+			Output: common.MapStr{
+				"field1": "Hello G%ünter",
+			},
+			error: false,
+		},
+		{
+			description: "missing field",
+			config: urlDecodeConfig{
+				Fields: []fromTo{{
+					From: "field2", To: "field3",
+				}},
+				IgnoreMissing: false,
+				FailOnError:   true,
+			},
+			Input: common.MapStr{
+				"field1": "correct%20data",
+			},
+			Output: common.MapStr{
+				"field1": "correct%20data",
+				"error": common.MapStr{
+					"message": "failed to decode fields in urldecode processor: could not fetch value for key: field2, Error: key not found",
+				},
+			},
+			error: true,
+		},
+		{
+			description: "missing field ignore",
+			config: urlDecodeConfig{
+				Fields: []fromTo{{
+					From: "field2", To: "field3",
+				}},
+				IgnoreMissing: true,
+				FailOnError:   true,
+			},
+			Input: common.MapStr{
+				"field1": "correct%20data",
+			},
+			Output: common.MapStr{
+				"field1": "correct%20data",
+			},
+			error: false,
+		},
+	}
+
+	for _, test := range testCases {
+		test := test
+		t.Run(test.description, func(t *testing.T) {
+			//t.Parallel()
+
+			f := &urlDecode{
+				log:    logp.NewLogger("urldecode"),
+				config: test.config,
+			}
+
+			event := &beat.Event{
+				Fields: test.Input,
+			}
+
+			newEvent, err := f.Run(event)
+			if !test.error {
+				assert.Nil(t, err)
+			} else {
+				assert.NotNil(t, err)
+			}
+
+			assert.Equal(t, test.Output, newEvent.Fields)
+
+		})
+	}
+
+}

From e3e032c97397093b5a4946bf1eff6b14059991fe Mon Sep 17 00:00:00 2001
From: Chao <oleooo@126.com>
Date: Mon, 6 Apr 2020 01:58:13 +0800
Subject: [PATCH 2/7] update reference yml files

---
 auditbeat/auditbeat.reference.yml             | 10 +++++++++
 filebeat/filebeat.reference.yml               | 10 +++++++++
 heartbeat/heartbeat.reference.yml             | 10 +++++++++
 journalbeat/journalbeat.reference.yml         | 10 +++++++++
 libbeat/_meta/config.reference.yml.tmpl       | 10 +++++++++
 metricbeat/metricbeat.reference.yml           | 10 +++++++++
 packetbeat/packetbeat.reference.yml           | 10 +++++++++
 winlogbeat/winlogbeat.reference.yml           | 10 +++++++++
 x-pack/auditbeat/auditbeat.reference.yml      | 10 +++++++++
 x-pack/filebeat/filebeat.reference.yml        | 22 ++++++++++++++-----
 .../functionbeat/functionbeat.reference.yml   | 10 +++++++++
 x-pack/metricbeat/metricbeat.reference.yml    | 10 +++++++++
 x-pack/winlogbeat/winlogbeat.reference.yml    | 10 +++++++++
 13 files changed, 136 insertions(+), 6 deletions(-)

diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml
index 581e7bbe95d..4913b1f4780 100644
--- a/auditbeat/auditbeat.reference.yml
+++ b/auditbeat/auditbeat.reference.yml
@@ -378,6 +378,16 @@ auditbeat.modules:
 #    max_bytes: 1024
 #    fail_on_error: false
 #    ignore_missing: true
+#
+# The following example URL-decodes the value of field1 to field2
+#
+#processors:
+#- urldecode:
+#    fields:
+#      - from: "field1"
+#        to: "field2"
+#    ignore_missing: false
+#    fail_on_error: true
 
 #============================= Elastic Cloud ==================================
 
diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml
index 9649e8e1487..e22d16362bf 100644
--- a/filebeat/filebeat.reference.yml
+++ b/filebeat/filebeat.reference.yml
@@ -1084,6 +1084,16 @@ filebeat.inputs:
 #    max_bytes: 1024
 #    fail_on_error: false
 #    ignore_missing: true
+#
+# The following example URL-decodes the value of field1 to field2
+#
+#processors:
+#- urldecode:
+#    fields:
+#      - from: "field1"
+#        to: "field2"
+#    ignore_missing: false
+#    fail_on_error: true
 
 #============================= Elastic Cloud ==================================
 
diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml
index ba6e08ffe68..33863fae1a2 100644
--- a/heartbeat/heartbeat.reference.yml
+++ b/heartbeat/heartbeat.reference.yml
@@ -522,6 +522,16 @@ heartbeat.scheduler:
 #    max_bytes: 1024
 #    fail_on_error: false
 #    ignore_missing: true
+#
+# The following example URL-decodes the value of field1 to field2
+#
+#processors:
+#- urldecode:
+#    fields:
+#      - from: "field1"
+#        to: "field2"
+#    ignore_missing: false
+#    fail_on_error: true
 
 #============================= Elastic Cloud ==================================
 
diff --git a/journalbeat/journalbeat.reference.yml b/journalbeat/journalbeat.reference.yml
index 0afd73547a8..7f5a33d5b9c 100644
--- a/journalbeat/journalbeat.reference.yml
+++ b/journalbeat/journalbeat.reference.yml
@@ -316,6 +316,16 @@ setup.template.settings:
 #    max_bytes: 1024
 #    fail_on_error: false
 #    ignore_missing: true
+#
+# The following example URL-decodes the value of field1 to field2
+#
+#processors:
+#- urldecode:
+#    fields:
+#      - from: "field1"
+#        to: "field2"
+#    ignore_missing: false
+#    fail_on_error: true
 
 #============================= Elastic Cloud ==================================
 
diff --git a/libbeat/_meta/config.reference.yml.tmpl b/libbeat/_meta/config.reference.yml.tmpl
index 995e7bf3df4..65e6d8aa992 100644
--- a/libbeat/_meta/config.reference.yml.tmpl
+++ b/libbeat/_meta/config.reference.yml.tmpl
@@ -259,6 +259,16 @@
 #    max_bytes: 1024
 #    fail_on_error: false
 #    ignore_missing: true
+#
+# The following example URL-decodes the value of field1 to field2
+#
+#processors:
+#- urldecode:
+#    fields:
+#      - from: "field1"
+#        to: "field2"
+#    ignore_missing: false
+#    fail_on_error: true
 
 #============================= Elastic Cloud ==================================
 
diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml
index 7d0b9219e9a..aa46c1709f5 100644
--- a/metricbeat/metricbeat.reference.yml
+++ b/metricbeat/metricbeat.reference.yml
@@ -1114,6 +1114,16 @@ metricbeat.modules:
 #    max_bytes: 1024
 #    fail_on_error: false
 #    ignore_missing: true
+#
+# The following example URL-decodes the value of field1 to field2
+#
+#processors:
+#- urldecode:
+#    fields:
+#      - from: "field1"
+#        to: "field2"
+#    ignore_missing: false
+#    fail_on_error: true
 
 #============================= Elastic Cloud ==================================
 
diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml
index 4c1e8f7fb50..2b036da5808 100644
--- a/packetbeat/packetbeat.reference.yml
+++ b/packetbeat/packetbeat.reference.yml
@@ -805,6 +805,16 @@ packetbeat.ignore_outgoing: false
 #    max_bytes: 1024
 #    fail_on_error: false
 #    ignore_missing: true
+#
+# The following example URL-decodes the value of field1 to field2
+#
+#processors:
+#- urldecode:
+#    fields:
+#      - from: "field1"
+#        to: "field2"
+#    ignore_missing: false
+#    fail_on_error: true
 
 #============================= Elastic Cloud ==================================
 
diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml
index 23097261f64..d7e87ea2826 100644
--- a/winlogbeat/winlogbeat.reference.yml
+++ b/winlogbeat/winlogbeat.reference.yml
@@ -301,6 +301,16 @@ winlogbeat.event_logs:
 #    max_bytes: 1024
 #    fail_on_error: false
 #    ignore_missing: true
+#
+# The following example URL-decodes the value of field1 to field2
+#
+#processors:
+#- urldecode:
+#    fields:
+#      - from: "field1"
+#        to: "field2"
+#    ignore_missing: false
+#    fail_on_error: true
 
 #============================= Elastic Cloud ==================================
 
diff --git a/x-pack/auditbeat/auditbeat.reference.yml b/x-pack/auditbeat/auditbeat.reference.yml
index e21dd537f52..ab6c189656a 100644
--- a/x-pack/auditbeat/auditbeat.reference.yml
+++ b/x-pack/auditbeat/auditbeat.reference.yml
@@ -434,6 +434,16 @@ auditbeat.modules:
 #    max_bytes: 1024
 #    fail_on_error: false
 #    ignore_missing: true
+#
+# The following example URL-decodes the value of field1 to field2
+#
+#processors:
+#- urldecode:
+#    fields:
+#      - from: "field1"
+#        to: "field2"
+#    ignore_missing: false
+#    fail_on_error: true
 
 #============================= Elastic Cloud ==================================
 
diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml
index 838301aa98d..dc40882f29d 100644
--- a/x-pack/filebeat/filebeat.reference.yml
+++ b/x-pack/filebeat/filebeat.reference.yml
@@ -326,7 +326,7 @@ filebeat.modules:
 #------------------------------- Coredns Module -------------------------------
 - module: coredns
   # Fileset for native deployment
-  log: 
+  log:
     enabled: true
 
     # Set custom paths for the log files. If left empty,
@@ -335,7 +335,7 @@ filebeat.modules:
 
 #----------------------------- Crowdstrike Module -----------------------------
 - module: crowdstrike
-  
+
   falcon:
     enabled: true
 
@@ -380,7 +380,7 @@ filebeat.modules:
 #------------------------------ Envoyproxy Module ------------------------------
 - module: envoyproxy
   # Fileset for native deployment
-  log: 
+  log:
     enabled: true
 
     # Set custom paths for the log files. If left empty,
@@ -596,7 +596,7 @@ filebeat.modules:
 
     # URL of the MISP REST API
     #var.url
-   
+
     # You can also pass SSL options. For example:
     #var.ssl: |-
     #  {
@@ -762,7 +762,7 @@ filebeat.modules:
     # URL of the Okta REST API
     #var.url
 
-    # Disable SSL verification 
+    # Disable SSL verification
     #var.ssl: |-
     #  {
     #    "verification_mode": "none"
@@ -893,7 +893,7 @@ filebeat.modules:
   http:
     enabled: true
   intel:
-    enabled: true    
+    enabled: true
   irc:
     enabled: true
   kerberos:
@@ -1729,6 +1729,16 @@ filebeat.inputs:
 #    max_bytes: 1024
 #    fail_on_error: false
 #    ignore_missing: true
+#
+# The following example URL-decodes the value of field1 to field2
+#
+#processors:
+#- urldecode:
+#    fields:
+#      - from: "field1"
+#        to: "field2"
+#    ignore_missing: false
+#    fail_on_error: true
 
 #============================= Elastic Cloud ==================================
 
diff --git a/x-pack/functionbeat/functionbeat.reference.yml b/x-pack/functionbeat/functionbeat.reference.yml
index 7a85d3e805e..0e945a8dea2 100644
--- a/x-pack/functionbeat/functionbeat.reference.yml
+++ b/x-pack/functionbeat/functionbeat.reference.yml
@@ -644,6 +644,16 @@ functionbeat.provider.gcp.functions:
 #    max_bytes: 1024
 #    fail_on_error: false
 #    ignore_missing: true
+#
+# The following example URL-decodes the value of field1 to field2
+#
+#processors:
+#- urldecode:
+#    fields:
+#      - from: "field1"
+#        to: "field2"
+#    ignore_missing: false
+#    fail_on_error: true
 
 #============================= Elastic Cloud ==================================
 
diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml
index 462338ad399..e1f2bb16f0b 100644
--- a/x-pack/metricbeat/metricbeat.reference.yml
+++ b/x-pack/metricbeat/metricbeat.reference.yml
@@ -1522,6 +1522,16 @@ metricbeat.modules:
 #    max_bytes: 1024
 #    fail_on_error: false
 #    ignore_missing: true
+#
+# The following example URL-decodes the value of field1 to field2
+#
+#processors:
+#- urldecode:
+#    fields:
+#      - from: "field1"
+#        to: "field2"
+#    ignore_missing: false
+#    fail_on_error: true
 
 #============================= Elastic Cloud ==================================
 
diff --git a/x-pack/winlogbeat/winlogbeat.reference.yml b/x-pack/winlogbeat/winlogbeat.reference.yml
index 435fc9b323a..e4265a3b3ec 100644
--- a/x-pack/winlogbeat/winlogbeat.reference.yml
+++ b/x-pack/winlogbeat/winlogbeat.reference.yml
@@ -304,6 +304,16 @@ winlogbeat.event_logs:
 #    max_bytes: 1024
 #    fail_on_error: false
 #    ignore_missing: true
+#
+# The following example URL-decodes the value of field1 to field2
+#
+#processors:
+#- urldecode:
+#    fields:
+#      - from: "field1"
+#        to: "field2"
+#    ignore_missing: false
+#    fail_on_error: true
 
 #============================= Elastic Cloud ==================================
 

From e47bf4a3ff874d2a2ababdd9772ef5e972a29200 Mon Sep 17 00:00:00 2001
From: Chao <oleooo@126.com>
Date: Mon, 6 Apr 2020 02:07:02 +0800
Subject: [PATCH 3/7] update doc with PR number

---
 CHANGELOG.next.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
index 758ec3d8bd3..ea114076dc4 100644
--- a/CHANGELOG.next.asciidoc
+++ b/CHANGELOG.next.asciidoc
@@ -175,7 +175,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Update supported versions of `redis` output. {pull}17198[17198]
 - Update documentation for system.process.memory fields to include clarification on Windows os's. {pull}17268[17268]
 - Add optional regex based cid extractor to `add_kubernetes_metadata` processor. {pull}17360[17360]
-- Add `urldecode` processor to for decoding URL encoded fields.
+- Add `urldecode` processor to for decoding URL-encoded fields. {pull}17505[17505]
 
 *Auditbeat*
 

From 924bd3746bfa7243247bfb01f95c007821d41e48 Mon Sep 17 00:00:00 2001
From: Chao <oleooo@126.com>
Date: Mon, 6 Apr 2020 02:35:28 +0800
Subject: [PATCH 4/7] remove unexpected format

---
 x-pack/filebeat/filebeat.reference.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml
index dc40882f29d..dc47bdd07a8 100644
--- a/x-pack/filebeat/filebeat.reference.yml
+++ b/x-pack/filebeat/filebeat.reference.yml
@@ -326,7 +326,7 @@ filebeat.modules:
 #------------------------------- Coredns Module -------------------------------
 - module: coredns
   # Fileset for native deployment
-  log:
+  log: 
     enabled: true
 
     # Set custom paths for the log files. If left empty,
@@ -335,7 +335,7 @@ filebeat.modules:
 
 #----------------------------- Crowdstrike Module -----------------------------
 - module: crowdstrike
-
+  
   falcon:
     enabled: true
 
@@ -380,7 +380,7 @@ filebeat.modules:
 #------------------------------ Envoyproxy Module ------------------------------
 - module: envoyproxy
   # Fileset for native deployment
-  log:
+  log: 
     enabled: true
 
     # Set custom paths for the log files. If left empty,
@@ -596,7 +596,7 @@ filebeat.modules:
 
     # URL of the MISP REST API
     #var.url
-
+   
     # You can also pass SSL options. For example:
     #var.ssl: |-
     #  {
@@ -762,7 +762,7 @@ filebeat.modules:
     # URL of the Okta REST API
     #var.url
 
-    # Disable SSL verification
+    # Disable SSL verification 
     #var.ssl: |-
     #  {
     #    "verification_mode": "none"
@@ -893,7 +893,7 @@ filebeat.modules:
   http:
     enabled: true
   intel:
-    enabled: true
+    enabled: true    
   irc:
     enabled: true
   kerberos:

From 627ae785e9283b6a14e816c9fb4c627451b43ec5 Mon Sep 17 00:00:00 2001
From: Chao <oleooo@126.com>
Date: Tue, 7 Apr 2020 14:24:47 +0800
Subject: [PATCH 5/7] update from feedback

---
 libbeat/processors/actions/docs/urldecode.asciidoc | 6 +++---
 libbeat/processors/actions/urldecode.go            | 8 +++-----
 2 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/libbeat/processors/actions/docs/urldecode.asciidoc b/libbeat/processors/actions/docs/urldecode.asciidoc
index ba675e922e7..6a544749d2c 100644
--- a/libbeat/processors/actions/docs/urldecode.asciidoc
+++ b/libbeat/processors/actions/docs/urldecode.asciidoc
@@ -6,10 +6,10 @@
 ++++
 
 The `urldecode` processor specifies a list of fields to decode from URL encoded format. Under the `fields`
-key, each entry contains a `from: old-key` and a `to: new-key` pair, where:
+key, each entry contains a `from: source-field` and a `to: target-field` pair, where:
 
-* `from` is the original field name
-* `to` is the target field name
+* `from` is the source field name
+* `to` is the target field name (defaults to the `from` value)
 
 [source,yaml]
 -------
diff --git a/libbeat/processors/actions/urldecode.go b/libbeat/processors/actions/urldecode.go
index 5e1f04a1d25..0f61b59986c 100644
--- a/libbeat/processors/actions/urldecode.go
+++ b/libbeat/processors/actions/urldecode.go
@@ -37,7 +37,7 @@ type urlDecode struct {
 }
 
 type urlDecodeConfig struct {
-	Fields        []fromTo `config:"fields"`
+	Fields        []fromTo `config:"fields" validate:"required"`
 	IgnoreMissing bool     `config:"ignore_missing"`
 	FailOnError   bool     `config:"fail_on_error"`
 }
@@ -56,8 +56,7 @@ func NewURLDecode(c *common.Config) (processors.Processor, error) {
 		FailOnError:   true,
 	}
 
-	err := c.Unpack(&config)
-	if err != nil {
+	if err := c.Unpack(&config); err != nil {
 		return nil, fmt.Errorf("failed to unpack the configuration of urldecode processor: %s", err)
 	}
 
@@ -110,8 +109,7 @@ func (p *urlDecode) decodeField(from string, to string, event *beat.Event) error
 	}
 
 	target := to
-	// If to is empty
-	if to == "" || from == to {
+	if to == "" {
 		target = from
 	}
 

From 9e9457042d863b8a2fc0f3a0c1182ae8d998c1d9 Mon Sep 17 00:00:00 2001
From: Chao <oleooo@126.com>
Date: Tue, 7 Apr 2020 14:48:02 +0800
Subject: [PATCH 6/7] move urldecode processor into its own package

---
 libbeat/cmd/instance/imports_common.go                     | 1 +
 libbeat/docs/processors-list.asciidoc                      | 2 +-
 .../{actions => urldecode}/docs/urldecode.asciidoc         | 0
 libbeat/processors/{actions => urldecode}/urldecode.go     | 7 ++++++-
 .../processors/{actions => urldecode}/urldecode_test.go    | 2 +-
 5 files changed, 9 insertions(+), 3 deletions(-)
 rename libbeat/processors/{actions => urldecode}/docs/urldecode.asciidoc (100%)
 rename libbeat/processors/{actions => urldecode}/urldecode.go (97%)
 rename libbeat/processors/{actions => urldecode}/urldecode_test.go (99%)

diff --git a/libbeat/cmd/instance/imports_common.go b/libbeat/cmd/instance/imports_common.go
index 5e993f6a2a0..a2b2569d61c 100644
--- a/libbeat/cmd/instance/imports_common.go
+++ b/libbeat/cmd/instance/imports_common.go
@@ -36,5 +36,6 @@ import (
 	_ "github.com/elastic/beats/v7/libbeat/processors/fingerprint"
 	_ "github.com/elastic/beats/v7/libbeat/processors/registered_domain"
 	_ "github.com/elastic/beats/v7/libbeat/processors/translate_sid"
+	_ "github.com/elastic/beats/v7/libbeat/processors/urldecode"
 	_ "github.com/elastic/beats/v7/libbeat/publisher/includes" // Register publisher pipeline modules
 )
diff --git a/libbeat/docs/processors-list.asciidoc b/libbeat/docs/processors-list.asciidoc
index 5f78fc0dbea..169657aecc4 100644
--- a/libbeat/docs/processors-list.asciidoc
+++ b/libbeat/docs/processors-list.asciidoc
@@ -208,7 +208,7 @@ ifndef::no_translate_sid_processor[]
 include::{libbeat-processors-dir}/translate_sid/docs/translate_sid.asciidoc[]
 endif::[]
 ifndef::no_urldecode_processor[]
-include::{libbeat-processors-dir}/actions/docs/urldecode.asciidoc[]
+include::{libbeat-processors-dir}/urldecode/docs/urldecode.asciidoc[]
 endif::[]
 
 //# end::processors-include[]
diff --git a/libbeat/processors/actions/docs/urldecode.asciidoc b/libbeat/processors/urldecode/docs/urldecode.asciidoc
similarity index 100%
rename from libbeat/processors/actions/docs/urldecode.asciidoc
rename to libbeat/processors/urldecode/docs/urldecode.asciidoc
diff --git a/libbeat/processors/actions/urldecode.go b/libbeat/processors/urldecode/urldecode.go
similarity index 97%
rename from libbeat/processors/actions/urldecode.go
rename to libbeat/processors/urldecode/urldecode.go
index 0f61b59986c..96b155b8356 100644
--- a/libbeat/processors/actions/urldecode.go
+++ b/libbeat/processors/urldecode/urldecode.go
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-package actions
+package urldecode
 
 import (
 	"fmt"
@@ -42,6 +42,11 @@ type urlDecodeConfig struct {
 	FailOnError   bool     `config:"fail_on_error"`
 }
 
+type fromTo struct {
+	From string `config:"from"`
+	To   string `config:"to"`
+}
+
 func init() {
 	processors.RegisterPlugin("urldecode",
 		checks.ConfigChecked(NewURLDecode,
diff --git a/libbeat/processors/actions/urldecode_test.go b/libbeat/processors/urldecode/urldecode_test.go
similarity index 99%
rename from libbeat/processors/actions/urldecode_test.go
rename to libbeat/processors/urldecode/urldecode_test.go
index 36fb84337ff..bf10244965d 100644
--- a/libbeat/processors/actions/urldecode_test.go
+++ b/libbeat/processors/urldecode/urldecode_test.go
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-package actions
+package urldecode
 
 import (
 	"testing"

From 0347640dadb6e5fc7da04abd81b54b529b3fa738 Mon Sep 17 00:00:00 2001
From: Chao <oleooo@126.com>
Date: Tue, 7 Apr 2020 17:49:27 +0800
Subject: [PATCH 7/7] update from feedback

---
 libbeat/processors/urldecode/urldecode.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/libbeat/processors/urldecode/urldecode.go b/libbeat/processors/urldecode/urldecode.go
index 96b155b8356..6fc9cb8386d 100644
--- a/libbeat/processors/urldecode/urldecode.go
+++ b/libbeat/processors/urldecode/urldecode.go
@@ -43,19 +43,19 @@ type urlDecodeConfig struct {
 }
 
 type fromTo struct {
-	From string `config:"from"`
+	From string `config:"from" validate:"required"`
 	To   string `config:"to"`
 }
 
 func init() {
 	processors.RegisterPlugin("urldecode",
-		checks.ConfigChecked(NewURLDecode,
+		checks.ConfigChecked(New,
 			checks.RequireFields("fields"),
 			checks.AllowedFields("fields", "ignore_missing", "fail_on_error")))
-	jsprocessor.RegisterPlugin("URLDecode", NewURLDecode)
+	jsprocessor.RegisterPlugin("URLDecode", New)
 }
 
-func NewURLDecode(c *common.Config) (processors.Processor, error) {
+func New(c *common.Config) (processors.Processor, error) {
 	config := urlDecodeConfig{
 		IgnoreMissing: false,
 		FailOnError:   true,