Replace copy_from with templated value

To ensure compatibility with Elasticsearch versions <7.13 this removes usage of `copy_from` in `set` processors. Relates elastic#26629
andrewkroh · Jun 30, 2021 · 1c838a1 · 1c838a1
1 parent 92143fe
commit 1c838a1
Show file tree

Hide file tree

Showing 11 changed files with 13 additions and 12 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -280,6 +280,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix bug in `httpjson` that prevented `first_event` getting updated. {pull}26407[26407]
 - Fix bug in the Syslog input that misparsed rfc5424 days starting with 0. {pull}26419[26419]
 - Do not close filestream harvester if an unexpected error is returned when close.on_state_change.* is enabled. {pull}26411[26411]
+- Fix Elasticsearch compatability for modules that use `copy_from` in `set` processors. {issue}26629[26629]
 
 *Filebeat*
 

diff --git a/filebeat/module/elasticsearch/audit/ingest/pipeline-json.yml b/filebeat/module/elasticsearch/audit/ingest/pipeline-json.yml
@@ -156,7 +156,7 @@ processors:
       path: elasticsearch.audit
   - set:
       field: http.request.id
-      copy_from: elasticsearch.audit.request.id
+      value: '{{{elasticsearch.audit.request.id}}}'
       ignore_empty_value: true
   - dot_expander:
       field: cluster.name

diff --git a/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml b/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml
@@ -73,7 +73,7 @@ processors:
       value: ""
   - set:
       field: http.request.id
-      copy_from: nginx.ingress_controller.http.request.id
+      value: '{{{nginx.ingress_controller.http.request.id}}}'
       ignore_empty_value: true
       ignore_failure: true
   - script:

diff --git a/x-pack/filebeat/module/cyberarkpas/audit/ingest/pipeline.yml b/x-pack/filebeat/module/cyberarkpas/audit/ingest/pipeline.yml
@@ -994,7 +994,7 @@ processors:
       on_failure:
         - set:
             field: source.domain
-            copy_from: source.address
+            value: '{{{source.address}}}'
   - convert:
       field: destination.address
       target_field: destination.ip
@@ -1003,7 +1003,7 @@ processors:
       on_failure:
         - set:
             field: destination.domain
-            copy_from: destination.address
+            value: '{{{destination.address}}}'
 
   #
   # Populate related.ip

diff --git a/x-pack/filebeat/module/threatintel/abuseurl/ingest/pipeline.yml b/x-pack/filebeat/module/threatintel/abuseurl/ingest/pipeline.yml
@@ -45,7 +45,7 @@ processors:
     remove_if_successful: true
 - set:
     field: threatintel.indicator.url.full
-    copy_from: threatintel.indicator.url.original
+    value: '{{{threatintel.indicator.url.original}}}'
     ignore_empty_value: true
 - rename:
     field: threatintel.abuseurl.host

diff --git a/x-pack/filebeat/module/threatintel/anomali/ingest/pipeline.yml b/x-pack/filebeat/module/threatintel/anomali/ingest/pipeline.yml
@@ -84,7 +84,7 @@ processors:
     if: ctx?.threatintel?.indicator?.type == 'url'
 - set:
     field: threatintel.indicator.url.full
-    copy_from: threatintel.indicator.url.original
+    value: '{{{threatintel.indicator.url.original}}}'
     ignore_empty_value: true
 - rename:
     field: _tmp.threatvalue

diff --git a/x-pack/filebeat/module/threatintel/anomalithreatstream/ingest/pipeline.yml b/x-pack/filebeat/module/threatintel/anomalithreatstream/ingest/pipeline.yml
@@ -265,7 +265,7 @@ processors:
 
 - set:
     field: threatintel.indicator.url.full
-    copy_from: threatintel.indicator.url.original
+    value: '{{{threatintel.indicator.url.original}}}'
     ignore_empty_value: true
 
 - rename:

diff --git a/x-pack/filebeat/module/threatintel/misp/ingest/pipeline.yml b/x-pack/filebeat/module/threatintel/misp/ingest/pipeline.yml
@@ -139,7 +139,7 @@ processors:
 
 - set:
     field: threatintel.indicator.url.full
-    copy_from: threatintel.indicator.url.original
+    value: '{{{threatintel.indicator.url.original}}}'
     ignore_empty_value: true
     if: "ctx?.threatintel?.indicator?.type == 'url' && ctx?.threatintel?.misp?.attribute?.type != 'uri'"
 

diff --git a/x-pack/filebeat/module/threatintel/otx/ingest/pipeline.yml b/x-pack/filebeat/module/threatintel/otx/ingest/pipeline.yml
@@ -87,7 +87,7 @@ processors:
     if: ctx?.threatintel?.indicator?.type == 'url'
 - set:
     field: threatintel.indicator.url.full
-    copy_from: threatintel.indicator.url.original
+    value: '{{{threatintel.indicator.url.original}}}'
     ignore_empty_value: true
     if: "ctx?.threatintel?.otx?.type == 'URL'"
 

diff --git a/x-pack/filebeat/module/threatintel/recordedfuture/ingest/pipeline.yml b/x-pack/filebeat/module/threatintel/recordedfuture/ingest/pipeline.yml
@@ -122,7 +122,7 @@ processors:
 
   - set:
       field: json.ip_range
-      copy_from: json.entity.name
+      value: '{{{json.entity.name}}}'
       if: 'ctx.json.entity?.type == "IpAddress" && ctx.json.entity.name != null && ctx.json.entity.name.contains("/")'
   - set:
       field: json.ip_range
@@ -134,7 +134,7 @@ processors:
       if: 'ctx.threatintel?.indicator?.type == "ipv6-addr" && ctx.json.entity.name != null && !ctx.json.entity.name.contains("/")'
   - set:
       field: json.ip_range
-      copy_from: json.entity.name
+      value: '{{{json.entity.name}}}'
       if: 'ctx.json.entity?.type == "IpAddress" && ctx.json.entity.name != null && ctx.json.entity.name.contains("/")'
 
   - rename:

diff --git a/x-pack/filebeat/module/zoom/webhook/ingest/meeting.yml b/x-pack/filebeat/module/zoom/webhook/ingest/meeting.yml
@@ -51,7 +51,7 @@ processors:
     if: ctx?.url?.original != null
 - set:
     field: url.full
-    copy_from: url.original
+    value: '{{{url.original}}}'
     ignore_failure: true
     if: ctx?.url?.original != null
 #