[Filebeat] [auditd]: Support EXECVE events with truncated argument li…

…st (#30382)

This modifies Filebeat's auditd pipeline to support parsing of EXECVE
records with truncated argument lists.

When such a log is found, the arguments will be appended to process.args
with a leading entry informing about the truncation.

This is to prevent a mapping explosion in Filebeat when a lot of these
logs are ingested with the previous pipeline version.

CHANGELOG.next.asciidoc

@@ -51,10 +51,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...main[Check the HEAD dif
 - Fix using log_group_name_prefix in aws-cloudwatch input. {pull}29695[29695]
 - Fix multiple instances of the same module configured within `filebeat.modules` in filebeat.yml. {issue}29649[29649] {pull}29952[29952]
 - aws-s3: fix race condition in states used by s3-poller. {issue}30123[30123] {pull}30131[30131]
-- Report the starting offset of the line in `log.offset` when using `filestream` instead of the end to be ECS compliant. {pull}30445[30445]
-
-*Filebeat*
 - Fix broken Kafka input {issue}29746[29746] {pull}30277[30277]
+- Report the starting offset of the line in `log.offset` when using `filestream` instead of the end to be ECS compliant. {pull}30445[30445]
+- auditd: Prevent mapping explosion when truncated EXECVE records are ingested. {pull}30382[30382]
 
 *Heartbeat*
 - Fix missing mapping for `service.name`. {pull}30324[30324]

filebeat/module/auditd/log/ingest/pipeline.yml

@@ -2102,28 +2102,56 @@ processors:
     ignore_failure: true
     field: auditd.log.msg
     target_field: message
+
+# The processor below populates process.args list from argN fields.
+#
+# It handles the common case of a complete record: Contains argc=N and a0 to aN-1,
+# and the truncated case: Contains aI, aI+1, ..., aN-1, for I>0, and no argc.
 - script:
     lang: painless
-    description: Extracts process information from execve calls
-    if: 'ctx.process?.args_count != null && ctx.auditd?.log != null'
+    description: Extracts process arguments from EXECVE calls.
+    if: 'ctx.auditd?.log?.record_type == "EXECVE"'
     source: >-
-      long argc = ctx.process.args_count;
-      List args = new ArrayList();
-      def[] fmt = new def[] {0};
-      for (long i=0; i<argc; i++) {
-        fmt[0] = i;
-        String fld = String.format("a%d", fmt);
-        def arg = ctx.auditd.log.remove(fld);
-        if (arg == null) break;
-        args.add(arg);
+      /* Want to capture all aNN fields, including aN_len and aN[x] */
+      Pattern argRegex = /^a([0-9]+)(.*)$/;
+
+      List keys = ctx.auditd.log.keySet().stream()
+                  /* From List of keys to list of matchers */
+                  .map(x -> argRegex.matcher(x))
+                  /* Drop elements that didn't match the regex */
+                  .filter(x -> x.matches())
+                  /* Must save to a list because it needs to remove keys in auditd.log,
+                     which cannot be done while streaming from this source */
+                  .collect(Collectors.toList());
+
+      List args = keys.stream()
+                  /* List<Matcher> to List<[Matcher, Value for given key]>
+                     with side effect of removing the key */
+                  .map(x -> [x, ctx.auditd.log.remove(x.group(0))])
+                  /* Drop elements that end in _len, just wanted to remove them */
+                  .filter(x -> x[0].group(2) != "_len")
+                  /* List<Matcher, Value> to List<[Int, Value]>
+                     where the Int is the argument index */
+                  .map(x -> [Integer.parseInt(x[0].group(1)), x[1]])
+                  /* Sort by numeric argument index */
+                  .sorted((lhs, rhs) -> lhs[0].compareTo(rhs[0]))
+                  /* Save as List<[Index, Value]> */
+                  .collect(Collectors.toList());
+
+      if (args.isEmpty()) return;
+      if (ctx.process == null) ctx.process = new HashMap();
+      ctx.process.args = args.stream().map(x -> x[1]).collect(Collectors.toList());
+      def firstIndex = args[0][0];
+      if (firstIndex == 0) {
+        ctx.process.executable = ctx.process.args[0];
+      } else {
+        ctx.process.args.add(0, "[... " + firstIndex + " truncated arguments ...]");
       }
-      if (args.size() == 0) return;
-      ctx.process.put("args", args);
-      ctx.process.put("executable", args[0]);
+
     on_failure:
       - append:
           field: error.message
-          value: "failed extracting process arguments: {{ _ingest.on_failure_message }}"
+          value: "failed extracting process arguments: {{{ _ingest.on_failure_message }}}"
 
 - rename:
     ignore_failure: true

filebeat/module/auditd/log/test/rare.log

@@ -0,0 +1,4 @@
+type=EXECVE msg=audit(1643025668.518:5009988): a8_len=1 a8[0]=8 a9=9 a10=10 a11[1]=11
+type=EXECVE msg=audit(1643025668.518:5009988): a10517="/redacted/path/for/arg10517" a10518="/redacted/path/for/arg10518" a10519="/redacted/path/for/arg10519" a10520="/redacted/path/for/arg10520" a10521="/redacted/path/for/arg10521" a10522="/redacted/path/for/arg10522" a10523="/redacted/path/for/arg10523" a10524="/redacted/path/for/arg10524" a10525="/redacted/path/for/arg10525" a10526="/redacted/path/for/arg10526" a10527="/redacted/path/for/arg10527" a10528="/redacted/path/for/arg10528" a10529="/redacted/path/for/arg10529" a10530="/redacted/path/for/arg10530" a10531="/redacted/path/for/arg10531" a10532="/redacted/path/for/arg10532" a10533="/redacted/path/for/arg10533" a10534="/redacted/path/for/arg10534" a10535="/redacted/path/for/arg10535" a10536="/redacted/path/for/arg10536" a10537="/redacted/path/for/arg10537" a10538="/redacted/path/for/arg10538" a10539="/redacted/path/for/arg10539" a10540="/redacted/path/for/arg10540" a10541="/redacted/path/for/arg10541" a10542="/redacted/path/for/arg10542" a10543="/redacted/path/for/arg10543" a10544="/redacted/path/for/arg10544" a10545="/redacted/path/for/arg10545" a10546="/redacted/path/for/arg10546" a10547="/redacted/path/for/arg10547" a10548="/redacted/path/for/arg10548" a10549="/redacted/path/for/arg10549" a10550="/redacted/path/for/arg10550" a10551="/redacted/path/for/arg10551" a10552="/redacted/path/for/arg10552" a10553="/redacted/path/for/arg10553" a10554="/redacted/path/for/arg10554" a10555="/redacted/path/for/arg10555" a10556="/redacted/path/for/arg10556" a10557="/redacted/path/for/arg10557" a10558="/redacted/path/for/arg10558" a10559="/redacted/path/for/arg10559" a10560="/redacted/path/for/arg10560" a10561="/redacted/path/for/arg10561" a10562="/redacted/path/for/arg10562" a10563="/redacted/path/for/arg10563" a10564="/redacted/path/for/arg10564" a10565="/redacted/path/for/arg10565" a10566="/redacted/path/for/arg10566" a10567="/redacted/path/for/arg10567" a10568="/redacted/path/for/arg10568" a10569="/redacted/path/for/arg10569" a10570="/redacted/path/for/arg10570" a10571="/redacted/path/for/arg10571" a10572="/redacted/path/for/arg10572" a10573="/redacted/path/for/arg10573" a10574="/redacted/path/for/arg10574" a10575="/redacted/path/for/arg10575" a10576="/redacted/path/for/arg10576" a10577="/redacted/path/for/arg10577" a10578="/redacted/path/for/arg10578" a10579="/redacted/path/for/arg10579" a10580="/redacted/path/for/arg10580" a10581="/redacted/path/for/arg10581" a10582="/redacted/path/for/arg10582" a10583="/redacted/path/for/arg10583" a10584="/redacted/path/for/arg10584" a10585="/redacted/path/for/arg10585" a10586="/redacted/path/for/arg10586" a10587_len=95 a10587[0]="/redacted/path"
+type=EXECVE msg=audit(1643025668.518:5009988): a10091="/redacted/path/for/arg10091" a10092="/redacted/path/for/arg10092" a10093="/redacted/path/for/arg10093" a10094="/redacted/path/for/arg10094" a10095="/redacted/path/for/arg10095" a10096="/redacted/path/for/arg10096" a10097="/redacted/path/for/arg10097" a10098="/redacted/path/for/arg10098" a10099="/redacted/path/for/arg10099" a10100="/redacted/path/for/arg10100" a10101="/redacted/path/for/arg10101" a10102="/redacted/path/for/arg10102" a10103="/redacted/path/for/arg10103" a10104="/redacted/path/for/arg10104" a10105="/redacted/path/for/arg10105" a10106="/redacted/path/for/arg10106" a10107="/redacted/path/for/arg10107" a10108="/redacted/path/for/arg10108" a10109="/redacted/path/for/arg10109" a10110="/redacted/path/for/arg10110" a10111="/redacted/path/for/arg10111" a10112="/redacted/path/for/arg10112" a10113="/redacted/path/for/arg10113" a10114="/redacted/path/for/arg10114" a10115="/redacted/path/for/arg10115" a10116="/redacted/path/for/arg10116" a10117="/redacted/path/for/arg10117" a10118="/redacted/path/for/arg10118" a10119="/redacted/path/for/arg10119" a10120="/redacted/path/for/arg10120" a10121="/redacted/path/for/arg10121" a10122="/redacted/path/for/arg10122" a10123="/redacted/path/for/arg10123" a10124="/redacted/path/for/arg10124" a10125="/redacted/path/for/arg10125" a10126="/redacted/path/for/arg10126" a10127="/redacted/path/for/arg10127" a10128="/redacted/path/for/arg10128" a10129="/redacted/path/for/arg10129" a10130="/redacted/path/for/arg10130" a10131="/redacted/path/for/arg10131" a10132="/redacted/path/for/arg10132" a10133="/redacted/path/for/arg10133" a10134="/redacted/path/for/arg10134" a10135="/redacted/path/for/arg10135" a10136="/redacted/path/for/arg10136" a10137="/redacted/path/for/arg10137" a10138="/redacted/path/for/arg10138" a10139="/redacted/path/for/arg10139" a10140="/redacted/path/for/arg10140" a10141="/redacted/path/for/arg10141" a10142="/redacted/path/for/arg10142" a10143="/redacted/path/for/arg10143" a10144="/redacted/path/for/arg10144" a10145="/redacted/path/for/arg10145" a10146="/redacted/path/for/arg10146" a10147="/redacted/path/for/arg10147" a10148="/redacted/path/for/arg10148" a10149="/redacted/path/for/arg10149" a10150="/redacted/path/for/arg10150" a10151="/redacted/path/for/arg10151" a10152="/redacted/path/for/arg10152" a10153="/redacted/path/for/arg10153" a10154="/redacted/path/for/arg10154" a10155="/redacted/path/for/arg10155" a10156="/redacted/path/for/arg10156" a10157="/redacted/path/for/arg10157" a10158="/redacted/path/for/arg10158" a10159="/redacted/path/for/arg10159" a10160="/redacted/path/for/arg10160" a10161_len=95 a10161[0]="/redacted/path"
+type=EXECVE msg=audit(1644323462.830:9381969):  a73[1]="arg73.gz" a74="/redacted/path/for/arg74" a75="/redacted/path/for/arg75" a76="/redacted/path/for/arg76" a77="/redacted/path/for/arg77" a78="/redacted/path/for/arg78" a79="/redacted/path/for/arg79" a80="/redacted/path/for/arg80" a81="/redacted/path/for/arg81" a82="/redacted/path/for/arg82" a83="/redacted/path/for/arg83" a84="/redacted/path/for/arg84" a85="/redacted/path/for/arg85" a86="/redacted/path/for/arg86" a87="/redacted/path/for/arg87"