ECS support for Grok processor (elastic#76885)

libs/grok/src/main/java/org/elasticsearch/grok/Grok.java

@@ -22,6 +22,7 @@
 import java.io.InputStreamReader;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
@@ -31,10 +32,14 @@
 import java.util.function.Consumer;
 
 public final class Grok {
+
+    public static final String[] ECS_COMPATIBILITY_MODES = {"disabled", "v1"};
+
     /**
      * Patterns built in to the grok library.
      */
-    public static final Map<String, String> BUILTIN_PATTERNS = loadBuiltinPatterns();
+    private static Map<String, String> LEGACY_PATTERNS;
+    private static Map<String, String> ECS_V1_PATTERNS;
 
     private static final String NAME_GROUP = "name";
     private static final String SUBNAME_GROUP = "subname";
@@ -296,16 +301,51 @@ public List<GrokCaptureConfig> captureConfig() {
     /**
      * Load built-in patterns.
      */
-    private static Map<String, String> loadBuiltinPatterns() {
-        String[] patternNames = new String[] {
+    public static synchronized Map<String, String> getBuiltinPatterns(boolean ecsCompatibility) {
+        if (ecsCompatibility) {
+            if (ECS_V1_PATTERNS == null) {
+                ECS_V1_PATTERNS = loadPatterns(ecsCompatibility);
+            }
+            return ECS_V1_PATTERNS;
+        } else {
+            if (LEGACY_PATTERNS == null) {
+                LEGACY_PATTERNS = loadPatterns(ecsCompatibility);
+            }
+            return LEGACY_PATTERNS;
+        }
+    }
+
+    public static Map<String, String> getBuiltinPatterns(String ecsCompatibility) {
+        if (isValidEcsCompatibilityMode(ecsCompatibility)) {
+            return getBuiltinPatterns(ECS_COMPATIBILITY_MODES[1].equals(ecsCompatibility));
+        } else {
+            throw new IllegalArgumentException("unsupported ECS compatibility mode [" + ecsCompatibility + "]");
+        }
+    }
+
+    public static boolean isValidEcsCompatibilityMode(String ecsCompatibility) {
+        return Arrays.asList(ECS_COMPATIBILITY_MODES).contains(ecsCompatibility);
+    }
+
+    private static Map<String, String> loadPatterns(boolean ecsCompatibility) {
+        String[] legacyPatternNames = {
             "aws", "bacula", "bind", "bro", "exim", "firewalls", "grok-patterns", "haproxy",
             "httpd", "java", "junos", "linux-syslog", "maven", "mcollective-patterns", "mongodb", "nagios",
             "postgresql", "rails", "redis", "ruby", "squid"
         };
+        String[] ecsPatternNames = {
+            "aws", "bacula", "bind", "bro", "exim", "firewalls", "grok-patterns", "haproxy",
+            "httpd", "java", "junos", "linux-syslog", "maven", "mcollective", "mongodb", "nagios",
+            "postgresql", "rails", "redis", "ruby", "squid", "zeek"
+        };
+
+        String[] patternNames = ecsCompatibility ? ecsPatternNames : legacyPatternNames;
+        String directory = ecsCompatibility ? "/patterns/ecs-v1/" : "/patterns/legacy/";
+
         Map<String, String> builtinPatterns = new LinkedHashMap<>();
         for (String pattern : patternNames) {
             try {
-                try(InputStream is = Grok.class.getResourceAsStream("/patterns/" + pattern)) {
+                try (InputStream is = Grok.class.getResourceAsStream(directory + pattern)) {
                     loadPatterns(builtinPatterns, is);
                 }
             } catch (IOException e) {

libs/grok/src/main/resources/patterns/ecs-v1/aws

@@ -0,0 +1,28 @@
+S3_REQUEST_LINE (?:%{WORD:http.request.method} %{NOTSPACE:url.original}(?: HTTP/%{NUMBER:http.version})?)
+
+S3_ACCESS_LOG %{WORD:aws.s3access.bucket_owner} %{NOTSPACE:aws.s3access.bucket} \%{HTTPDATE:timestamp}\ (?:-|%{IP:client.ip}) (?:-|%{NOTSPACE:client.user.id}) %{NOTSPACE:aws.s3access.request_id} %{NOTSPACE:aws.s3access.operation} (?:-|%{NOTSPACE:aws.s3access.key}) (?:-|"%{S3_REQUEST_LINE:aws.s3access.request_uri}") (?:-|%{INT:http.response.status_code:int}) (?:-|%{NOTSPACE:aws.s3access.error_code}) (?:-|%{INT:aws.s3access.bytes_sent:long}) (?:-|%{INT:aws.s3access.object_size:long}) (?:-|%{INT:aws.s3access.total_time:int}) (?:-|%{INT:aws.s3access.turn_around_time:int}) "(?:-|%{DATA:http.request.referrer})" "(?:-|%{DATA:user_agent.original})" (?:-|%{NOTSPACE:aws.s3access.version_id})(?: (?:-|%{NOTSPACE:aws.s3access.host_id}) (?:-|%{NOTSPACE:aws.s3access.signature_version}) (?:-|%{NOTSPACE:tls.cipher}) (?:-|%{NOTSPACE:aws.s3access.authentication_type}) (?:-|%{NOTSPACE:aws.s3access.host_header}) (?:-|%{NOTSPACE:aws.s3access.tls_version}))?
+# :long - %{INT:aws.s3access.bytes_sent:int}
+# :long - %{INT:aws.s3access.object_size:int}
+
+ELB_URIHOST %{IPORHOST:url.domain}(?::%{POSINT:url.port:int})?
+ELB_URIPATHQUERY %{URIPATH:url.path}(?:\?%{URIQUERY:url.query})?
+# deprecated - old name:
+ELB_URIPATHPARAM %{ELB_URIPATHQUERY}
+ELB_URI %{URIPROTO:url.scheme}://(?:%{USER:url.username}(?::^@*)?@)?(?:%{ELB_URIHOST})?(?:%{ELB_URIPATHQUERY})?
+
+ELB_REQUEST_LINE (?:%{WORD:http.request.method} %{ELB_URI:url.original}(?: HTTP/%{NUMBER:http.version})?)
+
+# pattern supports 'regular' HTTP ELB format
+ELB_V1_HTTP_LOG %{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE:aws.elb.name} %{IP:source.ip}:%{INT:source.port:int} (?:-|(?:%{IP:aws.elb.backend.ip}:%{INT:aws.elb.backend.port:int})) (?:-1|%{NUMBER:aws.elb.request_processing_time.sec:float}) (?:-1|%{NUMBER:aws.elb.backend_processing_time.sec:float}) (?:-1|%{NUMBER:aws.elb.response_processing_time.sec:float}) %{INT:http.response.status_code:int} (?:-|%{INT:aws.elb.backend.http.response.status_code:int}) %{INT:http.request.body.bytes:long} %{INT:http.response.body.bytes:long} "%{ELB_REQUEST_LINE}"(?: "(?:-|%{DATA:user_agent.original})" (?:-|%{NOTSPACE:tls.cipher}) (?:-|%{NOTSPACE:aws.elb.ssl_protocol}))?
+# :long - %{INT:http.request.body.bytes:int}
+# :long - %{INT:http.response.body.bytes:int}
+
+ELB_ACCESS_LOG %{ELB_V1_HTTP_LOG}
+
+# pattern used to match a shorted format, that's why we have the optional part (starting with *http.version*) at the end
+CLOUDFRONT_ACCESS_LOG (?<timestamp>%{YEAR}-%{MONTHNUM}-%{MONTHDAY}\t%{TIME})\t%{WORD:aws.cloudfront.x_edge_location}\t(?:-|%{INT:destination.bytes:long})\t%{IPORHOST:source.ip}\t%{WORD:http.request.method}\t%{HOSTNAME:url.domain}\t%{NOTSPACE:url.path}\t(?:(?:000)|%{INT:http.response.status_code:int})\t(?:-|%{DATA:http.request.referrer})\t%{DATA:user_agent.original}\t(?:-|%{DATA:url.query})\t(?:-|%{DATA:aws.cloudfront.http.request.cookie})\t%{WORD:aws.cloudfront.x_edge_result_type}\t%{NOTSPACE:aws.cloudfront.x_edge_request_id}\t%{HOSTNAME:aws.cloudfront.http.request.host}\t%{URIPROTO:network.protocol}\t(?:-|%{INT:source.bytes:long})\t%{NUMBER:aws.cloudfront.time_taken:float}\t(?:-|%{IP:network.forwarded_ip})\t(?:-|%{DATA:aws.cloudfront.ssl_protocol})\t(?:-|%{NOTSPACE:tls.cipher})\t%{WORD:aws.cloudfront.x_edge_response_result_type}(?:\t(?:-|HTTP/%{NUMBER:http.version})\t(?:-|%{DATA:aws.cloudfront.fle_status})\t(?:-|%{DATA:aws.cloudfront.fle_encrypted_fields})\t%{INT:source.port:int}\t%{NUMBER:aws.cloudfront.time_to_first_byte:float}\t(?:-|%{DATA:aws.cloudfront.x_edge_detailed_result_type})\t(?:-|%{NOTSPACE:http.request.mime_type})\t(?:-|%{INT:aws.cloudfront.http.request.size:long})\t(?:-|%{INT:aws.cloudfront.http.request.range.start:long})\t(?:-|%{INT:aws.cloudfront.http.request.range.end:long}))?
+# :long - %{INT:destination.bytes:int}
+# :long - %{INT:source.bytes:int}
+# :long - %{INT:aws.cloudfront.http.request.size:int}
+# :long - %{INT:aws.cloudfront.http.request.range.start:int}
+# :long - %{INT:aws.cloudfront.http.request.range.end:int}

libs/grok/src/main/resources/patterns/ecs-v1/bacula

@@ -0,0 +1,53 @@
+BACULA_TIMESTAMP %{MONTHDAY}-%{MONTH}(?:-%{YEAR})? %{HOUR}:%{MINUTE}
+BACULA_HOST %{HOSTNAME}
+BACULA_VOLUME %{USER}
+BACULA_DEVICE %{USER}
+BACULA_DEVICEPATH %{UNIXPATH}
+BACULA_CAPACITY %{INT}{1,3}(,%{INT}{3})*
+BACULA_VERSION %{USER}
+BACULA_JOB %{USER}
+
+BACULA_LOG_MAX_CAPACITY User defined maximum volume capacity %{BACULA_CAPACITY:bacula.volume.max_capacity} exceeded on device \"%{BACULA_DEVICE:bacula.volume.device}\" \(%{BACULA_DEVICEPATH:bacula.volume.path}\).?
+BACULA_LOG_END_VOLUME End of medium on Volume \"%{BACULA_VOLUME:bacula.volume.name}\" Bytes=%{BACULA_CAPACITY:bacula.volume.bytes} Blocks=%{BACULA_CAPACITY:bacula.volume.blocks} at %{BACULA_TIMESTAMP:bacula.timestamp}.
+BACULA_LOG_NEW_VOLUME Created new Volume \"%{BACULA_VOLUME:bacula.volume.name}\" in catalog.
+BACULA_LOG_NEW_LABEL Labeled new Volume \"%{BACULA_VOLUME:bacula.volume.name}\" on (?:file )?device \"%{BACULA_DEVICE:bacula.volume.device}\" \(%{BACULA_DEVICEPATH:bacula.volume.path}\).
+BACULA_LOG_WROTE_LABEL Wrote label to prelabeled Volume \"%{BACULA_VOLUME:bacula.volume.name}\" on device \"%{BACULA_DEVICE:bacula.volume.device}\" \(%{BACULA_DEVICEPATH:bacula.volume.path}\)
+BACULA_LOG_NEW_MOUNT New volume \"%{BACULA_VOLUME:bacula.volume.name}\" mounted on device \"%{BACULA_DEVICE:bacula.volume.device}\" \(%{BACULA_DEVICEPATH:bacula.volume.path}\) at %{BACULA_TIMESTAMP:bacula.timestamp}.
+BACULA_LOG_NOOPEN \s*Cannot open %{DATA}: ERR=%{GREEDYDATA:error.message}
+BACULA_LOG_NOOPENDIR \s*Could not open directory \"?%{DATA:file.path}\"?: ERR=%{GREEDYDATA:error.message}
+BACULA_LOG_NOSTAT \s*Could not stat %{DATA:file.path}: ERR=%{GREEDYDATA:error.message}
+BACULA_LOG_NOJOBS There are no more Jobs associated with Volume \"%{BACULA_VOLUME:bacula.volume.name}\". Marking it purged.
+BACULA_LOG_ALL_RECORDS_PRUNED .*?All records pruned from Volume \"%{BACULA_VOLUME:bacula.volume.name}\"; marking it \"Purged\"
+BACULA_LOG_BEGIN_PRUNE_JOBS Begin pruning Jobs older than %{INT} month %{INT} days .
+BACULA_LOG_BEGIN_PRUNE_FILES Begin pruning Files.
+BACULA_LOG_PRUNED_JOBS Pruned %{INT} Jobs* for client %{BACULA_HOST:bacula.client.name} from catalog.
+BACULA_LOG_PRUNED_FILES Pruned Files from %{INT} Jobs* for client %{BACULA_HOST:bacula.client.name} from catalog.
+BACULA_LOG_ENDPRUNE End auto prune.
+BACULA_LOG_STARTJOB Start Backup JobId %{INT}, Job=%{BACULA_JOB:bacula.job.name}
+BACULA_LOG_STARTRESTORE Start Restore Job %{BACULA_JOB:bacula.job.name}
+BACULA_LOG_USEDEVICE Using Device \"%{BACULA_DEVICE:bacula.volume.device}\"
+BACULA_LOG_DIFF_FS \s*%{UNIXPATH} is a different filesystem. Will not descend from %{UNIXPATH} into it.
+BACULA_LOG_JOBEND Job write elapsed time = %{DATA:bacula.job.elapsed_time}, Transfer rate = %{NUMBER} (K|M|G)? Bytes/second
+BACULA_LOG_NOPRUNE_JOBS No Jobs found to prune.
+BACULA_LOG_NOPRUNE_FILES No Files found to prune.
+BACULA_LOG_VOLUME_PREVWRITTEN Volume \"?%{BACULA_VOLUME:bacula.volume.name}\"? previously written, moving to end of data.
+BACULA_LOG_READYAPPEND Ready to append to end of Volume \"%{BACULA_VOLUME:bacula.volume.name}\" size=%{INT:bacula.volume.size:long}
+# :long - %{INT:bacula.volume.size:int}
+BACULA_LOG_CANCELLING Cancelling duplicate JobId=%{INT:bacula.job.other_id}.
+BACULA_LOG_MARKCANCEL JobId %{INT:bacula.job.id}, Job %{BACULA_JOB:bacula.job.name} marked to be canceled.
+BACULA_LOG_CLIENT_RBJ shell command: run ClientRunBeforeJob \"%{GREEDYDATA:bacula.job.client_run_before_command}\"
+BACULA_LOG_VSS (Generate )?VSS (Writer)?
+BACULA_LOG_MAXSTART Fatal [eE]rror: Job canceled because max start delay time exceeded.
+BACULA_LOG_DUPLICATE Fatal [eE]rror: JobId %{INT:bacula.job.other_id} already running. Duplicate job not allowed.
+BACULA_LOG_NOJOBSTAT Fatal [eE]rror: No Job status returned from FD.
+BACULA_LOG_FATAL_CONN Fatal [eE]rror: bsock.c:133 Unable to connect to (Client: %{BACULA_HOST:bacula.client.name}|Storage daemon) on %{IPORHOST:client.address}:%{POSINT:client.port:int}. ERR=%{GREEDYDATA:error.message}
+BACULA_LOG_NO_CONNECT Warning: bsock.c:127 Could not connect to (Client: %{BACULA_HOST:bacula.client.name}|Storage daemon) on %{IPORHOST:client.address}:%{POSINT:client.port:int}. ERR=%{GREEDYDATA:error.message}
+BACULA_LOG_NO_AUTH Fatal error: Unable to authenticate with File daemon at \"?%{IPORHOST:client.address}(?::%{POSINT:client.port:int})?\"?. Possible causes:
+BACULA_LOG_NOSUIT No prior or suitable Full backup found in catalog. Doing FULL backup.
+BACULA_LOG_NOPRIOR No prior Full backup Job record found.
+
+BACULA_LOG_JOB (Error: )?Bacula %{BACULA_HOST} %{BACULA_VERSION} \(%{BACULA_VERSION}\):
+
+BACULA_LOG %{BACULA_TIMESTAMP:timestamp} %{BACULA_HOST:host.hostname}(?: JobId %{INT:bacula.job.id})?:? (%{BACULA_LOG_MAX_CAPACITY}|%{BACULA_LOG_END_VOLUME}|%{BACULA_LOG_NEW_VOLUME}|%{BACULA_LOG_NEW_LABEL}|%{BACULA_LOG_WROTE_LABEL}|%{BACULA_LOG_NEW_MOUNT}|%{BACULA_LOG_NOOPEN}|%{BACULA_LOG_NOOPENDIR}|%{BACULA_LOG_NOSTAT}|%{BACULA_LOG_NOJOBS}|%{BACULA_LOG_ALL_RECORDS_PRUNED}|%{BACULA_LOG_BEGIN_PRUNE_JOBS}|%{BACULA_LOG_BEGIN_PRUNE_FILES}|%{BACULA_LOG_PRUNED_JOBS}|%{BACULA_LOG_PRUNED_FILES}|%{BACULA_LOG_ENDPRUNE}|%{BACULA_LOG_STARTJOB}|%{BACULA_LOG_STARTRESTORE}|%{BACULA_LOG_USEDEVICE}|%{BACULA_LOG_DIFF_FS}|%{BACULA_LOG_JOBEND}|%{BACULA_LOG_NOPRUNE_JOBS}|%{BACULA_LOG_NOPRUNE_FILES}|%{BACULA_LOG_VOLUME_PREVWRITTEN}|%{BACULA_LOG_READYAPPEND}|%{BACULA_LOG_CANCELLING}|%{BACULA_LOG_MARKCANCEL}|%{BACULA_LOG_CLIENT_RBJ}|%{BACULA_LOG_VSS}|%{BACULA_LOG_MAXSTART}|%{BACULA_LOG_DUPLICATE}|%{BACULA_LOG_NOJOBSTAT}|%{BACULA_LOG_FATAL_CONN}|%{BACULA_LOG_NO_CONNECT}|%{BACULA_LOG_NO_AUTH}|%{BACULA_LOG_NOSUIT}|%{BACULA_LOG_JOB}|%{BACULA_LOG_NOPRIOR})
+# old (deprecated) name :
+BACULA_LOGLINE %{BACULA_LOG}

libs/grok/src/main/resources/patterns/ecs-v1/bind

@@ -0,0 +1,13 @@
+BIND9_TIMESTAMP %{MONTHDAY}[-]%{MONTH}[-]%{YEAR} %{TIME}
+
+BIND9_DNSTYPE (?:A|AAAA|CAA|CDNSKEY|CDS|CERT|CNAME|CSYNC|DLV|DNAME|DNSKEY|DS|HINFO|LOC|MX|NAPTR|NS|NSEC|NSEC3|OPENPGPKEY|PTR|RRSIG|RP|SIG|SMIMEA|SOA|SRV|TSIG|TXT|URI)
+BIND9_CATEGORY (?:queries)
+
+# dns.question.class is static - only 'IN' is supported by Bind9
+# bind.log.question.name is expected to be a 'duplicate' (same as the dns.question.name capture)
+BIND9_QUERYLOGBASE client(:? @0x(?:[0-9A-Fa-f]+))? %{IP:client.ip}#%{POSINT:client.port:int} \(%{GREEDYDATA:bind.log.question.name}\): query: %{GREEDYDATA:dns.question.name} (?<dns.question.class>IN) %{BIND9_DNSTYPE:dns.question.type}(:? %{DATA:bind.log.question.flags})? \(%{IP:server.ip}\)
+
+# for query-logging category and severity are always fixed as "queries: info: "
+BIND9_QUERYLOG %{BIND9_TIMESTAMP:timestamp} %{BIND9_CATEGORY:bing.log.category}: %{LOGLEVEL:log.level}: %{BIND9_QUERYLOGBASE}
+
+BIND9 %{BIND9_QUERYLOG}

libs/grok/src/main/resources/patterns/ecs-v1/bro

@@ -0,0 +1,30 @@
+# supports the 'old' BRO log files, for updated Zeek log format see the patters/ecs-v1/zeek
+# https://www.bro.org/sphinx/script-reference/log-files.html
+
+BRO_BOOL [TF]
+BRO_DATA [^\t]+
+
+# http.log - old format (before the Zeek rename) :
+BRO_HTTP %{NUMBER:timestamp}\t%{NOTSPACE:zeek.session_id}\t%{IP:source.ip}\t%{INT:source.port:int}\t%{IP:destination.ip}\t%{INT:destination.port:int}\t%{INT:zeek.http.trans_depth:int}\t(?:-|%{WORD:http.request.method})\t(?:-|%{BRO_DATA:url.domain})\t(?:-|%{BRO_DATA:url.original})\t(?:-|%{BRO_DATA:http.request.referrer})\t(?:-|%{BRO_DATA:user_agent.original})\t(?:-|%{NUMBER:http.request.body.bytes:long})\t(?:-|%{NUMBER:http.response.body.bytes:long})\t(?:-|%{POSINT:http.response.status_code:int})\t(?:-|%{DATA:zeek.http.status_msg})\t(?:-|%{POSINT:zeek.http.info_code:int})\t(?:-|%{DATA:zeek.http.info_msg})\t(?:-|%{BRO_DATA:zeek.http.filename})\t(?:\(empty\)|%{BRO_DATA:zeek.http.tags})\t(?:-|%{BRO_DATA:url.username})\t(?:-|%{BRO_DATA:url.password})\t(?:-|%{BRO_DATA:zeek.http.proxied})\t(?:-|%{BRO_DATA:zeek.http.orig_fuids})\t(?:-|%{BRO_DATA:http.request.mime_type})\t(?:-|%{BRO_DATA:zeek.http.resp_fuids})\t(?:-|%{BRO_DATA:http.response.mime_type})
+# :long - %{NUMBER:http.request.body.bytes:int}
+# :long - %{NUMBER:http.response.body.bytes:int}
+
+# dns.log - old format
+BRO_DNS %{NUMBER:timestamp}\t%{NOTSPACE:zeek.session_id}\t%{IP:source.ip}\t%{INT:source.port:int}\t%{IP:destination.ip}\t%{INT:destination.port:int}\t%{WORD:network.transport}\t(?:-|%{INT:dns.id:int})\t(?:-|%{BRO_DATA:dns.question.name})\t(?:-|%{INT:zeek.dns.qclass:int})\t(?:-|%{BRO_DATA:zeek.dns.qclass_name})\t(?:-|%{INT:zeek.dns.qtype:int})\t(?:-|%{BRO_DATA:dns.question.type})\t(?:-|%{INT:zeek.dns.rcode:int})\t(?:-|%{BRO_DATA:dns.response_code})\t(?:-|%{BRO_BOOL:zeek.dns.AA})\t(?:-|%{BRO_BOOL:zeek.dns.TC})\t(?:-|%{BRO_BOOL:zeek.dns.RD})\t(?:-|%{BRO_BOOL:zeek.dns.RA})\t(?:-|%{NONNEGINT:zeek.dns.Z:int})\t(?:-|%{BRO_DATA:zeek.dns.answers})\t(?:-|%{DATA:zeek.dns.TTLs})\t(?:-|%{BRO_BOOL:zeek.dns.rejected})
+
+# conn.log - old bro, also supports 'newer' format (optional *zeek.connection.local_resp* flag) compared to non-ecs mode
+BRO_CONN %{NUMBER:timestamp}\t%{NOTSPACE:zeek.session_id}\t%{IP:source.ip}\t%{INT:source.port:int}\t%{IP:destination.ip}\t%{INT:destination.port:int}\t%{WORD:network.transport}\t(?:-|%{BRO_DATA:network.protocol})\t(?:-|%{NUMBER:zeek.connection.duration:float})\t(?:-|%{INT:zeek.connection.orig_bytes:long})\t(?:-|%{INT:zeek.connection.resp_bytes:long})\t(?:-|%{BRO_DATA:zeek.connection.state})\t(?:-|%{BRO_BOOL:zeek.connection.local_orig})\t(?:(?:-|%{BRO_BOOL:zeek.connection.local_resp})\t)?(?:-|%{INT:zeek.connection.missed_bytes:long})\t(?:-|%{BRO_DATA:zeek.connection.history})\t(?:-|%{INT:source.packets:long})\t(?:-|%{INT:source.bytes:long})\t(?:-|%{INT:destination.packets:long})\t(?:-|%{INT:destination.bytes:long})\t(?:\(empty\)|%{BRO_DATA:zeek.connection.tunnel_parents})
+# :long - %{INT:zeek.connection.orig_bytes:int}
+# :long - %{INT:zeek.connection.resp_bytes:int}
+# :long - %{INT:zeek.connection.missed_bytes:int}
+# :long - %{INT:source.packets:int}
+# :long - %{INT:source.bytes:int}
+# :long - %{INT:destination.packets:int}
+# :long - %{INT:destination.bytes:int}
+
+# files.log - old format
+BRO_FILES %{NUMBER:timestamp}\t%{NOTSPACE:zeek.files.fuid}\t(?:-|%{IP:server.ip})\t(?:-|%{IP:client.ip})\t(?:-|%{BRO_DATA:zeek.files.session_ids})\t(?:-|%{BRO_DATA:zeek.files.source})\t(?:-|%{INT:zeek.files.depth:int})\t(?:-|%{BRO_DATA:zeek.files.analyzers})\t(?:-|%{BRO_DATA:file.mime_type})\t(?:-|%{BRO_DATA:file.name})\t(?:-|%{NUMBER:zeek.files.duration:float})\t(?:-|%{BRO_DATA:zeek.files.local_orig})\t(?:-|%{BRO_BOOL:zeek.files.is_orig})\t(?:-|%{INT:zeek.files.seen_bytes:long})\t(?:-|%{INT:file.size:long})\t(?:-|%{INT:zeek.files.missing_bytes:long})\t(?:-|%{INT:zeek.files.overflow_bytes:long})\t(?:-|%{BRO_BOOL:zeek.files.timedout})\t(?:-|%{BRO_DATA:zeek.files.parent_fuid})\t(?:-|%{BRO_DATA:file.hash.md5})\t(?:-|%{BRO_DATA:file.hash.sha1})\t(?:-|%{BRO_DATA:file.hash.sha256})\t(?:-|%{BRO_DATA:zeek.files.extracted})
+# :long - %{INT:zeek.files.seen_bytes:int}
+# :long - %{INT:file.size:int}
+# :long - %{INT:zeek.files.missing_bytes:int}
+# :long - %{INT:zeek.files.overflow_bytes:int}

libs/grok/src/main/resources/patterns/ecs-v1/exim

@@ -0,0 +1,26 @@
+EXIM_MSGID [0-9A-Za-z]{6}-[0-9A-Za-z]{6}-[0-9A-Za-z]{2}
+# <=     message arrival
+# =>     normal message delivery
+# ->     additional address in same delivery
+# *>     delivery suppressed by -N
+# **     delivery failed; address bounced
+# ==     delivery deferred; temporary problem
+EXIM_FLAGS (?:<=|=>|->|\*>|\*\*|==|<>|>>)
+EXIM_DATE (:?%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME})
+EXIM_PID \[%{POSINT:process.pid:int}\]
+EXIM_QT ((\d+y)?(\d+w)?(\d+d)?(\d+h)?(\d+m)?(\d+s)?)
+EXIM_EXCLUDE_TERMS (Message is frozen|(Start|End) queue run| Warning: | retry time not reached | no (IP address|host name) found for (IP address|host) | unexpected disconnection while reading SMTP command | no immediate delivery: |another process is handling this message)
+EXIM_REMOTE_HOST (H=(%{NOTSPACE:source.address} )?(\(%{NOTSPACE:exim.log.remote_address}\) )?\%{IP:source.ip}\](?::%{POSINT:source.port:int})?)
+EXIM_INTERFACE (I=\[%{IP:destination.ip}\](?::%{NUMBER:destination.port:int}))
+EXIM_PROTOCOL (P=%{NOTSPACE:network.protocol})
+EXIM_MSG_SIZE (S=%{NUMBER:exim.log.message.size:int})
+EXIM_HEADER_ID (id=%{NOTSPACE:exim.log.header_id})
+EXIM_QUOTED_CONTENT (?:\\.|[^\\"])*
+EXIM_SUBJECT (T="%{EXIM_QUOTED_CONTENT:exim.log.message.subject}")
+
+EXIM_UNKNOWN_FIELD (?:[A-Za-z0-9]{1,4}=(?:%{QUOTEDSTRING}|%{NOTSPACE}))
+EXIM_NAMED_FIELDS (?: (?:%{EXIM_REMOTE_HOST}|%{EXIM_INTERFACE}|%{EXIM_PROTOCOL}|%{EXIM_MSG_SIZE}|%{EXIM_HEADER_ID}|%{EXIM_SUBJECT}|%{EXIM_UNKNOWN_FIELD}))*
+
+EXIM_MESSAGE_ARRIVAL %{EXIM_DATE:timestamp} (?:%{EXIM_PID} )?%{EXIM_MSGID:exim.log.message.id} (?<exim.log.flags><=) (?<exim.log.status>[a-z:] )?%{EMAILADDRESS:exim.log.sender.email}%{EXIM_NAMED_FIELDS}(?:(?: from <?%{DATA:exim.log.sender.original}>?)? for %{EMAILADDRESS:exim.log.recipient.email})?
+
+EXIM %{EXIM_MESSAGE_ARRIVAL}