diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index ed4d7cfb671..ebb0999888a 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -417,6 +417,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add support to merge registry updates in the filestream input across multiple ACKed batches in case of backpressure in the registry or disk. {pull}25976[25976] - Add support to `decode_cef` for MAC addresses that do not contain separator characters. {issue}27050[27050] {pull}27109[27109] - Add new `hmac` template function for httpjson input {pull}27168[27168] +- Add `timezone` config option to the `decode_cef` processor. {issue}27232[27232] {pull}27727[27727] +- Add `timezone` config option to the `syslog` input. {pull}27727[27727] - Update `tags` and `threatintel.indicator.provider` fields in `threatintel.anomali` ingest pipeline {issue}24746[24746] {pull}27141[27141] - Move AWS module and filesets to GA. {pull}27428[27428] - update ecs.version to ECS 1.11.0. {pull}27107[27107] diff --git a/NOTICE.txt b/NOTICE.txt index a8062dd598a..ae5b5b74793 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -9,69 +9,6 @@ Third party libraries used by the Elastic Beats project: ================================================================================ --------------------------------------------------------------------------------- -Dependency : 4d63.com/tz -Version: v1.1.1-0.20191124060701-6d37baae851b -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/4d63.com/tz@v1.1.1-0.20191124060701-6d37baae851b/LICENSE: - -MIT License - -Copyright (c) 2018 Leigh McCulloch - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - --------------------------------------------------------------------------------- - -zoneinfo.go generated from /lib/time/zoneinfo.zip from Go. - -Copyright (c) 2009 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -------------------------------------------------------------------------------- Dependency : cloud.google.com/go Version: v0.51.0 @@ -19439,35 +19376,6 @@ Contents of probable licence file $GOMODCACHE/k8s.io/client-go@v0.19.4/LICENSE: Indirect dependencies --------------------------------------------------------------------------------- -Dependency : 4d63.com/embedfiles -Version: v0.0.0-20190311033909-995e0740726f -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/4d63.com/embedfiles@v0.0.0-20190311033909-995e0740726f/LICENSE: - -Copyright (c) 2017, Leigh McCulloch - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - - -------------------------------------------------------------------------------- Dependency : bazil.org/fuse Version: v0.0.0-20160811212531-371fbbdaa898 diff --git a/filebeat/docs/inputs/input-syslog.asciidoc b/filebeat/docs/inputs/input-syslog.asciidoc index a1146afab71..e43eabea378 100644 --- a/filebeat/docs/inputs/input-syslog.asciidoc +++ b/filebeat/docs/inputs/input-syslog.asciidoc @@ -7,7 +7,8 @@ Syslog ++++ -The `syslog` input reads Syslog events as specified by RFC 3164 and RFC 5424, over TCP, UDP, or a Unix stream socket. +The `syslog` input reads Syslog events as specified by RFC 3164 and RFC 5424, +over TCP, UDP, or a Unix stream socket. Example configurations: @@ -40,12 +41,21 @@ Example configurations: ==== Configuration options -The `syslog` input configuration includes format, protocol specific options, and the -<<{beatname_lc}-input-{type}-common-options>> described later. +The `syslog` input configuration includes format, protocol specific options, and +the <<{beatname_lc}-input-{type}-common-options>> described later. ===== `format` -The syslog variant to use, `rfc3164` or `rfc5424`. To automatically detect the format from the log entries, set this option to `auto`. The default is `rfc3164`. +The syslog variant to use, `rfc3164` or `rfc5424`. To automatically detect the +format from the log entries, set this option to `auto`. The default is +`rfc3164`. + +===== `timezone` + +IANA time zone name (e.g. `America/New_York`) or fixed time offset (e.g. +`+0200`) to use when parsing syslog timestamps that do not contain a time zone. +`Local` may be specified to use the machine's local time zone. Defaults to +`Local`. ===== Protocol `udp`: diff --git a/filebeat/docs/modules/cef.asciidoc b/filebeat/docs/modules/cef.asciidoc index cb5af4a9230..95220bbfd5d 100644 --- a/filebeat/docs/modules/cef.asciidoc +++ b/filebeat/docs/modules/cef.asciidoc @@ -46,6 +46,13 @@ A list of tags to include in events. Including `forwarded` indicates that the events did not originate on this host and causes `host.name` to not be added to events. Defaults to `[cef, forwarded]`. +*`var.timezone`*:: + +IANA time zone name (e.g. `America/New_York`) or fixed time offset (e.g. +`+0200`) to use when parsing times from the CEF message that do not contain a +time zone. `Local` may be specified to use the machine's local time zone. +Defaults to `UTC`. + [float] ==== Forcepoint NGFW Security Management Center diff --git a/filebeat/input/syslog/config.go b/filebeat/input/syslog/config.go index 08bc5f166d7..b50db762d72 100644 --- a/filebeat/input/syslog/config.go +++ b/filebeat/input/syslog/config.go @@ -30,6 +30,7 @@ import ( "github.com/elastic/beats/v7/filebeat/inputsource/udp" "github.com/elastic/beats/v7/filebeat/inputsource/unix" "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/cfgtype" "github.com/elastic/beats/v7/libbeat/logp" ) @@ -37,6 +38,7 @@ type config struct { harvester.ForwarderConfig `config:",inline"` Format syslogFormat `config:"format"` Protocol common.ConfigNamespace `config:"protocol"` + Timezone *cfgtype.Timezone `config:"timezone"` } type syslogFormat int @@ -59,7 +61,8 @@ var defaultConfig = config{ ForwarderConfig: harvester.ForwarderConfig{ Type: "syslog", }, - Format: syslogFormatRFC3164, + Format: syslogFormatRFC3164, + Timezone: cfgtype.MustNewTimezone("Local"), } type syslogTCP struct { diff --git a/filebeat/input/syslog/input.go b/filebeat/input/syslog/input.go index 6e66e1e0c72..b055a729050 100644 --- a/filebeat/input/syslog/input.go +++ b/filebeat/input/syslog/input.go @@ -179,7 +179,7 @@ func GetCbByConfig(cfg config, forwarder *harvester.Forwarder, log *logp.Logger) case syslogFormatRFC5424: return func(data []byte, metadata inputsource.NetworkMetadata) { - ev := parseAndCreateEvent5424(data, metadata, time.Local, log) + ev := parseAndCreateEvent5424(data, metadata, cfg.Timezone.Location(), log) forwarder.Send(ev) } @@ -187,9 +187,9 @@ func GetCbByConfig(cfg config, forwarder *harvester.Forwarder, log *logp.Logger) return func(data []byte, metadata inputsource.NetworkMetadata) { var ev beat.Event if IsRFC5424Format(data) { - ev = parseAndCreateEvent5424(data, metadata, time.Local, log) + ev = parseAndCreateEvent5424(data, metadata, cfg.Timezone.Location(), log) } else { - ev = parseAndCreateEvent3164(data, metadata, time.Local, log) + ev = parseAndCreateEvent3164(data, metadata, cfg.Timezone.Location(), log) } forwarder.Send(ev) } @@ -198,7 +198,7 @@ func GetCbByConfig(cfg config, forwarder *harvester.Forwarder, log *logp.Logger) } return func(data []byte, metadata inputsource.NetworkMetadata) { - ev := parseAndCreateEvent3164(data, metadata, time.Local, log) + ev := parseAndCreateEvent3164(data, metadata, cfg.Timezone.Location(), log) forwarder.Send(ev) } } @@ -287,7 +287,7 @@ func parseAndCreateEvent3164(data []byte, metadata inputsource.NetworkMetadata, "message": string(data), }) } - return createEvent(ev, metadata, time.Local, log) + return createEvent(ev, metadata, timezone, log) } func parseAndCreateEvent5424(data []byte, metadata inputsource.NetworkMetadata, timezone *time.Location, log *logp.Logger) beat.Event { @@ -299,7 +299,7 @@ func parseAndCreateEvent5424(data []byte, metadata inputsource.NetworkMetadata, "message": string(data), }) } - return createEvent(ev, metadata, time.Local, log) + return createEvent(ev, metadata, timezone, log) } func newBeatEvent(timestamp time.Time, metadata inputsource.NetworkMetadata, fields common.MapStr) beat.Event { diff --git a/go.mod b/go.mod index ac30acdd69b..96f3ae0b310 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,6 @@ module github.com/elastic/beats/v7 go 1.16 require ( - 4d63.com/tz v1.1.1-0.20191124060701-6d37baae851b cloud.google.com/go v0.51.0 cloud.google.com/go/bigquery v1.0.1 cloud.google.com/go/pubsub v1.0.1 diff --git a/go.sum b/go.sum index 7723ce06b73..0a75a5b5eee 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,3 @@ -4d63.com/embedfiles v0.0.0-20190311033909-995e0740726f h1:oyYjGRBNq1TxAIG8aHqtxlvqUfzdZf+MbcRb/oweNfY= -4d63.com/embedfiles v0.0.0-20190311033909-995e0740726f/go.mod h1:HxEsUxoVZyRxsZML/S6e2xAuieFMlGO0756ncWx1aXE= -4d63.com/tz v1.1.1-0.20191124060701-6d37baae851b h1:+TO4EgK74+Qo/ilRDiF2WpY09Jk9VSJSLe3wEn+dJBw= -4d63.com/tz v1.1.1-0.20191124060701-6d37baae851b/go.mod h1:SHGqVdL7hd2ZaX2T9uEiOZ/OFAUfCCLURdLPJsd8ZNs= bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= diff --git a/libbeat/common/cfgtype/timezone.go b/libbeat/common/cfgtype/timezone.go new file mode 100644 index 00000000000..db49eb4d506 --- /dev/null +++ b/libbeat/common/cfgtype/timezone.go @@ -0,0 +1,94 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package cfgtype + +import ( + "encoding/json" + "time" + + "github.com/pkg/errors" + + // Embed the timezone database so this code works across platforms. + _ "time/tzdata" +) + +var fixedOffsetFormats = []string{"-07", "-0700", "-07:00"} + +// Timezone maps time instants to the zone in use at that time. Typically, the +// Timezone represents the collection of time offsets in use in a geographical +// area. For many Locations the time offset varies depending on whether daylight +// savings time is in use at the time instant. +type Timezone time.Location + +// NewTimezone returns a new timezone. +func NewTimezone(tz string) (*Timezone, error) { + loc, err := loadLocation(tz) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse timezone %q", tz) + } + return (*Timezone)(loc), nil +} + +// MustNewTimezone returns a new timezone. If tz is invalid it panics. +func MustNewTimezone(tz string) *Timezone { + timestamp, err := NewTimezone(tz) + if err != nil { + panic(err) + } + return timestamp +} + +// Location returns a *time.Location. If timezone is nil it returns *time.UTC. +func (tz *Timezone) Location() *time.Location { + if tz == nil { + return time.UTC + } + return (*time.Location)(tz) +} + +// MarshalJSON implements json.Marshaler interface. +func (tz *Timezone) MarshalJSON() ([]byte, error) { + if tz == nil { + return []byte("null"), nil + } + return json.Marshal(tz.Location().String()) +} + +// Unpack converts a time zone name or offset to Timezone. If using a fixed +// offset then the format must be [+-]HHMM (e.g +0800 or -0530). +func (tz *Timezone) Unpack(v string) error { + timezone, err := NewTimezone(v) + if err != nil { + return err + } + *tz = *timezone + return nil +} + +func loadLocation(timezone string) (*time.Location, error) { + for _, format := range fixedOffsetFormats { + t, err := time.Parse(format, timezone) + if err == nil { + name, offset := t.Zone() + return time.FixedZone(name, offset), nil + } + } + + // Handle IANA time zones. + return time.LoadLocation(timezone) +} diff --git a/libbeat/common/cfgtype/timezone_test.go b/libbeat/common/cfgtype/timezone_test.go new file mode 100644 index 00000000000..bcc762d74f3 --- /dev/null +++ b/libbeat/common/cfgtype/timezone_test.go @@ -0,0 +1,63 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package cfgtype + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestTimezoneUnpack(t *testing.T) { + testCases := []struct { + ZoneName string + }{ + {"America/New_York"}, + {"Local"}, + {"+0500"}, + {"-0500"}, + {"+05:00"}, + {"-05:00"}, + {"+05"}, + {"-05"}, + {"UTC"}, + } + + for _, tc := range testCases { + t.Run(tc.ZoneName, func(t *testing.T) { + tz := &Timezone{} + err := tz.Unpack(tc.ZoneName) + require.NoError(t, err) + }) + } +} + +func TestTimezoneUnpackFixedZone(t *testing.T) { + tz := &Timezone{} + err := tz.Unpack("+0530") + require.NoError(t, err) + + now := time.Time{} + loc := tz.Location() + offset := now.In(loc) + offsetHour := offset.Hour() + offsetMinute := offset.Minute() + require.Equal(t, 5, offsetHour) + require.Equal(t, 30, offsetMinute) +} diff --git a/libbeat/common/datetime.go b/libbeat/common/datetime.go index c0eac0cc6f5..fec3d8ebc0e 100644 --- a/libbeat/common/datetime.go +++ b/libbeat/common/datetime.go @@ -61,7 +61,7 @@ func ParseTime(timespec string) (Time, error) { } func (t Time) String() string { - return time.Time(t).Format(TsLayout) + return time.Time(t).UTC().Format(TsLayout) } // MustParseTime is a convenience equivalent of the ParseTime function diff --git a/libbeat/processors/timestamp/config.go b/libbeat/processors/timestamp/config.go index 7b0fa5f2fc8..7bfc5c04b98 100644 --- a/libbeat/processors/timestamp/config.go +++ b/libbeat/processors/timestamp/config.go @@ -17,15 +17,17 @@ package timestamp +import "github.com/elastic/beats/v7/libbeat/common/cfgtype" + type config struct { - Field string `config:"field" validate:"required"` // Source field containing time time to be parsed. - TargetField string `config:"target_field"` // Target field for the parsed time value. The target value is always written as UTC. Defaults to @timestamp. - Layouts []string `config:"layouts" validate:"required"` // Timestamp layouts that define the expected time value format. - Timezone string `config:"timezone"` // Timezone (e.g. America/New_York) to use when parsing a timestamp not containing a timezone. - IgnoreMissing bool `config:"ignore_missing"` // Ignore errors when the source field is missing. - IgnoreFailure bool `config:"ignore_failure"` // Ignore errors when parsing the timestamp. - TestTimestamps []string `config:"test"` // A list of timestamps that must parse successfully when loading the processor. - ID string `config:"id"` // An identifier for this processor. Useful for debugging. + Field string `config:"field" validate:"required"` // Source field containing time time to be parsed. + TargetField string `config:"target_field"` // Target field for the parsed time value. The target value is always written as UTC. Defaults to @timestamp. + Layouts []string `config:"layouts" validate:"required"` // Timestamp layouts that define the expected time value format. + Timezone *cfgtype.Timezone `config:"timezone"` // IANA time zone (e.g. America/New_York) or fixed offset to use when parsing a timestamp not containing a timezone. + IgnoreMissing bool `config:"ignore_missing"` // Ignore errors when the source field is missing. + IgnoreFailure bool `config:"ignore_failure"` // Ignore errors when parsing the timestamp. + TestTimestamps []string `config:"test"` // A list of timestamps that must parse successfully when loading the processor. + ID string `config:"id"` // An identifier for this processor. Useful for debugging. } func defaultConfig() config { diff --git a/libbeat/processors/timestamp/docs/timestamp.asciidoc b/libbeat/processors/timestamp/docs/timestamp.asciidoc index eb5ba628995..2c9f67af6bf 100644 --- a/libbeat/processors/timestamp/docs/timestamp.asciidoc +++ b/libbeat/processors/timestamp/docs/timestamp.asciidoc @@ -41,7 +41,7 @@ If a layout does not contain a year then the current year in the specified | `field` | yes | | Source field containing the time to be parsed. | | `target_field` | no | @timestamp | Target field for the parsed time value. The target value is always written as UTC. | | `layouts` | yes | | Timestamp layouts that define the expected time value format. In addition layouts, `UNIX` and `UNIX_MS` are accepted. | -| `timezone` | no | UTC | Time zone (e.g. America/New_York) to use when parsing a timestamp not containing a time zone. | +| `timezone` | no | UTC | IANA time zone name (e.g. `America/New_York`) or fixed time offset (e.g. `+0200`) to use when parsing times that do not contain a time zone. `Local` may be specified to use the machine's local time zone.| | `ignore_missing` | no | false | Ignore errors when the source field is missing. | | `ignore_failure` | no | false | Ignore all errors produced by the processor. | | `test` | no | | A list of timestamps that must parse successfully when loading the processor. | diff --git a/libbeat/processors/timestamp/timestamp.go b/libbeat/processors/timestamp/timestamp.go index 5c1a3e47d12..7b147bbee1a 100644 --- a/libbeat/processors/timestamp/timestamp.go +++ b/libbeat/processors/timestamp/timestamp.go @@ -21,7 +21,6 @@ import ( "fmt" "time" - "4d63.com/tz" "github.com/pkg/errors" "github.com/elastic/beats/v7/libbeat/beat" @@ -57,16 +56,11 @@ func New(cfg *common.Config) (processors.Processor, error) { } func newFromConfig(c config) (*processor, error) { - loc, err := loadLocation(c.Timezone) - if err != nil { - return nil, errors.Wrap(err, "failed to load timezone") - } - p := &processor{ config: c, log: logp.NewLogger(logName), isDebug: logp.IsDebug(logName), - tz: loc, + tz: c.Timezone.Location(), } if c.ID != "" { p.log = p.log.With("instance_id", c.ID) @@ -84,21 +78,6 @@ func newFromConfig(c config) (*processor, error) { return p, nil } -var timezoneFormats = []string{"-07", "-0700", "-07:00"} - -func loadLocation(timezone string) (*time.Location, error) { - for _, format := range timezoneFormats { - t, err := time.Parse(format, timezone) - if err == nil { - name, offset := t.Zone() - return time.FixedZone(name, offset), nil - } - } - - // Rest of location formats - return tz.LoadLocation(timezone) -} - func (p *processor) String() string { return fmt.Sprintf("timestamp=[field=%s, target_field=%v, timezone=%v, layouts=%v]", p.Field, p.TargetField, p.tz, p.Layouts) diff --git a/libbeat/processors/timestamp/timestamp_test.go b/libbeat/processors/timestamp/timestamp_test.go index 4b7e8dd0993..466db497cfd 100644 --- a/libbeat/processors/timestamp/timestamp_test.go +++ b/libbeat/processors/timestamp/timestamp_test.go @@ -27,6 +27,7 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/cfgtype" "github.com/elastic/beats/v7/libbeat/logp" ) @@ -117,7 +118,7 @@ func TestParseNoYear(t *testing.T) { c := defaultConfig() c.Field = "ts" c.Layouts = append(c.Layouts, time.StampMilli) - c.Timezone = "EST" + c.Timezone = cfgtype.MustNewTimezone("EST") p, err := newFromConfig(c) if err != nil { @@ -261,7 +262,7 @@ func TestTimezone(t *testing.T) { Error: true, }, "non-existing location": { - Timezone: "Kalimdor/Orgrimmar", + Timezone: "Equatorial/Kundu", Error: true, }, "incomplete offset": { @@ -272,12 +273,13 @@ func TestTimezone(t *testing.T) { for title, c := range cases { t.Run(title, func(t *testing.T) { - config := defaultConfig() - config.Field = "ts" - config.Timezone = c.Timezone - config.Layouts = append(config.Layouts, time.ANSIC) + config := common.MustNewConfigFrom(map[string]interface{}{ + "field": "ts", + "timezone": c.Timezone, + "layouts": []string{time.ANSIC}, + }) - processor, err := newFromConfig(config) + processor, err := New(config) if c.Error { require.Error(t, err) return @@ -290,7 +292,7 @@ func TestTimezone(t *testing.T) { event := &beat.Event{ Fields: common.MapStr{ - config.Field: originalTimestamp, + "ts": originalTimestamp, }, } diff --git a/x-pack/filebeat/module/cef/_meta/docs.asciidoc b/x-pack/filebeat/module/cef/_meta/docs.asciidoc index 365a07b933a..475e51f98fd 100644 --- a/x-pack/filebeat/module/cef/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/cef/_meta/docs.asciidoc @@ -41,6 +41,13 @@ A list of tags to include in events. Including `forwarded` indicates that the events did not originate on this host and causes `host.name` to not be added to events. Defaults to `[cef, forwarded]`. +*`var.timezone`*:: + +IANA time zone name (e.g. `America/New_York`) or fixed time offset (e.g. +`+0200`) to use when parsing times from the CEF message that do not contain a +time zone. `Local` may be specified to use the machine's local time zone. +Defaults to `UTC`. + [float] ==== Forcepoint NGFW Security Management Center diff --git a/x-pack/filebeat/module/cef/log/config/input.yml b/x-pack/filebeat/module/cef/log/config/input.yml index 64e1dd7a27e..a42adbd2813 100644 --- a/x-pack/filebeat/module/cef/log/config/input.yml +++ b/x-pack/filebeat/module/cef/log/config/input.yml @@ -24,6 +24,9 @@ processors: - {from: "message", to: "event.original"} - decode_cef: field: event.original +{{ if .timezone }} + timezone: '{{ .timezone }}' +{{ end }} - community_id: - add_fields: target: '' diff --git a/x-pack/filebeat/module/cef/log/manifest.yml b/x-pack/filebeat/module/cef/log/manifest.yml index d1314088d69..f8e09c82de6 100644 --- a/x-pack/filebeat/module/cef/log/manifest.yml +++ b/x-pack/filebeat/module/cef/log/manifest.yml @@ -14,6 +14,7 @@ var: default: syslog - name: internal_zones - name: external_zones + - name: timezone ingest_pipeline: - ingest/pipeline.yml diff --git a/x-pack/filebeat/processors/decode_cef/cef/cef.go b/x-pack/filebeat/processors/decode_cef/cef/cef.go index e3bc284cd9c..4c0f983aa96 100644 --- a/x-pack/filebeat/processors/decode_cef/cef/cef.go +++ b/x-pack/filebeat/processors/decode_cef/cef/cef.go @@ -134,7 +134,7 @@ func (e *Event) Unpack(data string, opts ...Option) error { // Mark the data type and do the actual conversion. field.Type = mapping.Type - field.Interface, err = ToType(field.String, mapping.Type) + field.Interface, err = toType(field.String, mapping.Type, &settings) if err != nil { // Drop the key because the field value is invalid. delete(e.Extensions, key) diff --git a/x-pack/filebeat/processors/decode_cef/cef/cef_test.go b/x-pack/filebeat/processors/decode_cef/cef/cef_test.go index fb67afbe657..a0dc7c0c46c 100644 --- a/x-pack/filebeat/processors/decode_cef/cef/cef_test.go +++ b/x-pack/filebeat/processors/decode_cef/cef/cef_test.go @@ -411,7 +411,7 @@ func LongField(v int64) *Field { } func UndocumentedField(v string) *Field { return &Field{String: v} } func TimestampField(v string) *Field { - ts, err := toTimestamp(v) + ts, err := toTimestamp(v, nil) if err != nil { panic(err) } diff --git a/x-pack/filebeat/processors/decode_cef/cef/option.go b/x-pack/filebeat/processors/decode_cef/cef/option.go index 9fae32f011c..2a2e26dcb88 100644 --- a/x-pack/filebeat/processors/decode_cef/cef/option.go +++ b/x-pack/filebeat/processors/decode_cef/cef/option.go @@ -4,6 +4,10 @@ package cef +import ( + "time" +) + // Option controls Setting used in unpacking messages. type Option interface { Apply(*Settings) @@ -12,6 +16,8 @@ type Option interface { // Settings for unpacking messages. type Settings struct { fullExtensionNames bool + + timezone *time.Location } type withFullExtensionNames struct{} @@ -25,3 +31,17 @@ func (w withFullExtensionNames) Apply(s *Settings) { func WithFullExtensionNames() Option { return withFullExtensionNames{} } + +type withTimezone struct { + timezone *time.Location +} + +func (w withTimezone) Apply(s *Settings) { + s.timezone = w.timezone +} + +// WithTimezone causes CEF timestamps that do not contain a timezone to be +// parsed in the specified timezone. +func WithTimezone(timezone *time.Location) Option { + return withTimezone{timezone} +} diff --git a/x-pack/filebeat/processors/decode_cef/cef/types.go b/x-pack/filebeat/processors/decode_cef/cef/types.go index e5573aa46c4..20ca869a40e 100644 --- a/x-pack/filebeat/processors/decode_cef/cef/types.go +++ b/x-pack/filebeat/processors/decode_cef/cef/types.go @@ -32,8 +32,8 @@ const ( TimestampType ) -// ToType converts the given value string value to the specified data type. -func ToType(value string, typ DataType) (interface{}, error) { +// toType converts the given value string value to the specified data type. +func toType(value string, typ DataType, settings *Settings) (interface{}, error) { switch typ { case StringType: return value, nil @@ -52,7 +52,7 @@ func ToType(value string, typ DataType) (interface{}, error) { case MACAddressType: return toMACAddress(value) case TimestampType: - return toTimestamp(value) + return toTimestamp(value, settings) default: return nil, errors.Errorf("invalid data type: %v", typ) } @@ -166,15 +166,21 @@ var timeLayouts = []string{ "Jan _2 2006 15:04:05", } -func toTimestamp(v string) (common.Time, error) { +func toTimestamp(v string, settings *Settings) (common.Time, error) { if unixMs, err := toLong(v); err == nil { return common.Time(time.Unix(0, unixMs*int64(time.Millisecond))), nil } + // Use this timezone when one is not included in the time string. + defaultLocation := time.UTC + if settings != nil && settings.timezone != nil { + defaultLocation = settings.timezone + } + for _, layout := range timeLayouts { - ts, err := time.ParseInLocation(layout, v, time.UTC) + ts, err := time.ParseInLocation(layout, v, defaultLocation) if err == nil { - // Use current year if no year is zero. + // Use current year if year is zero. if ts.Year() == 0 { currentYear := time.Now().In(ts.Location()).Year() ts = ts.AddDate(currentYear, 0, 0) diff --git a/x-pack/filebeat/processors/decode_cef/cef/types_test.go b/x-pack/filebeat/processors/decode_cef/cef/types_test.go index 6c93eb3317f..feb788e24fb 100644 --- a/x-pack/filebeat/processors/decode_cef/cef/types_test.go +++ b/x-pack/filebeat/processors/decode_cef/cef/types_test.go @@ -6,8 +6,10 @@ package cef import ( "testing" + "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestToTimestamp(t *testing.T) { @@ -61,11 +63,23 @@ func TestToTimestamp(t *testing.T) { } for _, timeValue := range times { - _, err := toTimestamp(timeValue) + _, err := toTimestamp(timeValue, nil) assert.NoError(t, err, timeValue) } } +func TestToTimestampWithTimezone(t *testing.T) { + const offsetHour, offsetMin = 2, 15 // +0215 + + ct, err := toTimestamp("Jun 23 10:30:03.004", &Settings{timezone: time.FixedZone("", offsetHour*60*60+offsetMin*60)}) + require.NoError(t, err) + + // 2021-06-23 08:15:03.004 +0000 UTC + ts := time.Time(ct).UTC() + assert.Equal(t, 10-offsetHour, ts.Hour()) + assert.Equal(t, 30-offsetMin, ts.Minute()) +} + func TestToMACAddress(t *testing.T) { var macs = []string{ // EUI-48 (with and without separators). diff --git a/x-pack/filebeat/processors/decode_cef/config.go b/x-pack/filebeat/processors/decode_cef/config.go index 361a66672da..00139c400e4 100644 --- a/x-pack/filebeat/processors/decode_cef/config.go +++ b/x-pack/filebeat/processors/decode_cef/config.go @@ -4,13 +4,16 @@ package decode_cef +import "github.com/elastic/beats/v7/libbeat/common/cfgtype" + type config struct { - Field string `config:"field"` // Source field containing the CEF message. - TargetField string `config:"target_field"` // Target field for the CEF object. - IgnoreMissing bool `config:"ignore_missing"` // Ignore missing source field. - IgnoreFailure bool `config:"ignore_failure"` // Ignore failures when the source field does not contain a CEF message. Parse errors do not cause failures, but are added to error.message. - ID string `config:"id"` // Instance ID for debugging purposes. - ECS bool `config:"ecs"` // Generate ECS fields. + Field string `config:"field"` // Source field containing the CEF message. + TargetField string `config:"target_field"` // Target field for the CEF object. + IgnoreMissing bool `config:"ignore_missing"` // Ignore missing source field. + IgnoreFailure bool `config:"ignore_failure"` // Ignore failures when the source field does not contain a CEF message. Parse errors do not cause failures, but are added to error.message. + ID string `config:"id"` // Instance ID for debugging purposes. + ECS bool `config:"ecs"` // Generate ECS fields. + Timezone *cfgtype.Timezone `config:"timezone"` // Timezone used when parsing timestamps that do not contain a time zone or offset. } func defaultConfig() config { diff --git a/x-pack/filebeat/processors/decode_cef/decode_cef.go b/x-pack/filebeat/processors/decode_cef/decode_cef.go index 1538fe2f417..51b2d62aad3 100644 --- a/x-pack/filebeat/processors/decode_cef/decode_cef.go +++ b/x-pack/filebeat/processors/decode_cef/decode_cef.go @@ -86,7 +86,7 @@ func (p *processor) Run(event *beat.Event) (*beat.Event, error) { // If the version < 0 after parsing then none of the data is valid so return here. var ce cef.Event - if err = ce.Unpack(cefData, cef.WithFullExtensionNames()); ce.Version < 0 && err != nil { + if err = ce.Unpack(cefData, cef.WithFullExtensionNames(), cef.WithTimezone(p.Timezone.Location())); ce.Version < 0 && err != nil { if p.IgnoreFailure { return event, nil } diff --git a/x-pack/filebeat/processors/decode_cef/docs/decode_cef.asciidoc b/x-pack/filebeat/processors/decode_cef/docs/decode_cef.asciidoc index 4666100a39e..9910e4d39f4 100644 --- a/x-pack/filebeat/processors/decode_cef/docs/decode_cef.asciidoc +++ b/x-pack/filebeat/processors/decode_cef/docs/decode_cef.asciidoc @@ -33,6 +33,7 @@ The `decode_cef` processor has the following configuration settings. | `target_field` | no | cef | Target field where the parsed CEF object will be written. | | `ecs` | no | true | Generate Elastic Common Schema (ECS) fields from the CEF data. Certain CEF header and extension values will be used to populate ECS fields. | +| `timezone` | no | UTC | IANA time zone name (e.g. `America/New_York`) or fixed time offset (e.g. `+0200`) to use when parsing times that do not contain a time zone. `Local` may be specified to use the machine's local time zone.| | `ignore_missing` | no | false | Ignore errors when the source field is missing. | | `ignore_failure` | no | false | Ignore failures when the source field does not contain a CEF message. | | `id` | no | | An identifier for this processor instance. Useful for debugging. |