diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index c64f64174489..f0a29e27c855 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -64,6 +64,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d *Winlogbeat* +- Fix evtx parsing failures. {issue}30621[30621] {pull}30942[30942] + *Functionbeat* diff --git a/winlogbeat/Jenkinsfile.yml b/winlogbeat/Jenkinsfile.yml index 4eafb1807577..7fc1e1607aa8 100644 --- a/winlogbeat/Jenkinsfile.yml +++ b/winlogbeat/Jenkinsfile.yml @@ -24,7 +24,16 @@ stages: crosscompile: make: "make -C winlogbeat crosscompile" stage: mandatory +<<<<<<< HEAD windows: +======= + windows-2022: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-2022" + stage: mandatory + windows-2019: +>>>>>>> 34bdc3d468 (winlogbeat: fix event handling for Windows 2022 (#30942)) mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2019" diff --git a/winlogbeat/beater/winlogbeat.go b/winlogbeat/beater/winlogbeat.go index 568bb911539d..e20467b35b08 100644 --- a/winlogbeat/beater/winlogbeat.go +++ b/winlogbeat/beater/winlogbeat.go @@ -91,7 +91,11 @@ func (eb *Winlogbeat) init(b *beat.Beat) error { if err != nil { return fmt.Errorf("failed to create new event log: %w", err) } +<<<<<<< HEAD eb.log.Debugf("Initialized EventLog]", eventLog.Name()) +======= + eb.log.Debugf("initialized WinEventLog[%s]", eventLog.Name()) +>>>>>>> 34bdc3d468 (winlogbeat: fix event handling for Windows 2022 (#30942)) logger, err := newEventLogger(b.Info, eventLog, config, eb.log) if err != nil { diff --git a/winlogbeat/eventlog/wineventlog.go b/winlogbeat/eventlog/wineventlog.go index 7ec830f220a5..9a9f52627fcd 100644 --- a/winlogbeat/eventlog/wineventlog.go +++ b/winlogbeat/eventlog/wineventlog.go @@ -55,6 +55,15 @@ const ( eventLoggingAPIName = "eventlogging" ) +func init() { + // Register wineventlog API if it is available. + available, _ := win.IsAvailable() + if available { + Register(winEventLogAPIName, 0, newWinEventLog, win.Channels) + Register(eventLoggingAPIName, 1, newEventLogging, win.Channels) + } +} + type winEventLogConfig struct { ConfigCommon `config:",inline"` BatchReadSize int `config:"batch_read_size"` // Maximum number of events that Read will return. @@ -160,6 +169,7 @@ type winEventLog struct { lastRead checkpoint.EventLogState // Record number of the last read event. render func(event win.EvtHandle, out io.Writer) error // Function for rendering the event to XML. + message func(event win.EvtHandle) (string, error) // Message fallback function. renderBuf []byte // Buffer used for rendering event. outputBuf *sys.ByteBuffer // Buffer for receiving XML cache *messageFilesCache // Cached mapping of source name to event message file handles. @@ -198,7 +208,11 @@ func (l *winEventLog) openChannel(bookmark win.EvtHandle) error { if err != nil { return nil } +<<<<<<< HEAD defer windows.CloseHandle(signalEvent) +======= + defer windows.CloseHandle(signalEvent) //nolint:errcheck // This is just a resource release. +>>>>>>> 34bdc3d468 (winlogbeat: fix event handling for Windows 2022 (#30942)) var flags win.EvtSubscribeFlag if bookmark > 0 { @@ -273,11 +287,20 @@ func (l *winEventLog) Read() ([]Record, error) { }() detailf("%s EventHandles returned %d handles", l.logPrefix, len(handles)) +<<<<<<< HEAD var records []Record for _, h := range handles { l.outputBuf.Reset() err := l.render(h, l.outputBuf) if bufErr, ok := err.(sys.InsufficientBufferError); ok { +======= + var records []Record //nolint:prealloc // This linter gives bad advice and does not take into account conditionals in loops. + for _, h := range handles { + l.outputBuf.Reset() + err := l.render(h, l.outputBuf) + var bufErr sys.InsufficientBufferError + if errors.As(err, &bufErr) { +>>>>>>> 34bdc3d468 (winlogbeat: fix event handling for Windows 2022 (#30942)) detailf("%s Increasing render buffer size to %d", l.logPrefix, bufErr.RequiredSize) l.renderBuf = make([]byte, bufErr.RequiredSize) @@ -299,6 +322,12 @@ func (l *winEventLog) Read() ([]Record, error) { if r.Offset.Bookmark, err = l.createBookmarkFromEvent(h); err != nil { logp.Warn("%s failed creating bookmark: %v", l.logPrefix, err) } + if r.Message == "" { + r.Message, err = l.message(h) + if err != nil { + logp.Err("%s error salvaging message: %v", l.logPrefix, err) + } + } records = append(records, r) l.lastRead = r.Offset } @@ -314,7 +343,11 @@ func (l *winEventLog) Close() error { func (l *winEventLog) eventHandles(maxRead int) ([]win.EvtHandle, int, error) { handles, err := win.EventHandles(l.subscription, maxRead) +<<<<<<< HEAD switch err { +======= + switch err { //nolint:errorlint // This is an errno or nil. +>>>>>>> 34bdc3d468 (winlogbeat: fix event handling for Windows 2022 (#30942)) case nil: if l.maxRead > maxRead { debugf("%s Recovered from RPC_S_INVALID_BOUND error (errno 1734) "+ @@ -366,13 +399,15 @@ func (l *winEventLog) buildRecordFromXML(x []byte, recoveredErr error) (Record, e.RenderErr = append(e.RenderErr, recoveredErr.Error()) } + // Get basic string values for raw fields. + winevent.EnrichRawValuesWithNames(nil, &e) if e.Level == "" { // Fallback on LevelRaw if the Level is not set in the RenderingInfo. e.Level = win.EventLevel(e.LevelRaw).String() } if logp.IsDebug(detailSelector) { - detailf("%s XML=%s Event=%+v", l.logPrefix, string(x), e) + detailf("%s XML=%s Event=%+v", l.logPrefix, x, e) } r := Record{ @@ -474,6 +509,9 @@ func newWinEventLog(options *common.Config) (EventLog, error) { return win.RenderEvent(event, c.EventLanguage, l.renderBuf, l.cache.get, out) } } + l.message = func(event win.EvtHandle) (string, error) { + return win.Message(event, l.renderBuf, l.cache.get) + } return l, nil } @@ -488,12 +526,3 @@ func (l *winEventLog) createBookmarkFromEvent(evtHandle win.EvtHandle) (string, win.Close(bmHandle) return string(l.outputBuf.Bytes()), err } - -func init() { - // Register wineventlog API if it is available. - available, _ := win.IsAvailable() - if available { - Register(winEventLogAPIName, 0, newWinEventLog, win.Channels) - Register(eventLoggingAPIName, 1, newEventLogging, win.Channels) - } -} diff --git a/winlogbeat/eventlog/wineventlog_experimental.go b/winlogbeat/eventlog/wineventlog_experimental.go index 87eb4b328026..1871501bca23 100644 --- a/winlogbeat/eventlog/wineventlog_experimental.go +++ b/winlogbeat/eventlog/wineventlog_experimental.go @@ -21,6 +21,10 @@ package eventlog import ( +<<<<<<< HEAD +======= + "fmt" +>>>>>>> 34bdc3d468 (winlogbeat: fix event handling for Windows 2022 (#30942)) "io" "os" "path/filepath" @@ -42,6 +46,14 @@ const ( winEventLogExpAPIName = "wineventlog-experimental" ) +func init() { + // Register wineventlog API if it is available. + available, _ := win.IsAvailable() + if available { + Register(winEventLogExpAPIName, 10, newWinEventLogExp, win.Channels) + } +} + // winEventLogExp implements the EventLog interface for reading from the Windows // Event Log API. type winEventLogExp struct { @@ -99,7 +111,11 @@ func (l *winEventLogExp) openChannel(bookmark win.Bookmark) (win.EvtHandle, erro if err != nil { return win.NilHandle, err } +<<<<<<< HEAD defer windows.CloseHandle(signalEvent) +======= + defer windows.CloseHandle(signalEvent) //nolint:errcheck // This is just a resource release. +>>>>>>> 34bdc3d468 (winlogbeat: fix event handling for Windows 2022 (#30942)) var flags win.EvtSubscribeFlag if bookmark > 0 { @@ -116,6 +132,20 @@ func (l *winEventLogExp) openChannel(bookmark win.Bookmark) (win.EvtHandle, erro l.query, // Query - nil means all events win.EvtHandle(bookmark), // Bookmark - for resuming from a specific event flags) +<<<<<<< HEAD +======= + + switch err { //nolint:errorlint // This is an errno or nil. + case nil: + return h, nil + case win.ERROR_NOT_FOUND, win.ERROR_EVT_QUERY_RESULT_STALE, win.ERROR_EVT_QUERY_RESULT_INVALID_POSITION: + // The bookmarked event was not found, we retry the subscription from the start. + incrementMetric(readErrors, err) + return win.Subscribe(0, signalEvent, "", l.query, 0, win.EvtSubscribeStartAtOldestRecord) + default: + return 0, err + } +>>>>>>> 34bdc3d468 (winlogbeat: fix event handling for Windows 2022 (#30942)) } func (l *winEventLogExp) openFile(state checkpoint.EventLogState, bookmark win.Bookmark) (win.EvtHandle, error) { @@ -198,6 +228,10 @@ func (l *winEventLogExp) processHandle(h win.EvtHandle) (*Record, error) { evt.RenderErr = append(evt.RenderErr, err.Error()) } +<<<<<<< HEAD +======= + //nolint:godox // Bad linter! Keep to have a record of feature disparity between non-experimental vs experimental. +>>>>>>> 34bdc3d468 (winlogbeat: fix event handling for Windows 2022 (#30942)) // TODO: Need to add XML when configured. r := &Record{ @@ -305,11 +339,3 @@ func newWinEventLogExp(options *common.Config) (EventLog, error) { return l, nil } - -func init() { - // Register wineventlog API if it is available. - available, _ := win.IsAvailable() - if available { - Register(winEventLogExpAPIName, 10, newWinEventLogExp, win.Channels) - } -} diff --git a/winlogbeat/eventlog/wineventlog_test.go b/winlogbeat/eventlog/wineventlog_test.go index c5d92ce18739..da67c61b3e11 100644 --- a/winlogbeat/eventlog/wineventlog_test.go +++ b/winlogbeat/eventlog/wineventlog_test.go @@ -31,6 +31,7 @@ import ( "github.com/andrewkroh/sys/windows/svc/eventlog" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/winlogbeat/checkpoint" @@ -187,13 +188,27 @@ func testWindowsEventLog(t *testing.T, api string) { const messageSize = 256 // Originally 31800, such a large value resulted in an empty eventlog under Win10. const totalEvents = 1000 for i := 0; i < totalEvents; i++ { - safeWriteEvent(t, writer, eventlog.Info, uint32(i%1000), []string{strconv.Itoa(i) + " " + randomSentence(messageSize)}) + safeWriteEvent(t, writer, eventlog.Info, uint32(i%1000)+1, []string{strconv.Itoa(i) + " " + randomSentence(messageSize)}) } openLog := func(t testing.TB, config map[string]interface{}) EventLog { return openLog(t, api, nil, config) } + t.Run("has_message", func(t *testing.T) { + log := openLog(t, map[string]interface{}{"name": providerName, "batch_read_size": 1}) + defer log.Close() + + for i := 0; i < 10; i++ { + records, err := log.Read() + require.NotEmpty(t, records) + require.NoError(t, err) + + r := records[0] + require.NotEmpty(t, r.Message, "message field is empty: errors:%v\nrecord:%#v", r.Event.RenderErr, r) + } + }) + // Test reading from an event log using a custom XML query. t.Run("custom_xml_query", func(t *testing.T) { cfg := map[string]interface{}{ @@ -307,16 +322,18 @@ func createLog(t testing.TB, messageFiles ...string) (log *eventlog.Log, tearDow } if existed { - wineventlog.EvtClearLog(wineventlog.NilHandle, name, "") + wineventlog.EvtClearLog(wineventlog.NilHandle, name, "") //nolint:errcheck // This is just a resource release. } log, err = eventlog.Open(source) + //nolint:errcheck // This is just a resource release. if err != nil { eventlog.RemoveSource(name, source) eventlog.RemoveProvider(name) t.Fatal(err) } + //nolint:errcheck // This is just a resource release. tearDown = func() { log.Close() wineventlog.EvtClearLog(wineventlog.NilHandle, name, "") @@ -343,7 +360,7 @@ func safeWriteEvent(t testing.TB, log *eventlog.Log, etype uint16, eid uint32, m // setLogSize set the maximum number of bytes that an event log can hold. func setLogSize(t testing.TB, provider string, sizeBytes int) { - output, err := exec.Command("wevtutil.exe", "sl", "/ms:"+strconv.Itoa(sizeBytes), provider).CombinedOutput() + output, err := exec.Command("wevtutil.exe", "sl", "/ms:"+strconv.Itoa(sizeBytes), provider).CombinedOutput() //nolint:gosec // No possibility of command injection. if err != nil { t.Fatal("Failed to set log size", err, string(output)) } diff --git a/winlogbeat/sys/winevent/event.go b/winlogbeat/sys/winevent/event.go index 9c342e73f480..4054626ed8fb 100644 --- a/winlogbeat/sys/winevent/event.go +++ b/winlogbeat/sys/winevent/event.go @@ -44,11 +44,6 @@ var ( const ( keywordAuditFailure = 0x10000000000000 keywordAuditSuccess = 0x20000000000000 - - // keywordClassic indicates the log was published with the "classic" event - // logging API. - // https://docs.microsoft.com/en-us/dotnet/api/system.diagnostics.eventing.reader.standardeventkeywords?view=netframework-4.8 - keywordClassic = 0x80000000000000 ) // UnmarshalXML unmarshals the given XML into a new Event. @@ -67,7 +62,7 @@ type Event struct { Version Version `xml:"System>Version"` LevelRaw uint8 `xml:"System>Level"` TaskRaw uint16 `xml:"System>Task"` - OpcodeRaw uint8 `xml:"System>Opcode"` + OpcodeRaw *uint8 `xml:"System>Opcode,omitempty"` KeywordsRaw HexInt64 `xml:"System>Keywords"` TimeCreated TimeCreated `xml:"System>TimeCreated"` RecordID uint64 `xml:"System>EventRecordID"` @@ -258,7 +253,10 @@ func (u *UserData) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { u.Name = se.Name u.Pairs = in.Pairs - d.Skip() + err = d.Skip() + if err != nil { + return err + } break } } @@ -309,8 +307,7 @@ func (v *Version) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { version, err := strconv.ParseUint(s, 10, 8) if err != nil { - // Ignore invalid version values. - return nil + return nil //nolint:nilerr // Ignore invalid version values. } *v = Version(version) @@ -341,20 +338,19 @@ func (v *HexInt64) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { func EnrichRawValuesWithNames(publisherMeta *WinMeta, event *Event) { // Keywords. Each bit in the value can represent a keyword. rawKeyword := int64(event.KeywordsRaw) - isClassic := keywordClassic&rawKeyword > 0 if len(event.Keywords) == 0 { for mask, keyword := range defaultWinMeta.Keywords { - if rawKeyword&mask > 0 { + if rawKeyword&mask != 0 { event.Keywords = append(event.Keywords, keyword) - rawKeyword -= mask + rawKeyword &^= mask } } if publisherMeta != nil { for mask, keyword := range publisherMeta.Keywords { - if rawKeyword&mask > 0 { + if rawKeyword&mask != 0 { event.Keywords = append(event.Keywords, keyword) - rawKeyword -= mask + rawKeyword &^= mask } } } @@ -363,10 +359,10 @@ func EnrichRawValuesWithNames(publisherMeta *WinMeta, event *Event) { var found bool if event.Opcode == "" { // Opcode (search in defaultWinMeta first). - if !isClassic { - event.Opcode, found = defaultWinMeta.Opcodes[event.OpcodeRaw] + if event.OpcodeRaw != nil { + event.Opcode, found = defaultWinMeta.Opcodes[*event.OpcodeRaw] if !found && publisherMeta != nil { - event.Opcode = publisherMeta.Opcodes[event.OpcodeRaw] + event.Opcode = publisherMeta.Opcodes[*event.OpcodeRaw] } } } diff --git a/winlogbeat/sys/winevent/event_test.go b/winlogbeat/sys/winevent/event_test.go index 4ed391b91bec..3cc743272d8f 100644 --- a/winlogbeat/sys/winevent/event_test.go +++ b/winlogbeat/sys/winevent/event_test.go @@ -20,7 +20,6 @@ package winevent import ( "encoding/json" "encoding/xml" - "fmt" "strings" "testing" "time" @@ -94,6 +93,7 @@ func TestXML(t *testing.T) { EventIdentifier: EventIdentifier{ID: 91}, LevelRaw: 4, TaskRaw: 9, + OpcodeRaw: new(uint8), // The value in the XML is 0. KeywordsRaw: 0x8020000000000000, TimeCreated: TimeCreated{allXMLTimeCreated}, RecordID: 100, @@ -166,7 +166,7 @@ func TestXML(t *testing.T) { if err != nil { t.Error(err) } - fmt.Println(string(json)) + t.Logf("%s", json) } } } diff --git a/winlogbeat/sys/wineventlog/format_message.go b/winlogbeat/sys/wineventlog/format_message.go index d953c210b562..642eaa69965b 100644 --- a/winlogbeat/sys/wineventlog/format_message.go +++ b/winlogbeat/sys/wineventlog/format_message.go @@ -21,9 +21,9 @@ package wineventlog import ( + "fmt" "unsafe" - "github.com/pkg/errors" "golang.org/x/sys/windows" "github.com/elastic/beats/v7/winlogbeat/sys" @@ -80,8 +80,8 @@ func evtFormatMessage(metadataHandle EvtHandle, eventHandle EvtHandle, messageID // Determine the buffer size needed (given in WCHARs). var bufferUsed uint32 err := _EvtFormatMessage(metadataHandle, eventHandle, messageID, valuesCount, valuesPtr, messageFlag, 0, nil, &bufferUsed) - if err != windows.ERROR_INSUFFICIENT_BUFFER { - return "", errors.Wrap(err, "failed in EvtFormatMessage") + if err != windows.ERROR_INSUFFICIENT_BUFFER { //nolint:errorlint // This is an errno. + return "", fmt.Errorf("failed in EvtFormatMessage: %w", err) } // Get a buffer from the pool and adjust its length. @@ -89,16 +89,17 @@ func evtFormatMessage(metadataHandle EvtHandle, eventHandle EvtHandle, messageID defer bb.Free() bb.Reserve(int(bufferUsed * 2)) - err = _EvtFormatMessage(metadataHandle, eventHandle, messageID, valuesCount, valuesPtr, messageFlag, uint32(bb.Len()/2), bb.PtrAt(0), &bufferUsed) - if err != nil { - switch err { - // Ignore some errors so it can tolerate missing or mismatched parameter values. - case windows.ERROR_EVT_UNRESOLVED_VALUE_INSERT: - case windows.ERROR_EVT_UNRESOLVED_PARAMETER_INSERT: - case windows.ERROR_EVT_MAX_INSERTS_REACHED: - default: - return "", errors.Wrap(err, "failed in EvtFormatMessage") - } + err = _EvtFormatMessage(metadataHandle, eventHandle, messageID, valuesCount, valuesPtr, messageFlag, uint32(bb.Len()), bb.PtrAt(0), &bufferUsed) + switch err { //nolint:errorlint // This is an errno or nil. + case nil: // OK + + // Ignore some errors so it can tolerate missing or mismatched parameter values. + case windows.ERROR_EVT_UNRESOLVED_VALUE_INSERT, + windows.ERROR_EVT_UNRESOLVED_PARAMETER_INSERT, + windows.ERROR_EVT_MAX_INSERTS_REACHED: + + default: + return "", fmt.Errorf("failed in EvtFormatMessage: %w", err) } return sys.UTF16BytesToString(bb.Bytes()) diff --git a/winlogbeat/sys/wineventlog/query_test.go b/winlogbeat/sys/wineventlog/query_test.go index f31f98ac6c95..8c3a5031f127 100644 --- a/winlogbeat/sys/wineventlog/query_test.go +++ b/winlogbeat/sys/wineventlog/query_test.go @@ -49,7 +49,7 @@ func TestIgnoreOlderQuery(t *testing.T) { q, err := Query{Log: "Application", IgnoreOlder: time.Hour}.Build() if assert.NoError(t, err) { assert.Equal(t, expected, q) - fmt.Println(q) + t.Log(q) } } @@ -64,7 +64,7 @@ func TestEventIDQuery(t *testing.T) { q, err := Query{Log: "Application", EventID: "1, 1-100, -75"}.Build() if assert.NoError(t, err) { assert.Equal(t, expected, q) - fmt.Println(q) + t.Log(q) } } @@ -78,7 +78,7 @@ func TestLevelQuery(t *testing.T) { q, err := Query{Log: "Application", Level: "Verbose"}.Build() if assert.NoError(t, err) { assert.Equal(t, expected, q) - fmt.Println(q) + t.Log(q) } } @@ -92,7 +92,7 @@ func TestProviderQuery(t *testing.T) { q, err := Query{Log: "Application", Provider: []string{"mysrc"}}.Build() if assert.NoError(t, err) { assert.Equal(t, expected, q) - fmt.Println(q) + t.Log(q) } } @@ -112,7 +112,7 @@ func TestCombinedQuery(t *testing.T) { }.Build() if assert.NoError(t, err) { assert.Equal(t, expected, q) - fmt.Println(q) + t.Log(q) } } @@ -126,6 +126,6 @@ func TestQueryNoParams(t *testing.T) { q, err := Query{Log: "Application"}.Build() if assert.NoError(t, err) { assert.Equal(t, expected, q) - fmt.Println(q) + t.Log(q) } } diff --git a/winlogbeat/sys/wineventlog/renderer.go b/winlogbeat/sys/wineventlog/renderer.go index 8a16567e7ad1..310eab450f62 100644 --- a/winlogbeat/sys/wineventlog/renderer.go +++ b/winlogbeat/sys/wineventlog/renderer.go @@ -30,7 +30,6 @@ import ( "unsafe" "github.com/cespare/xxhash/v2" - "github.com/pkg/errors" "go.uber.org/multierr" "golang.org/x/sys/windows" @@ -48,10 +47,10 @@ const ( // Renderer is used for converting event log handles into complete events. type Renderer struct { - // Cache of publisher metadata. Maps publisher names to stored metadata. - metadataCache map[string]*PublisherMetadataStore // Mutex to guard the metadataCache. The other members are immutable. mutex sync.RWMutex + // Cache of publisher metadata. Maps publisher names to stored metadata. + metadataCache map[string]*PublisherMetadataStore session EvtHandle // Session handle if working with remote log. systemContext EvtHandle // Render context for system values. @@ -63,12 +62,12 @@ type Renderer struct { func NewRenderer(session EvtHandle, log *logp.Logger) (*Renderer, error) { systemContext, err := _EvtCreateRenderContext(0, 0, EvtRenderContextSystem) if err != nil { - return nil, errors.Wrap(err, "failed in EvtCreateRenderContext for system context") + return nil, fmt.Errorf("failed in EvtCreateRenderContext for system context: %w", err) } userContext, err := _EvtCreateRenderContext(0, 0, EvtRenderContextUser) if err != nil { - return nil, errors.Wrap(err, "failed in EvtCreateRenderContext for user context") + return nil, fmt.Errorf("failed in EvtCreateRenderContext for user context: %w", err) } return &Renderer{ @@ -99,7 +98,7 @@ func (r *Renderer) Render(handle EvtHandle) (*winevent.Event, error) { event := &winevent.Event{} if err := r.renderSystem(handle, event); err != nil { - return nil, errors.Wrap(err, "failed to render system properties") + return nil, fmt.Errorf("failed to render system properties: %w", err) } // From this point on it will return both the event and any errors. It's @@ -117,7 +116,7 @@ func (r *Renderer) Render(handle EvtHandle) (*winevent.Event, error) { eventData, fingerprint, err := r.renderUser(handle, event) if err != nil { - errs = append(errs, errors.Wrap(err, "failed to render event data")) + errs = append(errs, fmt.Errorf("failed to render event data: %w", err)) } // Load cached event metadata or try to bootstrap it from the event's XML. @@ -127,7 +126,7 @@ func (r *Renderer) Render(handle EvtHandle) (*winevent.Event, error) { r.addEventData(eventMeta, eventData, event) if event.Message, err = r.formatMessage(md, eventMeta, handle, eventData, uint16(event.EventIdentifier.ID)); err != nil { - errs = append(errs, errors.Wrap(err, "failed to get the event message string")) + errs = append(errs, fmt.Errorf("failed to get the event message string: %w", err)) } if len(errs) > 0 { @@ -165,8 +164,8 @@ func (r *Renderer) getPublisherMetadata(publisher string) (*PublisherMetadataSto // Return an empty store on error (can happen in cases where the // log was forwarded and the provider doesn't exist on collector). md = NewEmptyPublisherMetadataStore(publisher, r.log) - err = errors.Wrapf(err, "failed to load publisher metadata for %v "+ - "(returning an empty metadata store)", publisher) + err = fmt.Errorf("failed to load publisher metadata for %v "+ + "(returning an empty metadata store): %w", publisher, err) } r.metadataCache[publisher] = md } else { @@ -180,11 +179,11 @@ func (r *Renderer) getPublisherMetadata(publisher string) (*PublisherMetadataSto func (r *Renderer) renderSystem(handle EvtHandle, event *winevent.Event) error { bb, propertyCount, err := r.render(r.systemContext, handle) if err != nil { - return errors.Wrap(err, "failed to get system values") + return fmt.Errorf("failed to get system values: %w", err) } defer bb.Free() - for i := 0; i < int(propertyCount); i++ { + for i := 0; i < propertyCount; i++ { property := EvtSystemPropertyID(i) offset := i * int(sizeofEvtVariant) evtVar := (*EvtVariant)(unsafe.Pointer(bb.PtrAt(offset))) @@ -194,6 +193,7 @@ func (r *Renderer) renderSystem(handle EvtHandle, event *winevent.Event) error { continue } + //nolint:errcheck // Bad linter! switch property { case EvtSystemProviderName: event.Provider.Name = data.(string) @@ -208,7 +208,10 @@ func (r *Renderer) renderSystem(handle EvtHandle, event *winevent.Event) error { case EvtSystemTask: event.TaskRaw = data.(uint16) case EvtSystemOpcode: - event.OpcodeRaw = data.(uint8) + if event.OpcodeRaw == nil { + event.OpcodeRaw = new(uint8) + } + *event.OpcodeRaw = data.(uint8) case EvtSystemKeywords: event.KeywordsRaw = winevent.HexInt64(data.(hexInt64)) case EvtSystemTimeCreated: @@ -247,7 +250,7 @@ func (r *Renderer) renderSystem(handle EvtHandle, event *winevent.Event) error { func (r *Renderer) renderUser(handle EvtHandle, event *winevent.Event) (values []interface{}, fingerprint uint64, err error) { bb, propertyCount, err := r.render(r.userContext, handle) if err != nil { - return nil, 0, errors.Wrap(err, "failed to get user values") + return nil, 0, fmt.Errorf("failed to get user values: %w", err) } defer bb.Free() @@ -264,7 +267,7 @@ func (r *Renderer) renderUser(handle EvtHandle, event *winevent.Event) (values [ for i := 0; i < propertyCount; i++ { offset := i * int(sizeofEvtVariant) evtVar := (*EvtVariant)(unsafe.Pointer(bb.PtrAt(offset))) - binary.Write(argumentHash, binary.LittleEndian, uint32(evtVar.Type)) + binary.Write(argumentHash, binary.LittleEndian, uint32(evtVar.Type)) //nolint:errcheck // Hash writes never fail. values[i], err = evtVar.Data(bb.Bytes()) if err != nil { @@ -287,8 +290,8 @@ func (r *Renderer) render(context EvtHandle, eventHandle EvtHandle) (*sys.Pooled var bufferUsed, propertyCount uint32 err := _EvtRender(context, eventHandle, EvtRenderEventValues, 0, nil, &bufferUsed, &propertyCount) - if err != nil && err != windows.ERROR_INSUFFICIENT_BUFFER { - return nil, 0, errors.Wrap(err, "failed in EvtRender") + if err != nil && err != windows.ERROR_INSUFFICIENT_BUFFER { //nolint:errorlint // This is an errno or nil. + return nil, 0, fmt.Errorf("failed in EvtRender: %w", err) } if propertyCount == 0 { @@ -301,7 +304,7 @@ func (r *Renderer) render(context EvtHandle, eventHandle EvtHandle) (*sys.Pooled err = _EvtRender(context, eventHandle, EvtRenderEventValues, uint32(bb.Len()), bb.PtrAt(0), &bufferUsed, &propertyCount) if err != nil { bb.Free() - return nil, 0, errors.Wrap(err, "failed in EvtRender") + return nil, 0, fmt.Errorf("failed in EvtRender: %w", err) } return bb, int(propertyCount), nil @@ -390,7 +393,7 @@ func (r *Renderer) formatMessageFromTemplate(msgTmpl *template.Template, values defer bb.Free() if err := msgTmpl.Execute(bb, values); err != nil { - return "", errors.Wrapf(err, "failed to execute template with data=%#v template=%v", values, msgTmpl.Root.String()) + return "", fmt.Errorf("failed to execute template with data=%#v template=%v: %w", values, msgTmpl.Root.String(), err) } return string(bb.Bytes()), nil diff --git a/winlogbeat/sys/wineventlog/renderer_test.go b/winlogbeat/sys/wineventlog/renderer_test.go index ea0c179cd1ea..a283bc0f0570 100644 --- a/winlogbeat/sys/wineventlog/renderer_test.go +++ b/winlogbeat/sys/wineventlog/renderer_test.go @@ -41,7 +41,7 @@ import ( ) func TestRenderer(t *testing.T) { - logp.TestingSetup() + logp.TestingSetup() //nolint:errcheck // Bad linter! Never returns a non-nil error when called without options. t.Run(filepath.Base(sysmon9File), func(t *testing.T) { log := openLog(t, sysmon9File) @@ -86,7 +86,8 @@ func TestRenderer(t *testing.T) { assert.Equal(t, e.Keywords, []string{"Audit Success"}) - assert.EqualValues(t, 0, e.OpcodeRaw) + assert.NotNil(t, 0, e.OpcodeRaw) + assert.EqualValues(t, 0, *e.OpcodeRaw) assert.Equal(t, "Info", e.Opcode) assert.EqualValues(t, 0, e.LevelRaw) @@ -131,7 +132,7 @@ func TestRenderer(t *testing.T) { assert.Equal(t, e.Keywords, []string{"Classic"}) - assert.EqualValues(t, 0, e.OpcodeRaw) + assert.EqualValues(t, (*uint8)(nil), e.OpcodeRaw) assert.Equal(t, "", e.Opcode) assert.EqualValues(t, 4, e.LevelRaw) @@ -197,7 +198,7 @@ func renderAllEvents(t *testing.T, log EvtHandle, renderer *Renderer, ignoreMiss // setLogSize set the maximum number of bytes that an event log can hold. func setLogSize(t testing.TB, provider string, sizeBytes int) { - output, err := exec.Command("wevtutil.exe", "sl", "/ms:"+strconv.Itoa(sizeBytes), provider).CombinedOutput() + output, err := exec.Command("wevtutil.exe", "sl", "/ms:"+strconv.Itoa(sizeBytes), provider).CombinedOutput() //nolint:gosec // No possibility of command injection. if err != nil { t.Fatal("failed to set log size", err, string(output)) } diff --git a/winlogbeat/sys/wineventlog/testdata/application-windows-error-reporting.xml b/winlogbeat/sys/wineventlog/testdata/application-windows-error-reporting.xml new file mode 100644 index 000000000000..0e768eaeebde --- /dev/null +++ b/winlogbeat/sys/wineventlog/testdata/application-windows-error-reporting.xml @@ -0,0 +1,18 @@ +1001400x80000000000000420107Applicationvagrant0WindowsWcpOtherFailure3Not available010.0.17763.850:3inc\auto_hive.hWindows::Rtl::AutoHive::Unload358c00001210xaad0d4fb +\\?\C:\Windows\Logs\CBS\CBS.log +\\?\C:\Windows\Logs\CBS\CbsPersist_20200212163557.log +\\?\C:\Windows\Logs\CBS\CbsPersist_20200211235949.log +\\?\C:\Windows\Logs\CBS\CbsPersist_20200211033558.cab +\\?\C:\Windows\Logs\CBS\CbsPersist_20200210020038.cab +\\?\C:\Windows\Logs\CBS\CbsPersist_20200209082850.cab +\\?\C:\Windows\servicing\Sessions\Sessions.xml +\\?\C:\Windows\WinSxs\pending.xml +\\?\C:\Windows\WinSxs\poqexec.log +\\?\C:\Windows\Logs\Cbs\FilterList.log +\\?\C:\ProgramData\Microsoft\Windows\WER\Temp\WERC5A1.tmp.WERInternalMetadata.xml +\\?\C:\ProgramData\Microsoft\Windows\WER\Temp\WERC7D5.tmp.xml +\\?\C:\ProgramData\Microsoft\Windows\WER\Temp\WERC7F3.tmp.csv +\\?\C:\ProgramData\Microsoft\Windows\WER\Temp\WERC9F8.tmp.txt +\\?\C:\ProgramData\Microsoft\Windows\WER\Temp\WERCA08.tmp.mdmp +\\?\C:\ProgramData\Microsoft\Windows\WER\ReportQueue\Critical_10.0.17763.850_3_b785171a54ee6e13bf912aeeb5bef5d9105e314b_00000000_cab_0c38cad1\memory.hdmp +\\?\C:\Windows\Temp\WERCAD4.tmp.WERDataCollectionStatus.txt\\?\C:\ProgramData\Microsoft\Windows\WER\ReportQueue\Critical_10.0.17763.850_3_b785171a54ee6e13bf912aeeb5bef5d9105e314b_00000000_cab_0c38cad105e9de0ad-0fa4-4daa-aec1-8127dc88e6c71000 diff --git a/winlogbeat/sys/wineventlog/testdata/ec1.evtx b/winlogbeat/sys/wineventlog/testdata/ec1.evtx new file mode 100644 index 000000000000..3e498db3ae0f Binary files /dev/null and b/winlogbeat/sys/wineventlog/testdata/ec1.evtx differ diff --git a/winlogbeat/sys/wineventlog/testdata/ec1.xml b/winlogbeat/sys/wineventlog/testdata/ec1.xml new file mode 100644 index 000000000000..d1607a648352 --- /dev/null +++ b/winlogbeat/sys/wineventlog/testdata/ec1.xml @@ -0,0 +1 @@ +100002000x80000000000000316ApplicationvagrantMy custom error event for the application log diff --git a/winlogbeat/sys/wineventlog/testdata/ec2.evtx b/winlogbeat/sys/wineventlog/testdata/ec2.evtx new file mode 100644 index 000000000000..7b8eb85263de Binary files /dev/null and b/winlogbeat/sys/wineventlog/testdata/ec2.evtx differ diff --git a/winlogbeat/sys/wineventlog/testdata/ec2.xml b/winlogbeat/sys/wineventlog/testdata/ec2.xml new file mode 100644 index 000000000000..5e2e8e38b7da --- /dev/null +++ b/winlogbeat/sys/wineventlog/testdata/ec2.xml @@ -0,0 +1 @@ +99902000x80000000000000317ApplicationvagrantWinword event 999 happened due to low diskspace diff --git a/winlogbeat/sys/wineventlog/testdata/ec3.evtx b/winlogbeat/sys/wineventlog/testdata/ec3.evtx new file mode 100644 index 000000000000..fb0bdc95fceb Binary files /dev/null and b/winlogbeat/sys/wineventlog/testdata/ec3.evtx differ diff --git a/winlogbeat/sys/wineventlog/testdata/ec3.xml b/winlogbeat/sys/wineventlog/testdata/ec3.xml new file mode 100644 index 000000000000..fdd0622ae59f --- /dev/null +++ b/winlogbeat/sys/wineventlog/testdata/ec3.xml @@ -0,0 +1 @@ +502000x800000000000001413SystemvagrantCatastrophe! diff --git a/winlogbeat/sys/wineventlog/testdata/ec3and4.evtx b/winlogbeat/sys/wineventlog/testdata/ec3and4.evtx new file mode 100644 index 000000000000..74e7cd1e2cf6 Binary files /dev/null and b/winlogbeat/sys/wineventlog/testdata/ec3and4.evtx differ diff --git a/winlogbeat/sys/wineventlog/testdata/ec3and4.xml b/winlogbeat/sys/wineventlog/testdata/ec3and4.xml new file mode 100644 index 000000000000..5844cfbfe918 --- /dev/null +++ b/winlogbeat/sys/wineventlog/testdata/ec3and4.xml @@ -0,0 +1,2 @@ +502000x800000000000001414SystemvagrantBackup failure +502000x800000000000001413SystemvagrantCatastrophe! diff --git a/winlogbeat/sys/wineventlog/testdata/ec4.evtx b/winlogbeat/sys/wineventlog/testdata/ec4.evtx new file mode 100644 index 000000000000..94547179581c Binary files /dev/null and b/winlogbeat/sys/wineventlog/testdata/ec4.evtx differ diff --git a/winlogbeat/sys/wineventlog/testdata/ec4.xml b/winlogbeat/sys/wineventlog/testdata/ec4.xml new file mode 100644 index 000000000000..232960fb4b80 --- /dev/null +++ b/winlogbeat/sys/wineventlog/testdata/ec4.xml @@ -0,0 +1 @@ +502000x800000000000001414SystemvagrantBackup failure diff --git a/winlogbeat/sys/wineventlog/testdata/experimental.evtx b/winlogbeat/sys/wineventlog/testdata/experimental.evtx new file mode 100644 index 000000000000..1fbfa0a461fb Binary files /dev/null and b/winlogbeat/sys/wineventlog/testdata/experimental.evtx differ diff --git a/winlogbeat/sys/wineventlog/testdata/experimental.xml b/winlogbeat/sys/wineventlog/testdata/experimental.xml new file mode 100644 index 000000000000..5f845b287cca --- /dev/null +++ b/winlogbeat/sys/wineventlog/testdata/experimental.xml @@ -0,0 +1,5 @@ +404000x8000000000000020050WinlogbeatTestGovagrant4 college quality neutral feather article article trolley attract bargain college arrange recover feather arrange percent wriggle wriggle feather college highway feather neutral quality manager manager recover arrange article arrange manager quality bargain +304000x8000000000000020049WinlogbeatTestGovagrant3 highway article bargain article college trolley percent college attract recover arrange attract manager highway trolley bargain recover trolley arrange manager bargain wriggle arrange manager trolley bargain recover highway bargain feather manager percent +204000x8000000000000020048WinlogbeatTestGovagrant2 wriggle college highway wriggle quality manager college article neutral bargain arrange quality highway percent attract attract arrange manager wriggle neutral highway article feather recover highway highway hunting arrange article manager neutral attract +104000x8000000000000020047WinlogbeatTestGovagrant1 feather bargain feather neutral bargain recover hunting quality attract neutral wriggle quality percent manager feather neutral attract neutral highway bargain bargain attract college article wriggle quality percent wriggle article recover hunting bargain +004000x8000000000000020046WinlogbeatTestGovagrant0 wriggle neutral trolley hunting highway attract college percent highway recover arrange bargain percent arrange quality arrange manager quality trolley feather percent hunting highway quality neutral arrange recover quality recover article bargain wriggle diff --git a/winlogbeat/sys/wineventlog/testdata/original.evtx b/winlogbeat/sys/wineventlog/testdata/original.evtx new file mode 100644 index 000000000000..a973a51a4c34 Binary files /dev/null and b/winlogbeat/sys/wineventlog/testdata/original.evtx differ diff --git a/winlogbeat/sys/wineventlog/testdata/original.xml b/winlogbeat/sys/wineventlog/testdata/original.xml new file mode 100644 index 000000000000..40ce93c2a1b2 --- /dev/null +++ b/winlogbeat/sys/wineventlog/testdata/original.xml @@ -0,0 +1,5 @@ +404000x8000000000000020055WinlogbeatTestGovagrant4 college quality neutral feather article article trolley attract bargain college arrange recover feather arrange percent wriggle wriggle feather college highway feather neutral quality manager manager recover arrange article arrange manager quality bargain +304000x8000000000000020054WinlogbeatTestGovagrant3 highway article bargain article college trolley percent college attract recover arrange attract manager highway trolley bargain recover trolley arrange manager bargain wriggle arrange manager trolley bargain recover highway bargain feather manager percent +204000x8000000000000020053WinlogbeatTestGovagrant2 wriggle college highway wriggle quality manager college article neutral bargain arrange quality highway percent attract attract arrange manager wriggle neutral highway article feather recover highway highway hunting arrange article manager neutral attract +104000x8000000000000020052WinlogbeatTestGovagrant1 feather bargain feather neutral bargain recover hunting quality attract neutral wriggle quality percent manager feather neutral attract neutral highway bargain bargain attract college article wriggle quality percent wriggle article recover hunting bargain +004000x8000000000000020051WinlogbeatTestGovagrant0 wriggle neutral trolley hunting highway attract college percent highway recover arrange bargain percent arrange quality arrange manager quality trolley feather percent hunting highway quality neutral arrange recover quality recover article bargain wriggle diff --git a/winlogbeat/sys/wineventlog/testdata/sysmon-9.01.xml b/winlogbeat/sys/wineventlog/testdata/sysmon-9.01.xml new file mode 100644 index 000000000000..82a6ff6dca4e --- /dev/null +++ b/winlogbeat/sys/wineventlog/testdata/sysmon-9.01.xml @@ -0,0 +1,32 @@ +244200x800000000000000032Microsoft-Windows-Sysmon/Operationalvagrant-2012-r22019-03-18 16:57:52.433{42f11c3b-ccaa-5c8f-0000-0010b4e22700}1600C:\Program Files (x86)\Google\Chrome\Application\chrome.exeC:\Users\vagrant\AppData\Local\Google\Chrome\User Data\Default\Storage\ext\gfdkimpbcpahaombhbimeihdjnejgicl\def\ee4a6e45-bffd-49f4-98ae-32aebcc890b5.tmp2019-03-18 16:52:05.3392019-03-18 16:57:52.417 +244200x800000000000000031Microsoft-Windows-Sysmon/Operationalvagrant-2012-r22019-03-18 16:57:52.433{42f11c3b-ccaa-5c8f-0000-0010b4e22700}1600C:\Program Files (x86)\Google\Chrome\Application\chrome.exeC:\Users\vagrant\AppData\Local\Google\Chrome\User Data\Default\Storage\ext\nmmhkkegccagdldgiimedpiccmgmieda\def\ecb9c915-c4c2-4600-a920-f2bc302990a8.tmp2019-03-18 16:52:08.4962019-03-18 16:57:52.417 +534500x800000000000000030Microsoft-Windows-Sysmon/Operationalvagrant-2012-r22019-03-18 16:57:52.433{42f11c3b-ccab-5c8f-0000-001064eb2700}2680C:\Program Files (x86)\Google\Chrome\Application\chrome.exe +244200x800000000000000029Microsoft-Windows-Sysmon/Operationalvagrant-2012-r22019-03-18 16:57:52.417{42f11c3b-ccaa-5c8f-0000-0010b4e22700}1600C:\Program Files (x86)\Google\Chrome\Application\chrome.exeC:\Users\vagrant\AppData\Local\Google\Chrome\User Data\Default\37ed32e9-3c5f-4663-8457-c70743e9456d.tmp2019-03-18 16:51:54.9802019-03-18 16:57:52.417 +244200x800000000000000028Microsoft-Windows-Sysmon/Operationalvagrant-2012-r22019-03-18 16:57:52.417{42f11c3b-ccaa-5c8f-0000-0010b4e22700}1600C:\Program Files (x86)\Google\Chrome\Application\chrome.exeC:\Users\vagrant\AppData\Local\Google\Chrome\User Data\Default\1450fedf-ac4c-4e35-b371-ed5d3bbe4776.tmp2019-03-18 16:52:05.0282019-03-18 16:57:52.402 +244200x800000000000000027Microsoft-Windows-Sysmon/Operationalvagrant-2012-r22019-03-18 16:57:52.417{42f11c3b-ccaa-5c8f-0000-0010b4e22700}1600C:\Program Files (x86)\Google\Chrome\Application\chrome.exeC:\Users\vagrant\AppData\Local\Google\Chrome\User Data\162d4140-cfab-4d05-9c92-bca60515a622.tmp2019-03-18 16:52:04.9802019-03-18 16:57:52.402 +244200x800000000000000026Microsoft-Windows-Sysmon/Operationalvagrant-2012-r22019-03-18 16:57:52.387{42f11c3b-ccaa-5c8f-0000-0010b4e22700}1600C:\Program Files (x86)\Google\Chrome\Application\chrome.exeC:\Users\vagrant\AppData\Local\Google\Chrome\User Data\fe823684-c940-49f2-a940-14b02cbafba9.tmp2019-03-18 16:52:04.9802019-03-18 16:57:52.387 +534500x800000000000000025Microsoft-Windows-Sysmon/Operationalvagrant-2012-r22019-03-18 16:57:52.364{42f11c3b-cccc-5c8f-0000-0010e8272900}3208C:\Program Files (x86)\Google\Chrome\Application\chrome.exe +534500x800000000000000024Microsoft-Windows-Sysmon/Operationalvagrant-2012-r22019-03-18 16:57:52.350{42f11c3b-ccc6-5c8f-0000-001005082900}4832C:\Program Files (x86)\Google\Chrome\Application\chrome.exe +354300x800000000000000023Microsoft-Windows-Sysmon/Operationalvagrant-2012-r22019-03-18 16:57:49.218{42f11c3b-6e19-5c8c-0000-0010eb030000}4SystemNT AUTHORITY\SYSTEMudptruefalse10.0.2.15vagrant-2012-r2.local.crowbird.com137netbios-nsfalse169.254.180.25137netbios-ns +354300x800000000000000022Microsoft-Windows-Sysmon/Operationalvagrant-2012-r22019-03-18 16:57:49.213{42f11c3b-6e19-5c8c-0000-0010eb030000}4SystemNT AUTHORITY\SYSTEMudptruefalse10.0.2.15vagrant-2012-r2.local.crowbird.com137netbios-nsfalse169.254.255.255137netbios-ns +354300x800000000000000021Microsoft-Windows-Sysmon/Operationalvagrant-2012-r22019-03-18 16:57:48.276{42f11c3b-6e19-5c8c-0000-0010eb030000}4SystemNT AUTHORITY\SYSTEMudptruefalse10.0.2.15vagrant-2012-r2.local.crowbird.com137netbios-nsfalse10.0.2.3137netbios-ns +354300x800000000000000020Microsoft-Windows-Sysmon/Operationalvagrant-2012-r22019-03-18 16:57:48.264{42f11c3b-6e19-5c8c-0000-0010eb030000}4SystemNT AUTHORITY\SYSTEMudptruefalse10.0.2.15vagrant-2012-r2.local.crowbird.com137netbios-nsfalse40.77.226.250137netbios-ns +354300x800000000000000019Microsoft-Windows-Sysmon/Operationalvagrant-2012-r22019-03-18 16:57:48.251{42f11c3b-0bad-5c8c-0000-0010dfbc0000}924C:\Windows\System32\svchost.exeNT AUTHORITY\NETWORK SERVICEudptruetruea9fe:b419:0:0:f880:2301:e0:ffff55717truee000:fc:0:0:0:0:0:05355llmnr +354300x800000000000000018Microsoft-Windows-Sysmon/Operationalvagrant-2012-r22019-03-18 16:57:48.251{42f11c3b-0bad-5c8c-0000-0010dfbc0000}924C:\Windows\System32\svchost.exeNT AUTHORITY\NETWORK SERVICEudptruetruefe80:0:0:0:616f:32fa:b04f:b41955717trueff02:0:0:0:0:0:1:35355llmnr +354300x800000000000000017Microsoft-Windows-Sysmon/Operationalvagrant-2012-r22019-03-18 16:57:48.251{42f11c3b-6e19-5c8c-0000-0010eb030000}4SystemNT AUTHORITY\SYSTEMudpfalsefalse169.254.255.255137netbios-nsfalse169.254.180.25137netbios-ns +354300x800000000000000016Microsoft-Windows-Sysmon/Operationalvagrant-2012-r22019-03-18 16:57:48.250{42f11c3b-6e19-5c8c-0000-0010eb030000}4SystemNT AUTHORITY\SYSTEMudptruefalse169.254.180.25137netbios-nsfalse169.254.255.255137netbios-ns +354300x800000000000000015Microsoft-Windows-Sysmon/Operationalvagrant-2012-r22019-03-18 16:57:48.250{42f11c3b-0bad-5c8c-0000-0010dfbc0000}924C:\Windows\System32\svchost.exeNT AUTHORITY\NETWORK SERVICEudptruetruea00:20f:0:0:18a2:6e00:e0:ffff55542truee000:fc:4300:6800:7200:6f00:6d00:65005355llmnr +354300x800000000000000014Microsoft-Windows-Sysmon/Operationalvagrant-2012-r22019-03-18 16:57:48.250{42f11c3b-0bad-5c8c-0000-0010dfbc0000}924C:\Windows\System32\svchost.exeNT AUTHORITY\NETWORK SERVICEudptruetruefe80:0:0:0:e488:b85c:5262:ff86vagrant-2012-r2.local.crowbird.com55542trueff02:0:0:0:0:0:1:35355llmnr +354300x800000000000000013Microsoft-Windows-Sysmon/Operationalvagrant-2012-r22019-03-18 16:57:48.250{42f11c3b-6e19-5c8c-0000-0010eb030000}4SystemNT AUTHORITY\SYSTEMudpfalsefalse10.0.2.255137netbios-nsfalse10.0.2.15vagrant-2012-r2.local.crowbird.com137netbios-ns +354300x800000000000000012Microsoft-Windows-Sysmon/Operationalvagrant-2012-r22019-03-18 16:57:48.250{42f11c3b-6e19-5c8c-0000-0010eb030000}4SystemNT AUTHORITY\SYSTEMudptruefalse10.0.2.15vagrant-2012-r2.local.crowbird.com137netbios-nsfalse10.0.2.255137netbios-ns +354300x800000000000000011Microsoft-Windows-Sysmon/Operationalvagrant-2012-r22019-03-18 16:57:48.214{42f11c3b-ccaa-5c8f-0000-0010b4e22700}1600C:\Program Files (x86)\Google\Chrome\Application\chrome.exeVAGRANT-2012-R2\vagranttcptruefalse10.0.2.15vagrant-2012-r2.local.crowbird.com1139false40.77.226.250443https +354300x800000000000000010Microsoft-Windows-Sysmon/Operationalvagrant-2012-r22019-03-18 16:57:48.148{42f11c3b-ccaa-5c8f-0000-0010b4e22700}1600C:\Program Files (x86)\Google\Chrome\Application\chrome.exeVAGRANT-2012-R2\vagranttcptruefalse10.0.2.15vagrant-2012-r2.local.crowbird.com1138false40.77.226.250443https +354300x80000000000000009Microsoft-Windows-Sysmon/Operationalvagrant-2012-r22019-03-18 16:57:48.070{42f11c3b-0bad-5c8c-0000-0010dfbc0000}924C:\Windows\System32\svchost.exeNT AUTHORITY\NETWORK SERVICEudpfalsefalse10.0.2.15vagrant-2012-r2.local.crowbird.com62141false10.0.2.353domain +354300x80000000000000008Microsoft-Windows-Sysmon/Operationalvagrant-2012-r22019-03-18 16:57:47.847{42f11c3b-0bad-5c8c-0000-0010dfbc0000}924C:\Windows\System32\svchost.exeNT AUTHORITY\NETWORK SERVICEudptruetruea00:20f:0:0:18a2:6e00:e0:ffff62141truea00:203:3000:3000:3000:3000:3000:330053domain +154100x80000000000000007Microsoft-Windows-Sysmon/Operationalvagrant-2012-r22019-03-18 16:57:39.012{42f11c3b-ce03-5c8f-0000-0010e9462a00}4508C:\Windows\System32\wbem\WmiPrvSE.exe6.3.9600.16384 (winblue_rtm.130821-1623)WMI Provider HostMicrosoft® Windows® Operating SystemMicrosoft CorporationC:\Windows\system32\wbem\wmiprvse.exe -EmbeddingC:\Windows\system32\NT AUTHORITY\SYSTEM{42f11c3b-6e1a-5c8c-0000-0020e7030000}0x3e70SystemSHA1=5A4C0E82FF95C9FB762D46A696EF9F1B68001C21{42f11c3b-6e1b-5c8c-0000-00102f610000}560C:\Windows\System32\svchost.exeC:\Windows\system32\svchost.exe -k DcomLaunch +534500x80000000000000006Microsoft-Windows-Sysmon/Operationalvagrant-2012-r22019-03-18 16:57:38.981{42f11c3b-cdf4-5c8f-0000-0010071e2a00}4648C:\Users\vagrant\Downloads\Sysmon.exe +534500x80000000000000005Microsoft-Windows-Sysmon/Operationalvagrant-2012-r22019-03-18 16:57:38.981{42f11c3b-cdf4-5c8f-0000-0010e61e2a00}4616C:\Users\vagrant\AppData\Local\Temp\Sysmon.exe +154100x80000000000000004Microsoft-Windows-Sysmon/Operationalvagrant-2012-r22019-03-18 16:57:37.964{42f11c3b-ce01-5c8f-0000-00102c412a00}5028C:\Windows\System32\wbem\unsecapp.exe6.3.9600.16384 (winblue_rtm.130821-1623)Sink to receive asynchronous callbacks for WMI client applicationMicrosoft® Windows® Operating SystemMicrosoft CorporationC:\Windows\system32\wbem\unsecapp.exe -EmbeddingC:\Windows\system32\NT AUTHORITY\SYSTEM{42f11c3b-6e1a-5c8c-0000-0020e7030000}0x3e70SystemSHA1=6DF8163A6320B80B60733F9D62E2F39B4B16B678{42f11c3b-6e1b-5c8c-0000-00102f610000}560C:\Windows\System32\svchost.exeC:\Windows\system32\svchost.exe -k DcomLaunch +154100x80000000000000003Microsoft-Windows-Sysmon/Operationalvagrant-2012-r22019-03-18 16:57:37.949{42f11c3b-ce01-5c8f-0000-0010c73e2a00}4860C:\Windows\Sysmon.exe9.01System activity monitorSysinternals SysmonSysinternals - www.sysinternals.comC:\Windows\Sysmon.exeC:\Windows\system32\NT AUTHORITY\SYSTEM{42f11c3b-6e1a-5c8c-0000-0020e7030000}0x3e70SystemSHA1=AC93C3B38E57A2715572933DBCB2A1C2892DBC5E{42f11c3b-6e1a-5c8c-0000-0010f14d0000}488C:\Windows\System32\services.exeC:\Windows\system32\services.exe +434400x80000000000000002Microsoft-Windows-Sysmon/Operationalvagrant-2012-r22019-03-18 16:57:38.011Started9.014.20 +16341600x80000000000000001Microsoft-Windows-Sysmon/Operationalvagrant-2012-r22019-03-18 16:57:37.933C:\Users\vagrant\Downloads\"C:\Users\vagrant\Downloads\Sysmon.exe" -i -n diff --git a/winlogbeat/sys/wineventlog/wineventlog_windows.go b/winlogbeat/sys/wineventlog/wineventlog_windows.go index fed3e42ccf18..3cad7d57548e 100644 --- a/winlogbeat/sys/wineventlog/wineventlog_windows.go +++ b/winlogbeat/sys/wineventlog/wineventlog_windows.go @@ -28,10 +28,9 @@ import ( "sort" "syscall" - "github.com/elastic/beats/v7/libbeat/common" - "golang.org/x/sys/windows" + "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/winlogbeat/sys" ) @@ -74,7 +73,7 @@ func Channels() ([]string, error) { if err != nil { return nil, err } - defer _EvtClose(handle) + defer _EvtClose(handle) //nolint:errcheck // This is just a resource release. var channels []string cpBuffer := make([]uint16, 512) @@ -83,7 +82,7 @@ loop: var used uint32 err := _EvtNextChannelPath(handle, uint32(len(cpBuffer)), &cpBuffer[0], &used) if err != nil { - errno, ok := err.(syscall.Errno) + errno, ok := err.(syscall.Errno) //nolint:errorlint // This is an errno or nil. if ok { switch errno { case ERROR_INSUFFICIENT_BUFFER: @@ -203,7 +202,7 @@ func EventHandles(subscription EvtHandle, maxHandles int) ([]EvtHandle, error) { // Munge ERROR_INVALID_OPERATION to ERROR_NO_MORE_ITEMS when no handles // were read. This happens you call the method and there are no events // to read (i.e. polling). - if err == ERROR_INVALID_OPERATION && numRead == 0 { + if err == ERROR_INVALID_OPERATION && numRead == 0 { //nolint:errorlint // This is an errno or nil. return nil, ERROR_NO_MORE_ITEMS } return nil, err @@ -241,12 +240,11 @@ func RenderEvent( // Only a single string is returned when rendering XML. err = FormatEventString(EvtFormatMessageXml, eventHandle, providerName, EvtHandle(publisherHandle), lang, renderBuf, out) - // Recover by rendering the XML without the RenderingInfo (message string). if err != nil { // Do not try to recover from InsufficientBufferErrors because these // can be retried with a larger buffer. - if _, ok := err.(sys.InsufficientBufferError); ok { + if errors.Is(err, sys.InsufficientBufferError{}) { return err } @@ -256,6 +254,26 @@ func RenderEvent( return err } +// Message reads the event data associated with the EvtHandle and renders +// and returns the message only. +func Message(h EvtHandle, buf []byte, pubHandleProvider func(string) sys.MessageFiles) (message string, err error) { + providerName, err := evtRenderProviderName(buf, h) + if err != nil { + return "", err + } + + var pub EvtHandle + if pubHandleProvider != nil { + messageFiles := pubHandleProvider(providerName) + if messageFiles.Err == nil { + // There is only ever a single handle when using the Windows Event + // Log API. + pub = EvtHandle(messageFiles.Handles[0].Handle) + } + } + return getMessageStringFromHandle(&PublisherMetadata{Handle: pub}, h, nil) +} + // RenderEventXML renders the event as XML. If the event is already rendered, as // in a forwarded event whose content type is "RenderedText", then the XML will // include the RenderingInfo (message). If the event is not rendered then the @@ -315,7 +333,7 @@ func CreateBookmarkFromXML(bookmarkXML string) (EvtHandle, error) { // CreateRenderContext creates a render context. Close must be called on // returned EvtHandle when finished with the handle. func CreateRenderContext(valuePaths []string, flag EvtRenderContextFlag) (EvtHandle, error) { - var paths []uintptr + paths := make([]uintptr, 0, len(valuePaths)) for _, path := range valuePaths { utf16, err := syscall.UTF16FromString(path) if err != nil { @@ -384,11 +402,12 @@ func FormatEventString( // Open a publisher handle if one was not provided. ph := publisherHandle if ph == 0 { - ph, err := OpenPublisherMetadata(0, publisher, lang) + var err error + ph, err = OpenPublisherMetadata(0, publisher, lang) if err != nil { return err } - defer _EvtClose(ph) + defer _EvtClose(ph) //nolint:errcheck // This is just a resource release. } // Create a buffer if one was not provided. @@ -396,7 +415,7 @@ func FormatEventString( if buffer == nil { err := _EvtFormatMessage(ph, eventHandle, 0, 0, 0, messageFlag, 0, nil, &bufferUsed) - if err != nil && err != ERROR_INSUFFICIENT_BUFFER { + if err != nil && err != ERROR_INSUFFICIENT_BUFFER { //nolint:errorlint // This is an errno or nil. return err } @@ -408,8 +427,8 @@ func FormatEventString( err := _EvtFormatMessage(ph, eventHandle, 0, 0, 0, messageFlag, uint32(len(buffer)/2), &buffer[0], &bufferUsed) bufferUsed *= 2 - if err == ERROR_INSUFFICIENT_BUFFER { - return sys.InsufficientBufferError{err, int(bufferUsed)} + if err == ERROR_INSUFFICIENT_BUFFER { //nolint:errorlint // This is an errno or nil. + return sys.InsufficientBufferError{Cause: err, RequiredSize: int(bufferUsed)} } if err != nil { return err @@ -426,7 +445,7 @@ func Publishers() ([]string, error) { if err != nil { return nil, fmt.Errorf("failed in EvtOpenPublisherEnum: %w", err) } - defer Close(publisherEnumerator) + defer Close(publisherEnumerator) //nolint:errcheck // This is just a resource release. var ( publishers []string @@ -437,7 +456,7 @@ func Publishers() ([]string, error) { loop: for { if err = _EvtNextPublisherId(publisherEnumerator, uint32(len(buffer)), &buffer[0], &bufferUsed); err != nil { - switch err { + switch err { //nolint:errorlint // This is an errno or nil. case ERROR_NO_MORE_ITEMS: break loop case ERROR_INSUFFICIENT_BUFFER: @@ -466,7 +485,7 @@ func offset(buffer []byte, reader io.Reader) (uint64, error) { var err error switch runtime.GOARCH { default: - return 0, fmt.Errorf("Unhandled architecture: %s", runtime.GOARCH) + return 0, fmt.Errorf("unhandled architecture: %s", runtime.GOARCH) case "amd64": err = binary.Read(reader, binary.LittleEndian, &dataPtr) if err != nil { @@ -488,9 +507,15 @@ func offset(buffer []byte, reader io.Reader) (uint64, error) { bufferPtr := uint64(reflect.ValueOf(&buffer[0]).Pointer()) offset := dataPtr - bufferPtr +<<<<<<< HEAD if offset < 0 || offset > uint64(len(buffer)) { return 0, fmt.Errorf("Invalid pointer %x. Cannot dereference an "+ "address outside of the buffer [%x:%x].", dataPtr, bufferPtr, +======= + if offset > uint64(len(buffer)) { + return 0, fmt.Errorf("invalid pointer %x: cannot dereference an "+ + "address outside of the buffer [%x:%x]", dataPtr, bufferPtr, +>>>>>>> 34bdc3d468 (winlogbeat: fix event handling for Windows 2022 (#30942)) bufferPtr+uint64(len(buffer))) } @@ -503,7 +528,7 @@ func readString(buffer []byte, reader io.Reader) (string, error) { offset, err := offset(buffer, reader) if err != nil { // Ignore NULL values. - if err == ErrorEvtVarTypeNull { + if err == ErrorEvtVarTypeNull { //nolint:errorlint // This is never wrapped. return "", nil } return "", err @@ -517,11 +542,11 @@ func evtRenderProviderName(renderBuf []byte, eventHandle EvtHandle) (string, err var bufferUsed, propertyCount uint32 err := _EvtRender(providerNameContext, eventHandle, EvtRenderEventValues, uint32(len(renderBuf)), &renderBuf[0], &bufferUsed, &propertyCount) - if err == ERROR_INSUFFICIENT_BUFFER { - return "", sys.InsufficientBufferError{err, int(bufferUsed)} + if err == ERROR_INSUFFICIENT_BUFFER { //nolint:errorlint // This is an errno or nil. + return "", sys.InsufficientBufferError{Cause: err, RequiredSize: int(bufferUsed)} } if err != nil { - return "", fmt.Errorf("evtRenderProviderName %v", err) + return "", fmt.Errorf("evtRenderProviderName: %w", err) } reader := bytes.NewReader(renderBuf) @@ -532,14 +557,15 @@ func renderXML(eventHandle EvtHandle, flag EvtRenderFlag, renderBuf []byte, out var bufferUsed, propertyCount uint32 err := _EvtRender(0, eventHandle, flag, uint32(len(renderBuf)), &renderBuf[0], &bufferUsed, &propertyCount) - if err == ERROR_INSUFFICIENT_BUFFER { - return sys.InsufficientBufferError{err, int(bufferUsed)} + if err == ERROR_INSUFFICIENT_BUFFER { //nolint:errorlint // This is an errno or nil. + return sys.InsufficientBufferError{Cause: err, RequiredSize: int(bufferUsed)} } if err != nil { return err } if int(bufferUsed) > len(renderBuf) { + //nolint:stylecheck // These are proper nouns. return fmt.Errorf("Windows EvtRender reported that wrote %d bytes "+ "to the buffer, but the buffer can only hold %d bytes", bufferUsed, len(renderBuf)) diff --git a/winlogbeat/sys/wineventlog/wineventlog_windows_test.go b/winlogbeat/sys/wineventlog/wineventlog_windows_test.go index ff4fe566e9c2..e6b494e3b242 100644 --- a/winlogbeat/sys/wineventlog/wineventlog_windows_test.go +++ b/winlogbeat/sys/wineventlog/wineventlog_windows_test.go @@ -19,79 +19,163 @@ package wineventlog import ( "bytes" + "encoding/xml" + "flag" + "fmt" + "io" "os" "path/filepath" + "reflect" + "strings" "testing" + "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/assert" -) - -var sysmonEvtx string -func init() { - var err error - sysmonEvtx, err = filepath.Abs("testdata/sysmon-9.01.evtx") - if err != nil { - panic(err) - } + "github.com/elastic/beats/v7/winlogbeat/sys/winevent" +) - if _, err = os.Lstat(sysmonEvtx); err != nil { - panic(err) - } -} +var updateXML = flag.Bool("update", false, "update XML golden files from evtx files in testdata") + +func TestWinEventLog(t *testing.T) { + for _, test := range []struct { + path string + events int + }{ + {path: "application-windows-error-reporting.evtx", events: 1}, + {path: "sysmon-9.01.evtx", events: 32}, + {path: "ec1.evtx", events: 1}, // eventcreate /id 1000 /t error /l application /d "My custom error event for the application log" + {path: "ec2.evtx", events: 1}, // eventcreate /id 999 /t error /l application /so WinWord /d "Winword event 999 happened due to low diskspace" + {path: "ec3.evtx", events: 1}, // eventcreate /id 5 /t error /l system /d "Catastrophe!" + {path: "ec4.evtx", events: 1}, // eventcreate /id 5 /t error /l system /so Backup /d "Backup failure" + {path: "ec3and4.evtx", events: 2}, // ec3 and ec3 exported as a single evtx. + {path: "original.evtx", events: 5}, // a capture from a short generation of the eventlog WindowsEventLogAPI test. + {path: "experimental.evtx", events: 5}, // a capture from a short generation of the eventlog WindowsEventLogAPIExperimental test. + } { + t.Run(test.path, func(t *testing.T) { + evtx, err := filepath.Abs(filepath.Join("testdata", test.path)) + if err != nil { + t.Fatal(err) + } + xmlPath := evtx[:len(evtx)-len("evtx")] + "xml" -func TestEvtOpenLog(t *testing.T) { - h, err := EvtOpenLog(0, sysmonEvtx, EvtOpenFilePath) - if err != nil { - t.Fatal(err) - } - defer Close(h) -} + if _, err = os.Lstat(evtx); err != nil { + t.Fatal(err) + } -func TestEvtQuery(t *testing.T) { - h, err := EvtQuery(0, sysmonEvtx, "", EvtQueryFilePath) - if err != nil { - t.Fatal(err) + t.Run("EvtOpenLog", func(t *testing.T) { + h, err := EvtOpenLog(0, evtx, EvtOpenFilePath) + if err != nil { + t.Fatal(err) + } + defer Close(h) //nolint:errcheck // This is just a resource release. + }) + + t.Run("EvtQuery", func(t *testing.T) { + h, err := EvtQuery(0, evtx, "", EvtQueryFilePath) + if err != nil { + t.Fatal(err) + } + defer Close(h) //nolint:errcheck // This is just a resource release. + }) + + t.Run("ReadEvtx", func(t *testing.T) { + // Open .evtx file. + h, err := EvtQuery(0, evtx, "", EvtQueryFilePath|EvtQueryReverseDirection) + if err != nil { + t.Fatal(err) + } + defer Close(h) //nolint:errcheck // This is just a resource release. + + // Get handles to events. + buf := make([]byte, 32*1024) + var out io.Writer + if *updateXML { + f, err := os.Create(xmlPath) + if err != nil { + t.Fatalf("failed to create golden file: %v", err) + } + defer f.Close() + out = f + } else { + out = &bytes.Buffer{} + } + var count int + for { + handles, err := EventHandles(h, 8) + if err == ERROR_NO_MORE_ITEMS { //nolint:errorlint // This is never wrapped. + t.Log(err) + break + } + if err != nil { + t.Fatal(err) + } + + // Read events. + for _, h := range handles { + if err = RenderEventXML(h, buf, out); err != nil { + t.Fatal(err) + } + Close(h) //nolint:errcheck // This is just a resource release. + fmt.Fprintln(out) + count++ + } + } + if !*updateXML { + f, err := os.Open(xmlPath) + if err != nil { + t.Fatalf("failed to read golden file: %v", err) + } + want, err := unmarshalXMLEvents(f) + if err != nil { + t.Fatalf("failed to unmarshal golden events: %v", err) + } + got, err := unmarshalXMLEvents(out.(*bytes.Buffer)) + if err != nil { + t.Fatalf("failed to unmarshal obtained events: %v", err) + } + if !reflect.DeepEqual(want, got) { + t.Errorf("unexpected result for %s: got:- want:+\n%s", test.path, cmp.Diff(want, got)) + } + } + + if count != test.events { + t.Errorf("expected to read %d events but got %d from %s", test.events, count, test.path) + } + }) + }) } - defer Close(h) } -func TestReadEvtx(t *testing.T) { - // Open .evtx file. - h, err := EvtQuery(0, sysmonEvtx, "", EvtQueryFilePath|EvtQueryReverseDirection) - if err != nil { - t.Fatal(err) - } - defer Close(h) - - // Get handles to events. - buf := make([]byte, 32*1024) - out := new(bytes.Buffer) - count := 0 +// unmarshalXMLEvents unmarshals a complete set of events from the XML data +// in the provided io.Reader. GUID values are canonicalised to lowercase. +func unmarshalXMLEvents(r io.Reader) ([]winevent.Event, error) { + var events []winevent.Event + decoder := xml.NewDecoder(r) for { - handles, err := EventHandles(h, 8) - if err == ERROR_NO_MORE_ITEMS { - t.Log(err) - break - } + var e winevent.Event + err := decoder.Decode(&e) if err != nil { - t.Fatal(err) - } - - // Read events. - for _, h := range handles { - out.Reset() - if err = RenderEventXML(h, buf, out); err != nil { - t.Fatal(err) + if err != io.EOF { //nolint:errorlint // This is never wrapped. + return nil, err } - Close(h) - count++ + break } + events = append(events, canonical(e)) } + return events, nil +} - if count != 32 { - t.Fatal("expected to read 32 events but got", count, "from", sysmonEvtx) +// canonical return e with its GUID values canonicalised to lower case. +// Different versions of Windows render these values in different cases; ¯\_(ツ)_/¯ +func canonical(e winevent.Event) winevent.Event { + e.Provider.GUID = strings.ToLower(e.Provider.GUID) + for i, kv := range e.EventData.Pairs { + if strings.Contains(strings.ToLower(kv.Key), "guid") { + e.EventData.Pairs[i].Value = strings.ToLower(kv.Value) + } } + return e } func TestChannels(t *testing.T) { diff --git a/x-pack/winlogbeat/Jenkinsfile.yml b/x-pack/winlogbeat/Jenkinsfile.yml index d972f712b0d8..e6863315e6b1 100644 --- a/x-pack/winlogbeat/Jenkinsfile.yml +++ b/x-pack/winlogbeat/Jenkinsfile.yml @@ -29,6 +29,19 @@ stages: platforms: ## override default labels in this specific stage. - "windows-2019" stage: mandatory +<<<<<<< HEAD +======= + windows-2022: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-2022" + stage: mandatory + windows-2019: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-2019" + stage: extended_win +>>>>>>> 34bdc3d468 (winlogbeat: fix event handling for Windows 2022 (#30942)) windows-2016: mage: "mage build unitTest" platforms: ## override default labels in this specific stage.