[Winlogbeat] Add support for custom XML queries (#29330) (#29474)

- Added new configuration field (xml_query) to support custom XML queries - This new configuration item will conflict with existing simple query configuration items (ignore_older, event_id, level, provider) - Validator has been updated to check for key conflicts and XML syntax, but does not check for correctness of XML schema. - Added unit tests for config validation - Added unit/system test for XML query runner (cherry picked from commit b5e9414) Co-authored-by: Taylor Swanson <[email protected]>
elastic · Dec 20, 2021 · a6794f1 · a6794f1
1 parent 5fea797
commit a6794f1
Show file tree

Hide file tree

Showing 11 changed files with 324 additions and 63 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -76,6 +76,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Winlogbeat*
 
+- Add support for custom XML queries {issue}1054[1054] {pull}29330[29330]
+
 
 *Elastic Log Driver*
 

diff --git a/winlogbeat/_meta/config/header.yml.tmpl b/winlogbeat/_meta/config/header.yml.tmpl
@@ -26,7 +26,9 @@
 # accompanying options. The YAML data type of event_logs is a list of
 # dictionaries.
 #
-# The supported keys are name (required), tags, fields, fields_under_root,
-# forwarded, ignore_older, level, event_id, provider, and include_xml. Please
-# visit the documentation for the complete details of each option.
+# The supported keys are name, id, xml_query, tags, fields, fields_under_root,
+# forwarded, ignore_older, level, event_id, provider, and include_xml.
+# The xml_query key requires an id and must not be used with the name,
+# ignore_older, level, event_id, or provider keys. Please visit the
+# documentation for the complete details of each option.
 # https://go.es.io/WinlogbeatConfig
diff --git a/winlogbeat/docs/winlogbeat-options.asciidoc b/winlogbeat/docs/winlogbeat-options.asciidoc
@@ -87,7 +87,6 @@ winlogbeat.shutdown_timeout: 30s
 A list of entries (called 'dictionaries' in YAML) that specify which event logs
 to monitor. Each entry in the list defines an event log to monitor as well as
 any information to be associated with the event log (filter, tags, and so on).
-The `name` field is the only required field for each event log.
 
 [source,yaml]
 --------------------------------------------------------------------------------
@@ -113,9 +112,9 @@ reading additional event log records.
 ==== `event_logs.name`
 
 The name of the event log to monitor. Each dictionary under `event_logs` must
-have a `name` field. You can get a list of available event logs by running
-`Get-EventLog *` in PowerShell.  Here is a sample of the output from the
-command:
+have a `name` field, except for those which use a custom XML query. You can
+get a list of available event logs by running `Get-EventLog *` in PowerShell.
+Here is a sample of the output from the command:
 
 [source,sh]
 --------------------------------------------------------------------------------
@@ -173,6 +172,28 @@ winlogbeat.event_logs:
   - name: 'C:\backup\sysmon-2019.08.evtx'
 --------------------------------------------------------------------------------
 
+The name key must not be used with custom XML queries.
+
+[float]
+==== `event_logs.id`
+
+A unique identifier for the event log. This key is required when using a custom
+XML query.
+
+It is used to uniquely identify the event log reader in the registry file. This is
+useful if multiple event logs are being set up to watch the same channel or file. If an
+ID is not given, the `event_logs.name` value will be used.
+
+This value must be unique.
+
+[source,yaml]
+--------------------------------------------------------------------------------
+winlogbeat.event_logs:
+  - name: Application
+    id: application-logs
+    ignore_older: 168h
+--------------------------------------------------------------------------------
+
 [float]
 ==== `event_logs.ignore_older`
 
@@ -335,6 +356,35 @@ Microsoft-Windows-Security-Auditing
 Microsoft-Windows-Eventlog
 --------------------------------------------------------------------------------
 
+[float]
+==== `event_logs.xml_query`
+
+Provide a custom XML query. This option is mutually exclusive with the `name`, `event_id`,
+`ignore_older`, `level`, and `provider` options. These options should be included in
+the XML query directly. Furthermore, an `id` must be provided. Custom XML queries
+provide more flexibility and advanced options than the simpler query options in {beatname_uc}.
+*{vista_and_newer}*
+
+Here is a configuration which will collect DHCP server events from multiple channels:
+
+[source,yaml]
+--------------------------------------------------------------------------------
+winlogbeat.event_logs:
+  - id: dhcp-server-logs
+    xml_query: >
+      <QueryList>
+        <Query Id="0" Path="DhcpAdminEvents">
+          <Select Path="DhcpAdminEvents">*</Select>
+          <Select Path="Microsoft-Windows-Dhcp-Server/FilterNotifications">*</Select>
+          <Select Path="Microsoft-Windows-Dhcp-Server/Operational">*</Select>
+        </Query>
+      </QueryList>
+--------------------------------------------------------------------------------
+
+XML queries may also be created in Windows Event Viewer using custom views. The query
+can be created using a graphical interface and the corresponding XML can be
+retrieved from the XML tab.
+
 [float]
 ==== `event_logs.include_xml`
 

diff --git a/winlogbeat/eventlog/factory.go b/winlogbeat/eventlog/factory.go
@@ -29,8 +29,10 @@ import (
 // EventLog. Each implementation is free to support additional configuration
 // options.
 type ConfigCommon struct {
-	API  string `config:"api"`  // Name of the API to use. Optional.
-	Name string `config:"name"` // Name of the event log or channel or file.
+	API      string `config:"api"`       // Name of the API to use. Optional.
+	Name     string `config:"name"`      // Name of the event log or channel or file.
+	ID       string `config:"id"`        // Identifier for the event log.
+	XMLQuery string `config:"xml_query"` // Custom query XML. Must not be used with the keys from eventlog.query.
 }
 
 type validator interface {

diff --git a/winlogbeat/eventlog/wineventlog.go b/winlogbeat/eventlog/wineventlog.go
@@ -21,6 +21,7 @@
 package eventlog
 
 import (
+	"encoding/xml"
 	"fmt"
 	"io"
 	"path/filepath"
@@ -113,7 +114,30 @@ type query struct {
 // any problems or nil.
 func (c *winEventLogConfig) Validate() error {
 	var errs multierror.Errors
-	if c.Name == "" {
+
+	if c.XMLQuery != "" {
+		if c.ID == "" {
+			errs = append(errs, fmt.Errorf("event log is missing an 'id'"))
+		}
+
+		// Check for XML syntax errors. This does not check the validity of the query itself.
+		if err := xml.Unmarshal([]byte(c.XMLQuery), &struct{}{}); err != nil {
+			errs = append(errs, fmt.Errorf("invalid xml_query: %w", err))
+		}
+
+		switch {
+		case c.Name != "":
+			errs = append(errs, fmt.Errorf("xml_query cannot be used with 'name'"))
+		case c.SimpleQuery.IgnoreOlder != 0:
+			errs = append(errs, fmt.Errorf("xml_query cannot be used with 'ignore_older'"))
+		case c.SimpleQuery.Level != "":
+			errs = append(errs, fmt.Errorf("xml_query cannot be used with 'level'"))
+		case c.SimpleQuery.EventID != "":
+			errs = append(errs, fmt.Errorf("xml_query cannot be used with 'event_id'"))
+		case len(c.SimpleQuery.Provider) != 0:
+			errs = append(errs, fmt.Errorf("xml_query cannot be used with 'provider'"))
+		}
+	} else if c.Name == "" {
 		errs = append(errs, fmt.Errorf("event log is missing a 'name'"))
 	}
 
@@ -128,6 +152,7 @@ var _ EventLog = &winEventLog{}
 type winEventLog struct {
 	config       winEventLogConfig
 	query        string
+	id           string                   // Identifier of this event log.
 	channelName  string                   // Name of the channel from which to read.
 	file         bool                     // Reading from file rather than channel.
 	subscription win.EvtHandle            // Handle to the subscription.
@@ -144,15 +169,15 @@ type winEventLog struct {
 
 // Name returns the name of the event log (i.e. Application, Security, etc.).
 func (l *winEventLog) Name() string {
-	return l.channelName
+	return l.id
 }
 
 func (l *winEventLog) Open(state checkpoint.EventLogState) error {
 	var bookmark win.EvtHandle
 	var err error
 	if len(state.Bookmark) > 0 {
 		bookmark, err = win.CreateBookmarkFromXML(state.Bookmark)
-	} else if state.RecordNumber > 0 {
+	} else if state.RecordNumber > 0 && l.channelName != "" {
 		bookmark, err = win.CreateBookmarkFromRecordID(l.channelName, state.RecordNumber)
 	}
 	if err != nil {
@@ -267,7 +292,7 @@ func (l *winEventLog) Read() ([]Record, error) {
 
 		r, _ := l.buildRecordFromXML(l.outputBuf.Bytes(), err)
 		r.Offset = checkpoint.EventLogState{
-			Name:         l.channelName,
+			Name:         l.id,
 			RecordNumber: r.RecordID,
 			Timestamp:    r.TimeCreated.SystemTime,
 		}
@@ -356,7 +381,7 @@ func (l *winEventLog) buildRecordFromXML(x []byte, recoveredErr error) (Record,
 	}
 
 	if l.file {
-		r.File = l.channelName
+		r.File = l.id
 	}
 
 	if includeXML {
@@ -374,20 +399,32 @@ func newEventLogging(options *common.Config) (EventLog, error) {
 // newWinEventLog creates and returns a new EventLog for reading event logs
 // using the Windows Event Log.
 func newWinEventLog(options *common.Config) (EventLog, error) {
+	var xmlQuery string
+	var err error
+
 	c := defaultWinEventLogConfig
-	if err := readConfig(options, &c); err != nil {
+	if err = readConfig(options, &c); err != nil {
 		return nil, err
 	}
 
-	query, err := win.Query{
-		Log:         c.Name,
-		IgnoreOlder: c.SimpleQuery.IgnoreOlder,
-		Level:       c.SimpleQuery.Level,
-		EventID:     c.SimpleQuery.EventID,
-		Provider:    c.SimpleQuery.Provider,
-	}.Build()
-	if err != nil {
-		return nil, err
+	id := c.ID
+	if id == "" {
+		id = c.Name
+	}
+
+	if c.XMLQuery != "" {
+		xmlQuery = c.XMLQuery
+	} else {
+		xmlQuery, err = win.Query{
+			Log:         c.Name,
+			IgnoreOlder: c.SimpleQuery.IgnoreOlder,
+			Level:       c.SimpleQuery.Level,
+			EventID:     c.SimpleQuery.EventID,
+			Provider:    c.SimpleQuery.Provider,
+		}.Build()
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	eventMetadataHandle := func(providerName, sourceName string) sys.MessageFiles {
@@ -411,15 +448,16 @@ func newWinEventLog(options *common.Config) (EventLog, error) {
 	}
 
 	l := &winEventLog{
+		id:          id,
 		config:      c,
-		query:       query,
+		query:       xmlQuery,
 		channelName: c.Name,
 		file:        filepath.IsAbs(c.Name),
 		maxRead:     c.BatchReadSize,
 		renderBuf:   make([]byte, renderBufferSize),
 		outputBuf:   sys.NewByteBuffer(renderBufferSize),
-		cache:       newMessageFilesCache(c.Name, eventMetadataHandle, freeHandle),
-		logPrefix:   fmt.Sprintf("WinEventLog[%s]", c.Name),
+		cache:       newMessageFilesCache(id, eventMetadataHandle, freeHandle),
+		logPrefix:   fmt.Sprintf("WinEventLog[%s]", id),
 	}
 
 	// Forwarded events should be rendered using RenderEventXML. It is more

diff --git a/winlogbeat/eventlog/wineventlog_experimental.go b/winlogbeat/eventlog/wineventlog_experimental.go
@@ -47,6 +47,7 @@ const (
 type winEventLogExp struct {
 	config      winEventLogConfig
 	query       string
+	id          string                   // Identifier of this event log.
 	channelName string                   // Name of the channel from which to read.
 	file        bool                     // Reading from file rather than channel.
 	maxRead     int                      // Maximum number returned in one Read.
@@ -59,7 +60,7 @@ type winEventLogExp struct {
 
 // Name returns the name of the event log (i.e. Application, Security, etc.).
 func (l *winEventLogExp) Name() string {
-	return l.channelName
+	return l.id
 }
 
 func (l *winEventLogExp) Open(state checkpoint.EventLogState) error {
@@ -205,11 +206,11 @@ func (l *winEventLogExp) processHandle(h win.EvtHandle) (*Record, error) {
 	}
 
 	if l.file {
-		r.File = l.channelName
+		r.File = l.id
 	}
 
 	r.Offset = checkpoint.EventLogState{
-		Name:         l.channelName,
+		Name:         l.id,
 		RecordNumber: r.RecordID,
 		Timestamp:    r.TimeCreated.SystemTime,
 	}
@@ -241,45 +242,60 @@ func (l *winEventLogExp) Close() error {
 // newWinEventLogExp creates and returns a new EventLog for reading event logs
 // using the Windows Event Log.
 func newWinEventLogExp(options *common.Config) (EventLog, error) {
+	var xmlQuery string
+	var err error
+	var isFile bool
+	var log *logp.Logger
+
 	cfgwarn.Experimental("The %s event log reader is experimental.", winEventLogExpAPIName)
 
 	c := winEventLogConfig{BatchReadSize: 512}
 	if err := readConfig(options, &c); err != nil {
 		return nil, err
 	}
 
-	queryLog := c.Name
-	isFile := false
-	if info, err := os.Stat(c.Name); err == nil && info.Mode().IsRegular() {
-		path, err := filepath.Abs(c.Name)
+	id := c.ID
+	if id == "" {
+		id = c.Name
+	}
+
+	if c.XMLQuery != "" {
+		xmlQuery = c.XMLQuery
+		log = logp.NewLogger("wineventlog").With("id", id)
+	} else {
+		queryLog := c.Name
+		if info, err := os.Stat(c.Name); err == nil && info.Mode().IsRegular() {
+			path, err := filepath.Abs(c.Name)
+			if err != nil {
+				return nil, err
+			}
+			isFile = true
+			queryLog = "file://" + path
+		}
+
+		xmlQuery, err = win.Query{
+			Log:         queryLog,
+			IgnoreOlder: c.SimpleQuery.IgnoreOlder,
+			Level:       c.SimpleQuery.Level,
+			EventID:     c.SimpleQuery.EventID,
+			Provider:    c.SimpleQuery.Provider,
+		}.Build()
 		if err != nil {
 			return nil, err
 		}
-		isFile = true
-		queryLog = "file://" + path
-	}
 
-	query, err := win.Query{
-		Log:         queryLog,
-		IgnoreOlder: c.SimpleQuery.IgnoreOlder,
-		Level:       c.SimpleQuery.Level,
-		EventID:     c.SimpleQuery.EventID,
-		Provider:    c.SimpleQuery.Provider,
-	}.Build()
-	if err != nil {
-		return nil, err
+		log = logp.NewLogger("wineventlog").With("id", id).With("channel", c.Name)
 	}
 
-	log := logp.NewLogger("wineventlog").With("channel", c.Name)
-
 	renderer, err := win.NewRenderer(win.NilHandle, log)
 	if err != nil {
 		return nil, err
 	}
 
 	l := &winEventLogExp{
 		config:      c,
-		query:       query,
+		query:       xmlQuery,
+		id:          id,
 		channelName: c.Name,
 		file:        isFile,
 		maxRead:     c.BatchReadSize,
Original file line number	Diff line number	Diff line change
Expand Up		@@ -76,6 +76,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d

		Winlogbeat

		- Add support for custom XML queries {issue}1054[1054] {pull}29330[29330]


		Elastic Log Driver

Expand Down