wintest: new package to provide support for winlogbeat ingest node pi…

…peline testing (#31833)

- powershell: fix regexp constraints in event 800 parameter detail processing
- security: fix documentation
- security: fix sidlist processing
- security: fix access mask and access list processing

CHANGELOG.next.asciidoc

@@ -71,6 +71,8 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff]
 *Winlogbeat*
 
 - Sysmon: Drop fields with "-" value (unset) {pull}31556[31556]
+- Powershell: Fix processing of parameter details. {pull}31833[31833]
+- Security: Fix processing of sidlist, access list and access mask. {pull}31833[31833]
 
 *Functionbeat*
 

go.mod

@@ -163,6 +163,7 @@ require (
 	github.com/elastic/elastic-agent-autodiscover v0.1.1
 	github.com/elastic/elastic-agent-libs v0.2.5
 	github.com/elastic/elastic-agent-system-metrics v0.4.1
+	github.com/elastic/go-elasticsearch/v8 v8.2.0
 	github.com/shirou/gopsutil/v3 v3.21.12
 	go.elastic.co/apm/module/apmelasticsearch/v2 v2.0.0
 	go.elastic.co/apm/module/apmhttp/v2 v2.0.0
@@ -208,6 +209,7 @@ require (
 	github.com/docker/go-metrics v0.0.1 // indirect
 	github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect
 	github.com/eapache/queue v1.1.0 // indirect
+	github.com/elastic/elastic-transport-go/v8 v8.1.0 // indirect
 	github.com/envoyproxy/go-control-plane v0.10.1 // indirect
 	github.com/envoyproxy/protoc-gen-validate v0.6.2 // indirect
 	github.com/evanphx/json-patch v4.12.0+incompatible // indirect

go.sum

@@ -544,11 +544,15 @@ github.com/elastic/elastic-agent-libs v0.2.5 h1:8+sYCW/kkWQe5KegGLMYYT3ELXUwibMc
 github.com/elastic/elastic-agent-libs v0.2.5/go.mod h1:chO3rtcLyGlKi9S0iGVZhYCzDfdDsAQYBc+ui588AFE=
 github.com/elastic/elastic-agent-system-metrics v0.4.1 h1:1bKgU0Y2F4PBLSCX2LmJbRd4wWoq5DOvXc9ysuXBVpI=
 github.com/elastic/elastic-agent-system-metrics v0.4.1/go.mod h1:tF/f9Off38nfzTZHIVQ++FkXrDm9keFhFpJ+3pQ00iI=
+github.com/elastic/elastic-transport-go/v8 v8.1.0 h1:NeqEz1ty4RQz+TVbUrpSU7pZ48XkzGWQj02k5koahIE=
+github.com/elastic/elastic-transport-go/v8 v8.1.0/go.mod h1:87Tcz8IVNe6rVSLdBux1o/PEItLtyabHU3naC7IoqKI=
 github.com/elastic/fsevents v0.0.0-20181029231046-e1d381a4d270 h1:cWPqxlPtir4RoQVCpGSRXmLqjEHpJKbR60rxh1nQZY4=
 github.com/elastic/fsevents v0.0.0-20181029231046-e1d381a4d270/go.mod h1:Msl1pdboCbArMF/nSCDUXgQuWTeoMmE/z8607X+k7ng=
 github.com/elastic/glog v1.0.1-0.20210831205241-7d8b5c89dfc4/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
 github.com/elastic/go-concert v0.2.0 h1:GAQrhRVXprnNjtvTP9pWJ1d4ToEA4cU5ci7TwTa20xg=
 github.com/elastic/go-concert v0.2.0/go.mod h1:HWjpO3IAEJUxOeaJOWXWEp7imKd27foxz9V5vegC/38=
+github.com/elastic/go-elasticsearch/v8 v8.2.0 h1:oagGcb1gqxT7yWpQ3E7wMP3NhGRamsKVd7kRdbuI+/Y=
+github.com/elastic/go-elasticsearch/v8 v8.2.0/go.mod h1:yY52i2Vj0unLz+N3Nwx1gM5LXwoj3h2dgptNGBYkMLA=
 github.com/elastic/go-libaudit/v2 v2.3.1-0.20220523121157-87f0a814a1c0 h1:UaX9gwFak4RyXlTCEOXONNvmZxBk0MAcXA0kCvlSxy4=
 github.com/elastic/go-libaudit/v2 v2.3.1-0.20220523121157-87f0a814a1c0/go.mod h1:GOkMRbzKV7ePrMOy+k6gGF0QvulQ16Cr38b60oirv8U=
 github.com/elastic/go-licenser v0.4.0 h1:jLq6A5SilDS/Iz1ABRkO6BHy91B9jBora8FwGRsDqUI=

winlogbeat/_meta/fields.common.yml

@@ -45,6 +45,24 @@
             The name of the computer that generated the record. When using Windows
             event forwarding, this name can differ from `agent.hostname`.
 
+        - name: computerObject.domain
+          required: false
+          type: keyword
+          description: >
+            The domain of the account that was added, modified or deleted in the event.
+
+        - name: computerObject.id
+          required: false
+          type: keyword
+          description: >
+            A globally unique identifier that identifies the target device.
+
+        - name: computerObject.name
+          required: false
+          type: keyword
+          description: >
+            The account name that was added, modified or deleted in the event.
+
         - name: event_data
           type: object
           object_type: keyword
@@ -366,6 +384,30 @@
           description: >
             The event creation time.
 
+        - name: trustAttribute
+          required: false
+          type: keyword
+          description: >
+            The decimal value of attributes for new trust created to a domain.
+
+        - name: trustDirection
+          required: false
+          type: keyword
+          description: >
+            The direction of new trust created to a domain.
+
+            Possible values are `TRUST_DIRECTION_DISABLED`, `TRUST_DIRECTION_INBOUND`,
+            `TRUST_DIRECTION_OUTBOUND` and `TRUST_DIRECTION_BIDIRECTIONAL`
+
+        - name: trustType
+          required: false
+          type: keyword
+          description: >
+            The account name that was added, modified or deleted in the event.
+
+            Possible values are `TRUST_TYPE_DOWNLEVEL`, `TRUST_TYPE_UPLEVEL`,
+            `TRUST_TYPE_MIT` and `TRUST_TYPE_DCE`
+
         - name: process.thread.id
           type: long
           required: false

winlogbeat/beater/winlogbeat.go

@@ -109,7 +109,8 @@ func (eb *Winlogbeat) init(b *beat.Beat) error {
 		if err != nil {
 			return err
 		}
-		return module.UploadPipelines(b.Info, esClient, overwritePipelines)
+		_, err = module.UploadPipelines(b.Info, esClient, overwritePipelines)
+		return err
 	}
 	return nil
 }
@@ -137,7 +138,8 @@ func (eb *Winlogbeat) Run(b *beat.Beat) error {
 
 	if b.Config.Output.Name() == "elasticsearch" {
 		callback := func(esClient *eslegclient.Connection) error {
-			return module.UploadPipelines(b.Info, esClient, eb.config.OverwritePipelines)
+			_, err := module.UploadPipelines(b.Info, esClient, eb.config.OverwritePipelines)
+			return err
 		}
 		_, err := elasticsearch.RegisterConnectCallback(callback)
 		if err != nil {

winlogbeat/docs/fields.asciidoc

@@ -16313,6 +16313,42 @@ required: True
 
 --
 
+*`winlog.computerObject.domain`*::
++
+--
+The domain of the account that was added, modified or deleted in the event.
+
+
+type: keyword
+
+required: False
+
+--
+
+*`winlog.computerObject.id`*::
++
+--
+A globally unique identifier that identifies the target device.
+
+
+type: keyword
+
+required: False
+
+--
+
+*`winlog.computerObject.name`*::
++
+--
+The account name that was added, modified or deleted in the event.
+
+
+type: keyword
+
+required: False
+
+--
+
 *`winlog.event_data`*::
 +
 --
@@ -17241,6 +17277,44 @@ required: False
 
 --
 
+*`winlog.trustAttribute`*::
++
+--
+The decimal value of attributes for new trust created to a domain.
+
+
+type: keyword
+
+required: False
+
+--
+
+*`winlog.trustDirection`*::
++
+--
+The direction of new trust created to a domain.
+Possible values are `TRUST_DIRECTION_DISABLED`, `TRUST_DIRECTION_INBOUND`, `TRUST_DIRECTION_OUTBOUND` and `TRUST_DIRECTION_BIDIRECTIONAL`
+
+
+type: keyword
+
+required: False
+
+--
+
+*`winlog.trustType`*::
++
+--
+The account name that was added, modified or deleted in the event.
+Possible values are `TRUST_TYPE_DOWNLEVEL`, `TRUST_TYPE_UPLEVEL`, `TRUST_TYPE_MIT` and `TRUST_TYPE_DCE`
+
+
+type: keyword
+
+required: False
+
+--
+
 *`winlog.process.thread.id`*::
 +
 --

winlogbeat/module/pipeline.go

@@ -52,11 +52,12 @@ type pipeline struct {
 
 // UploadPipelines reads all pipelines embedded in the Winlogbeat executable
 // and adapts the pipeline for a given ES version, converts to JSON if
-// necessary and creates or updates ingest pipeline in ES.
-func UploadPipelines(info beat.Info, esClient *eslegclient.Connection, overwritePipelines bool) error {
+// necessary and creates or updates ingest pipeline in ES. The IDs of pipelines
+// uploaded to ES are returned in loaded.
+func UploadPipelines(info beat.Info, esClient *eslegclient.Connection, overwritePipelines bool) (loaded []string, err error) {
 	pipelines, err := readAll(info)
 	if err != nil {
-		return err
+		return nil, err
 	}
 	return load(esClient, pipelines, overwritePipelines)
 }
@@ -94,7 +95,7 @@ func ExportPipelines(info beat.Info, version version.V, directory string) error
 // with load.
 func readAll(info beat.Info) (pipelines []pipeline, err error) {
 	p, err := readDir(".", info)
-	if err == errNoFS {
+	if err == errNoFS { //nolint:errorlint // Bad linter! This is never wrapped.
 		return nil, nil
 	}
 	return p, err
@@ -118,7 +119,7 @@ func readDir(dir string, info beat.Info) (pipelines []pipeline, err error) {
 			continue
 		}
 		p, err := readFile(path.Join(dir, de.Name()), info)
-		if err == errNoFS {
+		if err == errNoFS { //nolint:errorlint // Bad linter! This is never wrapped.
 			continue
 		}
 		if err != nil {
@@ -149,11 +150,11 @@ func readFile(filename string, info beat.Info) (p pipeline, err error) {
 }
 
 // load uses esClient to load pipelines to Elasticsearch cluster.
-// Will only overwrite existing pipelines if overwritePipelines is
-// true.  An error in loading one of the pipelines will cause the
+// The IDs of loaded pipelines will be returned in loaded.
+// load will only overwrite existing pipelines if overwritePipelines is
+// true. An error in loading one of the pipelines will cause the
 // successfully loaded ones to be deleted.
-func load(esClient *eslegclient.Connection, pipelines []pipeline, overwritePipelines bool) (err error) {
-	var pipelineIDsLoaded []string
+func load(esClient *eslegclient.Connection, pipelines []pipeline, overwritePipelines bool) (loaded []string, err error) {
 	log := logp.NewLogger(logName)
 
 	for _, pipeline := range pipelines {
@@ -162,20 +163,20 @@ func load(esClient *eslegclient.Connection, pipelines []pipeline, overwritePipel
 			err = fmt.Errorf("error loading pipeline %s: %w", pipeline.id, err)
 			break
 		}
-		pipelineIDsLoaded = append(pipelineIDsLoaded, pipeline.id)
+		loaded = append(loaded, pipeline.id)
 	}
 
 	if err != nil {
 		errs := multierror.Errors{err}
-		for _, pipelineID := range pipelineIDsLoaded {
-			err = fileset.DeletePipeline(esClient, pipelineID)
+		for _, id := range loaded {
+			err = fileset.DeletePipeline(esClient, id)
 			if err != nil {
 				errs = append(errs, err)
 			}
 		}
-		return errs.Err()
+		return nil, errs.Err()
 	}
-	return nil
+	return loaded, nil
 }
 
 func applyTemplates(prefix string, version string, filename string, original []byte) (converted map[string]interface{}, err error) {

x-pack/winlogbeat/cmd/root.go

@@ -7,11 +7,13 @@ package cmd
 import (
 	"github.com/elastic/beats/v7/libbeat/cmd"
 	winlogbeatCmd "github.com/elastic/beats/v7/winlogbeat/cmd"
-	"github.com/elastic/beats/v7/x-pack/winlogbeat/module"
 
 	// Register fields.
 	_ "github.com/elastic/beats/v7/x-pack/libbeat/include"
 	_ "github.com/elastic/beats/v7/x-pack/winlogbeat/include"
+
+	// Enable pipelines.
+	_ "github.com/elastic/beats/v7/x-pack/winlogbeat/module"
 )
 
 // Name of this beat.
@@ -25,5 +27,4 @@ func init() {
 	settings.ElasticLicensed = true
 	RootCmd = winlogbeatCmd.Initialize(settings)
 	RootCmd.ExportCmd.AddCommand(GenExportPipelineCmd(settings))
-	module.Init()
 }

x-pack/winlogbeat/magefile.go

@@ -8,33 +8,83 @@
 package main
 
 import (
+	"context"
+	"fmt"
+
 	"github.com/magefile/mage/mg"
 
 	devtools "github.com/elastic/beats/v7/dev-tools/mage"
+	"github.com/elastic/beats/v7/dev-tools/mage/target/test"
 
-	// mage:import
+	//mage:import
 	_ "github.com/elastic/beats/v7/dev-tools/mage/target/common"
-	// mage:import
+	//mage:import
 	_ "github.com/elastic/beats/v7/dev-tools/mage/target/build"
-	// mage:import
+	//mage:import
 	_ "github.com/elastic/beats/v7/dev-tools/mage/target/pkg"
-	// mage:import
+	//mage:import
 	_ "github.com/elastic/beats/v7/dev-tools/mage/target/dashboards"
-	// mage:import
-	_ "github.com/elastic/beats/v7/dev-tools/mage/target/test"
-	// mage:import
-	"github.com/elastic/beats/v7/dev-tools/mage/target/unittest"
-	// mage:import
+	//mage:import
 	winlogbeat "github.com/elastic/beats/v7/winlogbeat/scripts/mage"
 )
 
 func init() {
-	unittest.RegisterGoTestDeps(winlogbeat.Update.Fields)
-
 	winlogbeat.SelectLogic = devtools.XPackProject
 	devtools.BeatLicense = "Elastic License"
+
+	RegisterGoTestDeps(winlogbeat.Update.Fields)
+	test.RegisterDeps(UnitTest)
+}
+
+var goTestDeps, pythonTestDeps []interface{}
+
+// RegisterGoTestDeps registers dependencies of the GoUnitTest target.
+func RegisterGoTestDeps(deps ...interface{}) {
+	goTestDeps = append(goTestDeps, deps...)
+}
+
+// RegisterPythonTestDeps registers dependencies of the PythonUnitTest target.
+func RegisterPythonTestDeps(deps ...interface{}) {
+	pythonTestDeps = append(pythonTestDeps, deps...)
+}
+
+// UnitTest executes the unit tests (Go and Python).
+func UnitTest() {
+	mg.SerialDeps(GoUnitTest, PythonUnitTest)
 }
 
 // Update is an alias for update:all. This is a workaround for
 // https://github.com/magefile/mage/issues/217.
 func Update() { mg.Deps(winlogbeat.Update.All) }
+
+// GoUnitTest executes the Go unit tests.
+// Use TEST_COVERAGE=true to enable code coverage profiling.
+// Use RACE_DETECTOR=true to enable the race detector.
+func GoUnitTest(ctx context.Context) error {
+	mg.SerialCtxDeps(ctx, goTestDeps...)
+	args := devtools.DefaultGoTestUnitArgs()
+	// The module unit tests depend on a running docker container to provide
+	// the ES instance to run the processor pipeline. In the absence of a
+	// test supervisor or a single test executable to ensure that only a
+	// single container is running, or additional logic to ensure no network
+	// collisions, we ensure that only one test package is running at a time.
+	args.ExtraFlags = append(args.ExtraFlags, "-p", "1")
+	return devtools.GoTest(ctx, args)
+}
+
+// PythonUnitTest executes the python system tests.
+func PythonUnitTest() error {
+	mg.SerialDeps(pythonTestDeps...)
+	mg.Deps(devtools.BuildSystemTestBinary)
+	return devtools.PythonTest(devtools.DefaultPythonTestUnitArgs())
+}
+
+// PythonVirtualEnv creates the testing virtual environment and prints its location.
+func PythonVirtualEnv() error {
+	venv, err := devtools.PythonVirtualenv(true)
+	if err != nil {
+		return err
+	}
+	fmt.Println(venv)
+	return nil
+}

x-pack/winlogbeat/module/.gitignore

@@ -0,0 +1 @@
+build