x-pack/packetbeat: install Npcap at start-up when required (#29112)

This add automated installation of an OEM Npcap provided by embedding in the
packetbeat executable. The installation is configurable to allow users to retain
their own version of Npcap if they have one and to specify the location of the
install. To simplify support this configurablity will not be included in the
next release, but is comitted here to allow it to be reverted back out to make it
easier to add if the decision is made to do that later.

(cherry picked from commit 72a43be)

# Conflicts:
#	go.mod

CHANGELOG.next.asciidoc

@@ -181,6 +181,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Packetbeat*
 
+- Add automated OEM Npcap installation handling. {pull}29112[29112]
+
 *Functionbeat*
 
 

NOTICE.txt

@@ -16728,6 +16728,43 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
+--------------------------------------------------------------------------------
+Dependency : golang.org/x/mod
+Version: v0.5.1
+Licence type (autodetected): BSD-3-Clause
+--------------------------------------------------------------------------------
+
+Contents of probable licence file $GOMODCACHE/golang.org/x/[email protected]/LICENSE:
+
+Copyright (c) 2009 The Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
 --------------------------------------------------------------------------------
 Dependency : golang.org/x/net
 Version: v0.0.0-20211020060615-d418f374d309
@@ -34313,43 +34350,6 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 
 
---------------------------------------------------------------------------------
-Dependency : golang.org/x/mod
-Version: v0.5.1
-Licence type (autodetected): BSD-3-Clause
---------------------------------------------------------------------------------
-
-Contents of probable licence file $GOMODCACHE/golang.org/x/[email protected]/LICENSE:
-
-Copyright (c) 2009 The Go Authors. All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
-   * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-   * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
-   * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-
 --------------------------------------------------------------------------------
 Dependency : golang.org/x/term
 Version: v0.0.0-20210220032956-6a3ed077a48d
@@ -36969,4 +36969,3 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
-

dev-tools/mage/config.go

@@ -149,6 +149,7 @@ func makeConfigTemplate(destination string, mode os.FileMode, confParams ConfigF
 	params := map[string]interface{}{
 		"GOOS":                           EnvOr("DEV_OS", "linux"),
 		"GOARCH":                         EnvOr("DEV_ARCH", "amd64"),
+		"BeatLicense":                    BeatLicense,
 		"Reference":                      false,
 		"Docker":                         false,
 		"ExcludeConsole":                 false,

dev-tools/mage/crossbuild.go

@@ -183,7 +183,7 @@ func CrossBuild(options ...CrossBuildOption) error {
 		builder := GolangCrossBuilder{buildPlatform.Name, params.Target, params.InDir, params.ImageSelector}
 		if params.Serial {
 			if err := builder.Build(); err != nil {
-				return errors.Wrapf(err, "failed cross-building target=%v for platform=%v %v", params.ImageSelector,
+				return errors.Wrapf(err, "failed cross-building target=%s for platform=%s",
 					params.Target, buildPlatform.Name)
 			}
 		} else {
@@ -321,8 +321,11 @@ func (b GolangCrossBuilder) Build() error {
 		"-v", repoInfo.RootDir+":"+mountPoint,
 		"-w", workDir,
 		image,
+
+		// Arguments for docker crossbuild entrypoint. For details see
+		// https://github.com/elastic/golang-crossbuild/blob/main/go1.17/base/rootfs/entrypoint.go.
 		"--build-cmd", buildCmd+" "+b.Target,
-		"-p", b.Platform,
+		"--platforms", b.Platform,
 	)
 
 	return dockerRun(args...)

dev-tools/mage/settings.go

@@ -209,6 +209,7 @@ BeatUser         = {{.BeatUser}}
 VersionQualifier = {{.Qualifier}}
 PLATFORMS        = {{.PLATFORMS}}
 PACKAGES         = {{.PACKAGES}}
+CI               = {{.CI}}
 
 ## Functions
 

dev-tools/notice/NOTICE.txt.tmpl

@@ -26,5 +26,5 @@ Third party libraries used by the Elastic Beats project:
 {{ "=" | line }}
 Indirect dependencies
 
-{{ template "depInfo" .Indirect }}
-{{ end }}
+{{ template "depInfo" .Indirect -}}
+{{- end}}

dev-tools/packaging/package_test.go

@@ -23,6 +23,7 @@ package dev_tools
 import (
 	"archive/tar"
 	"archive/zip"
+	"bufio"
 	"bytes"
 	"compress/gzip"
 	"encoding/json"
@@ -169,7 +170,7 @@ func checkTar(t *testing.T, file string) {
 }
 
 func checkZip(t *testing.T, file string) {
-	p, err := readZip(file)
+	p, err := readZip(t, file, checkNpcapNotices)
 	if err != nil {
 		t.Error(err)
 		return
@@ -183,6 +184,62 @@ func checkZip(t *testing.T, file string) {
 	checkLicensesPresent(t, "", p)
 }
 
+const (
+	npcapSettings   = "Windows Npcap installation settings"
+	npcapGrant      = `Insecure.Com LLC \(“The Nmap Project”\) has granted Elasticsearch`
+	npcapLicense    = `Dependency : Npcap \(https://nmap.org/npcap/\)`
+	libpcapLicense  = `Dependency : Libpcap \(http://www.tcpdump.org/\)`
+	winpcapLicense  = `Dependency : Winpcap \(https://www.winpcap.org/\)`
+	radiotapLicense = `Dependency : ieee80211_radiotap.h Header File`
+)
+
+var (
+	// These reflect the order that the licenses and notices appear in the relevant files.
+	npcapConfigPattern = regexp.MustCompile(
+		"(?s)" + npcapSettings +
+			".*" + npcapGrant,
+	)
+	npcapLicensePattern = regexp.MustCompile(
+		"(?s)" + npcapLicense +
+			".*" + libpcapLicense +
+			".*" + winpcapLicense +
+			".*" + radiotapLicense,
+	)
+)
+
+func checkNpcapNotices(pkg, file string, contents io.Reader) error {
+	if !strings.Contains(pkg, "packetbeat") {
+		return nil
+	}
+
+	wantNotices := strings.Contains(pkg, "windows") && !strings.Contains(pkg, "oss")
+
+	// If the packetbeat README.md is made to be generated
+	// conditionally then it should also be checked here.
+	pkg = filepath.Base(pkg)
+	file, err := filepath.Rel(pkg[:len(pkg)-len(filepath.Ext(pkg))], file)
+	if err != nil {
+		return err
+	}
+	switch file {
+	case "packetbeat.yml", "packetbeat.reference.yml":
+		if npcapConfigPattern.MatchReader(bufio.NewReader(contents)) != wantNotices {
+			if wantNotices {
+				return fmt.Errorf("Npcap config section not found in config file %s in %s", file, pkg)
+			}
+			return fmt.Errorf("unexpected Npcap config section found in config file %s in %s", file, pkg)
+		}
+	case "NOTICE.txt":
+		if npcapLicensePattern.MatchReader(bufio.NewReader(contents)) != wantNotices {
+			if wantNotices {
+				return fmt.Errorf("Npcap license section not found in %s file in %s", file, pkg)
+			}
+			return fmt.Errorf("unexpected Npcap license section found in %s file in %s", file, pkg)
+		}
+	}
+	return nil
+}
+
 func checkDocker(t *testing.T, file string) {
 	p, info, err := readDocker(file)
 	if err != nil {
@@ -623,7 +680,11 @@ func readTarContents(tarName string, data io.Reader) (*packageFile, error) {
 	return p, nil
 }
 
-func readZip(zipFile string) (*packageFile, error) {
+// inspector is a file contents inspector. It vets the contents of the file
+// within a package for a requirement and returns an error if it is not met.
+type inspector func(pkg, file string, contents io.Reader) error
+
+func readZip(t *testing.T, zipFile string, inspectors ...inspector) (*packageFile, error) {
 	r, err := zip.OpenReader(zipFile)
 	if err != nil {
 		return nil, err
@@ -636,6 +697,18 @@ func readZip(zipFile string) (*packageFile, error) {
 			File: f.Name,
 			Mode: f.Mode(),
 		}
+		for _, inspect := range inspectors {
+			r, err := f.Open()
+			if err != nil {
+				t.Errorf("failed to open %s in %s: %v", f.Name, zipFile, err)
+				break
+			}
+			err = inspect(zipFile, f.Name, r)
+			if err != nil {
+				t.Error(err)
+			}
+			r.Close()
+		}
 	}
 
 	return p, nil
@@ -740,7 +813,6 @@ func readDockerManifest(r io.Reader) (*dockerManifest, error) {
 	err = json.Unmarshal(data, &manifests)
 	if err != nil {
 		return nil, err
-
 	}
 
 	if len(manifests) != 1 {

go.mod

@@ -165,6 +165,7 @@ require (
 	go.uber.org/zap v1.14.0
 	golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e
 	golang.org/x/lint v0.0.0-20210508222113-6edffad5e616
+	golang.org/x/mod v0.5.1
 	golang.org/x/net v0.0.0-20211020060615-d418f374d309
 	golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
@@ -268,9 +269,13 @@ require (
 	github.com/xdg/stringprep v1.0.3 // indirect
 	go.elastic.co/fastjson v1.1.0 // indirect
 	go.opencensus.io v0.23.0 // indirect
+<<<<<<< HEAD
 	go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee // indirect
 	golang.org/x/mod v0.5.1 // indirect
 	golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d // indirect
+=======
+	golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect
+>>>>>>> 72a43be9e (x-pack/packetbeat: install Npcap at start-up when required (#29112))
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect

packetbeat/README.md

@@ -39,3 +39,17 @@ If you are sure you found a bug or have a feature request, open an issue on
 
 We love contributions from our community! Please read the
 [CONTRIBUTING.md](../CONTRIBUTING.md) file.
+
+## LICENSE NOTICE for Windows users of Packetbeat
+
+The default distribution of Packetbeat for Windows comes bundled with the Npcap
+library. This is not available in the OSS-only distribution of Packetbeat.
+
+**Restrictions on Distribution**
+
+Insecure.Com LLC (“The Nmap Project”) has granted Elasticsearch BV and its
+affiliates the right to include Npcap with this distribution of Packetbeat.
+You may not distribute this version of Packetbeat or any other package from
+Elastic that includes Npcap. If you wish to distribute Npcap, or any package
+that includes Npcap, you should reach out to The Nmap Project to obtain a
+distribution license. See https://nmap.org/npcap/ for more details.

packetbeat/_meta/config/beat.reference.yml.tmpl

@@ -57,6 +57,8 @@ packetbeat.interfaces.internal_networks:
 # can stay enabled even after beat is shut down.
 #packetbeat.interfaces.auto_promisc_mode: true
 
+{{- template "windows_npcap.yml.tmpl" .}}
+
 {{header "Flows"}}
 
 packetbeat.flows:

packetbeat/_meta/config/beat.yml.tmpl

@@ -23,6 +23,8 @@ packetbeat.interfaces.device: {{ call .device .GOOS }}
 packetbeat.interfaces.internal_networks:
   - private
 
+{{- template "windows_npcap.yml.tmpl" .}}
+
 {{header "Flows"}}
 
 # Set `enabled: false` or comment out all options to disable flows reporting.

packetbeat/_meta/config/windows_npcap.yml.tmpl

@@ -0,0 +1,36 @@
+{{if and (eq .BeatLicense "Elastic License") (eq .GOOS "windows")}}
+
+{{header "Windows Npcap installation settings"}}
+
+# Windows Npcap installation options. These options specify how the Npcap packet
+# capture library for Windows should be obtained and installed.
+# Npcap installation is only available in the default distribution of Packetbeat
+# for Windows and is not available in the OSS-only distribution of Packetbeat.
+#
+# LICENSE NOTICE
+#
+# Restrictions on Distribution
+#
+# Insecure.Com LLC (“The Nmap Project”) has granted Elasticsearch BV and its
+# affiliates the right to include Npcap with this distribution of Packetbeat.
+# You may not distribute this version of Packetbeat or any other package from
+# Elastic that includes Npcap. If you wish to distribute Npcap, or any package
+# that includes Npcap, you should reach out to The Nmap Project to obtain a
+# distribution license. See https://nmap.org/npcap/ for more details.
+#
+#npcap:
+#  # install_destination allows configuration of the location that the Npcap will
+#  # place the Npcap library and associated files. See https://nmap.org/npcap/guide/npcap-users-guide.html#npcap-installation-uninstall-options.
+#  install_destination: ""
+#  install_timeout: 120s
+#  # ignore_missing_registry specifies that failure to query the registry server
+#  # will be ignored with a logged warning.
+#  ignore_missing_registry: false
+#  # By default Npcap will be installed only when a newer version of Npcap is available.
+#  # force_reinstall forces a new installation of Npcap in all cases.
+#  force_reinstall: false
+#  # If a specific local version of Npcap is required installation by packetbeat
+#  # can be blocked by setting never_install to true. No action is taken if this
+#  # option is set to true.
+#  never_install: false
+{{- end -}}