x-pack/packetbeat: install Npcap at start-up when required (elastic#2…

…9112)

This add automated installation of an OEM Npcap provided by embedding in the
packetbeat executable. The installation is configurable to allow users to retain
their own version of Npcap if they have one and to specify the location of the
install. To simplify support this configurablity will not be included in the
next release, but is comitted here to allow it to be reverted back out to make it
easier to add if the decision is made to do that later.

CHANGELOG.next.asciidoc

@@ -196,6 +196,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Packetbeat*
 
+- Add automated OEM Npcap installation handling. {pull}29112[29112]
+
 *Functionbeat*
 
 

NOTICE.txt

@@ -16728,6 +16728,43 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
+--------------------------------------------------------------------------------
+Dependency : golang.org/x/mod
+Version: v0.5.1
+Licence type (autodetected): BSD-3-Clause
+--------------------------------------------------------------------------------
+
+Contents of probable licence file $GOMODCACHE/golang.org/x/[email protected]/LICENSE:
+
+Copyright (c) 2009 The Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
 --------------------------------------------------------------------------------
 Dependency : golang.org/x/net
 Version: v0.0.0-20211020060615-d418f374d309
@@ -34519,43 +34556,6 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 
 
---------------------------------------------------------------------------------
-Dependency : golang.org/x/mod
-Version: v0.5.1
-Licence type (autodetected): BSD-3-Clause
---------------------------------------------------------------------------------
-
-Contents of probable licence file $GOMODCACHE/golang.org/x/[email protected]/LICENSE:
-
-Copyright (c) 2009 The Go Authors. All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
-   * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-   * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
-   * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-
 --------------------------------------------------------------------------------
 Dependency : golang.org/x/term
 Version: v0.0.0-20210615171337-6886f2dfbf5b
@@ -37624,4 +37624,3 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
-

dev-tools/mage/config.go

@@ -149,6 +149,7 @@ func makeConfigTemplate(destination string, mode os.FileMode, confParams ConfigF
 	params := map[string]interface{}{
 		"GOOS":                           EnvOr("DEV_OS", "linux"),
 		"GOARCH":                         EnvOr("DEV_ARCH", "amd64"),
+		"BeatLicense":                    BeatLicense,
 		"Reference":                      false,
 		"Docker":                         false,
 		"ExcludeConsole":                 false,

dev-tools/mage/crossbuild.go

@@ -183,7 +183,7 @@ func CrossBuild(options ...CrossBuildOption) error {
 		builder := GolangCrossBuilder{buildPlatform.Name, params.Target, params.InDir, params.ImageSelector}
 		if params.Serial {
 			if err := builder.Build(); err != nil {
-				return errors.Wrapf(err, "failed cross-building target=%v for platform=%v %v", params.ImageSelector,
+				return errors.Wrapf(err, "failed cross-building target=%s for platform=%s",
 					params.Target, buildPlatform.Name)
 			}
 		} else {
@@ -321,8 +321,11 @@ func (b GolangCrossBuilder) Build() error {
 		"-v", repoInfo.RootDir+":"+mountPoint,
 		"-w", workDir,
 		image,
+
+		// Arguments for docker crossbuild entrypoint. For details see
+		// https://github.com/elastic/golang-crossbuild/blob/main/go1.17/base/rootfs/entrypoint.go.
 		"--build-cmd", buildCmd+" "+b.Target,
-		"-p", b.Platform,
+		"--platforms", b.Platform,
 	)
 
 	return dockerRun(args...)

dev-tools/mage/settings.go

@@ -209,6 +209,7 @@ BeatUser         = {{.BeatUser}}
 VersionQualifier = {{.Qualifier}}
 PLATFORMS        = {{.PLATFORMS}}
 PACKAGES         = {{.PACKAGES}}
+CI               = {{.CI}}
 
 ## Functions
 

dev-tools/notice/NOTICE.txt.tmpl

@@ -26,5 +26,5 @@ Third party libraries used by the Elastic Beats project:
 {{ "=" | line }}
 Indirect dependencies
 
-{{ template "depInfo" .Indirect }}
-{{ end }}
+{{ template "depInfo" .Indirect -}}
+{{- end}}

dev-tools/packaging/package_test.go

@@ -23,6 +23,7 @@ package dev_tools
 import (
 	"archive/tar"
 	"archive/zip"
+	"bufio"
 	"bytes"
 	"compress/gzip"
 	"encoding/json"
@@ -169,7 +170,7 @@ func checkTar(t *testing.T, file string) {
 }
 
 func checkZip(t *testing.T, file string) {
-	p, err := readZip(file)
+	p, err := readZip(t, file, checkNpcapNotices)
 	if err != nil {
 		t.Error(err)
 		return
@@ -183,6 +184,62 @@ func checkZip(t *testing.T, file string) {
 	checkLicensesPresent(t, "", p)
 }
 
+const (
+	npcapSettings   = "Windows Npcap installation settings"
+	npcapGrant      = `Insecure.Com LLC \(“The Nmap Project”\) has granted Elasticsearch`
+	npcapLicense    = `Dependency : Npcap \(https://nmap.org/npcap/\)`
+	libpcapLicense  = `Dependency : Libpcap \(http://www.tcpdump.org/\)`
+	winpcapLicense  = `Dependency : Winpcap \(https://www.winpcap.org/\)`
+	radiotapLicense = `Dependency : ieee80211_radiotap.h Header File`
+)
+
+var (
+	// These reflect the order that the licenses and notices appear in the relevant files.
+	npcapConfigPattern = regexp.MustCompile(
+		"(?s)" + npcapSettings +
+			".*" + npcapGrant,
+	)
+	npcapLicensePattern = regexp.MustCompile(
+		"(?s)" + npcapLicense +
+			".*" + libpcapLicense +
+			".*" + winpcapLicense +
+			".*" + radiotapLicense,
+	)
+)
+
+func checkNpcapNotices(pkg, file string, contents io.Reader) error {
+	if !strings.Contains(pkg, "packetbeat") {
+		return nil
+	}
+
+	wantNotices := strings.Contains(pkg, "windows") && !strings.Contains(pkg, "oss")
+
+	// If the packetbeat README.md is made to be generated
+	// conditionally then it should also be checked here.
+	pkg = filepath.Base(pkg)
+	file, err := filepath.Rel(pkg[:len(pkg)-len(filepath.Ext(pkg))], file)
+	if err != nil {
+		return err
+	}
+	switch file {
+	case "packetbeat.yml", "packetbeat.reference.yml":
+		if npcapConfigPattern.MatchReader(bufio.NewReader(contents)) != wantNotices {
+			if wantNotices {
+				return fmt.Errorf("Npcap config section not found in config file %s in %s", file, pkg)
+			}
+			return fmt.Errorf("unexpected Npcap config section found in config file %s in %s", file, pkg)
+		}
+	case "NOTICE.txt":
+		if npcapLicensePattern.MatchReader(bufio.NewReader(contents)) != wantNotices {
+			if wantNotices {
+				return fmt.Errorf("Npcap license section not found in %s file in %s", file, pkg)
+			}
+			return fmt.Errorf("unexpected Npcap license section found in %s file in %s", file, pkg)
+		}
+	}
+	return nil
+}
+
 func checkDocker(t *testing.T, file string) {
 	p, info, err := readDocker(file)
 	if err != nil {
@@ -623,7 +680,11 @@ func readTarContents(tarName string, data io.Reader) (*packageFile, error) {
 	return p, nil
 }
 
-func readZip(zipFile string) (*packageFile, error) {
+// inspector is a file contents inspector. It vets the contents of the file
+// within a package for a requirement and returns an error if it is not met.
+type inspector func(pkg, file string, contents io.Reader) error
+
+func readZip(t *testing.T, zipFile string, inspectors ...inspector) (*packageFile, error) {
 	r, err := zip.OpenReader(zipFile)
 	if err != nil {
 		return nil, err
@@ -636,6 +697,18 @@ func readZip(zipFile string) (*packageFile, error) {
 			File: f.Name,
 			Mode: f.Mode(),
 		}
+		for _, inspect := range inspectors {
+			r, err := f.Open()
+			if err != nil {
+				t.Errorf("failed to open %s in %s: %v", f.Name, zipFile, err)
+				break
+			}
+			err = inspect(zipFile, f.Name, r)
+			if err != nil {
+				t.Error(err)
+			}
+			r.Close()
+		}
 	}
 
 	return p, nil
@@ -740,7 +813,6 @@ func readDockerManifest(r io.Reader) (*dockerManifest, error) {
 	err = json.Unmarshal(data, &manifests)
 	if err != nil {
 		return nil, err
-
 	}
 
 	if len(manifests) != 1 {