diff --git a/.buildkite/scripts/steps/integration_tests.sh b/.buildkite/scripts/steps/integration_tests.sh index 4143e2413e5..2a129193267 100755 --- a/.buildkite/scripts/steps/integration_tests.sh +++ b/.buildkite/scripts/steps/integration_tests.sh @@ -19,7 +19,7 @@ else OVERRIDE_TEST_AGENT_VERSION="" fi # PACKAGE -AGENT_PACKAGE_VERSION="${OVERRIDE_AGENT_PACKAGE_VERSION}" DEV=true EXTERNAL=true SNAPSHOT=true PLATFORMS=linux/amd64,linux/arm64,windows/amd64 PACKAGES=tar.gz,zip,rpm,deb mage package +AGENT_PACKAGE_VERSION="${OVERRIDE_AGENT_PACKAGE_VERSION}" DEV=true EXTERNAL=true SNAPSHOT=true PLATFORMS=linux/amd64,linux/arm64,windows/amd64 PACKAGES=tar.gz,zip mage package # Run integration tests set +e diff --git a/dev-tools/mage/pkg.go b/dev-tools/mage/pkg.go index b13e3d7390f..e00bbbdb661 100644 --- a/dev-tools/mage/pkg.go +++ b/dev-tools/mage/pkg.go @@ -34,7 +34,7 @@ func Package() error { // platforms := updateWithDarwinUniversal(Platforms) platforms := Platforms - tasks := make(map[string][]interface{}) + var tasks []interface{} for _, target := range platforms { for _, pkg := range Packages { if pkg.OS != target.GOOS() || pkg.Arch != "" && pkg.Arch != target.Arch() { @@ -94,15 +94,12 @@ func Package() error { spec = spec.Evaluate() - tasks[target.GOOS()+"-"+target.Arch()] = append(tasks[target.GOOS()+"-"+target.Arch()], packageBuilder{target, spec, pkgType}.Build) + tasks = append(tasks, packageBuilder{target, spec, pkgType}.Build) } } } - for k, v := range tasks { - fmt.Printf(">> package: Building %s\n", k) - Parallel(v...) - } + Parallel(tasks...) return nil } diff --git a/pkg/testing/define/define.go b/pkg/testing/define/define.go index 20eac6c8f85..72105caa461 100644 --- a/pkg/testing/define/define.go +++ b/pkg/testing/define/define.go @@ -151,7 +151,7 @@ func runOrSkip(t *testing.T, req Requirements, local bool) *Info { panic("failed to get OS information") } if !req.runtimeAllowed(runtime.GOOS, runtime.GOARCH, osInfo.Version, osInfo.Platform) { - t.Skipf("platform: %s, architecture: %s, version: %s, and distro: %s combination is not supported by test. required: %v", runtime.GOOS, runtime.GOARCH, osInfo.Version, osInfo.Platform, req.OS) + t.Skip("platform, architecture, version, and distro not supported by test") return nil } namespace, err := getNamespace(t, local) diff --git a/pkg/testing/fetch_test.go b/pkg/testing/fetch_test.go deleted file mode 100644 index bb20c718f21..00000000000 --- a/pkg/testing/fetch_test.go +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package testing - -import ( - "errors" - gtesting "testing" -) - -func TestGetPackageSuffix(t *gtesting.T) { - tests := map[string]struct { - os string - arch string - format string - expected string - err error - }{ - "windows zip": {os: "windows", arch: "amd64", format: "zip", expected: "windows-x86_64.zip", err: nil}, - "windows msi": {os: "windows", arch: "amd64", format: "msi", expected: "", err: ErrUnsupportedPlatform}, - "linux deb": {os: "linux", arch: "amd64", format: "deb", expected: "amd64.deb", err: nil}, - "linux rpm": {os: "linux", arch: "amd64", format: "rpm", expected: "x86_64.rpm", err: nil}, - "linux tar.gz": {os: "linux", arch: "amd64", format: "targz", expected: "linux-x86_64.tar.gz", err: nil}, - "linux pkg.tar.zst": {os: "linux", arch: "amd64", format: "pkg.tar.zst", expected: "", err: ErrUnsupportedPlatform}, - "darwin arm64": {os: "darwin", arch: "arm64", format: "targz", expected: "darwin-aarch64.tar.gz", err: nil}, - } - for name, tc := range tests { - t.Run(name, func(t *gtesting.T) { - got, err := GetPackageSuffix(tc.os, tc.arch, tc.format) - if !errors.Is(err, tc.err) { - t.Fatalf("wrong error. expected: %v got: %v", tc.err, err) - } - if got != tc.expected { - t.Fatalf("wrong output. expected: %s got: %s", tc.expected, got) - } - }) - } -} diff --git a/pkg/testing/fetcher.go b/pkg/testing/fetcher.go index efd2ce5c641..baa734495f9 100644 --- a/pkg/testing/fetcher.go +++ b/pkg/testing/fetcher.go @@ -31,23 +31,19 @@ var ( // packageArchMap provides a mapping for the endings of the builds of Elastic Agent based on the // operating system and architecture. var packageArchMap = map[string]string{ - "linux-amd64-targz": "linux-x86_64.tar.gz", - "linux-amd64-deb": "amd64.deb", - "linux-amd64-rpm": "x86_64.rpm", - "linux-arm64-targz": "linux-arm64.tar.gz", - "linux-arm64-deb": "arm64.deb", - "linux-arm64-rpm": "aarch64.rpm", - "windows-amd64-zip": "windows-x86_64.zip", - "darwin-amd64-targz": "darwin-x86_64.tar.gz", - "darwin-arm64-targz": "darwin-aarch64.tar.gz", + "linux-amd64": "linux-x86_64.tar.gz", + "linux-arm64": "linux-arm64.tar.gz", + "windows-amd64": "windows-x86_64.zip", + "darwin-amd64": "darwin-x86_64.tar.gz", + "darwin-arm64": "darwin-aarch64.tar.gz", } // GetPackageSuffix returns the suffix ending for the builds of Elastic Agent based on the // operating system and architecture. -func GetPackageSuffix(operatingSystem string, architecture string, packageFormat string) (string, error) { - suffix, ok := packageArchMap[fmt.Sprintf("%s-%s-%s", operatingSystem, architecture, packageFormat)] +func GetPackageSuffix(operatingSystem string, architecture string) (string, error) { + suffix, ok := packageArchMap[fmt.Sprintf("%s-%s", operatingSystem, architecture)] if !ok { - return "", fmt.Errorf("%w: %s/%s/%s", ErrUnsupportedPlatform, operatingSystem, architecture, packageFormat) + return "", fmt.Errorf("%w: %s/%s", ErrUnsupportedPlatform, operatingSystem, architecture) } return suffix, nil } @@ -72,7 +68,7 @@ type Fetcher interface { // // The extraction is handled by the caller. This should only download the file // and place it into the directory. - Fetch(ctx context.Context, operatingSystem string, architecture string, version string, packageFormat string) (FetcherResult, error) + Fetch(ctx context.Context, operatingSystem string, architecture string, version string) (FetcherResult, error) } // fetchCache is global to all tests, reducing the time required to fetch the needed artifacts @@ -109,12 +105,6 @@ func splitFileType(name string) (string, string, error) { if strings.HasSuffix(name, ".zip") { return strings.TrimSuffix(name, ".zip"), ".zip", nil } - if strings.HasSuffix(name, ".deb") { - return strings.TrimSuffix(name, ".deb"), ".deb", nil - } - if strings.HasSuffix(name, ".rpm") { - return strings.TrimSuffix(name, ".rpm"), ".rpm", nil - } return "", "", fmt.Errorf("unknown file extension type: %s", filepath.Ext(name)) } diff --git a/pkg/testing/fetcher_artifact.go b/pkg/testing/fetcher_artifact.go index 3f3c1ec8ce8..ec99477182c 100644 --- a/pkg/testing/fetcher_artifact.go +++ b/pkg/testing/fetcher_artifact.go @@ -59,8 +59,8 @@ func (f *artifactFetcher) Name() string { } // Fetch fetches the Elastic Agent and places the resulting binary at the path. -func (f *artifactFetcher) Fetch(ctx context.Context, operatingSystem string, architecture string, version string, packageFormat string) (FetcherResult, error) { - suffix, err := GetPackageSuffix(operatingSystem, architecture, packageFormat) +func (f *artifactFetcher) Fetch(ctx context.Context, operatingSystem string, architecture string, version string) (FetcherResult, error) { + suffix, err := GetPackageSuffix(operatingSystem, architecture) if err != nil { return nil, err } diff --git a/pkg/testing/fetcher_artifact_test.go b/pkg/testing/fetcher_artifact_test.go index f2f0aae51e2..42d9585c606 100644 --- a/pkg/testing/fetcher_artifact_test.go +++ b/pkg/testing/fetcher_artifact_test.go @@ -29,7 +29,7 @@ func TestArtifactFetcher_Default(t *testing.T) { af.doer = newFakeHttpClient(t) tmp := t.TempDir() - res, err := f.Fetch(context.Background(), "linux", "amd64", "8.12.0", "targz") + res, err := f.Fetch(context.Background(), "linux", "amd64", "8.12.0") require.NoError(t, err) err = res.Fetch(context.Background(), t, tmp) @@ -46,7 +46,7 @@ func TestArtifactFetcher_Snapshot(t *testing.T) { af.doer = newFakeHttpClient(t) tmp := t.TempDir() - res, err := f.Fetch(context.Background(), "linux", "amd64", "8.13.0-SNAPSHOT", "targz") + res, err := f.Fetch(context.Background(), "linux", "amd64", "8.13.0-SNAPSHOT") require.NoError(t, err) err = res.Fetch(context.Background(), t, tmp) @@ -64,7 +64,7 @@ func TestArtifactFetcher_SnapshotOnly(t *testing.T) { af.doer = newFakeHttpClient(t) tmp := t.TempDir() - res, err := f.Fetch(context.Background(), "linux", "amd64", "8.13.0", "targz") + res, err := f.Fetch(context.Background(), "linux", "amd64", "8.13.0") require.NoError(t, err) err = res.Fetch(context.Background(), t, tmp) @@ -82,7 +82,7 @@ func TestArtifactFetcher_Build(t *testing.T) { af.doer = newFakeHttpClient(t) tmp := t.TempDir() - res, err := f.Fetch(context.Background(), "linux", "amd64", "8.13.0-SNAPSHOT+l5snflwr", "targz") + res, err := f.Fetch(context.Background(), "linux", "amd64", "8.13.0-SNAPSHOT+l5snflwr") require.NoError(t, err) err = res.Fetch(context.Background(), t, tmp) diff --git a/pkg/testing/fetcher_http.go b/pkg/testing/fetcher_http.go index 5a30a6e4e86..b56086b022e 100644 --- a/pkg/testing/fetcher_http.go +++ b/pkg/testing/fetcher_http.go @@ -45,8 +45,8 @@ func (h HttpFetcher) Name() string { return fmt.Sprintf("httpFetcher-%s", sanitizeFetcherName(h.baseURL)) } -func (h HttpFetcher) Fetch(ctx context.Context, operatingSystem string, architecture string, version string, packageFormat string) (FetcherResult, error) { - suffix, err := GetPackageSuffix(operatingSystem, architecture, packageFormat) +func (h HttpFetcher) Fetch(ctx context.Context, operatingSystem string, architecture string, version string) (FetcherResult, error) { + suffix, err := GetPackageSuffix(operatingSystem, architecture) if err != nil { return nil, err } diff --git a/pkg/testing/fetcher_http_test.go b/pkg/testing/fetcher_http_test.go index 869cd57095c..72f2d9c0a1f 100644 --- a/pkg/testing/fetcher_http_test.go +++ b/pkg/testing/fetcher_http_test.go @@ -24,7 +24,6 @@ func TestHttpFetcher_Fetch(t *testing.T) { operatingSystem string architecture string version string - pkgFormat string } tests := []struct { name string @@ -39,7 +38,6 @@ func TestHttpFetcher_Fetch(t *testing.T) { operatingSystem: "linux", architecture: "arm64", version: "1.2.3", - pkgFormat: "targz", }, want: &httpFetcherResult{ baseURL: "https://artifacts.elastic.co/downloads/beats/elastic-agent/", @@ -54,7 +52,6 @@ func TestHttpFetcher_Fetch(t *testing.T) { operatingSystem: "windows", architecture: "amd64", version: "1.2.3", - pkgFormat: "zip", }, want: &httpFetcherResult{ baseURL: "http://somehost.somedomain/some/path/here", @@ -72,7 +69,7 @@ func TestHttpFetcher_Fetch(t *testing.T) { h := NewHttpFetcher(opts...) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - got, err := h.Fetch(ctx, tt.args.operatingSystem, tt.args.architecture, tt.args.version, tt.args.pkgFormat) + got, err := h.Fetch(ctx, tt.args.operatingSystem, tt.args.architecture, tt.args.version) if !tt.wantErr(t, err, fmt.Sprintf("Fetch(%v, %v, %v, %v)", ctx, tt.args.operatingSystem, tt.args.architecture, tt.args.version)) { return } diff --git a/pkg/testing/fetcher_local.go b/pkg/testing/fetcher_local.go index 4962283e2cd..4a8e4993d13 100644 --- a/pkg/testing/fetcher_local.go +++ b/pkg/testing/fetcher_local.go @@ -55,8 +55,8 @@ func (f *localFetcher) Name() string { } // Fetch fetches the Elastic Agent and places the resulting binary at the path. -func (f *localFetcher) Fetch(_ context.Context, operatingSystem string, architecture string, version string, packageFormat string) (FetcherResult, error) { - suffix, err := GetPackageSuffix(operatingSystem, architecture, packageFormat) +func (f *localFetcher) Fetch(_ context.Context, operatingSystem string, architecture string, version string) (FetcherResult, error) { + suffix, err := GetPackageSuffix(operatingSystem, architecture) if err != nil { return nil, err } diff --git a/pkg/testing/fetcher_local_test.go b/pkg/testing/fetcher_local_test.go index 1223b84ebb7..78f2bd5b02c 100644 --- a/pkg/testing/fetcher_local_test.go +++ b/pkg/testing/fetcher_local_test.go @@ -28,13 +28,9 @@ func TestLocalFetcher(t *testing.T) { snapshotContentHash := []byte("snapshot contents hash") noSnapshotContent := []byte("not snapshot contents") noSnapshotContentHash := []byte("not snapshot contents hash") - pkgFormat := "targz" - if runtime.GOOS == "windows" { - pkgFormat = "zip" - } testdata := t.TempDir() - suffix, err := GetPackageSuffix(runtime.GOOS, runtime.GOARCH, pkgFormat) + suffix, err := GetPackageSuffix(runtime.GOOS, runtime.GOARCH) require.NoError(t, err) snapshotPath := fmt.Sprintf("elastic-agent-%s-SNAPSHOT-%s", baseVersion, suffix) @@ -96,12 +92,8 @@ func TestLocalFetcher(t *testing.T) { tmp := t.TempDir() f := LocalFetcher(testdata, tc.opts...) - pkgFormat := "targz" - if runtime.GOOS == "windows" { - pkgFormat = "zip" - } got, err := f.Fetch( - context.Background(), runtime.GOOS, runtime.GOARCH, tc.version, pkgFormat) + context.Background(), runtime.GOOS, runtime.GOARCH, tc.version) require.NoError(t, err) err = got.Fetch(context.Background(), t, tmp) diff --git a/pkg/testing/fixture.go b/pkg/testing/fixture.go index 1f74eac5ad2..f4990f591c0 100644 --- a/pkg/testing/fixture.go +++ b/pkg/testing/fixture.go @@ -41,7 +41,6 @@ type Fixture struct { fetcher Fetcher operatingSystem string architecture string - packageFormat string logOutput bool allowErrs bool connectTimout time.Duration @@ -84,14 +83,6 @@ func WithOSArchitecture(operatingSystem string, architecture string) FixtureOpt } } -// WithPackageFormat changes the package format to use for the fixture. -// By default, targz is picked except for windows which uses zip -func WithPackageFormat(packageFormat string) FixtureOpt { - return func(f *Fixture) { - f.packageFormat = packageFormat - } -} - // WithLogOutput instructs the fixture to log all Elastic Agent output to the test log. // By default, the Elastic Agent output will not be logged to the test logger. func WithLogOutput() FixtureOpt { @@ -148,10 +139,6 @@ func NewFixture(t *testing.T, version string, opts ...FixtureOpt) (*Fixture, err if !ok { return nil, errors.New("unable to determine callers file path") } - pkgFormat := "targz" - if runtime.GOOS == "windows" { - pkgFormat = "zip" - } f := &Fixture{ t: t, version: version, @@ -159,7 +146,6 @@ func NewFixture(t *testing.T, version string, opts ...FixtureOpt) (*Fixture, err fetcher: ArtifactFetcher(), operatingSystem: runtime.GOOS, architecture: runtime.GOARCH, - packageFormat: pkgFormat, connectTimout: 15 * time.Second, // default to elastic-agent, can be changed by a set FixtureOpt below binaryName: "elastic-agent", @@ -267,11 +253,6 @@ func (f *Fixture) SrcPackage(ctx context.Context) (string, error) { return f.srcPackage, nil } -// PackageFormat returns the package format for the fixture -func (f *Fixture) PackageFormat() string { - return f.packageFormat -} - func ExtractArtifact(l Logger, artifactFile, outputDir string) error { filename := filepath.Base(artifactFile) _, ext, err := splitFileType(filename) @@ -290,11 +271,6 @@ func ExtractArtifact(l Logger, artifactFile, outputDir string) error { if err != nil { return fmt.Errorf("failed to unzip %s: %w", artifactFile, err) } - case ".deb", "rpm": - err := copy.Copy(artifactFile, filepath.Join(outputDir, filepath.Base(artifactFile))) - if err != nil { - return fmt.Errorf("failed to copy %s to %s: %w", artifactFile, outputDir, err) - } } l.Logf("Completed extraction of artifact %s to %s", filename, outputDir) return nil @@ -837,9 +813,6 @@ func (f *Fixture) binaryPath() string { workDir = filepath.Join(paths.DefaultBasePath, "Elastic", "Agent") } } - if f.packageFormat == "deb" { - workDir = "/usr/bin" - } defaultBin := "elastic-agent" if f.binaryName != "" { defaultBin = f.binaryName @@ -867,7 +840,7 @@ func (f *Fixture) fetch(ctx context.Context) (string, error) { cache.dir = dir } - res, err := f.fetcher.Fetch(ctx, f.operatingSystem, f.architecture, f.version, f.packageFormat) + res, err := f.fetcher.Fetch(ctx, f.operatingSystem, f.architecture, f.version) if err != nil { return "", err } diff --git a/pkg/testing/fixture_install.go b/pkg/testing/fixture_install.go index 23edbde1ebe..651a1eac573 100644 --- a/pkg/testing/fixture_install.go +++ b/pkg/testing/fixture_install.go @@ -13,7 +13,6 @@ import ( "io" "io/fs" "os" - "os/exec" "path/filepath" "runtime" "strconv" @@ -133,32 +132,11 @@ func (f *Fixture) Install(ctx context.Context, installOpts *InstallOpts, opts .. // check for running agents before installing, but proceed anyway assert.Empty(f.t, getElasticAgentProcesses(f.t), "there should be no running agent at beginning of Install()") - switch f.packageFormat { - case "targz", "zip": - return f.installNoPkgManager(ctx, installOpts, opts) - case "deb": - return f.installDeb(ctx, installOpts, opts) - default: - return nil, fmt.Errorf("package format %s isn't supported yet", f.packageFormat) - } -} - -// installNoPkgManager installs the prepared Elastic Agent binary from -// the tgz or zip archive and registers a t.Cleanup function to -// uninstall the agent if it hasn't been uninstalled. It also takes -// care of collecting a diagnostics when AGENT_COLLECT_DIAG=true or -// the test has failed. -// It returns: -// - the combined output of Install command stdout and stderr -// - an error if any. -func (f *Fixture) installNoPkgManager(ctx context.Context, installOpts *InstallOpts, opts []process.CmdOption) ([]byte, error) { - f.t.Logf("[test %s] Inside fixture installNoPkgManager function", f.t.Name()) + installArgs := []string{"install"} if installOpts == nil { // default options when not provided installOpts = &InstallOpts{} } - - installArgs := []string{"install"} installOptsArgs, err := installOpts.toCmdArgs(f.operatingSystem) if err != nil { return nil, err @@ -361,80 +339,6 @@ func getProcesses(t *gotesting.T, regex string) []runningProcess { return processes } -// installDeb installs the prepared Elastic Agent binary from the deb -// package and registers a t.Cleanup function to uninstall the agent if -// it hasn't been uninstalled. It also takes care of collecting a -// diagnostics when AGENT_COLLECT_DIAG=true or the test has failed. -// It returns: -// - the combined output of Install command stdout and stderr -// - an error if any. -func (f *Fixture) installDeb(ctx context.Context, installOpts *InstallOpts, opts []process.CmdOption) ([]byte, error) { - f.t.Logf("[test %s] Inside fixture installDeb function", f.t.Name()) - //Prepare so that the f.srcPackage string is populated - err := f.EnsurePrepared(ctx) - if err != nil { - return nil, fmt.Errorf("failed to prepare: %w", err) - } - - // sudo apt install the deb - out, err := exec.CommandContext(ctx, "sudo", "apt", "install", f.srcPackage).CombinedOutput() // #nosec G204 -- Need to pass in name of package - if err != nil { - return out, fmt.Errorf("apt install failed: %w output:%s", err, string(out)) - } - - f.t.Cleanup(func() { - f.t.Logf("[test %s] Inside fixture installDeb cleanup function", f.t.Name()) - uninstallCtx, uninstallCancel := context.WithTimeout(context.Background(), 5*time.Minute) - defer uninstallCancel() - // stop elastic-agent, non fatal if error, might have been stopped before this. - f.t.Logf("running 'sudo systemctl stop elastic-agent'") - out, err := exec.CommandContext(uninstallCtx, "sudo", "systemctl", "stop", "elastic-agent").CombinedOutput() - if err != nil { - f.t.Logf("error systemctl stop elastic-agent: %s, output: %s", err, string(out)) - } - // apt-get purge elastic-agent - f.t.Logf("running 'sudo apt-get -y -q purge elastic-agent'") - out, err = exec.CommandContext(uninstallCtx, "sudo", "apt-get", "-y", "-q", "purge", "elastic-agent").CombinedOutput() - if err != nil { - f.t.Logf("failed to apt-get purge elastic-agent: %s, output: %s", err, string(out)) - f.t.FailNow() - } - }) - - // start elastic-agent - out, err = exec.CommandContext(ctx, "sudo", "systemctl", "start", "elastic-agent").CombinedOutput() - if err != nil { - return out, fmt.Errorf("systemctl start elastic-agent failed: %w", err) - } - - // apt install doesn't enroll, so need to do that - enrollArgs := []string{"elastic-agent", "enroll"} - if installOpts.Force { - enrollArgs = append(enrollArgs, "--force") - } - if installOpts.Insecure { - enrollArgs = append(enrollArgs, "--insecure") - } - if installOpts.ProxyURL != "" { - enrollArgs = append(enrollArgs, "--proxy-url="+installOpts.ProxyURL) - } - if installOpts.DelayEnroll { - enrollArgs = append(enrollArgs, "--delay-enroll") - } - if installOpts.EnrollOpts.URL != "" { - enrollArgs = append(enrollArgs, "--url", installOpts.EnrollOpts.URL) - } - if installOpts.EnrollOpts.EnrollmentToken != "" { - enrollArgs = append(enrollArgs, "--enrollment-token", installOpts.EnrollOpts.EnrollmentToken) - } - out, err = exec.CommandContext(ctx, "sudo", enrollArgs...).CombinedOutput() - if err != nil { - return out, fmt.Errorf("elastic-agent enroll failed: %w, output: %s args: %v", err, string(out), enrollArgs) - } - - return nil, nil -} - type UninstallOpts struct { Force bool // --force UninstallToken string @@ -455,30 +359,6 @@ func (i UninstallOpts) toCmdArgs() []string { // Uninstall uninstalls the installed Elastic Agent binary func (f *Fixture) Uninstall(ctx context.Context, uninstallOpts *UninstallOpts, opts ...process.CmdOption) ([]byte, error) { - switch f.packageFormat { - case "targz", "zip": - return f.uninstallNoPkgManager(ctx, uninstallOpts, opts) - case "deb": - return f.uninstallDeb(ctx, uninstallOpts, opts) - default: - return nil, fmt.Errorf("uninstall of package format '%s' not supported yet", f.packageFormat) - } -} - -func (f *Fixture) uninstallDeb(ctx context.Context, uninstallOpts *UninstallOpts, opts []process.CmdOption) ([]byte, error) { - // stop elastic-agent, non fatal if error, might have been stopped before this. - out, err := exec.CommandContext(ctx, "sudo", "systemctl", "stop", "elastic-agent").CombinedOutput() - if err != nil { - f.t.Logf("error systemctl stop elastic-agent: %s, output: %s", err, string(out)) - } - out, err = exec.CommandContext(ctx, "sudo", "apt-get", "-y", "-q", "purge", "elastic-agent").CombinedOutput() - if err != nil { - return out, fmt.Errorf("error removing apt: %w", err) - } - return out, nil -} - -func (f *Fixture) uninstallNoPkgManager(ctx context.Context, uninstallOpts *UninstallOpts, opts []process.CmdOption) ([]byte, error) { if !f.installed { return nil, ErrNotInstalled } diff --git a/pkg/testing/runner/debian.go b/pkg/testing/runner/debian.go index a3995efd602..268d42fb604 100644 --- a/pkg/testing/runner/debian.go +++ b/pkg/testing/runner/debian.go @@ -87,7 +87,7 @@ func (DebianRunner) Prepare(ctx context.Context, sshClient SSHClient, logger Log } // Copy places the required files on the host. -func (DebianRunner) Copy(ctx context.Context, sshClient SSHClient, logger Logger, repoArchive string, builds []Build) error { +func (DebianRunner) Copy(ctx context.Context, sshClient SSHClient, logger Logger, repoArchive string, build Build) error { // copy the archive and extract it on the host logger.Logf("Copying repo") destRepoName := filepath.Base(repoArchive) @@ -97,14 +97,12 @@ func (DebianRunner) Copy(ctx context.Context, sshClient SSHClient, logger Logger } // remove build paths, on cases where the build path is different from agent. - for _, build := range builds { - for _, remoteBuildPath := range []string{build.Path, build.SHA512Path} { - relativeAgentDir := filepath.Join("agent", remoteBuildPath) - _, _, err := sshClient.Exec(ctx, "sudo", []string{"rm", "-rf", relativeAgentDir}, nil) - // doesn't need to be a fatal error. - if err != nil { - logger.Logf("error removing build dir %s: %w", relativeAgentDir, err) - } + for _, remoteBuildPath := range []string{build.Path, build.SHA512Path} { + relativeAgentDir := filepath.Join("agent", remoteBuildPath) + _, _, err := sshClient.Exec(ctx, "sudo", []string{"rm", "-rf", relativeAgentDir}, nil) + // doesn't need to be a fatal error. + if err != nil { + logger.Logf("error removing build dir %s: %w", relativeAgentDir, err) } } @@ -134,58 +132,56 @@ func (DebianRunner) Copy(ctx context.Context, sshClient SSHClient, logger Logger // determine if the build needs to be replaced on the host // if it already exists and the SHA512 are the same contents, then // there is no reason to waste time uploading the build - for _, build := range builds { - copyBuild := true - localSHA512, err := os.ReadFile(build.SHA512Path) + copyBuild := true + localSHA512, err := os.ReadFile(build.SHA512Path) + if err != nil { + return fmt.Errorf("failed to read local SHA52 contents %s: %w", build.SHA512Path, err) + } + hostSHA512Path := filepath.Base(build.SHA512Path) + hostSHA512, err := sshClient.GetFileContents(ctx, hostSHA512Path) + if err == nil { + if string(localSHA512) == string(hostSHA512) { + logger.Logf("Skipping copy agent build %s; already the same", filepath.Base(build.Path)) + copyBuild = false + } + } + + if copyBuild { + // ensure the existing copies are removed first + toRemove := filepath.Base(build.Path) + stdOut, errOut, err = sshClient.Exec(ctx, + "sudo", []string{"rm", "-f", toRemove}, nil) if err != nil { - return fmt.Errorf("failed to read local SHA52 contents %s: %w", build.SHA512Path, err) + return fmt.Errorf("failed to remove %q: %w (stdout: %q, stderr: %q)", + toRemove, err, stdOut, errOut) } - hostSHA512Path := filepath.Base(build.SHA512Path) - hostSHA512, err := sshClient.GetFileContents(ctx, hostSHA512Path) - if err == nil { - if string(localSHA512) == string(hostSHA512) { - logger.Logf("Skipping copy agent build %s; already the same", filepath.Base(build.Path)) - copyBuild = false - } + + toRemove = filepath.Base(build.SHA512Path) + stdOut, errOut, err = sshClient.Exec(ctx, + "sudo", []string{"rm", "-f", toRemove}, nil) + if err != nil { + return fmt.Errorf("failed to remove %q: %w (stdout: %q, stderr: %q)", + toRemove, err, stdOut, errOut) } - if copyBuild { - // ensure the existing copies are removed first - toRemove := filepath.Base(build.Path) - stdOut, errOut, err = sshClient.Exec(ctx, - "sudo", []string{"rm", "-f", toRemove}, nil) - if err != nil { - return fmt.Errorf("failed to remove %q: %w (stdout: %q, stderr: %q)", - toRemove, err, stdOut, errOut) - } + logger.Logf("Copying agent build %s", filepath.Base(build.Path)) + } - toRemove = filepath.Base(build.SHA512Path) - stdOut, errOut, err = sshClient.Exec(ctx, - "sudo", []string{"rm", "-f", toRemove}, nil) + for _, buildPath := range []string{build.Path, build.SHA512Path} { + if copyBuild { + err = sshClient.Copy(buildPath, filepath.Base(buildPath)) if err != nil { - return fmt.Errorf("failed to remove %q: %w (stdout: %q, stderr: %q)", - toRemove, err, stdOut, errOut) + return fmt.Errorf("failed to SCP build %s: %w", filepath.Base(buildPath), err) } - - logger.Logf("Copying agent build %s", filepath.Base(build.Path)) } - - for _, buildPath := range []string{build.Path, build.SHA512Path} { - if copyBuild { - err = sshClient.Copy(buildPath, filepath.Base(buildPath)) - if err != nil { - return fmt.Errorf("failed to SCP build %s: %w", filepath.Base(buildPath), err) - } - } - insideAgentDir := filepath.Join("agent", buildPath) - stdOut, errOut, err = sshClient.Exec(ctx, "mkdir", []string{"-p", filepath.Dir(insideAgentDir)}, nil) - if err != nil { - return fmt.Errorf("failed to create %s directory: %w (stdout: %s, stderr: %s)", filepath.Dir(insideAgentDir), err, stdOut, errOut) - } - stdOut, errOut, err = sshClient.Exec(ctx, "ln", []string{filepath.Base(buildPath), insideAgentDir}, nil) - if err != nil { - return fmt.Errorf("failed to hard link %s to %s: %w (stdout: %s, stderr: %s)", filepath.Base(buildPath), insideAgentDir, err, stdOut, errOut) - } + insideAgentDir := filepath.Join("agent", buildPath) + stdOut, errOut, err = sshClient.Exec(ctx, "mkdir", []string{"-p", filepath.Dir(insideAgentDir)}, nil) + if err != nil { + return fmt.Errorf("failed to create %s directory: %w (stdout: %s, stderr: %s)", filepath.Dir(insideAgentDir), err, stdOut, errOut) + } + stdOut, errOut, err = sshClient.Exec(ctx, "ln", []string{filepath.Base(buildPath), insideAgentDir}, nil) + if err != nil { + return fmt.Errorf("failed to hard link %s to %s: %w (stdout: %s, stderr: %s)", filepath.Base(buildPath), insideAgentDir, err, stdOut, errOut) } } diff --git a/pkg/testing/runner/runner.go b/pkg/testing/runner/runner.go index 9ed999760ef..f6a304c1de5 100644 --- a/pkg/testing/runner/runner.go +++ b/pkg/testing/runner/runner.go @@ -24,7 +24,6 @@ import ( "golang.org/x/sync/errgroup" "k8s.io/utils/strings/slices" - "github.com/elastic/elastic-agent/pkg/testing" "github.com/elastic/elastic-agent/pkg/testing/define" ) @@ -66,7 +65,7 @@ type OSRunner interface { // Prepare prepares the runner to actual run on the host. Prepare(ctx context.Context, sshClient SSHClient, logger Logger, arch string, goVersion string) error // Copy places the required files on the host. - Copy(ctx context.Context, sshClient SSHClient, logger Logger, repoArchive string, builds []Build) error + Copy(ctx context.Context, sshClient SSHClient, logger Logger, repoArchive string, build Build) error // Run runs the actual tests and provides the result. Run(ctx context.Context, verbose bool, sshClient SSHClient, logger Logger, agentVersion string, prefix string, batch define.Batch, env map[string]string) (OSRunnerResult, error) // Diagnostics gathers any diagnostics from the host. @@ -362,7 +361,7 @@ func (r *Runner) runInstance(ctx context.Context, sshAuth ssh.AuthMethod, logger } // copy the required files (done every run) - err = batch.OS.Runner.Copy(ctx, client, logger, repoArchive, r.getBuilds(batch)) + err = batch.OS.Runner.Copy(ctx, client, logger, repoArchive, r.getBuild(batch)) if err != nil { logger.Logf("Failed to copy files instance: %s", err) return OSRunnerResult{}, fmt.Errorf("failed to copy files to instance %s: %w", instance.Name, err) @@ -420,13 +419,12 @@ func (r *Runner) validate() error { var requiredFiles []string for _, b := range r.batches { if !b.Skip { - for _, build := range r.getBuilds(b) { - if !slices.Contains(requiredFiles, build.Path) { - requiredFiles = append(requiredFiles, build.Path) - } - if !slices.Contains(requiredFiles, build.SHA512Path) { - requiredFiles = append(requiredFiles, build.SHA512Path) - } + build := r.getBuild(b) + if !slices.Contains(requiredFiles, build.Path) { + requiredFiles = append(requiredFiles, build.Path) + } + if !slices.Contains(requiredFiles, build.SHA512Path) { + requiredFiles = append(requiredFiles, build.SHA512Path) } } } @@ -445,32 +443,29 @@ func (r *Runner) validate() error { return nil } -// getBuilds returns the build for the batch. -func (r *Runner) getBuilds(b OSBatch) []Build { - builds := []Build{} - formats := []string{"targz", "zip", "rpm", "deb"} - for _, f := range formats { - arch := b.OS.Arch - if arch == define.AMD64 { - arch = "x86_64" - } - suffix, err := testing.GetPackageSuffix(b.OS.Type, b.OS.Arch, f) - if err != nil { - // Means that OS type & Arch doesn't support that package format - continue - } - packageName := filepath.Join(r.cfg.BuildDir, fmt.Sprintf("%s-%s-%s", "elastic-agent", r.cfg.AgentVersion, suffix)) - build := Build{ - Version: r.cfg.ReleaseVersion, - Type: b.OS.Type, - Arch: arch, - Path: packageName, - SHA512Path: packageName + ".sha512", - } - - builds = append(builds, build) +// getBuild returns the build for the batch. +func (r *Runner) getBuild(b OSBatch) Build { + arch := b.OS.Arch + if arch == define.AMD64 { + arch = "x86_64" + } + ext := "tar.gz" + if b.OS.Type == define.Windows { + ext = "zip" + } + hashExt := ".sha512" + name := "elastic-agent" + if r.cfg.BinaryName != "" { + name = r.cfg.BinaryName + } + packageName := filepath.Join(r.cfg.BuildDir, fmt.Sprintf("%s-%s-%s-%s.%s", name, r.cfg.AgentVersion, b.OS.Type, arch, ext)) + return Build{ + Version: r.cfg.ReleaseVersion, + Type: b.OS.Type, + Arch: arch, + Path: packageName, + SHA512Path: packageName + hashExt, } - return builds } // prepare prepares for the runner to run. diff --git a/pkg/testing/runner/windows.go b/pkg/testing/runner/windows.go index 2fcdc4f420c..8180ff9a4f3 100644 --- a/pkg/testing/runner/windows.go +++ b/pkg/testing/runner/windows.go @@ -69,7 +69,7 @@ func (WindowsRunner) Prepare(ctx context.Context, sshClient SSHClient, logger Lo } // Copy places the required files on the host. -func (WindowsRunner) Copy(ctx context.Context, sshClient SSHClient, logger Logger, repoArchive string, builds []Build) error { +func (WindowsRunner) Copy(ctx context.Context, sshClient SSHClient, logger Logger, repoArchive string, build Build) error { // copy the archive and extract it on the host (tar exists and can extract zip on windows) logger.Logf("Copying repo") destRepoName := filepath.Base(repoArchive) @@ -101,57 +101,55 @@ func (WindowsRunner) Copy(ctx context.Context, sshClient SSHClient, logger Logge // determine if the build needs to be replaced on the host // if it already exists and the SHA512 are the same contents, then // there is no reason to waste time uploading the build - for _, build := range builds { - copyBuild := true - localSHA512, err := os.ReadFile(build.SHA512Path) + copyBuild := true + localSHA512, err := os.ReadFile(build.SHA512Path) + if err != nil { + return fmt.Errorf("failed to read local SHA52 contents %s: %w", build.SHA512Path, err) + } + hostSHA512Path := filepath.Base(build.SHA512Path) + hostSHA512, err := sshClient.GetFileContents(ctx, hostSHA512Path, WithContentFetchCommand("type")) + if err == nil { + if string(localSHA512) == string(hostSHA512) { + logger.Logf("Skipping copy agent build %s; already the same", filepath.Base(build.Path)) + copyBuild = false + } + } + + if copyBuild { + // ensure the existing copies are removed first + toRemove := filepath.Base(build.Path) + stdOut, errOut, err = sshClient.Exec(ctx, + "del", []string{toRemove, "/f", "/q"}, nil) if err != nil { - return fmt.Errorf("failed to read local SHA52 contents %s: %w", build.SHA512Path, err) + return fmt.Errorf("failed to remove %q: %w (stdout: %q, stderr: %q)", + toRemove, err, stdOut, errOut) } - hostSHA512Path := filepath.Base(build.SHA512Path) - hostSHA512, err := sshClient.GetFileContents(ctx, hostSHA512Path, WithContentFetchCommand("type")) - if err == nil { - if string(localSHA512) == string(hostSHA512) { - logger.Logf("Skipping copy agent build %s; already the same", filepath.Base(build.Path)) - copyBuild = false - } + + toRemove = filepath.Base(build.SHA512Path) + stdOut, errOut, err = sshClient.Exec(ctx, + "del", []string{toRemove, "/f", "/q"}, nil) + if err != nil { + return fmt.Errorf("failed to remove %q: %w (stdout: %q, stderr: %q)", + toRemove, err, stdOut, errOut) } - if copyBuild { - // ensure the existing copies are removed first - toRemove := filepath.Base(build.Path) - stdOut, errOut, err = sshClient.Exec(ctx, - "del", []string{toRemove, "/f", "/q"}, nil) - if err != nil { - return fmt.Errorf("failed to remove %q: %w (stdout: %q, stderr: %q)", - toRemove, err, stdOut, errOut) - } + logger.Logf("Copying agent build %s", filepath.Base(build.Path)) + } - toRemove = filepath.Base(build.SHA512Path) - stdOut, errOut, err = sshClient.Exec(ctx, - "del", []string{toRemove, "/f", "/q"}, nil) + for _, buildPath := range []string{build.Path, build.SHA512Path} { + if copyBuild { + err = sshClient.Copy(buildPath, filepath.Base(buildPath)) if err != nil { - return fmt.Errorf("failed to remove %q: %w (stdout: %q, stderr: %q)", - toRemove, err, stdOut, errOut) + return fmt.Errorf("failed to SCP build %s: %w", filepath.Base(buildPath), err) } - - logger.Logf("Copying agent build %s", filepath.Base(build.Path)) } - - for _, buildPath := range []string{build.Path, build.SHA512Path} { - if copyBuild { - err = sshClient.Copy(buildPath, filepath.Base(buildPath)) - if err != nil { - return fmt.Errorf("failed to SCP build %s: %w", filepath.Base(buildPath), err) - } - } - insideAgentDir := filepath.Join("agent", buildPath) - // possible the build path already exists, 'mkdir' on windows will fail if it already exists - // error from this call is ignored because of it - _, _, _ = sshClient.Exec(ctx, "mkdir", []string{toWindowsPath(filepath.Dir(insideAgentDir))}, nil) - stdOut, errOut, err = sshClient.Exec(ctx, "mklink", []string{"/h", toWindowsPath(insideAgentDir), filepath.Base(buildPath)}, nil) - if err != nil { - return fmt.Errorf("failed to hard link %s to %s: %w (stdout: %s, stderr: %s)", filepath.Base(buildPath), toWindowsPath(insideAgentDir), err, stdOut, errOut) - } + insideAgentDir := filepath.Join("agent", buildPath) + // possible the build path already exists, 'mkdir' on windows will fail if it already exists + // error from this call is ignored because of it + _, _, _ = sshClient.Exec(ctx, "mkdir", []string{toWindowsPath(filepath.Dir(insideAgentDir))}, nil) + stdOut, errOut, err = sshClient.Exec(ctx, "mklink", []string{"/h", toWindowsPath(insideAgentDir), filepath.Base(buildPath)}, nil) + if err != nil { + return fmt.Errorf("failed to hard link %s to %s: %w (stdout: %s, stderr: %s)", filepath.Base(buildPath), toWindowsPath(insideAgentDir), err, stdOut, errOut) } } diff --git a/testing/integration/groups_test.go b/testing/integration/groups_test.go index 2c0f38b7e53..d6311cc7eb6 100644 --- a/testing/integration/groups_test.go +++ b/testing/integration/groups_test.go @@ -30,7 +30,4 @@ const ( // Upgrade group of tests. Used for testing upgrades. Upgrade = "upgrade" - - // Deb group of tests. Used for testing .deb packages install & upgrades - Deb = "deb" ) diff --git a/testing/integration/logs_ingestion_test.go b/testing/integration/logs_ingestion_test.go index 89de18df5fe..55756c1543b 100644 --- a/testing/integration/logs_ingestion_test.go +++ b/testing/integration/logs_ingestion_test.go @@ -99,74 +99,6 @@ func TestLogIngestionFleetManaged(t *testing.T) { }) } -func TestDebLogIngestFleetManaged(t *testing.T) { - info := define.Require(t, define.Requirements{ - Group: Deb, - Stack: &define.Stack{}, - OS: []define.OS{ - { - Type: define.Linux, - Distro: "ubuntu", - }, - }, - Local: false, - Sudo: true, - }) - - ctx, cancel := testcontext.WithDeadline(t, context.Background(), time.Now().Add(10*time.Minute)) - defer cancel() - - agentFixture, err := define.NewFixture(t, define.Version(), atesting.WithPackageFormat("deb")) - require.NoError(t, err) - - // 1. Create a policy in Fleet with monitoring enabled. - // To ensure there are no conflicts with previous test runs against - // the same ESS stack, we add the current time at the end of the policy - // name. This policy does not contain any integration. - t.Log("Enrolling agent in Fleet with a test policy") - createPolicyReq := kibana.AgentPolicy{ - Name: fmt.Sprintf("test-policy-enroll-%d", time.Now().Unix()), - Namespace: info.Namespace, - Description: "test policy for agent enrollment", - MonitoringEnabled: []kibana.MonitoringEnabledOption{ - kibana.MonitoringEnabledLogs, - kibana.MonitoringEnabledMetrics, - }, - AgentFeatures: []map[string]interface{}{ - { - "name": "test_enroll", - "enabled": true, - }, - }, - } - - installOpts := atesting.InstallOpts{ - NonInteractive: true, - Force: true, - } - - // 2. Install the Elastic-Agent with the policy that - // was just created. - policy, err := tools.InstallAgentWithPolicy( - ctx, - t, - installOpts, - agentFixture, - info.KibanaClient, - createPolicyReq) - require.NoError(t, err) - t.Logf("created policy: %s", policy.ID) - check.ConnectedToFleet(ctx, t, agentFixture, 5*time.Minute) - - t.Run("Monitoring logs are shipped", func(t *testing.T) { - testMonitoringLogsAreShipped(t, ctx, info, agentFixture, policy) - }) - - t.Run("Normal logs with flattened data_stream are shipped", func(t *testing.T) { - testFlattenedDatastreamFleetPolicy(t, ctx, info, policy) - }) -} - func testMonitoringLogsAreShipped( t *testing.T, ctx context.Context, @@ -211,7 +143,6 @@ func testMonitoringLogsAreShipped( "add_cloud_metadata: received error failed requesting openstack metadata: Get \\\"https://169.254.169.254/2009-04-04/meta-data/instance-type\\\": dial tcp 169.254.169.254:443: connect: connection refused", // okay for the cloud metadata to not work "add_cloud_metadata: received error failed with http status code 404", // okay for the cloud metadata to not work "add_cloud_metadata: received error failed fetching EC2 Identity Document: operation error ec2imds: GetInstanceIdentityDocument, http response error StatusCode: 404, request to EC2 IMDS failed", // okay for the cloud metadata to not work - "failed to invoke rollback watcher: failed to start Upgrade Watcher: fork/exec /var/lib/elastic-agent/elastic-agent: no such file or directory", //on debian this happens probably need to fix. }) }) t.Logf("error logs: Got %d documents", len(docs.Hits.Hits)) diff --git a/testing/integration/upgrade_fleet_test.go b/testing/integration/upgrade_fleet_test.go index 28d1e7d7e8e..8452605adfc 100644 --- a/testing/integration/upgrade_fleet_test.go +++ b/testing/integration/upgrade_fleet_test.go @@ -146,7 +146,7 @@ func testFleetAirGappedUpgrade(t *testing.T, stack *define.Info, unprivileged bo err = upgradeTo.Prepare(ctx) require.NoError(t, err) - s := newArtifactsServer(ctx, t, latest, upgradeTo.PackageFormat()) + s := newArtifactsServer(ctx, t, latest) host := "artifacts.elastic.co" simulateAirGapedEnvironment(t, host) @@ -392,14 +392,14 @@ func simulateAirGapedEnvironment(t *testing.T, host string) { }) } -func newArtifactsServer(ctx context.Context, t *testing.T, version string, packageFormat string) *httptest.Server { +func newArtifactsServer(ctx context.Context, t *testing.T, version string) *httptest.Server { fileServerDir := t.TempDir() downloadAt := filepath.Join(fileServerDir, "downloads", "beats", "elastic-agent", "beats", "elastic-agent") err := os.MkdirAll(downloadAt, 0700) require.NoError(t, err, "could not create directory structure for file server") fetcher := atesting.ArtifactFetcher() - fr, err := fetcher.Fetch(ctx, runtime.GOOS, runtime.GOARCH, version, packageFormat) + fr, err := fetcher.Fetch(ctx, runtime.GOOS, runtime.GOARCH, version) require.NoErrorf(t, err, "could not prepare fetcher to download agent %s", version) err = fr.Fetch(ctx, t, downloadAt)