Revert "add a DEB integration test (#4301)" (#4419)

This reverts commit 4fd31f1.

The Serverless Beats Tests step in our daily builds started failing with:

> Error: error running test: missing required Elastic Agent package builds for integration runner to execute: /tmp/beats-build/beats/x-pack/metricbeat/build/distributions/elastic-agent-8.14.0-SNAPSHOT-linux-x86_64.tar.gz, /tmp/beats-build/beats/x-pack/metricbeat/build/distributions/elastic-agent-8.14.0-SNAPSHOT-linux-x86_64.tar.gz.sha512, /tmp/beats-build/beats/x-pack/metricbeat/build/distributions/elastic-agent-8.14.0-SNAPSHOT-x86_64.rpm, /tmp/beats-build/beats/x-pack/metricbeat/build/distributions/elastic-agent-8.14.0-SNAPSHOT-x86_64.rpm.sha512, /tmp/beats-build/beats/x-pack/metricbeat/build/distributions/elastic-agent-8.14.0-SNAPSHOT-amd64.deb, /tmp/beats-build/beats/x-pack/metricbeat/build/distributions/elastic-agent-8.14.0-SNAPSHOT-amd64.deb.sha512
~/builds/bk-agent-prod-gcp-1710406561496364062/elastic/elastic-agent

The last successful build is https://buildkite.com/elastic/elastic-agent/builds/7707
The first failure is https://buildkite.com/elastic/elastic-agent/builds/7735

There were only 2 changes made between these, this change is one of them. Let's try to revert this first because it has the least impact.

.buildkite/scripts/steps/integration_tests.sh

@@ -19,7 +19,7 @@ else
  OVERRIDE_TEST_AGENT_VERSION=""
 fi
 # PACKAGE
-AGENT_PACKAGE_VERSION="${OVERRIDE_AGENT_PACKAGE_VERSION}" DEV=true EXTERNAL=true SNAPSHOT=true PLATFORMS=linux/amd64,linux/arm64,windows/amd64 PACKAGES=tar.gz,zip,rpm,deb mage package
+AGENT_PACKAGE_VERSION="${OVERRIDE_AGENT_PACKAGE_VERSION}" DEV=true EXTERNAL=true SNAPSHOT=true PLATFORMS=linux/amd64,linux/arm64,windows/amd64 PACKAGES=tar.gz,zip mage package
 
 # Run integration tests
 set +e

dev-tools/mage/pkg.go

@@ -34,7 +34,7 @@ func Package() error {
 	// platforms := updateWithDarwinUniversal(Platforms)
 	platforms := Platforms
 
-	tasks := make(map[string][]interface{})
+	var tasks []interface{}
 	for _, target := range platforms {
 		for _, pkg := range Packages {
 			if pkg.OS != target.GOOS() || pkg.Arch != "" && pkg.Arch != target.Arch() {
@@ -94,15 +94,12 @@ func Package() error {
 
 				spec = spec.Evaluate()
 
-				tasks[target.GOOS()+"-"+target.Arch()] = append(tasks[target.GOOS()+"-"+target.Arch()], packageBuilder{target, spec, pkgType}.Build)
+				tasks = append(tasks, packageBuilder{target, spec, pkgType}.Build)
 			}
 		}
 	}
 
-	for k, v := range tasks {
-		fmt.Printf(">> package: Building %s\n", k)
-		Parallel(v...)
-	}
+	Parallel(tasks...)
 	return nil
 }
 

pkg/testing/define/define.go

@@ -151,7 +151,7 @@ func runOrSkip(t *testing.T, req Requirements, local bool) *Info {
 		panic("failed to get OS information")
 	}
 	if !req.runtimeAllowed(runtime.GOOS, runtime.GOARCH, osInfo.Version, osInfo.Platform) {
-		t.Skipf("platform: %s, architecture: %s, version: %s, and distro: %s combination is not supported by test.  required: %v", runtime.GOOS, runtime.GOARCH, osInfo.Version, osInfo.Platform, req.OS)
+		t.Skip("platform, architecture, version, and distro not supported by test")
 		return nil
 	}
 	namespace, err := getNamespace(t, local)

pkg/testing/fetcher.go

@@ -31,23 +31,19 @@ var (
 // packageArchMap provides a mapping for the endings of the builds of Elastic Agent based on the
 // operating system and architecture.
 var packageArchMap = map[string]string{
-	"linux-amd64-targz":  "linux-x86_64.tar.gz",
-	"linux-amd64-deb":    "amd64.deb",
-	"linux-amd64-rpm":    "x86_64.rpm",
-	"linux-arm64-targz":  "linux-arm64.tar.gz",
-	"linux-arm64-deb":    "arm64.deb",
-	"linux-arm64-rpm":    "aarch64.rpm",
-	"windows-amd64-zip":  "windows-x86_64.zip",
-	"darwin-amd64-targz": "darwin-x86_64.tar.gz",
-	"darwin-arm64-targz": "darwin-aarch64.tar.gz",
+	"linux-amd64":   "linux-x86_64.tar.gz",
+	"linux-arm64":   "linux-arm64.tar.gz",
+	"windows-amd64": "windows-x86_64.zip",
+	"darwin-amd64":  "darwin-x86_64.tar.gz",
+	"darwin-arm64":  "darwin-aarch64.tar.gz",
 }
 
 // GetPackageSuffix returns the suffix ending for the builds of Elastic Agent based on the
 // operating system and architecture.
-func GetPackageSuffix(operatingSystem string, architecture string, packageFormat string) (string, error) {
-	suffix, ok := packageArchMap[fmt.Sprintf("%s-%s-%s", operatingSystem, architecture, packageFormat)]
+func GetPackageSuffix(operatingSystem string, architecture string) (string, error) {
+	suffix, ok := packageArchMap[fmt.Sprintf("%s-%s", operatingSystem, architecture)]
 	if !ok {
-		return "", fmt.Errorf("%w: %s/%s/%s", ErrUnsupportedPlatform, operatingSystem, architecture, packageFormat)
+		return "", fmt.Errorf("%w: %s/%s", ErrUnsupportedPlatform, operatingSystem, architecture)
 	}
 	return suffix, nil
 }
@@ -72,7 +68,7 @@ type Fetcher interface {
 	//
 	// The extraction is handled by the caller. This should only download the file
 	// and place it into the directory.
-	Fetch(ctx context.Context, operatingSystem string, architecture string, version string, packageFormat string) (FetcherResult, error)
+	Fetch(ctx context.Context, operatingSystem string, architecture string, version string) (FetcherResult, error)
 }
 
 // fetchCache is global to all tests, reducing the time required to fetch the needed artifacts
@@ -109,12 +105,6 @@ func splitFileType(name string) (string, string, error) {
 	if strings.HasSuffix(name, ".zip") {
 		return strings.TrimSuffix(name, ".zip"), ".zip", nil
 	}
-	if strings.HasSuffix(name, ".deb") {
-		return strings.TrimSuffix(name, ".deb"), ".deb", nil
-	}
-	if strings.HasSuffix(name, ".rpm") {
-		return strings.TrimSuffix(name, ".rpm"), ".rpm", nil
-	}
 	return "", "", fmt.Errorf("unknown file extension type: %s", filepath.Ext(name))
 }
 

pkg/testing/fetcher_artifact.go

@@ -59,8 +59,8 @@ func (f *artifactFetcher) Name() string {
 }
 
 // Fetch fetches the Elastic Agent and places the resulting binary at the path.
-func (f *artifactFetcher) Fetch(ctx context.Context, operatingSystem string, architecture string, version string, packageFormat string) (FetcherResult, error) {
-	suffix, err := GetPackageSuffix(operatingSystem, architecture, packageFormat)
+func (f *artifactFetcher) Fetch(ctx context.Context, operatingSystem string, architecture string, version string) (FetcherResult, error) {
+	suffix, err := GetPackageSuffix(operatingSystem, architecture)
 	if err != nil {
 		return nil, err
 	}

pkg/testing/fetcher_artifact_test.go

@@ -29,7 +29,7 @@ func TestArtifactFetcher_Default(t *testing.T) {
 	af.doer = newFakeHttpClient(t)
 
 	tmp := t.TempDir()
-	res, err := f.Fetch(context.Background(), "linux", "amd64", "8.12.0", "targz")
+	res, err := f.Fetch(context.Background(), "linux", "amd64", "8.12.0")
 	require.NoError(t, err)
 
 	err = res.Fetch(context.Background(), t, tmp)
@@ -46,7 +46,7 @@ func TestArtifactFetcher_Snapshot(t *testing.T) {
 	af.doer = newFakeHttpClient(t)
 
 	tmp := t.TempDir()
-	res, err := f.Fetch(context.Background(), "linux", "amd64", "8.13.0-SNAPSHOT", "targz")
+	res, err := f.Fetch(context.Background(), "linux", "amd64", "8.13.0-SNAPSHOT")
 	require.NoError(t, err)
 
 	err = res.Fetch(context.Background(), t, tmp)
@@ -64,7 +64,7 @@ func TestArtifactFetcher_SnapshotOnly(t *testing.T) {
 	af.doer = newFakeHttpClient(t)
 
 	tmp := t.TempDir()
-	res, err := f.Fetch(context.Background(), "linux", "amd64", "8.13.0", "targz")
+	res, err := f.Fetch(context.Background(), "linux", "amd64", "8.13.0")
 	require.NoError(t, err)
 
 	err = res.Fetch(context.Background(), t, tmp)
@@ -82,7 +82,7 @@ func TestArtifactFetcher_Build(t *testing.T) {
 	af.doer = newFakeHttpClient(t)
 
 	tmp := t.TempDir()
-	res, err := f.Fetch(context.Background(), "linux", "amd64", "8.13.0-SNAPSHOT+l5snflwr", "targz")
+	res, err := f.Fetch(context.Background(), "linux", "amd64", "8.13.0-SNAPSHOT+l5snflwr")
 	require.NoError(t, err)
 
 	err = res.Fetch(context.Background(), t, tmp)

pkg/testing/fetcher_http.go

@@ -45,8 +45,8 @@ func (h HttpFetcher) Name() string {
 	return fmt.Sprintf("httpFetcher-%s", sanitizeFetcherName(h.baseURL))
 }
 
-func (h HttpFetcher) Fetch(ctx context.Context, operatingSystem string, architecture string, version string, packageFormat string) (FetcherResult, error) {
-	suffix, err := GetPackageSuffix(operatingSystem, architecture, packageFormat)
+func (h HttpFetcher) Fetch(ctx context.Context, operatingSystem string, architecture string, version string) (FetcherResult, error) {
+	suffix, err := GetPackageSuffix(operatingSystem, architecture)
 	if err != nil {
 		return nil, err
 	}

pkg/testing/fetcher_http_test.go

@@ -24,7 +24,6 @@ func TestHttpFetcher_Fetch(t *testing.T) {
 		operatingSystem string
 		architecture    string
 		version         string
-		pkgFormat       string
 	}
 	tests := []struct {
 		name    string
@@ -39,7 +38,6 @@ func TestHttpFetcher_Fetch(t *testing.T) {
 				operatingSystem: "linux",
 				architecture:    "arm64",
 				version:         "1.2.3",
-				pkgFormat:       "targz",
 			},
 			want: &httpFetcherResult{
 				baseURL:     "https://artifacts.elastic.co/downloads/beats/elastic-agent/",
@@ -54,7 +52,6 @@ func TestHttpFetcher_Fetch(t *testing.T) {
 				operatingSystem: "windows",
 				architecture:    "amd64",
 				version:         "1.2.3",
-				pkgFormat:       "zip",
 			},
 			want: &httpFetcherResult{
 				baseURL:     "http://somehost.somedomain/some/path/here",
@@ -72,7 +69,7 @@ func TestHttpFetcher_Fetch(t *testing.T) {
 			h := NewHttpFetcher(opts...)
 			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 			defer cancel()
-			got, err := h.Fetch(ctx, tt.args.operatingSystem, tt.args.architecture, tt.args.version, tt.args.pkgFormat)
+			got, err := h.Fetch(ctx, tt.args.operatingSystem, tt.args.architecture, tt.args.version)
 			if !tt.wantErr(t, err, fmt.Sprintf("Fetch(%v, %v, %v, %v)", ctx, tt.args.operatingSystem, tt.args.architecture, tt.args.version)) {
 				return
 			}

pkg/testing/fetcher_local.go

@@ -55,8 +55,8 @@ func (f *localFetcher) Name() string {
 }
 
 // Fetch fetches the Elastic Agent and places the resulting binary at the path.
-func (f *localFetcher) Fetch(_ context.Context, operatingSystem string, architecture string, version string, packageFormat string) (FetcherResult, error) {
-	suffix, err := GetPackageSuffix(operatingSystem, architecture, packageFormat)
+func (f *localFetcher) Fetch(_ context.Context, operatingSystem string, architecture string, version string) (FetcherResult, error) {
+	suffix, err := GetPackageSuffix(operatingSystem, architecture)
 	if err != nil {
 		return nil, err
 	}

pkg/testing/fetcher_local_test.go

@@ -28,13 +28,9 @@ func TestLocalFetcher(t *testing.T) {
 	snapshotContentHash := []byte("snapshot contents hash")
 	noSnapshotContent := []byte("not snapshot contents")
 	noSnapshotContentHash := []byte("not snapshot contents hash")
-	pkgFormat := "targz"
-	if runtime.GOOS == "windows" {
-		pkgFormat = "zip"
-	}
 
 	testdata := t.TempDir()
-	suffix, err := GetPackageSuffix(runtime.GOOS, runtime.GOARCH, pkgFormat)
+	suffix, err := GetPackageSuffix(runtime.GOOS, runtime.GOARCH)
 	require.NoError(t, err)
 
 	snapshotPath := fmt.Sprintf("elastic-agent-%s-SNAPSHOT-%s", baseVersion, suffix)
@@ -96,12 +92,8 @@ func TestLocalFetcher(t *testing.T) {
 		tmp := t.TempDir()
 
 		f := LocalFetcher(testdata, tc.opts...)
-		pkgFormat := "targz"
-		if runtime.GOOS == "windows" {
-			pkgFormat = "zip"
-		}
 		got, err := f.Fetch(
-			context.Background(), runtime.GOOS, runtime.GOARCH, tc.version, pkgFormat)
+			context.Background(), runtime.GOOS, runtime.GOARCH, tc.version)
 		require.NoError(t, err)
 
 		err = got.Fetch(context.Background(), t, tmp)

pkg/testing/fixture.go

@@ -41,7 +41,6 @@ type Fixture struct {
 	fetcher         Fetcher
 	operatingSystem string
 	architecture    string
-	packageFormat   string
 	logOutput       bool
 	allowErrs       bool
 	connectTimout   time.Duration
@@ -84,14 +83,6 @@ func WithOSArchitecture(operatingSystem string, architecture string) FixtureOpt
 	}
 }
 
-// WithPackageFormat changes the package format to use for the fixture.
-// By default, targz is picked except for windows which uses zip
-func WithPackageFormat(packageFormat string) FixtureOpt {
-	return func(f *Fixture) {
-		f.packageFormat = packageFormat
-	}
-}
-
 // WithLogOutput instructs the fixture to log all Elastic Agent output to the test log.
 // By default, the Elastic Agent output will not be logged to the test logger.
 func WithLogOutput() FixtureOpt {
@@ -148,18 +139,13 @@ func NewFixture(t *testing.T, version string, opts ...FixtureOpt) (*Fixture, err
 	if !ok {
 		return nil, errors.New("unable to determine callers file path")
 	}
-	pkgFormat := "targz"
-	if runtime.GOOS == "windows" {
-		pkgFormat = "zip"
-	}
 	f := &Fixture{
 		t:               t,
 		version:         version,
 		caller:          caller,
 		fetcher:         ArtifactFetcher(),
 		operatingSystem: runtime.GOOS,
 		architecture:    runtime.GOARCH,
-		packageFormat:   pkgFormat,
 		connectTimout:   15 * time.Second,
 		// default to elastic-agent, can be changed by a set FixtureOpt below
 		binaryName: "elastic-agent",
@@ -267,11 +253,6 @@ func (f *Fixture) SrcPackage(ctx context.Context) (string, error) {
 	return f.srcPackage, nil
 }
 
-// PackageFormat returns the package format for the  fixture
-func (f *Fixture) PackageFormat() string {
-	return f.packageFormat
-}
-
 func ExtractArtifact(l Logger, artifactFile, outputDir string) error {
 	filename := filepath.Base(artifactFile)
 	_, ext, err := splitFileType(filename)
@@ -290,11 +271,6 @@ func ExtractArtifact(l Logger, artifactFile, outputDir string) error {
 		if err != nil {
 			return fmt.Errorf("failed to unzip %s: %w", artifactFile, err)
 		}
-	case ".deb", "rpm":
-		err := copy.Copy(artifactFile, filepath.Join(outputDir, filepath.Base(artifactFile)))
-		if err != nil {
-			return fmt.Errorf("failed to copy %s to %s: %w", artifactFile, outputDir, err)
-		}
 	}
 	l.Logf("Completed extraction of artifact %s to %s", filename, outputDir)
 	return nil
@@ -837,9 +813,6 @@ func (f *Fixture) binaryPath() string {
 			workDir = filepath.Join(paths.DefaultBasePath, "Elastic", "Agent")
 		}
 	}
-	if f.packageFormat == "deb" {
-		workDir = "/usr/bin"
-	}
 	defaultBin := "elastic-agent"
 	if f.binaryName != "" {
 		defaultBin = f.binaryName
@@ -867,7 +840,7 @@ func (f *Fixture) fetch(ctx context.Context) (string, error) {
 		cache.dir = dir
 	}
 
-	res, err := f.fetcher.Fetch(ctx, f.operatingSystem, f.architecture, f.version, f.packageFormat)
+	res, err := f.fetcher.Fetch(ctx, f.operatingSystem, f.architecture, f.version)
 	if err != nil {
 		return "", err
 	}