From 112a7d8ceba614fb02c2bf1d09a6d12a8cd645ef Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Fri, 23 Sep 2022 13:42:24 +0200 Subject: [PATCH 1/4] initial --- magefile.go | 214 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 206 insertions(+), 8 deletions(-) diff --git a/magefile.go b/magefile.go index 34f8749..6449ad0 100644 --- a/magefile.go +++ b/magefile.go @@ -8,8 +8,12 @@ package main import ( + "archive/tar" + "archive/zip" + "compress/gzip" "context" "fmt" + "io" "os" "path/filepath" "runtime" @@ -29,7 +33,10 @@ import ( ) const ( - GoreleaserRepo = "github.com/goreleaser/goreleaser@v1.6.3" + GoreleaserRepo = "github.com/goreleaser/goreleaser@v1.6.3" + platformsEnvName = "PLATFORMS" + specFileName = devtools.ProjectName + ".spec" + configFileName = devtools.ProjectName + ".yml" ) // Aliases are shortcuts to long target names. @@ -52,7 +59,7 @@ func (Build) All() { } // Clean removes the build directory. -func (Build) Clean(test string) { +func (Build) Clean() { os.RemoveAll("build") // nolint:errcheck //not required } @@ -95,7 +102,7 @@ func (Build) Binary() error { } } - platform := os.Getenv("PLATFORMS") + platform := os.Getenv(platformsEnvName) if platform != "" && devtools.PlatformFiles[platform] == nil { return errors.Errorf("Platform %s not recognized, only supported options: all, darwin, linux, windows, darwin/amd64, darwin/arm64, linux/386, linux/amd64, linux/arm64, windows/386, windows/amd64", platform) } @@ -177,11 +184,7 @@ func CheckBinaries(platform string, version string) error { return errors.New("No selected platform files found") } for _, platform := range selectedPlatformFiles { - var execName = devtools.ProjectName - if strings.Contains(platform, "windows") { - execName += ".exe" - } - binary := filepath.Join(path, fmt.Sprintf("%s-%s-%s", devtools.ProjectName, version, platform), execName) + binary := binaryName(path, version, platform) if _, err := os.Stat(binary); err != nil { return errors.Wrap(err, "Build: binary check failed") } @@ -215,3 +218,198 @@ func Notice() error { // See https://github.com/golang/go/issues/43994#issuecomment-770053099 return gotool.Mod.Tidy() } + +// PACKAGE + +// Package runs all the checks including licence, notice, gomod, git changes, followed by binary build. +func Package() { + // these are not allowed in parallel + mg.SerialDeps( + Build.Clean, + CheckLicense, + Notice, + mage.Deps.CheckModuleTidy, + mage.CheckNoChanges, + Build.Binary, + ) + + version, err := fullVersion() + if err != nil { + panic(err) + } + + platforms := devtools.PlatformFiles[os.Getenv(platformsEnvName)] + // current by default + if platforms == nil { + platforms = devtools.PlatformFiles[runtime.GOOS+"/"+runtime.GOARCH] + } + + commonFiles := []string{ + specFileName, + configFileName, + } + archivePath := filepath.Join("build", "packages") + if err := os.MkdirAll(archivePath, 0755); err != nil { + panic(err) + } + + for _, platform := range platforms { + isWindows := platform == "windows-x86" || platform == "windows-x86_64" + + // grab binary + binaryPath := binaryFilePath(version, platform) + + // prepare package + var err error + if isWindows { + archiveFileName := devtools.ProjectName + platform + ".zip" + err = prepareZipArchive(archivePath, archiveFileName, append(commonFiles, binaryPath)) + } else { + archiveFileName := devtools.ProjectName + platform + ".tar.gz" + err = prepareTarArchive(archivePath, archiveFileName, append(commonFiles, binaryPath)) + } + + if err != nil { + panic(err) + } + + } +} + +func prepareZipArchive(path, archiveFileName string, files []string) error { + // remove if it exists + _ = os.Remove(path) + + archiveFullName := filepath.Join(path, archiveFileName) + archiveFile, err := os.Create(archiveFullName) + if err != nil { + return errors.Wrapf(err, "failed to create %q", archiveFullName) + } + defer archiveFile.Close() + + zipWriter := zip.NewWriter(archiveFile) + defer zipWriter.Close() + + addFile := func(filePath string, zipWriter *zip.Writer) error { + file, err := os.Open(filePath) + if err != nil { + return errors.Wrap(err, "failed opening a file") + } + defer file.Close() + + stat, err := file.Stat() + if err != nil { + return errors.Wrap(err, "failed retrieving stat info for a file") + } + + header, err := zip.FileInfoHeader(stat) + header.Method = zip.Deflate + + hWriter, err := zipWriter.CreateHeader(header) + if err != nil { + return errors.Wrap(err, "failed writing zip header") + } + + if _, err := io.Copy(hWriter, file); err != nil { + return errors.Wrap(err, "failed adding a file into an archive") + } + return nil + } + + for _, filePath := range files { + if err := addFile(filePath, zipWriter); err != nil { + return errors.Wrapf(err, "failed adding file %q into zip archive", filePath) + } + } + + return nil +} + +func prepareTarArchive(path, archiveFileName string, files []string) error { + // remove if it exists + _ = os.Remove(path) + + archiveFullName := filepath.Join(path, archiveFileName) + archiveFile, err := os.Create(archiveFullName) + if err != nil { + return errors.Wrapf(err, "failed to create %q", archiveFullName) + } + defer archiveFile.Close() + + gzipWriter := gzip.NewWriter(archiveFile) + defer gzipWriter.Close() + + tarWriter := tar.NewWriter(gzipWriter) + defer tarWriter.Close() + + addFile := func(filePath string, tarWriter *tar.Writer) error { + file, err := os.Open(filePath) + if err != nil { + return errors.Wrap(err, "failed opening a file") + } + defer file.Close() + + stat, err := file.Stat() + if err != nil { + return errors.Wrap(err, "failed retrieving stat info for a file") + } + + header := &tar.Header{ + Name: filePath, + Size: stat.Size(), + Mode: int64(stat.Mode()), + ModTime: stat.ModTime(), + } + + if err := tarWriter.WriteHeader(header); err != nil { + return errors.Wrap(err, "failed writing tar header") + } + + if _, err := io.Copy(tarWriter, file); err != nil { + return errors.Wrap(err, "failed adding a file into an archive") + } + + return nil + } + + for _, filePath := range files { + if err := addFile(filePath, tarWriter); err != nil { + return errors.Wrapf(err, "failed adding file %q into tar archive", filePath) + } + } + + return nil +} + +func fullVersion() (string, error) { + version := tools.DefaultBeatVersion + if versionQualifier, versionQualified := os.LookupEnv("VERSION_QUALIFIER"); versionQualified { + version += fmt.Sprintf("-%s", versionQualifier) + } + + if snapshotEnv := os.Getenv("SNAPSHOT"); snapshotEnv != "" { + isSnapshot, err := strconv.ParseBool(snapshotEnv) + if err != nil { + return "", err + } + if isSnapshot { + version += fmt.Sprintf("-%s", "SNAPSHOT") + } + } + + return version, nil +} + +func binaryFilePath(version, platform string) string { + path := filepath.Join("build", "binaries") + return binaryName(path, version, platform) +} + +func binaryName(path, version, platform string) string { + var execName = devtools.ProjectName + if strings.Contains(platform, "windows") { + execName += ".exe" + } + + return filepath.Join(path, fmt.Sprintf("%s-%s-%s", devtools.ProjectName, version, platform), execName) +} From 62850624a5c817d46df9a7699a5b636ea6a7eac9 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Mon, 26 Sep 2022 09:07:01 +0200 Subject: [PATCH 2/4] added spec file and updated docs --- README.md | 1 + elastic-agent-shipper.spec.yml | 16 ++++++ magefile.go | 95 ++++++++++++++++++++++------------ 3 files changed, 80 insertions(+), 32 deletions(-) create mode 100644 elastic-agent-shipper.spec.yml diff --git a/README.md b/README.md index 575db53..33ee6ea 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,7 @@ by running `mage -l` from the root of the repository. The most commonly used com * `mage build` to build the data shipper binary. * `mage check` to check license files and dependencies. * `mage lint` to lint the source code using [golangci-lint](https://golangci-lint.run/). +* `mage package` to produce release artifacts. * `go test ./...` to run all tests. Run mage commands with the `-v` flag for more detailed output, for example `mage -v check`. diff --git a/elastic-agent-shipper.spec.yml b/elastic-agent-shipper.spec.yml new file mode 100644 index 0000000..17586f0 --- /dev/null +++ b/elastic-agent-shipper.spec.yml @@ -0,0 +1,16 @@ +version: 2 +outputs: + - name: elasticsearch + description: Elasticsearch + platforms: &platforms + - linux/amd64 + - linux/arm64 + - darwin/amd64 + - darwin/arm64 + - windows/amd64 + - container/amd64 + - container/arm64 + - name: logstash + description: Logstash + platforms: *platforms + \ No newline at end of file diff --git a/magefile.go b/magefile.go index 6449ad0..fcaf771 100644 --- a/magefile.go +++ b/magefile.go @@ -12,8 +12,10 @@ import ( "archive/zip" "compress/gzip" "context" + "crypto/sha512" "fmt" "io" + "io/ioutil" "os" "path/filepath" "runtime" @@ -35,7 +37,7 @@ import ( const ( GoreleaserRepo = "github.com/goreleaser/goreleaser@v1.6.3" platformsEnvName = "PLATFORMS" - specFileName = devtools.ProjectName + ".spec" + specFileName = devtools.ProjectName + ".spec.yml" configFileName = devtools.ProjectName + ".yml" ) @@ -225,11 +227,6 @@ func Notice() error { func Package() { // these are not allowed in parallel mg.SerialDeps( - Build.Clean, - CheckLicense, - Notice, - mage.Deps.CheckModuleTidy, - mage.CheckNoChanges, Build.Binary, ) @@ -244,42 +241,74 @@ func Package() { platforms = devtools.PlatformFiles[runtime.GOOS+"/"+runtime.GOARCH] } - commonFiles := []string{ - specFileName, - configFileName, - } - archivePath := filepath.Join("build", "packages") + archivePath := filepath.Join("build", "distributions") if err := os.MkdirAll(archivePath, 0755); err != nil { panic(err) } for _, platform := range platforms { - isWindows := platform == "windows-x86" || platform == "windows-x86_64" + isWindows := strings.Contains(platform, "windows") // grab binary binaryPath := binaryFilePath(version, platform) // prepare package var err error + var archiveFileName string + targetDir := fmt.Sprintf("%s-%s", devtools.ProjectName, platform) if isWindows { - archiveFileName := devtools.ProjectName + platform + ".zip" - err = prepareZipArchive(archivePath, archiveFileName, append(commonFiles, binaryPath)) + archiveFileName = fmt.Sprintf("%s-%s.zip", devtools.ProjectName, platform) + err = prepareZipArchive( + archivePath, + targetDir, + archiveFileName, + map[string]string{ + specFileName: specFileName, + configFileName: configFileName, + execName(platform): binaryPath, + }, + ) } else { - archiveFileName := devtools.ProjectName + platform + ".tar.gz" - err = prepareTarArchive(archivePath, archiveFileName, append(commonFiles, binaryPath)) + archiveFileName = fmt.Sprintf("%s-%s.tar.gz", devtools.ProjectName, platform) + err = prepareTarArchive( + archivePath, + targetDir, + archiveFileName, + map[string]string{ + specFileName: specFileName, + configFileName: configFileName, + execName(platform): binaryPath, + }, + ) } if err != nil { panic(err) } + // generate sha sum + archiveFullPath := filepath.Join(archivePath, archiveFileName) + archive, err := os.Open(archiveFullPath) + if err != nil { + panic(errors.Wrapf(err, "failed opening archive %q", archiveFileName)) + } + + h := sha512.New() + if _, err := io.Copy(h, archive); err != nil { + panic(errors.Wrapf(err, "failed computing hash for archive %q", archiveFileName)) + } + + shaFile := archiveFullPath + ".sha512" + content := fmt.Sprintf("%x %s\n", h.Sum(nil), archiveFileName) + if err := ioutil.WriteFile(shaFile, []byte(content), 0644); err != nil { + panic(errors.Wrapf(err, "failed writing hash for archive %q", archiveFileName)) + } + + archive.Close() } } -func prepareZipArchive(path, archiveFileName string, files []string) error { - // remove if it exists - _ = os.Remove(path) - +func prepareZipArchive(path, targetDir, archiveFileName string, files map[string]string) error { archiveFullName := filepath.Join(path, archiveFileName) archiveFile, err := os.Create(archiveFullName) if err != nil { @@ -290,7 +319,7 @@ func prepareZipArchive(path, archiveFileName string, files []string) error { zipWriter := zip.NewWriter(archiveFile) defer zipWriter.Close() - addFile := func(filePath string, zipWriter *zip.Writer) error { + addFile := func(fileName, filePath string, zipWriter *zip.Writer) error { file, err := os.Open(filePath) if err != nil { return errors.Wrap(err, "failed opening a file") @@ -304,6 +333,7 @@ func prepareZipArchive(path, archiveFileName string, files []string) error { header, err := zip.FileInfoHeader(stat) header.Method = zip.Deflate + header.Name = filepath.Join(targetDir, fileName) hWriter, err := zipWriter.CreateHeader(header) if err != nil { @@ -316,8 +346,8 @@ func prepareZipArchive(path, archiveFileName string, files []string) error { return nil } - for _, filePath := range files { - if err := addFile(filePath, zipWriter); err != nil { + for fileName, filePath := range files { + if err := addFile(fileName, filePath, zipWriter); err != nil { return errors.Wrapf(err, "failed adding file %q into zip archive", filePath) } } @@ -325,10 +355,7 @@ func prepareZipArchive(path, archiveFileName string, files []string) error { return nil } -func prepareTarArchive(path, archiveFileName string, files []string) error { - // remove if it exists - _ = os.Remove(path) - +func prepareTarArchive(path, targetDir, archiveFileName string, files map[string]string) error { archiveFullName := filepath.Join(path, archiveFileName) archiveFile, err := os.Create(archiveFullName) if err != nil { @@ -342,7 +369,7 @@ func prepareTarArchive(path, archiveFileName string, files []string) error { tarWriter := tar.NewWriter(gzipWriter) defer tarWriter.Close() - addFile := func(filePath string, tarWriter *tar.Writer) error { + addFile := func(fileName, filePath string, tarWriter *tar.Writer) error { file, err := os.Open(filePath) if err != nil { return errors.Wrap(err, "failed opening a file") @@ -355,7 +382,7 @@ func prepareTarArchive(path, archiveFileName string, files []string) error { } header := &tar.Header{ - Name: filePath, + Name: filepath.Join(targetDir, fileName), Size: stat.Size(), Mode: int64(stat.Mode()), ModTime: stat.ModTime(), @@ -372,8 +399,8 @@ func prepareTarArchive(path, archiveFileName string, files []string) error { return nil } - for _, filePath := range files { - if err := addFile(filePath, tarWriter); err != nil { + for fileName, filePath := range files { + if err := addFile(fileName, filePath, tarWriter); err != nil { return errors.Wrapf(err, "failed adding file %q into tar archive", filePath) } } @@ -406,10 +433,14 @@ func binaryFilePath(version, platform string) string { } func binaryName(path, version, platform string) string { + return filepath.Join(path, fmt.Sprintf("%s-%s-%s", devtools.ProjectName, version, platform), execName(platform)) +} + +func execName(platform string) string { var execName = devtools.ProjectName if strings.Contains(platform, "windows") { execName += ".exe" } - return filepath.Join(path, fmt.Sprintf("%s-%s-%s", devtools.ProjectName, version, platform), execName) + return execName } From 63e27aec98726d93a31547bf594817aefa67b73d Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Mon, 26 Sep 2022 15:28:32 +0200 Subject: [PATCH 3/4] added command --- elastic-agent-shipper.spec.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/elastic-agent-shipper.spec.yml b/elastic-agent-shipper.spec.yml index 17586f0..cdd89f3 100644 --- a/elastic-agent-shipper.spec.yml +++ b/elastic-agent-shipper.spec.yml @@ -10,7 +10,9 @@ outputs: - windows/amd64 - container/amd64 - container/arm64 + command: {} - name: logstash description: Logstash platforms: *platforms + command: {} \ No newline at end of file From 2f2739ede5c5a7c916c94febb28d125cf47027b9 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Tue, 27 Sep 2022 09:19:05 +0200 Subject: [PATCH 4/4] switch errors to std --- elastic-agent-shipper.spec.yml | 4 ---- magefile.go | 38 +++++++++++++++++----------------- 2 files changed, 19 insertions(+), 23 deletions(-) diff --git a/elastic-agent-shipper.spec.yml b/elastic-agent-shipper.spec.yml index cdd89f3..c323af0 100644 --- a/elastic-agent-shipper.spec.yml +++ b/elastic-agent-shipper.spec.yml @@ -11,8 +11,4 @@ outputs: - container/amd64 - container/arm64 command: {} - - name: logstash - description: Logstash - platforms: *platforms - command: {} \ No newline at end of file diff --git a/magefile.go b/magefile.go index fcaf771..c554309 100644 --- a/magefile.go +++ b/magefile.go @@ -13,6 +13,7 @@ import ( "compress/gzip" "context" "crypto/sha512" + "errors" "fmt" "io" "io/ioutil" @@ -25,7 +26,6 @@ import ( "github.com/elastic/elastic-agent-libs/dev-tools/mage" devtools "github.com/elastic/elastic-agent-shipper/dev-tools/common" "github.com/elastic/elastic-agent-shipper/tools" - "github.com/pkg/errors" //mage:import @@ -106,7 +106,7 @@ func (Build) Binary() error { platform := os.Getenv(platformsEnvName) if platform != "" && devtools.PlatformFiles[platform] == nil { - return errors.Errorf("Platform %s not recognized, only supported options: all, darwin, linux, windows, darwin/amd64, darwin/arm64, linux/386, linux/amd64, linux/arm64, windows/386, windows/amd64", platform) + return fmt.Errorf("Platform %s not recognized, only supported options: all, darwin, linux, windows, darwin/amd64, darwin/arm64, linux/386, linux/amd64, linux/arm64, windows/386, windows/amd64", platform) } switch platform { case "windows", "linux", "darwin": @@ -127,7 +127,7 @@ func (Build) Binary() error { fmt.Println(">> build: Building binary for", platform) //nolint:forbidigo // it's ok to use fmt.println in mage err := sh.RunWithV(env, "goreleaser", args...) if err != nil { - return errors.Wrapf(err, "Build failed on %s", platform) + return fmt.Errorf("Build failed on %s: %w", platform, err) } return CheckBinaries(platform, env["DEFAULT_VERSION"]) @@ -188,7 +188,7 @@ func CheckBinaries(platform string, version string) error { for _, platform := range selectedPlatformFiles { binary := binaryName(path, version, platform) if _, err := os.Stat(binary); err != nil { - return errors.Wrap(err, "Build: binary check failed") + return fmt.Errorf("Build: binary check failed: %w", err) } } @@ -290,18 +290,18 @@ func Package() { archiveFullPath := filepath.Join(archivePath, archiveFileName) archive, err := os.Open(archiveFullPath) if err != nil { - panic(errors.Wrapf(err, "failed opening archive %q", archiveFileName)) + panic(fmt.Errorf("failed opening archive %q: %w", archiveFileName, err)) } h := sha512.New() if _, err := io.Copy(h, archive); err != nil { - panic(errors.Wrapf(err, "failed computing hash for archive %q", archiveFileName)) + panic(fmt.Errorf("failed computing hash for archive %q: %w", archiveFileName, err)) } shaFile := archiveFullPath + ".sha512" content := fmt.Sprintf("%x %s\n", h.Sum(nil), archiveFileName) if err := ioutil.WriteFile(shaFile, []byte(content), 0644); err != nil { - panic(errors.Wrapf(err, "failed writing hash for archive %q", archiveFileName)) + panic(fmt.Errorf("failed writing hash for archive %q: %w", archiveFileName, err)) } archive.Close() @@ -312,7 +312,7 @@ func prepareZipArchive(path, targetDir, archiveFileName string, files map[string archiveFullName := filepath.Join(path, archiveFileName) archiveFile, err := os.Create(archiveFullName) if err != nil { - return errors.Wrapf(err, "failed to create %q", archiveFullName) + return fmt.Errorf("failed to create %q: %w", archiveFullName, err) } defer archiveFile.Close() @@ -322,13 +322,13 @@ func prepareZipArchive(path, targetDir, archiveFileName string, files map[string addFile := func(fileName, filePath string, zipWriter *zip.Writer) error { file, err := os.Open(filePath) if err != nil { - return errors.Wrap(err, "failed opening a file") + return fmt.Errorf("failed opening a file: %w", err) } defer file.Close() stat, err := file.Stat() if err != nil { - return errors.Wrap(err, "failed retrieving stat info for a file") + return fmt.Errorf("failed retrieving stat info for a file: %w", err) } header, err := zip.FileInfoHeader(stat) @@ -337,18 +337,18 @@ func prepareZipArchive(path, targetDir, archiveFileName string, files map[string hWriter, err := zipWriter.CreateHeader(header) if err != nil { - return errors.Wrap(err, "failed writing zip header") + return fmt.Errorf("failed writing zip header: %w", err) } if _, err := io.Copy(hWriter, file); err != nil { - return errors.Wrap(err, "failed adding a file into an archive") + return fmt.Errorf("failed adding a file into an archive: %w", err) } return nil } for fileName, filePath := range files { if err := addFile(fileName, filePath, zipWriter); err != nil { - return errors.Wrapf(err, "failed adding file %q into zip archive", filePath) + return fmt.Errorf("failed adding file %q into zip archive: %w", filePath, err) } } @@ -359,7 +359,7 @@ func prepareTarArchive(path, targetDir, archiveFileName string, files map[string archiveFullName := filepath.Join(path, archiveFileName) archiveFile, err := os.Create(archiveFullName) if err != nil { - return errors.Wrapf(err, "failed to create %q", archiveFullName) + return fmt.Errorf("failed to create %q: %w", archiveFullName, err) } defer archiveFile.Close() @@ -372,13 +372,13 @@ func prepareTarArchive(path, targetDir, archiveFileName string, files map[string addFile := func(fileName, filePath string, tarWriter *tar.Writer) error { file, err := os.Open(filePath) if err != nil { - return errors.Wrap(err, "failed opening a file") + return fmt.Errorf("failed opening a file: %w", err) } defer file.Close() stat, err := file.Stat() if err != nil { - return errors.Wrap(err, "failed retrieving stat info for a file") + return fmt.Errorf("failed retrieving stat info for a file: %w", err) } header := &tar.Header{ @@ -389,11 +389,11 @@ func prepareTarArchive(path, targetDir, archiveFileName string, files map[string } if err := tarWriter.WriteHeader(header); err != nil { - return errors.Wrap(err, "failed writing tar header") + return fmt.Errorf("failed writing tar header: %w", err) } if _, err := io.Copy(tarWriter, file); err != nil { - return errors.Wrap(err, "failed adding a file into an archive") + return fmt.Errorf("failed adding a file into an archive: %w", err) } return nil @@ -401,7 +401,7 @@ func prepareTarArchive(path, targetDir, archiveFileName string, files map[string for fileName, filePath := range files { if err := addFile(fileName, filePath, tarWriter); err != nil { - return errors.Wrapf(err, "failed adding file %q into tar archive", filePath) + return fmt.Errorf("failed adding file %q into tar archive: %w", filePath, err) } }