From e61c571b726738c79aca9f5b235dab64d5db2e8f Mon Sep 17 00:00:00 2001 From: Vadym Popov Date: Tue, 1 Oct 2024 15:30:51 -0400 Subject: [PATCH 01/13] Add packaging utility for client tools auto updates --- lib/utils/packaging/archive.go | 124 +++++++++++++++ lib/utils/packaging/unarchive.go | 134 +++++++++++++++++ lib/utils/packaging/unarchive_test.go | 97 ++++++++++++ lib/utils/packaging/unarchive_unix.go | 182 +++++++++++++++++++++++ lib/utils/packaging/unarchive_windows.go | 44 ++++++ 5 files changed, 581 insertions(+) create mode 100644 lib/utils/packaging/archive.go create mode 100644 lib/utils/packaging/unarchive.go create mode 100644 lib/utils/packaging/unarchive_test.go create mode 100644 lib/utils/packaging/unarchive_unix.go create mode 100644 lib/utils/packaging/unarchive_windows.go diff --git a/lib/utils/packaging/archive.go b/lib/utils/packaging/archive.go new file mode 100644 index 0000000000000..ce9c5eb1f0bdc --- /dev/null +++ b/lib/utils/packaging/archive.go @@ -0,0 +1,124 @@ +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package packaging + +import ( + "archive/tar" + "archive/zip" + "compress/gzip" + "io" + "os" + "os/exec" + "path/filepath" + + "github.com/gravitational/teleport" + "github.com/gravitational/trace" +) + +// GenerateZipFile compresses the file into a `.zip` format. This format intended to be +// used only for windows platform and mocking paths for windows archive. +func GenerateZipFile(sourcePath, destPath string) error { + archive, err := os.Create(destPath) + if err != nil { + return trace.Wrap(err) + } + defer archive.Close() + + zipWriter := zip.NewWriter(archive) + defer zipWriter.Close() + + return filepath.Walk(sourcePath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + + zipFileWriter, err := zipWriter.Create(filepath.Base(path)) + if err != nil { + return trace.Wrap(err) + } + + _, err = io.Copy(zipFileWriter, file) + return trace.Wrap(err) + }) +} + +// GenerateTarGzFile compresses files into a `.tar.gz` format specifically in file +// structure related to linux packaging. +func GenerateTarGzFile(sourcePath, destPath string) error { + archive, err := os.Create(destPath) + if err != nil { + return trace.Wrap(err) + } + defer archive.Close() + + gzipWriter := gzip.NewWriter(archive) + defer gzipWriter.Close() + + tarWriter := tar.NewWriter(gzipWriter) + defer tarWriter.Close() + + return filepath.Walk(sourcePath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + + header, err := tar.FileInfoHeader(info, info.Name()) + if err != nil { + return err + } + header.Name = filepath.Join("teleport", filepath.Base(info.Name())) + if err := tarWriter.WriteHeader(header); err != nil { + return err + } + + _, err = io.Copy(tarWriter, file) + return trace.Wrap(err) + }) +} + +// GeneratePkgFile runs the macOS `pkgbuild` command to generate a .pkg file from the source. +func GeneratePkgFile(sourcePath, destPath, identifier string) error { + cmd := exec.Command("pkgbuild", + "--root", sourcePath, + "--identifier", identifier, + "--version", teleport.Version, + destPath, + ) + if _, err := cmd.CombinedOutput(); err != nil { + return err + } + + return nil +} diff --git a/lib/utils/packaging/unarchive.go b/lib/utils/packaging/unarchive.go new file mode 100644 index 0000000000000..ac2d40bc492ce --- /dev/null +++ b/lib/utils/packaging/unarchive.go @@ -0,0 +1,134 @@ +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package packaging + +import ( + "archive/zip" + "io" + "os" + "path/filepath" + "slices" + "strings" + + "github.com/gravitational/trace" +) + +const ( + reservedFreeDisk = 10 * 1024 * 1024 +) + +// Cleanup performs a cleanup pass to remove any old copies of apps. +func Cleanup(toolsDir, skipFileName, skipSuffix string) error { + err := filepath.Walk(toolsDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return trace.Wrap(err) + } + if !info.IsDir() { + return nil + } + if skipFileName == info.Name() { + return nil + } + if !strings.HasSuffix(info.Name(), skipSuffix) { + return nil + } + // Found a stale expanded package. + if err := os.RemoveAll(filepath.Join(toolsDir, info.Name())); err != nil { + return trace.Wrap(err) + } + return nil + }) + return trace.Wrap(err) +} + +func replaceZip(toolsDir string, archivePath string, hash string, apps []string) error { + f, err := os.Open(archivePath) + if err != nil { + return trace.Wrap(err) + } + defer f.Close() + fi, err := f.Stat() + if err != nil { + return trace.Wrap(err) + } + + zipReader, err := zip.NewReader(f, fi.Size()) + if err != nil { + return trace.Wrap(err) + } + + tempDir, err := os.MkdirTemp(toolsDir, hash) + if err != nil { + return trace.Wrap(err) + } + + for _, r := range zipReader.File { + // Skip over any files in the archive that are not defined apps. + if !slices.ContainsFunc(apps, func(s string) bool { + return strings.HasSuffix(r.Name, s) + }) { + continue + } + + // Verify that we have enough space for uncompressed file. + if err := checkFreeSpace(tempDir, r.UncompressedSize64); err != nil { + return trace.Wrap(err) + } + + rr, err := r.Open() + if err != nil { + return trace.Wrap(err) + } + defer rr.Close() + + dest := filepath.Join(tempDir, r.Name) + t, err := os.OpenFile(dest, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0755) + if err != nil { + return trace.Wrap(err) + } + defer t.Close() + + if _, err := io.Copy(t, rr); err != nil { + return trace.Wrap(err) + } + appPath := filepath.Join(toolsDir, r.Name) + if err := os.Remove(appPath); err != nil && !os.IsNotExist(err) { + return trace.Wrap(err) + } + if err := os.Symlink(dest, appPath); err != nil { + return trace.Wrap(err) + } + } + + return nil +} + +// checkFreeSpace verifies that we have enough requested space at specific directory. +func checkFreeSpace(path string, requested uint64) error { + free, err := freeDiskWithReserve(path) + if err != nil { + return trace.Errorf("failed to calculate free disk in %q: %v", path, err) + } + // Bail if there's not enough free disk space at the target. + if requested > free { + return trace.Errorf("%q needs %d additional bytes of disk space", path, requested-free) + } + + return nil +} diff --git a/lib/utils/packaging/unarchive_test.go b/lib/utils/packaging/unarchive_test.go new file mode 100644 index 0000000000000..d978e45ec5829 --- /dev/null +++ b/lib/utils/packaging/unarchive_test.go @@ -0,0 +1,97 @@ +//go:build !windows + +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package packaging + +import ( + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPackaging(t *testing.T) { + script := "#!/bin/sh\necho test" + + sourceDir, err := os.MkdirTemp(os.TempDir(), "source") + require.NoError(t, err) + + toolsDir, err := os.MkdirTemp(os.TempDir(), "dest") + require.NoError(t, err) + + t.Cleanup(func() { + require.NoError(t, os.RemoveAll(sourceDir)) + require.NoError(t, os.RemoveAll(toolsDir)) + }) + + // Create test script for packaging. + require.NoError(t, os.WriteFile(filepath.Join(sourceDir, "tsh"), []byte(script), 0o755)) + + t.Run("tar.gz", func(t *testing.T) { + archivePath := filepath.Join(toolsDir, "tsh.tar.gz") + err = GenerateTarGzFile(sourceDir, archivePath) + require.NoError(t, err) + require.FileExists(t, archivePath, "archive not created") + + err = replaceTarGz(toolsDir, archivePath, []string{"tsh"}) + require.NoError(t, err) + assert.FileExists(t, filepath.Join(toolsDir, "tsh"), "script not created") + + data, err := os.ReadFile(filepath.Join(toolsDir, "tsh")) + require.NoError(t, err) + assert.Equal(t, script, string(data)) + }) + + t.Run("pkg", func(t *testing.T) { + if runtime.GOOS != "darwin" { + t.Skip("unsupported platform") + } + archivePath := filepath.Join(toolsDir, "tsh.pkg") + err = GeneratePkgFile(sourceDir, archivePath, "com.example.pkgtest") + require.NoError(t, err) + require.FileExists(t, archivePath, "archive not created") + + err = replacePkg(toolsDir, archivePath, "hash", []string{"tsh"}) + require.NoError(t, err) + assert.FileExists(t, filepath.Join(toolsDir, "tsh"), "script not created") + + data, err := os.ReadFile(filepath.Join(toolsDir, "tsh")) + require.NoError(t, err) + assert.Equal(t, script, string(data)) + }) + + t.Run("zip", func(t *testing.T) { + archivePath := filepath.Join(toolsDir, "tsh.zip") + err = GenerateZipFile(sourceDir, archivePath) + require.NoError(t, err) + require.FileExists(t, archivePath, "archive not created") + + err = replaceZip(toolsDir, archivePath, "hash", []string{"tsh"}) + require.NoError(t, err) + assert.FileExists(t, filepath.Join(toolsDir, "tsh"), "script not created") + + data, err := os.ReadFile(filepath.Join(toolsDir, "tsh")) + require.NoError(t, err) + assert.Equal(t, script, string(data)) + }) +} diff --git a/lib/utils/packaging/unarchive_unix.go b/lib/utils/packaging/unarchive_unix.go new file mode 100644 index 0000000000000..673d58b84bc74 --- /dev/null +++ b/lib/utils/packaging/unarchive_unix.go @@ -0,0 +1,182 @@ +//go:build !windows + +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package packaging + +import ( + "archive/tar" + "compress/gzip" + "context" + "errors" + "io" + "log/slog" + "os" + "os/exec" + "path/filepath" + "runtime" + "slices" + "strings" + + "github.com/google/renameio/v2" + "github.com/gravitational/trace" + "golang.org/x/sys/unix" +) + +// Replace un-archives package from tools directory and replaces defined apps by symlinks. +func Replace(toolsDir string, archivePath string, hash string, apps []string) error { + switch runtime.GOOS { + case "darwin": + return replacePkg(toolsDir, archivePath, hash, apps) + default: + return replaceTarGz(toolsDir, archivePath, apps) + } +} + +func replaceTarGz(toolsDir string, archivePath string, apps []string) error { + tempDir := renameio.TempDir(toolsDir) + f, err := os.Open(archivePath) + if err != nil { + return trace.Wrap(err) + } + + gzipReader, err := gzip.NewReader(f) + if err != nil { + return trace.Wrap(err) + } + + tarReader := tar.NewReader(gzipReader) + for { + header, err := tarReader.Next() + if errors.Is(err, io.EOF) { + break + } + // Skip over any files in the archive that are not in apps. + if !slices.ContainsFunc(apps, func(s string) bool { + return strings.HasSuffix(header.Name, s) + }) { + if _, err := io.Copy(io.Discard, tarReader); err != nil { + slog.DebugContext(context.Background(), "failed to discard", "name", header.Name, "error", err) + } + continue + } + + // Verify that we have enough space for uncompressed file. + if err := checkFreeSpace(tempDir, uint64(header.Size)); err != nil { + return trace.Wrap(err) + } + + dest := filepath.Join(toolsDir, strings.TrimPrefix(header.Name, "teleport/")) + t, err := renameio.TempFile(tempDir, dest) + if err != nil { + return trace.Wrap(err) + } + if err := os.Chmod(t.Name(), 0o755); err != nil { + return trace.Wrap(err) + } + defer t.Cleanup() + + if _, err := io.Copy(t, tarReader); err != nil { + return trace.Wrap(err) + } + if err := t.CloseAtomicallyReplace(); err != nil { + return trace.Wrap(err) + } + } + return nil +} + +func replacePkg(toolsDir string, archivePath string, hash string, apps []string) error { + // Use "pkgutil" from the filesystem to expand the archive. In theory .pkg + // files are xz archives, however it's still safer to use "pkgutil" in-case + // Apple makes non-standard changes to the format. + // + // Full command: pkgutil --expand-full NAME.pkg DIRECTORY/ + pkgutil, err := exec.LookPath("pkgutil") + if err != nil { + return trace.Wrap(err) + } + expandPath := filepath.Join(toolsDir, hash+"-pkg") + out, err := exec.Command(pkgutil, "--expand-full", archivePath, expandPath).Output() + if err != nil { + slog.DebugContext(context.Background(), "failed to run pkgutil:", "output", out) + return trace.Wrap(err) + } + + err = filepath.Walk(expandPath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return trace.Wrap(err) + } + if info.IsDir() { + return nil + } + // Skip over any files in the archive that are not in apps. + if !slices.ContainsFunc(apps, func(s string) bool { + return strings.HasSuffix(info.Name(), s) + }) { + return nil + } + + // The first time a signed and notarized binary macOS application is run, + // execution is paused while it gets sent to Apple to verify. Once Apple + // approves the binary, the "com.apple.macl" extended attribute is added + // and the process is allowed to execute. This process is not concurrent, any + // other operations (like moving the application) on the application during + // this time will lead to the application being sent SIGKILL. + // + // Since apps have to be concurrent, execute app before performing any + // swap operations. This ensures that the "com.apple.macl" extended + // attribute is set and macOS will not send a SIGKILL to the process + // if multiple processes are trying to operate on it. + command := exec.Command(path, "version", "--client") + if err := command.Run(); err != nil { + return trace.Wrap(err) + } + + // Due to macOS applications not being a single binary (they are a + // directory), atomic operations are not possible. To work around this, use + // a symlink (which can be atomically swapped), then do a cleanup pass + // removing any stale copies of the expanded package. + newName := filepath.Join(toolsDir, filepath.Base(path)) + if err := renameio.Symlink(path, newName); err != nil { + return trace.Wrap(err) + } + + return nil + }) + + return trace.Wrap(err) +} + +// freeDiskWithReserve returns the available disk space. +func freeDiskWithReserve(dir string) (uint64, error) { + var stat unix.Statfs_t + err := unix.Statfs(dir, &stat) + if err != nil { + return 0, trace.Wrap(err) + } + if stat.Bsize < 0 { + return 0, trace.Errorf("invalid size") + } + avail := stat.Bavail * uint64(stat.Bsize) + if reservedFreeDisk > avail { + return 0, trace.Errorf("no free space left") + } + return avail - reservedFreeDisk, nil +} diff --git a/lib/utils/packaging/unarchive_windows.go b/lib/utils/packaging/unarchive_windows.go new file mode 100644 index 0000000000000..c51db8abdec4c --- /dev/null +++ b/lib/utils/packaging/unarchive_windows.go @@ -0,0 +1,44 @@ +//go:build windows + +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package packaging + +import ( + "github.com/gravitational/trace" + "golang.org/x/sys/windows" +) + +// Replace un-archives package from tools directory and replaces defined apps by symlinks. +func Replace(toolsDir string, archivePath string, hash string, apps []string) error { + return replaceZip(toolsDir, archivePath, hash, apps) +} + +// freeDiskWithReserve returns the available disk space. +func freeDiskWithReserve(dir string) (uint64, error) { + var avail uint64 + err := windows.GetDiskFreeSpaceEx(windows.StringToUTF16Ptr(dir), &avail, nil, nil) + if err != nil { + return 0, trace.Wrap(err) + } + if reservedFreeDisk > avail { + return 0, trace.Errorf("no free space left") + } + return avail - reservedFreeDisk, nil +} From 53c7b7c912beaac2b5556307ec5421e20dbcc783 Mon Sep 17 00:00:00 2001 From: Vadym Popov Date: Tue, 1 Oct 2024 20:44:17 -0400 Subject: [PATCH 02/13] Add error handling for close functions --- lib/utils/packaging/archive.go | 34 +++++--------- lib/utils/packaging/unarchive.go | 65 +++++++++++++++++---------- lib/utils/packaging/unarchive_test.go | 1 + lib/utils/packaging/unarchive_unix.go | 19 ++++---- 4 files changed, 65 insertions(+), 54 deletions(-) diff --git a/lib/utils/packaging/archive.go b/lib/utils/packaging/archive.go index ce9c5eb1f0bdc..37261682e5dce 100644 --- a/lib/utils/packaging/archive.go +++ b/lib/utils/packaging/archive.go @@ -38,12 +38,9 @@ func GenerateZipFile(sourcePath, destPath string) error { if err != nil { return trace.Wrap(err) } - defer archive.Close() zipWriter := zip.NewWriter(archive) - defer zipWriter.Close() - - return filepath.Walk(sourcePath, func(path string, info os.FileInfo, err error) error { + err = filepath.Walk(sourcePath, func(path string, info os.FileInfo, err error) error { if err != nil { return err } @@ -54,16 +51,17 @@ func GenerateZipFile(sourcePath, destPath string) error { if err != nil { return err } - defer file.Close() zipFileWriter, err := zipWriter.Create(filepath.Base(path)) if err != nil { - return trace.Wrap(err) + return trace.NewAggregate(err, file.Close()) } _, err = io.Copy(zipFileWriter, file) - return trace.Wrap(err) + return trace.NewAggregate(err, file.Close()) }) + + return trace.NewAggregate(err, zipWriter.Close(), archive.Close()) } // GenerateTarGzFile compresses files into a `.tar.gz` format specifically in file @@ -73,15 +71,9 @@ func GenerateTarGzFile(sourcePath, destPath string) error { if err != nil { return trace.Wrap(err) } - defer archive.Close() - gzipWriter := gzip.NewWriter(archive) - defer gzipWriter.Close() - tarWriter := tar.NewWriter(gzipWriter) - defer tarWriter.Close() - - return filepath.Walk(sourcePath, func(path string, info os.FileInfo, err error) error { + err = filepath.Walk(sourcePath, func(path string, info os.FileInfo, err error) error { if err != nil { return err } @@ -92,20 +84,21 @@ func GenerateTarGzFile(sourcePath, destPath string) error { if err != nil { return err } - defer file.Close() header, err := tar.FileInfoHeader(info, info.Name()) if err != nil { - return err + return trace.NewAggregate(err, file.Close()) } header.Name = filepath.Join("teleport", filepath.Base(info.Name())) if err := tarWriter.WriteHeader(header); err != nil { - return err + return trace.NewAggregate(err, file.Close()) } _, err = io.Copy(tarWriter, file) - return trace.Wrap(err) + return trace.NewAggregate(err, file.Close()) }) + + return trace.NewAggregate(err, tarWriter.Close(), gzipWriter.Close(), archive.Close()) } // GeneratePkgFile runs the macOS `pkgbuild` command to generate a .pkg file from the source. @@ -116,9 +109,6 @@ func GeneratePkgFile(sourcePath, destPath, identifier string) error { "--version", teleport.Version, destPath, ) - if _, err := cmd.CombinedOutput(); err != nil { - return err - } - return nil + return trace.Wrap(cmd.Run()) } diff --git a/lib/utils/packaging/unarchive.go b/lib/utils/packaging/unarchive.go index ac2d40bc492ce..7e714507d178d 100644 --- a/lib/utils/packaging/unarchive.go +++ b/lib/utils/packaging/unarchive.go @@ -62,61 +62,61 @@ func replaceZip(toolsDir string, archivePath string, hash string, apps []string) if err != nil { return trace.Wrap(err) } - defer f.Close() + addCloser, wrapClose := multiCloser() + addCloser(f.Close) + fi, err := f.Stat() if err != nil { - return trace.Wrap(err) + return wrapClose(err) } - zipReader, err := zip.NewReader(f, fi.Size()) if err != nil { - return trace.Wrap(err) + return wrapClose(err) } - tempDir, err := os.MkdirTemp(toolsDir, hash) if err != nil { - return trace.Wrap(err) + return wrapClose(err) } - for _, r := range zipReader.File { + for _, zipFile := range zipReader.File { // Skip over any files in the archive that are not defined apps. if !slices.ContainsFunc(apps, func(s string) bool { - return strings.HasSuffix(r.Name, s) + return strings.HasSuffix(zipFile.Name, s) }) { continue } - // Verify that we have enough space for uncompressed file. - if err := checkFreeSpace(tempDir, r.UncompressedSize64); err != nil { - return trace.Wrap(err) + // Verify that we have enough space for uncompressed zipFile. + if err := checkFreeSpace(tempDir, zipFile.UncompressedSize64); err != nil { + return wrapClose(err) } - rr, err := r.Open() + file, err := zipFile.Open() if err != nil { - return trace.Wrap(err) + return wrapClose(err) } - defer rr.Close() + addCloser(file.Close) - dest := filepath.Join(tempDir, r.Name) - t, err := os.OpenFile(dest, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0755) + dest := filepath.Join(tempDir, zipFile.Name) + destFile, err := os.OpenFile(dest, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0755) if err != nil { - return trace.Wrap(err) + return wrapClose(err) } - defer t.Close() + addCloser(destFile.Close) - if _, err := io.Copy(t, rr); err != nil { - return trace.Wrap(err) + if _, err := io.Copy(destFile, file); err != nil { + return wrapClose(err) } - appPath := filepath.Join(toolsDir, r.Name) + appPath := filepath.Join(toolsDir, zipFile.Name) if err := os.Remove(appPath); err != nil && !os.IsNotExist(err) { - return trace.Wrap(err) + return wrapClose(err) } if err := os.Symlink(dest, appPath); err != nil { - return trace.Wrap(err) + return wrapClose(err) } } - return nil + return wrapClose() } // checkFreeSpace verifies that we have enough requested space at specific directory. @@ -132,3 +132,20 @@ func checkFreeSpace(path string, requested uint64) error { return nil } + +// multiCloser creates functions for aggregating closing functions with reverse call +// and aggregating received error messages. +func multiCloser() (func(c func() error), func(errors ...error) error) { + var closers []func() error + return func(c func() error) { + closers = append(closers, c) + }, func(errors ...error) error { + closeErrors := make([]error, 0, len(closers)) + for i := len(closers) - 1; i >= 0; i-- { + if err := closers[i](); err != nil { + closeErrors = append(closeErrors, err) + } + } + return trace.NewAggregate(append(errors, closeErrors...)...) + } +} diff --git a/lib/utils/packaging/unarchive_test.go b/lib/utils/packaging/unarchive_test.go index d978e45ec5829..ed85882a2cc94 100644 --- a/lib/utils/packaging/unarchive_test.go +++ b/lib/utils/packaging/unarchive_test.go @@ -30,6 +30,7 @@ import ( "github.com/stretchr/testify/require" ) +// TestPackaging verifies unarchiving of all supported teleport package formats. func TestPackaging(t *testing.T) { script := "#!/bin/sh\necho test" diff --git a/lib/utils/packaging/unarchive_unix.go b/lib/utils/packaging/unarchive_unix.go index 673d58b84bc74..1fdb5253e4c18 100644 --- a/lib/utils/packaging/unarchive_unix.go +++ b/lib/utils/packaging/unarchive_unix.go @@ -55,10 +55,12 @@ func replaceTarGz(toolsDir string, archivePath string, apps []string) error { if err != nil { return trace.Wrap(err) } + addCloser, wrapClose := multiCloser() + addCloser(f.Close) gzipReader, err := gzip.NewReader(f) if err != nil { - return trace.Wrap(err) + return wrapClose(err) } tarReader := tar.NewReader(gzipReader) @@ -79,27 +81,28 @@ func replaceTarGz(toolsDir string, archivePath string, apps []string) error { // Verify that we have enough space for uncompressed file. if err := checkFreeSpace(tempDir, uint64(header.Size)); err != nil { - return trace.Wrap(err) + return wrapClose(err) } dest := filepath.Join(toolsDir, strings.TrimPrefix(header.Name, "teleport/")) t, err := renameio.TempFile(tempDir, dest) if err != nil { - return trace.Wrap(err) + return wrapClose(err) } if err := os.Chmod(t.Name(), 0o755); err != nil { - return trace.Wrap(err) + return wrapClose(err) } - defer t.Cleanup() + addCloser(t.Cleanup) if _, err := io.Copy(t, tarReader); err != nil { - return trace.Wrap(err) + return wrapClose(err) } if err := t.CloseAtomicallyReplace(); err != nil { - return trace.Wrap(err) + return wrapClose(err) } } - return nil + + return wrapClose() } func replacePkg(toolsDir string, archivePath string, hash string, apps []string) error { From a3fc5e06f50fa178caf7ebd4a006661903fb20d8 Mon Sep 17 00:00:00 2001 From: Vadym Popov Date: Thu, 3 Oct 2024 04:38:41 -0400 Subject: [PATCH 03/13] Move archive to existing utils package --- lib/utils/archive.go | 87 +++++++++++++++++ lib/utils/disk.go | 18 ++++ lib/utils/disk_windows.go | 18 +++- lib/utils/packaging/archive.go | 114 ----------------------- lib/utils/packaging/unarchive.go | 49 ++++------ lib/utils/packaging/unarchive_test.go | 18 ++-- lib/utils/packaging/unarchive_unix.go | 39 ++------ lib/utils/packaging/unarchive_windows.go | 18 ---- 8 files changed, 160 insertions(+), 201 deletions(-) delete mode 100644 lib/utils/packaging/archive.go diff --git a/lib/utils/archive.go b/lib/utils/archive.go index abab32e11dbc0..760044b15912e 100644 --- a/lib/utils/archive.go +++ b/lib/utils/archive.go @@ -20,10 +20,17 @@ package utils import ( "archive/tar" + "archive/zip" "bytes" "compress/gzip" + "io" "io/fs" + "os" + "os/exec" + "path/filepath" + "runtime" + "github.com/gravitational/teleport" "github.com/gravitational/trace" ) @@ -75,3 +82,83 @@ func CompressTarGzArchive(files []string, fileReader ReadStatFS) (*bytes.Buffer, return archiveBytes, nil } + +// CompressDirToZipFile compresses directory into a `.zip` format. +func CompressDirToZipFile(sourcePath, destPath string) error { + archive, err := os.Create(destPath) + if err != nil { + return trace.Wrap(err) + } + + zipWriter := zip.NewWriter(archive) + err = filepath.Walk(sourcePath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + file, err := os.Open(path) + if err != nil { + return err + } + zipFileWriter, err := zipWriter.Create(filepath.Base(path)) + if err != nil { + return trace.NewAggregate(err, file.Close()) + } + + _, err = io.Copy(zipFileWriter, file) + return trace.NewAggregate(err, file.Close()) + }) + + return trace.NewAggregate(err, zipWriter.Close(), archive.Close()) +} + +// CompressDirToTarGzFile compresses directory into a `.tar.gz` format. +func CompressDirToTarGzFile(sourcePath, destPath string) error { + archive, err := os.Create(destPath) + if err != nil { + return trace.Wrap(err) + } + gzipWriter := gzip.NewWriter(archive) + tarWriter := tar.NewWriter(gzipWriter) + err = filepath.Walk(sourcePath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + file, err := os.Open(path) + if err != nil { + return err + } + header, err := tar.FileInfoHeader(info, info.Name()) + if err != nil { + return trace.NewAggregate(err, file.Close()) + } + if err := tarWriter.WriteHeader(header); err != nil { + return trace.NewAggregate(err, file.Close()) + } + + _, err = io.Copy(tarWriter, file) + return trace.NewAggregate(err, file.Close()) + }) + + return trace.NewAggregate(err, tarWriter.Close(), gzipWriter.Close(), archive.Close()) +} + +// CompressDirToPkgFile runs for the macOS `pkgbuild` command to generate a .pkg file from the source. +func CompressDirToPkgFile(sourcePath, destPath, identifier string) error { + if runtime.GOOS != "darwin" { + return trace.BadParameter("only darwin packaging is supported") + } + cmd := exec.Command("pkgbuild", + "--root", sourcePath, + "--identifier", identifier, + "--version", teleport.Version, + destPath, + ) + + return trace.Wrap(cmd.Run()) +} diff --git a/lib/utils/disk.go b/lib/utils/disk.go index 78fba5457099b..9c8552051ff3e 100644 --- a/lib/utils/disk.go +++ b/lib/utils/disk.go @@ -31,6 +31,7 @@ import ( "syscall" "github.com/gravitational/trace" + "golang.org/x/sys/unix" ) // PercentUsed returns percentage of disk space used. The percentage of disk @@ -125,3 +126,20 @@ func CanUserWriteTo(path string) (bool, error) { return false, nil } + +// FreeDiskWithReserve returns the available disk space. +func FreeDiskWithReserve(dir string, reservedFreeDisk uint64) (uint64, error) { + var stat unix.Statfs_t + err := unix.Statfs(dir, &stat) + if err != nil { + return 0, trace.Wrap(err) + } + if stat.Bsize < 0 { + return 0, trace.Errorf("invalid size") + } + avail := stat.Bavail * uint64(stat.Bsize) + if reservedFreeDisk > avail { + return 0, trace.Errorf("no free space left") + } + return avail - reservedFreeDisk, nil +} diff --git a/lib/utils/disk_windows.go b/lib/utils/disk_windows.go index cde568b4c9589..79db4f46df0b0 100644 --- a/lib/utils/disk_windows.go +++ b/lib/utils/disk_windows.go @@ -21,7 +21,10 @@ package utils -import "github.com/gravitational/trace" +import ( + "github.com/gravitational/trace" + "golang.org/x/sys/windows" +) // PercentUsed is not supported on Windows. func PercentUsed(path string) (float64, error) { @@ -32,3 +35,16 @@ func PercentUsed(path string) (float64, error) { func CanUserWriteTo(path string) (bool, error) { return false, trace.NotImplemented("path permission checking is not supported on Windows") } + +// FreeDiskWithReserve returns the available disk space. +func FreeDiskWithReserve(dir string, reservedFreeDisk uint64) (uint64, error) { + var avail uint64 + err := windows.GetDiskFreeSpaceEx(windows.StringToUTF16Ptr(dir), &avail, nil, nil) + if err != nil { + return 0, trace.Wrap(err) + } + if reservedFreeDisk > avail { + return 0, trace.Errorf("no free space left") + } + return avail - reservedFreeDisk, nil +} diff --git a/lib/utils/packaging/archive.go b/lib/utils/packaging/archive.go deleted file mode 100644 index 37261682e5dce..0000000000000 --- a/lib/utils/packaging/archive.go +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Teleport - * Copyright (C) 2024 Gravitational, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package packaging - -import ( - "archive/tar" - "archive/zip" - "compress/gzip" - "io" - "os" - "os/exec" - "path/filepath" - - "github.com/gravitational/teleport" - "github.com/gravitational/trace" -) - -// GenerateZipFile compresses the file into a `.zip` format. This format intended to be -// used only for windows platform and mocking paths for windows archive. -func GenerateZipFile(sourcePath, destPath string) error { - archive, err := os.Create(destPath) - if err != nil { - return trace.Wrap(err) - } - - zipWriter := zip.NewWriter(archive) - err = filepath.Walk(sourcePath, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if info.IsDir() { - return nil - } - file, err := os.Open(path) - if err != nil { - return err - } - - zipFileWriter, err := zipWriter.Create(filepath.Base(path)) - if err != nil { - return trace.NewAggregate(err, file.Close()) - } - - _, err = io.Copy(zipFileWriter, file) - return trace.NewAggregate(err, file.Close()) - }) - - return trace.NewAggregate(err, zipWriter.Close(), archive.Close()) -} - -// GenerateTarGzFile compresses files into a `.tar.gz` format specifically in file -// structure related to linux packaging. -func GenerateTarGzFile(sourcePath, destPath string) error { - archive, err := os.Create(destPath) - if err != nil { - return trace.Wrap(err) - } - gzipWriter := gzip.NewWriter(archive) - tarWriter := tar.NewWriter(gzipWriter) - err = filepath.Walk(sourcePath, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if info.IsDir() { - return nil - } - file, err := os.Open(path) - if err != nil { - return err - } - - header, err := tar.FileInfoHeader(info, info.Name()) - if err != nil { - return trace.NewAggregate(err, file.Close()) - } - header.Name = filepath.Join("teleport", filepath.Base(info.Name())) - if err := tarWriter.WriteHeader(header); err != nil { - return trace.NewAggregate(err, file.Close()) - } - - _, err = io.Copy(tarWriter, file) - return trace.NewAggregate(err, file.Close()) - }) - - return trace.NewAggregate(err, tarWriter.Close(), gzipWriter.Close(), archive.Close()) -} - -// GeneratePkgFile runs the macOS `pkgbuild` command to generate a .pkg file from the source. -func GeneratePkgFile(sourcePath, destPath, identifier string) error { - cmd := exec.Command("pkgbuild", - "--root", sourcePath, - "--identifier", identifier, - "--version", teleport.Version, - destPath, - ) - - return trace.Wrap(cmd.Run()) -} diff --git a/lib/utils/packaging/unarchive.go b/lib/utils/packaging/unarchive.go index 7e714507d178d..79faf2cd60224 100644 --- a/lib/utils/packaging/unarchive.go +++ b/lib/utils/packaging/unarchive.go @@ -27,6 +27,8 @@ import ( "strings" "github.com/gravitational/trace" + + "github.com/gravitational/teleport/lib/utils" ) const ( @@ -62,20 +64,18 @@ func replaceZip(toolsDir string, archivePath string, hash string, apps []string) if err != nil { return trace.Wrap(err) } - addCloser, wrapClose := multiCloser() - addCloser(f.Close) fi, err := f.Stat() if err != nil { - return wrapClose(err) + return trace.NewAggregate(err, f.Close()) } zipReader, err := zip.NewReader(f, fi.Size()) if err != nil { - return wrapClose(err) + return trace.NewAggregate(err, f.Close()) } tempDir, err := os.MkdirTemp(toolsDir, hash) if err != nil { - return wrapClose(err) + return trace.NewAggregate(err, f.Close()) } for _, zipFile := range zipReader.File { @@ -85,43 +85,43 @@ func replaceZip(toolsDir string, archivePath string, hash string, apps []string) }) { continue } - // Verify that we have enough space for uncompressed zipFile. if err := checkFreeSpace(tempDir, zipFile.UncompressedSize64); err != nil { - return wrapClose(err) + return trace.NewAggregate(err, f.Close()) } file, err := zipFile.Open() if err != nil { - return wrapClose(err) + return trace.Wrap(err) } - addCloser(file.Close) dest := filepath.Join(tempDir, zipFile.Name) destFile, err := os.OpenFile(dest, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0755) if err != nil { - return wrapClose(err) + return trace.NewAggregate(err, file.Close(), f.Close()) } - addCloser(destFile.Close) if _, err := io.Copy(destFile, file); err != nil { - return wrapClose(err) + return trace.NewAggregate(err, destFile.Close(), file.Close(), f.Close()) } appPath := filepath.Join(toolsDir, zipFile.Name) if err := os.Remove(appPath); err != nil && !os.IsNotExist(err) { - return wrapClose(err) + return trace.NewAggregate(err, destFile.Close(), file.Close(), f.Close()) } if err := os.Symlink(dest, appPath); err != nil { - return wrapClose(err) + return trace.NewAggregate(err, destFile.Close(), file.Close(), f.Close()) + } + if err := trace.NewAggregate(destFile.Close(), file.Close()); err != nil { + return trace.NewAggregate(err, f.Close()) } } - return wrapClose() + return trace.Wrap(f.Close()) } // checkFreeSpace verifies that we have enough requested space at specific directory. func checkFreeSpace(path string, requested uint64) error { - free, err := freeDiskWithReserve(path) + free, err := utils.FreeDiskWithReserve(path, reservedFreeDisk) if err != nil { return trace.Errorf("failed to calculate free disk in %q: %v", path, err) } @@ -132,20 +132,3 @@ func checkFreeSpace(path string, requested uint64) error { return nil } - -// multiCloser creates functions for aggregating closing functions with reverse call -// and aggregating received error messages. -func multiCloser() (func(c func() error), func(errors ...error) error) { - var closers []func() error - return func(c func() error) { - closers = append(closers, c) - }, func(errors ...error) error { - closeErrors := make([]error, 0, len(closers)) - for i := len(closers) - 1; i >= 0; i-- { - if err := closers[i](); err != nil { - closeErrors = append(closeErrors, err) - } - } - return trace.NewAggregate(append(errors, closeErrors...)...) - } -} diff --git a/lib/utils/packaging/unarchive_test.go b/lib/utils/packaging/unarchive_test.go index ed85882a2cc94..b4bf298bc1cac 100644 --- a/lib/utils/packaging/unarchive_test.go +++ b/lib/utils/packaging/unarchive_test.go @@ -28,6 +28,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/gravitational/teleport/lib/utils" ) // TestPackaging verifies unarchiving of all supported teleport package formats. @@ -47,16 +49,18 @@ func TestPackaging(t *testing.T) { // Create test script for packaging. require.NoError(t, os.WriteFile(filepath.Join(sourceDir, "tsh"), []byte(script), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(sourceDir, "tctl"), []byte(script), 0o755)) t.Run("tar.gz", func(t *testing.T) { archivePath := filepath.Join(toolsDir, "tsh.tar.gz") - err = GenerateTarGzFile(sourceDir, archivePath) + err = utils.CompressDirToTarGzFile(sourceDir, archivePath) require.NoError(t, err) require.FileExists(t, archivePath, "archive not created") - err = replaceTarGz(toolsDir, archivePath, []string{"tsh"}) + err = replaceTarGz(toolsDir, archivePath, []string{"tsh", "tctl"}) require.NoError(t, err) assert.FileExists(t, filepath.Join(toolsDir, "tsh"), "script not created") + assert.FileExists(t, filepath.Join(toolsDir, "tctl"), "script not created") data, err := os.ReadFile(filepath.Join(toolsDir, "tsh")) require.NoError(t, err) @@ -68,13 +72,14 @@ func TestPackaging(t *testing.T) { t.Skip("unsupported platform") } archivePath := filepath.Join(toolsDir, "tsh.pkg") - err = GeneratePkgFile(sourceDir, archivePath, "com.example.pkgtest") + err = utils.CompressDirToPkgFile(sourceDir, archivePath, "com.example.pkgtest") require.NoError(t, err) require.FileExists(t, archivePath, "archive not created") - err = replacePkg(toolsDir, archivePath, "hash", []string{"tsh"}) + err = replacePkg(toolsDir, archivePath, "hash", []string{"tsh", "tctl"}) require.NoError(t, err) assert.FileExists(t, filepath.Join(toolsDir, "tsh"), "script not created") + assert.FileExists(t, filepath.Join(toolsDir, "tctl"), "script not created") data, err := os.ReadFile(filepath.Join(toolsDir, "tsh")) require.NoError(t, err) @@ -83,13 +88,14 @@ func TestPackaging(t *testing.T) { t.Run("zip", func(t *testing.T) { archivePath := filepath.Join(toolsDir, "tsh.zip") - err = GenerateZipFile(sourceDir, archivePath) + err = utils.CompressDirToZipFile(sourceDir, archivePath) require.NoError(t, err) require.FileExists(t, archivePath, "archive not created") - err = replaceZip(toolsDir, archivePath, "hash", []string{"tsh"}) + err = replaceZip(toolsDir, archivePath, "hash", []string{"tsh", "tctl"}) require.NoError(t, err) assert.FileExists(t, filepath.Join(toolsDir, "tsh"), "script not created") + assert.FileExists(t, filepath.Join(toolsDir, "tctl"), "script not created") data, err := os.ReadFile(filepath.Join(toolsDir, "tsh")) require.NoError(t, err) diff --git a/lib/utils/packaging/unarchive_unix.go b/lib/utils/packaging/unarchive_unix.go index 1fdb5253e4c18..3059cf5572dac 100644 --- a/lib/utils/packaging/unarchive_unix.go +++ b/lib/utils/packaging/unarchive_unix.go @@ -36,7 +36,6 @@ import ( "github.com/google/renameio/v2" "github.com/gravitational/trace" - "golang.org/x/sys/unix" ) // Replace un-archives package from tools directory and replaces defined apps by symlinks. @@ -55,12 +54,10 @@ func replaceTarGz(toolsDir string, archivePath string, apps []string) error { if err != nil { return trace.Wrap(err) } - addCloser, wrapClose := multiCloser() - addCloser(f.Close) gzipReader, err := gzip.NewReader(f) if err != nil { - return wrapClose(err) + return trace.NewAggregate(err, f.Close()) } tarReader := tar.NewReader(gzipReader) @@ -78,31 +75,32 @@ func replaceTarGz(toolsDir string, archivePath string, apps []string) error { } continue } - // Verify that we have enough space for uncompressed file. if err := checkFreeSpace(tempDir, uint64(header.Size)); err != nil { - return wrapClose(err) + return trace.NewAggregate(err, gzipReader.Close(), f.Close()) } dest := filepath.Join(toolsDir, strings.TrimPrefix(header.Name, "teleport/")) t, err := renameio.TempFile(tempDir, dest) if err != nil { - return wrapClose(err) + return trace.Wrap(err) } if err := os.Chmod(t.Name(), 0o755); err != nil { - return wrapClose(err) + return trace.NewAggregate(err, t.Cleanup(), gzipReader.Close(), f.Close()) } - addCloser(t.Cleanup) if _, err := io.Copy(t, tarReader); err != nil { - return wrapClose(err) + return trace.NewAggregate(err, t.Cleanup(), gzipReader.Close(), f.Close()) } if err := t.CloseAtomicallyReplace(); err != nil { - return wrapClose(err) + return trace.NewAggregate(err, t.Cleanup(), gzipReader.Close(), f.Close()) + } + if err := t.Cleanup(); err != nil { + return trace.Wrap(err) } } - return wrapClose() + return trace.NewAggregate(gzipReader.Close(), f.Close()) } func replacePkg(toolsDir string, archivePath string, hash string, apps []string) error { @@ -166,20 +164,3 @@ func replacePkg(toolsDir string, archivePath string, hash string, apps []string) return trace.Wrap(err) } - -// freeDiskWithReserve returns the available disk space. -func freeDiskWithReserve(dir string) (uint64, error) { - var stat unix.Statfs_t - err := unix.Statfs(dir, &stat) - if err != nil { - return 0, trace.Wrap(err) - } - if stat.Bsize < 0 { - return 0, trace.Errorf("invalid size") - } - avail := stat.Bavail * uint64(stat.Bsize) - if reservedFreeDisk > avail { - return 0, trace.Errorf("no free space left") - } - return avail - reservedFreeDisk, nil -} diff --git a/lib/utils/packaging/unarchive_windows.go b/lib/utils/packaging/unarchive_windows.go index c51db8abdec4c..6060852e6ceef 100644 --- a/lib/utils/packaging/unarchive_windows.go +++ b/lib/utils/packaging/unarchive_windows.go @@ -20,25 +20,7 @@ package packaging -import ( - "github.com/gravitational/trace" - "golang.org/x/sys/windows" -) - // Replace un-archives package from tools directory and replaces defined apps by symlinks. func Replace(toolsDir string, archivePath string, hash string, apps []string) error { return replaceZip(toolsDir, archivePath, hash, apps) } - -// freeDiskWithReserve returns the available disk space. -func freeDiskWithReserve(dir string) (uint64, error) { - var avail uint64 - err := windows.GetDiskFreeSpaceEx(windows.StringToUTF16Ptr(dir), &avail, nil, nil) - if err != nil { - return 0, trace.Wrap(err) - } - if reservedFreeDisk > avail { - return 0, trace.Errorf("no free space left") - } - return avail - reservedFreeDisk, nil -} From 7892864e84688dbaf24ba054b9c4751b47b13adf Mon Sep 17 00:00:00 2001 From: Vadym Popov Date: Thu, 3 Oct 2024 19:05:33 -0400 Subject: [PATCH 04/13] Move archive helpers to integration/helper CR changes --- integration/helpers/archive.go | 139 +++++++++++++++++++++++ lib/utils/archive.go | 87 -------------- lib/utils/packaging/unarchive.go | 37 +++--- lib/utils/packaging/unarchive_test.go | 8 +- lib/utils/packaging/unarchive_unix.go | 22 ++-- lib/utils/packaging/unarchive_windows.go | 4 +- 6 files changed, 178 insertions(+), 119 deletions(-) create mode 100644 integration/helpers/archive.go diff --git a/integration/helpers/archive.go b/integration/helpers/archive.go new file mode 100644 index 0000000000000..25978ae3dec44 --- /dev/null +++ b/integration/helpers/archive.go @@ -0,0 +1,139 @@ +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package helpers + +import ( + "archive/tar" + "archive/zip" + "compress/gzip" + "io" + "os" + "os/exec" + "path/filepath" + "runtime" + + "github.com/gravitational/teleport" + "github.com/gravitational/trace" +) + +// CompressDirToZipFile compresses directory into a `.zip` format. +func CompressDirToZipFile(sourcePath, destPath string) error { + archive, err := os.Create(destPath) + if err != nil { + return trace.Wrap(err) + } + + zipWriter := zip.NewWriter(archive) + err = filepath.Walk(sourcePath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return trace.Wrap(err) + } + if info.IsDir() { + return nil + } + file, err := os.Open(path) + if err != nil { + return trace.Wrap(err) + } + zipFileWriter, err := zipWriter.Create(filepath.Base(path)) + if err != nil { + _ = file.Close() + return trace.Wrap(err) + } + if _, err = io.Copy(zipFileWriter, file); err != nil { + _ = file.Close() + return trace.Wrap(err) + } + return trace.Wrap(file.Close()) + }) + if err != nil { + _ = archive.Close() + return trace.Wrap(err) + } + if err := zipWriter.Close(); err != nil { + return trace.Wrap(err) + } + + return trace.Wrap(archive.Close()) +} + +// CompressDirToTarGzFile compresses directory into a `.tar.gz` format. +func CompressDirToTarGzFile(sourcePath, destPath string) error { + archive, err := os.Create(destPath) + if err != nil { + return trace.Wrap(err) + } + gzipWriter := gzip.NewWriter(archive) + tarWriter := tar.NewWriter(gzipWriter) + err = filepath.Walk(sourcePath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + file, err := os.Open(path) + if err != nil { + return err + } + header, err := tar.FileInfoHeader(info, info.Name()) + if err != nil { + _ = file.Close() + return trace.Wrap(err) + } + if err := tarWriter.WriteHeader(header); err != nil { + _ = file.Close() + return trace.NewAggregate(err, file.Close()) + } + if _, err = io.Copy(tarWriter, file); err != nil { + _ = file.Close() + return trace.Wrap(err) + } + return trace.Wrap(file.Close()) + }) + if err != nil { + _ = archive.Close() + return trace.Wrap(err) + } + if err := tarWriter.Close(); err != nil { + _ = archive.Close() + return trace.Wrap(err) + } + if err := gzipWriter.Close(); err != nil { + _ = archive.Close() + return trace.Wrap(err) + } + + return trace.Wrap(archive.Close()) +} + +// CompressDirToPkgFile runs for the macOS `pkgbuild` command to generate a .pkg file from the source. +func CompressDirToPkgFile(sourcePath, destPath, identifier string) error { + if runtime.GOOS != "darwin" { + return trace.BadParameter("only darwin packaging is supported") + } + cmd := exec.Command("pkgbuild", + "--root", sourcePath, + "--identifier", identifier, + "--version", teleport.Version, + destPath, + ) + + return trace.Wrap(cmd.Run()) +} diff --git a/lib/utils/archive.go b/lib/utils/archive.go index 760044b15912e..abab32e11dbc0 100644 --- a/lib/utils/archive.go +++ b/lib/utils/archive.go @@ -20,17 +20,10 @@ package utils import ( "archive/tar" - "archive/zip" "bytes" "compress/gzip" - "io" "io/fs" - "os" - "os/exec" - "path/filepath" - "runtime" - "github.com/gravitational/teleport" "github.com/gravitational/trace" ) @@ -82,83 +75,3 @@ func CompressTarGzArchive(files []string, fileReader ReadStatFS) (*bytes.Buffer, return archiveBytes, nil } - -// CompressDirToZipFile compresses directory into a `.zip` format. -func CompressDirToZipFile(sourcePath, destPath string) error { - archive, err := os.Create(destPath) - if err != nil { - return trace.Wrap(err) - } - - zipWriter := zip.NewWriter(archive) - err = filepath.Walk(sourcePath, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if info.IsDir() { - return nil - } - file, err := os.Open(path) - if err != nil { - return err - } - zipFileWriter, err := zipWriter.Create(filepath.Base(path)) - if err != nil { - return trace.NewAggregate(err, file.Close()) - } - - _, err = io.Copy(zipFileWriter, file) - return trace.NewAggregate(err, file.Close()) - }) - - return trace.NewAggregate(err, zipWriter.Close(), archive.Close()) -} - -// CompressDirToTarGzFile compresses directory into a `.tar.gz` format. -func CompressDirToTarGzFile(sourcePath, destPath string) error { - archive, err := os.Create(destPath) - if err != nil { - return trace.Wrap(err) - } - gzipWriter := gzip.NewWriter(archive) - tarWriter := tar.NewWriter(gzipWriter) - err = filepath.Walk(sourcePath, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if info.IsDir() { - return nil - } - file, err := os.Open(path) - if err != nil { - return err - } - header, err := tar.FileInfoHeader(info, info.Name()) - if err != nil { - return trace.NewAggregate(err, file.Close()) - } - if err := tarWriter.WriteHeader(header); err != nil { - return trace.NewAggregate(err, file.Close()) - } - - _, err = io.Copy(tarWriter, file) - return trace.NewAggregate(err, file.Close()) - }) - - return trace.NewAggregate(err, tarWriter.Close(), gzipWriter.Close(), archive.Close()) -} - -// CompressDirToPkgFile runs for the macOS `pkgbuild` command to generate a .pkg file from the source. -func CompressDirToPkgFile(sourcePath, destPath, identifier string) error { - if runtime.GOOS != "darwin" { - return trace.BadParameter("only darwin packaging is supported") - } - cmd := exec.Command("pkgbuild", - "--root", sourcePath, - "--identifier", identifier, - "--version", teleport.Version, - destPath, - ) - - return trace.Wrap(cmd.Run()) -} diff --git a/lib/utils/packaging/unarchive.go b/lib/utils/packaging/unarchive.go index 79faf2cd60224..4d731ee021501 100644 --- a/lib/utils/packaging/unarchive.go +++ b/lib/utils/packaging/unarchive.go @@ -35,23 +35,23 @@ const ( reservedFreeDisk = 10 * 1024 * 1024 ) -// Cleanup performs a cleanup pass to remove any old copies of apps. -func Cleanup(toolsDir, skipFileName, skipSuffix string) error { - err := filepath.Walk(toolsDir, func(path string, info os.FileInfo, err error) error { +// RemoveWithSuffix removes all files in dir that have the provided suffix, except for files named `skipName` +func RemoveWithSuffix(dir, suffix, skipName string) error { + err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { if err != nil { return trace.Wrap(err) } if !info.IsDir() { return nil } - if skipFileName == info.Name() { + if skipName == info.Name() { return nil } - if !strings.HasSuffix(info.Name(), skipSuffix) { + if !strings.HasSuffix(info.Name(), suffix) { return nil } // Found a stale expanded package. - if err := os.RemoveAll(filepath.Join(toolsDir, info.Name())); err != nil { + if err := os.RemoveAll(path); err != nil { return trace.Wrap(err) } return nil @@ -64,18 +64,19 @@ func replaceZip(toolsDir string, archivePath string, hash string, apps []string) if err != nil { return trace.Wrap(err) } + defer f.Close() fi, err := f.Stat() if err != nil { - return trace.NewAggregate(err, f.Close()) + return trace.Wrap(err) } zipReader, err := zip.NewReader(f, fi.Size()) if err != nil { - return trace.NewAggregate(err, f.Close()) + return trace.Wrap(err) } tempDir, err := os.MkdirTemp(toolsDir, hash) if err != nil { - return trace.NewAggregate(err, f.Close()) + return trace.Wrap(err) } for _, zipFile := range zipReader.File { @@ -94,29 +95,33 @@ func replaceZip(toolsDir string, archivePath string, hash string, apps []string) if err != nil { return trace.Wrap(err) } + defer file.Close() dest := filepath.Join(tempDir, zipFile.Name) destFile, err := os.OpenFile(dest, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0755) if err != nil { - return trace.NewAggregate(err, file.Close(), f.Close()) + return trace.Wrap(err) } if _, err := io.Copy(destFile, file); err != nil { - return trace.NewAggregate(err, destFile.Close(), file.Close(), f.Close()) + _ = destFile.Close() + return trace.Wrap(err) } appPath := filepath.Join(toolsDir, zipFile.Name) if err := os.Remove(appPath); err != nil && !os.IsNotExist(err) { - return trace.NewAggregate(err, destFile.Close(), file.Close(), f.Close()) + _ = destFile.Close() + return trace.Wrap(err) } if err := os.Symlink(dest, appPath); err != nil { - return trace.NewAggregate(err, destFile.Close(), file.Close(), f.Close()) + _ = destFile.Close() + return trace.Wrap(err) } - if err := trace.NewAggregate(destFile.Close(), file.Close()); err != nil { - return trace.NewAggregate(err, f.Close()) + if err := destFile.Close(); err != nil { + return trace.Wrap(err) } } - return trace.Wrap(f.Close()) + return nil } // checkFreeSpace verifies that we have enough requested space at specific directory. diff --git a/lib/utils/packaging/unarchive_test.go b/lib/utils/packaging/unarchive_test.go index b4bf298bc1cac..6efe2b883deed 100644 --- a/lib/utils/packaging/unarchive_test.go +++ b/lib/utils/packaging/unarchive_test.go @@ -29,7 +29,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/gravitational/teleport/lib/utils" + "github.com/gravitational/teleport/integration/helpers" ) // TestPackaging verifies unarchiving of all supported teleport package formats. @@ -53,7 +53,7 @@ func TestPackaging(t *testing.T) { t.Run("tar.gz", func(t *testing.T) { archivePath := filepath.Join(toolsDir, "tsh.tar.gz") - err = utils.CompressDirToTarGzFile(sourceDir, archivePath) + err = helpers.CompressDirToTarGzFile(sourceDir, archivePath) require.NoError(t, err) require.FileExists(t, archivePath, "archive not created") @@ -72,7 +72,7 @@ func TestPackaging(t *testing.T) { t.Skip("unsupported platform") } archivePath := filepath.Join(toolsDir, "tsh.pkg") - err = utils.CompressDirToPkgFile(sourceDir, archivePath, "com.example.pkgtest") + err = helpers.CompressDirToPkgFile(sourceDir, archivePath, "com.example.pkgtest") require.NoError(t, err) require.FileExists(t, archivePath, "archive not created") @@ -88,7 +88,7 @@ func TestPackaging(t *testing.T) { t.Run("zip", func(t *testing.T) { archivePath := filepath.Join(toolsDir, "tsh.zip") - err = utils.CompressDirToZipFile(sourceDir, archivePath) + err = helpers.CompressDirToZipFile(sourceDir, archivePath) require.NoError(t, err) require.FileExists(t, archivePath, "archive not created") diff --git a/lib/utils/packaging/unarchive_unix.go b/lib/utils/packaging/unarchive_unix.go index 3059cf5572dac..64eca3cbb9e4c 100644 --- a/lib/utils/packaging/unarchive_unix.go +++ b/lib/utils/packaging/unarchive_unix.go @@ -38,8 +38,8 @@ import ( "github.com/gravitational/trace" ) -// Replace un-archives package from tools directory and replaces defined apps by symlinks. -func Replace(toolsDir string, archivePath string, hash string, apps []string) error { +// ReplaceToolsBinaries un-archives package from tools directory and replaces defined apps by symlinks. +func ReplaceToolsBinaries(toolsDir string, archivePath string, hash string, apps []string) error { switch runtime.GOOS { case "darwin": return replacePkg(toolsDir, archivePath, hash, apps) @@ -54,12 +54,12 @@ func replaceTarGz(toolsDir string, archivePath string, apps []string) error { if err != nil { return trace.Wrap(err) } + defer f.Close() gzipReader, err := gzip.NewReader(f) if err != nil { - return trace.NewAggregate(err, f.Close()) + return trace.Wrap(err) } - tarReader := tar.NewReader(gzipReader) for { header, err := tarReader.Next() @@ -77,7 +77,7 @@ func replaceTarGz(toolsDir string, archivePath string, apps []string) error { } // Verify that we have enough space for uncompressed file. if err := checkFreeSpace(tempDir, uint64(header.Size)); err != nil { - return trace.NewAggregate(err, gzipReader.Close(), f.Close()) + return trace.Wrap(err) } dest := filepath.Join(toolsDir, strings.TrimPrefix(header.Name, "teleport/")) @@ -86,21 +86,23 @@ func replaceTarGz(toolsDir string, archivePath string, apps []string) error { return trace.Wrap(err) } if err := os.Chmod(t.Name(), 0o755); err != nil { - return trace.NewAggregate(err, t.Cleanup(), gzipReader.Close(), f.Close()) + _ = t.Cleanup() + return trace.Wrap(err) } - if _, err := io.Copy(t, tarReader); err != nil { - return trace.NewAggregate(err, t.Cleanup(), gzipReader.Close(), f.Close()) + _ = t.Cleanup() + return trace.Wrap(err) } if err := t.CloseAtomicallyReplace(); err != nil { - return trace.NewAggregate(err, t.Cleanup(), gzipReader.Close(), f.Close()) + _ = t.Cleanup() + return trace.Wrap(err) } if err := t.Cleanup(); err != nil { return trace.Wrap(err) } } - return trace.NewAggregate(gzipReader.Close(), f.Close()) + return trace.Wrap(gzipReader.Close()) } func replacePkg(toolsDir string, archivePath string, hash string, apps []string) error { diff --git a/lib/utils/packaging/unarchive_windows.go b/lib/utils/packaging/unarchive_windows.go index 6060852e6ceef..cd6654db8c793 100644 --- a/lib/utils/packaging/unarchive_windows.go +++ b/lib/utils/packaging/unarchive_windows.go @@ -20,7 +20,7 @@ package packaging -// Replace un-archives package from tools directory and replaces defined apps by symlinks. -func Replace(toolsDir string, archivePath string, hash string, apps []string) error { +// ReplaceToolsBinaries un-archives package from tools directory and replaces defined apps by symlinks. +func ReplaceToolsBinaries(toolsDir string, archivePath string, hash string, apps []string) error { return replaceZip(toolsDir, archivePath, hash, apps) } From b9f8db6f8b5edb46fccb761ea353b8b8be63d2c2 Mon Sep 17 00:00:00 2001 From: Vadym Popov Date: Thu, 3 Oct 2024 19:36:43 -0400 Subject: [PATCH 05/13] CR changes --- integration/helpers/archive.go | 10 ++++------ lib/utils/packaging/unarchive.go | 7 ++++--- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/integration/helpers/archive.go b/integration/helpers/archive.go index 25978ae3dec44..2a9512f86976b 100644 --- a/integration/helpers/archive.go +++ b/integration/helpers/archive.go @@ -38,6 +38,7 @@ func CompressDirToZipFile(sourcePath, destPath string) error { if err != nil { return trace.Wrap(err) } + defer archive.Close() zipWriter := zip.NewWriter(archive) err = filepath.Walk(sourcePath, func(path string, info os.FileInfo, err error) error { @@ -63,14 +64,13 @@ func CompressDirToZipFile(sourcePath, destPath string) error { return trace.Wrap(file.Close()) }) if err != nil { - _ = archive.Close() return trace.Wrap(err) } if err := zipWriter.Close(); err != nil { return trace.Wrap(err) } - return trace.Wrap(archive.Close()) + return nil } // CompressDirToTarGzFile compresses directory into a `.tar.gz` format. @@ -79,6 +79,7 @@ func CompressDirToTarGzFile(sourcePath, destPath string) error { if err != nil { return trace.Wrap(err) } + defer archive.Close() gzipWriter := gzip.NewWriter(archive) tarWriter := tar.NewWriter(gzipWriter) err = filepath.Walk(sourcePath, func(path string, info os.FileInfo, err error) error { @@ -108,19 +109,16 @@ func CompressDirToTarGzFile(sourcePath, destPath string) error { return trace.Wrap(file.Close()) }) if err != nil { - _ = archive.Close() return trace.Wrap(err) } if err := tarWriter.Close(); err != nil { - _ = archive.Close() return trace.Wrap(err) } if err := gzipWriter.Close(); err != nil { - _ = archive.Close() return trace.Wrap(err) } - return trace.Wrap(archive.Close()) + return nil } // CompressDirToPkgFile runs for the macOS `pkgbuild` command to generate a .pkg file from the source. diff --git a/lib/utils/packaging/unarchive.go b/lib/utils/packaging/unarchive.go index 4d731ee021501..683faf22b61b4 100644 --- a/lib/utils/packaging/unarchive.go +++ b/lib/utils/packaging/unarchive.go @@ -102,23 +102,24 @@ func replaceZip(toolsDir string, archivePath string, hash string, apps []string) if err != nil { return trace.Wrap(err) } + defer destFile.Close() if _, err := io.Copy(destFile, file); err != nil { - _ = destFile.Close() return trace.Wrap(err) } appPath := filepath.Join(toolsDir, zipFile.Name) if err := os.Remove(appPath); err != nil && !os.IsNotExist(err) { - _ = destFile.Close() return trace.Wrap(err) } if err := os.Symlink(dest, appPath); err != nil { - _ = destFile.Close() return trace.Wrap(err) } if err := destFile.Close(); err != nil { return trace.Wrap(err) } + if err := file.Close(); err != nil { + return trace.Wrap(err) + } } return nil From ea036ae83820d899ebb2b36049694c141465dd89 Mon Sep 17 00:00:00 2001 From: Vadym Popov Date: Fri, 4 Oct 2024 02:27:03 -0400 Subject: [PATCH 06/13] CR changes --- integration/helpers/archive.go | 35 ++++++++++++++++++++-------------- lib/utils/disk.go | 35 +++++++++++++++++----------------- lib/utils/disk_windows.go | 12 ++++++------ 3 files changed, 44 insertions(+), 38 deletions(-) diff --git a/integration/helpers/archive.go b/integration/helpers/archive.go index 2a9512f86976b..06cddaa3ec6e5 100644 --- a/integration/helpers/archive.go +++ b/integration/helpers/archive.go @@ -33,12 +33,17 @@ import ( ) // CompressDirToZipFile compresses directory into a `.zip` format. -func CompressDirToZipFile(sourcePath, destPath string) error { +func CompressDirToZipFile(sourcePath, destPath string) (err error) { archive, err := os.Create(destPath) if err != nil { return trace.Wrap(err) } - defer archive.Close() + defer func() { + _ = archive.Close() + if err != nil { + _ = os.Remove(destPath) + } + }() zipWriter := zip.NewWriter(archive) err = filepath.Walk(sourcePath, func(path string, info os.FileInfo, err error) error { @@ -52,13 +57,12 @@ func CompressDirToZipFile(sourcePath, destPath string) error { if err != nil { return trace.Wrap(err) } + defer file.Close() zipFileWriter, err := zipWriter.Create(filepath.Base(path)) if err != nil { - _ = file.Close() return trace.Wrap(err) } if _, err = io.Copy(zipFileWriter, file); err != nil { - _ = file.Close() return trace.Wrap(err) } return trace.Wrap(file.Close()) @@ -66,7 +70,7 @@ func CompressDirToZipFile(sourcePath, destPath string) error { if err != nil { return trace.Wrap(err) } - if err := zipWriter.Close(); err != nil { + if err = zipWriter.Close(); err != nil { return trace.Wrap(err) } @@ -74,12 +78,17 @@ func CompressDirToZipFile(sourcePath, destPath string) error { } // CompressDirToTarGzFile compresses directory into a `.tar.gz` format. -func CompressDirToTarGzFile(sourcePath, destPath string) error { +func CompressDirToTarGzFile(sourcePath, destPath string) (err error) { archive, err := os.Create(destPath) if err != nil { return trace.Wrap(err) } - defer archive.Close() + defer func() { + _ = archive.Close() + if err != nil { + _ = os.Remove(destPath) + } + }() gzipWriter := gzip.NewWriter(archive) tarWriter := tar.NewWriter(gzipWriter) err = filepath.Walk(sourcePath, func(path string, info os.FileInfo, err error) error { @@ -93,17 +102,15 @@ func CompressDirToTarGzFile(sourcePath, destPath string) error { if err != nil { return err } + defer file.Close() header, err := tar.FileInfoHeader(info, info.Name()) if err != nil { - _ = file.Close() return trace.Wrap(err) } if err := tarWriter.WriteHeader(header); err != nil { - _ = file.Close() - return trace.NewAggregate(err, file.Close()) + return trace.Wrap(err) } if _, err = io.Copy(tarWriter, file); err != nil { - _ = file.Close() return trace.Wrap(err) } return trace.Wrap(file.Close()) @@ -111,10 +118,10 @@ func CompressDirToTarGzFile(sourcePath, destPath string) error { if err != nil { return trace.Wrap(err) } - if err := tarWriter.Close(); err != nil { + if err = tarWriter.Close(); err != nil { return trace.Wrap(err) } - if err := gzipWriter.Close(); err != nil { + if err = gzipWriter.Close(); err != nil { return trace.Wrap(err) } @@ -124,7 +131,7 @@ func CompressDirToTarGzFile(sourcePath, destPath string) error { // CompressDirToPkgFile runs for the macOS `pkgbuild` command to generate a .pkg file from the source. func CompressDirToPkgFile(sourcePath, destPath, identifier string) error { if runtime.GOOS != "darwin" { - return trace.BadParameter("only darwin packaging is supported") + return trace.BadParameter("only darwin platform is supported for pkg file") } cmd := exec.Command("pkgbuild", "--root", sourcePath, diff --git a/lib/utils/disk.go b/lib/utils/disk.go index 9c8552051ff3e..65172add5146c 100644 --- a/lib/utils/disk.go +++ b/lib/utils/disk.go @@ -31,7 +31,6 @@ import ( "syscall" "github.com/gravitational/trace" - "golang.org/x/sys/unix" ) // PercentUsed returns percentage of disk space used. The percentage of disk @@ -47,6 +46,23 @@ func PercentUsed(path string) (float64, error) { return Round(ratio * 100), nil } +// FreeDiskWithReserve returns the available disk space on the disk at dir, minus `reservedFreeDisk`. +func FreeDiskWithReserve(dir string, reservedFreeDisk uint64) (uint64, error) { + var stat syscall.Statfs_t + err := syscall.Statfs(dir, &stat) + if err != nil { + return 0, trace.Wrap(err) + } + if stat.Bsize < 0 { + return 0, trace.Errorf("invalid size") + } + avail := stat.Bavail * uint64(stat.Bsize) + if reservedFreeDisk > avail { + return 0, trace.Errorf("no free space left") + } + return avail - reservedFreeDisk, nil +} + // CanUserWriteTo attempts to check if a user has write access to certain path. // It also works around the program being run as root and tries to check // the permissions of the user who executed the program as root. @@ -126,20 +142,3 @@ func CanUserWriteTo(path string) (bool, error) { return false, nil } - -// FreeDiskWithReserve returns the available disk space. -func FreeDiskWithReserve(dir string, reservedFreeDisk uint64) (uint64, error) { - var stat unix.Statfs_t - err := unix.Statfs(dir, &stat) - if err != nil { - return 0, trace.Wrap(err) - } - if stat.Bsize < 0 { - return 0, trace.Errorf("invalid size") - } - avail := stat.Bavail * uint64(stat.Bsize) - if reservedFreeDisk > avail { - return 0, trace.Errorf("no free space left") - } - return avail - reservedFreeDisk, nil -} diff --git a/lib/utils/disk_windows.go b/lib/utils/disk_windows.go index 79db4f46df0b0..85b7a135cc3f8 100644 --- a/lib/utils/disk_windows.go +++ b/lib/utils/disk_windows.go @@ -31,12 +31,7 @@ func PercentUsed(path string) (float64, error) { return 0.0, trace.NotImplemented("disk usage not supported on Windows") } -// CanUserWriteTo is not supported on Windows. -func CanUserWriteTo(path string) (bool, error) { - return false, trace.NotImplemented("path permission checking is not supported on Windows") -} - -// FreeDiskWithReserve returns the available disk space. +// FreeDiskWithReserve returns the available disk space on the disk at dir, minus `reservedFreeDisk`. func FreeDiskWithReserve(dir string, reservedFreeDisk uint64) (uint64, error) { var avail uint64 err := windows.GetDiskFreeSpaceEx(windows.StringToUTF16Ptr(dir), &avail, nil, nil) @@ -48,3 +43,8 @@ func FreeDiskWithReserve(dir string, reservedFreeDisk uint64) (uint64, error) { } return avail - reservedFreeDisk, nil } + +// CanUserWriteTo is not supported on Windows. +func CanUserWriteTo(path string) (bool, error) { + return false, trace.NotImplemented("path permission checking is not supported on Windows") +} From 0c7673884cffb9c55e0f755e406215225fba0531 Mon Sep 17 00:00:00 2001 From: Vadym Popov Date: Mon, 7 Oct 2024 17:33:36 -0400 Subject: [PATCH 07/13] CR changes Replace creating directory with extract path as argument --- integration/helpers/archive.go | 16 +++++-- lib/utils/packaging/unarchive.go | 57 +++++++++++------------- lib/utils/packaging/unarchive_test.go | 12 +++-- lib/utils/packaging/unarchive_unix.go | 57 ++++++++++++------------ lib/utils/packaging/unarchive_windows.go | 7 +-- 5 files changed, 78 insertions(+), 71 deletions(-) diff --git a/integration/helpers/archive.go b/integration/helpers/archive.go index 06cddaa3ec6e5..e89db12362f55 100644 --- a/integration/helpers/archive.go +++ b/integration/helpers/archive.go @@ -22,7 +22,9 @@ import ( "archive/tar" "archive/zip" "compress/gzip" + "context" "io" + "log/slog" "os" "os/exec" "path/filepath" @@ -32,7 +34,8 @@ import ( "github.com/gravitational/trace" ) -// CompressDirToZipFile compresses directory into a `.zip` format. +// CompressDirToZipFile compresses a directory into `.zip` format, +// preserving the relative file path structure of the source directory. func CompressDirToZipFile(sourcePath, destPath string) (err error) { archive, err := os.Create(destPath) if err != nil { @@ -41,7 +44,9 @@ func CompressDirToZipFile(sourcePath, destPath string) (err error) { defer func() { _ = archive.Close() if err != nil { - _ = os.Remove(destPath) + if err := os.Remove(destPath); err != nil { + slog.ErrorContext(context.Background(), "failed to remove archive", "error", err) + } } }() @@ -77,7 +82,8 @@ func CompressDirToZipFile(sourcePath, destPath string) (err error) { return nil } -// CompressDirToTarGzFile compresses directory into a `.tar.gz` format. +// CompressDirToTarGzFile compresses a directory into .tar.gz format, +// preserving the relative file path structure of the source directory. func CompressDirToTarGzFile(sourcePath, destPath string) (err error) { archive, err := os.Create(destPath) if err != nil { @@ -86,7 +92,9 @@ func CompressDirToTarGzFile(sourcePath, destPath string) (err error) { defer func() { _ = archive.Close() if err != nil { - _ = os.Remove(destPath) + if err := os.Remove(destPath); err != nil { + slog.ErrorContext(context.Background(), "failed to remove archive", "error", err) + } } }() gzipWriter := gzip.NewWriter(archive) diff --git a/lib/utils/packaging/unarchive.go b/lib/utils/packaging/unarchive.go index 683faf22b61b4..ddf273f8032cd 100644 --- a/lib/utils/packaging/unarchive.go +++ b/lib/utils/packaging/unarchive.go @@ -59,7 +59,7 @@ func RemoveWithSuffix(dir, suffix, skipName string) error { return trace.Wrap(err) } -func replaceZip(toolsDir string, archivePath string, hash string, apps []string) error { +func replaceZip(toolsDir string, archivePath string, extractDir string, apps []string) error { f, err := os.Open(archivePath) if err != nil { return trace.Wrap(err) @@ -74,10 +74,6 @@ func replaceZip(toolsDir string, archivePath string, hash string, apps []string) if err != nil { return trace.Wrap(err) } - tempDir, err := os.MkdirTemp(toolsDir, hash) - if err != nil { - return trace.Wrap(err) - } for _, zipFile := range zipReader.File { // Skip over any files in the archive that are not defined apps. @@ -87,37 +83,36 @@ func replaceZip(toolsDir string, archivePath string, hash string, apps []string) continue } // Verify that we have enough space for uncompressed zipFile. - if err := checkFreeSpace(tempDir, zipFile.UncompressedSize64); err != nil { + if err := checkFreeSpace(extractDir, zipFile.UncompressedSize64); err != nil { return trace.NewAggregate(err, f.Close()) } - file, err := zipFile.Open() - if err != nil { - return trace.Wrap(err) - } - defer file.Close() + if err := func(zipFile *zip.File) error { + file, err := zipFile.Open() + if err != nil { + return trace.Wrap(err) + } + defer file.Close() - dest := filepath.Join(tempDir, zipFile.Name) - destFile, err := os.OpenFile(dest, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0755) - if err != nil { - return trace.Wrap(err) - } - defer destFile.Close() + dest := filepath.Join(extractDir, zipFile.Name) + destFile, err := os.OpenFile(dest, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0755) + if err != nil { + return trace.Wrap(err) + } + defer destFile.Close() - if _, err := io.Copy(destFile, file); err != nil { - return trace.Wrap(err) - } - appPath := filepath.Join(toolsDir, zipFile.Name) - if err := os.Remove(appPath); err != nil && !os.IsNotExist(err) { - return trace.Wrap(err) - } - if err := os.Symlink(dest, appPath); err != nil { - return trace.Wrap(err) - } - if err := destFile.Close(); err != nil { - return trace.Wrap(err) - } - if err := file.Close(); err != nil { + if _, err := io.Copy(destFile, file); err != nil { + return trace.Wrap(err) + } + appPath := filepath.Join(toolsDir, zipFile.Name) + if err := os.Remove(appPath); err != nil && !os.IsNotExist(err) { + return trace.Wrap(err) + } + if err := os.Symlink(dest, appPath); err != nil { + return trace.Wrap(err) + } + return nil + }(zipFile); err != nil { return trace.Wrap(err) } } diff --git a/lib/utils/packaging/unarchive_test.go b/lib/utils/packaging/unarchive_test.go index 6efe2b883deed..852da20048e7d 100644 --- a/lib/utils/packaging/unarchive_test.go +++ b/lib/utils/packaging/unarchive_test.go @@ -32,7 +32,7 @@ import ( "github.com/gravitational/teleport/integration/helpers" ) -// TestPackaging verifies unarchiving of all supported teleport package formats. +// TestPackaging verifies un-archiving of all supported teleport package formats. func TestPackaging(t *testing.T) { script := "#!/bin/sh\necho test" @@ -42,7 +42,11 @@ func TestPackaging(t *testing.T) { toolsDir, err := os.MkdirTemp(os.TempDir(), "dest") require.NoError(t, err) + extractDir, err := os.MkdirTemp(toolsDir, "extract") + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, os.RemoveAll(extractDir)) require.NoError(t, os.RemoveAll(sourceDir)) require.NoError(t, os.RemoveAll(toolsDir)) }) @@ -57,7 +61,7 @@ func TestPackaging(t *testing.T) { require.NoError(t, err) require.FileExists(t, archivePath, "archive not created") - err = replaceTarGz(toolsDir, archivePath, []string{"tsh", "tctl"}) + err = replaceTarGz(toolsDir, archivePath, extractDir, []string{"tsh", "tctl"}) require.NoError(t, err) assert.FileExists(t, filepath.Join(toolsDir, "tsh"), "script not created") assert.FileExists(t, filepath.Join(toolsDir, "tctl"), "script not created") @@ -76,7 +80,7 @@ func TestPackaging(t *testing.T) { require.NoError(t, err) require.FileExists(t, archivePath, "archive not created") - err = replacePkg(toolsDir, archivePath, "hash", []string{"tsh", "tctl"}) + err = replacePkg(toolsDir, archivePath, filepath.Join(extractDir, "apps"), []string{"tsh", "tctl"}) require.NoError(t, err) assert.FileExists(t, filepath.Join(toolsDir, "tsh"), "script not created") assert.FileExists(t, filepath.Join(toolsDir, "tctl"), "script not created") @@ -92,7 +96,7 @@ func TestPackaging(t *testing.T) { require.NoError(t, err) require.FileExists(t, archivePath, "archive not created") - err = replaceZip(toolsDir, archivePath, "hash", []string{"tsh", "tctl"}) + err = replaceZip(toolsDir, archivePath, extractDir, []string{"tsh", "tctl"}) require.NoError(t, err) assert.FileExists(t, filepath.Join(toolsDir, "tsh"), "script not created") assert.FileExists(t, filepath.Join(toolsDir, "tctl"), "script not created") diff --git a/lib/utils/packaging/unarchive_unix.go b/lib/utils/packaging/unarchive_unix.go index 64eca3cbb9e4c..c34038b4902ee 100644 --- a/lib/utils/packaging/unarchive_unix.go +++ b/lib/utils/packaging/unarchive_unix.go @@ -38,18 +38,18 @@ import ( "github.com/gravitational/trace" ) -// ReplaceToolsBinaries un-archives package from tools directory and replaces defined apps by symlinks. -func ReplaceToolsBinaries(toolsDir string, archivePath string, hash string, apps []string) error { +// ReplaceToolsBinaries extracts the package into `extractDir` and replaces the specified +// applications with symlinks in the tool's directory. +func ReplaceToolsBinaries(toolsDir string, archivePath string, extractDir string, apps []string) error { switch runtime.GOOS { case "darwin": - return replacePkg(toolsDir, archivePath, hash, apps) + return replacePkg(toolsDir, archivePath, extractDir, apps) default: - return replaceTarGz(toolsDir, archivePath, apps) + return replaceTarGz(toolsDir, archivePath, extractDir, apps) } } -func replaceTarGz(toolsDir string, archivePath string, apps []string) error { - tempDir := renameio.TempDir(toolsDir) +func replaceTarGz(toolsDir string, archivePath string, extractDir string, apps []string) error { f, err := os.Open(archivePath) if err != nil { return trace.Wrap(err) @@ -76,28 +76,28 @@ func replaceTarGz(toolsDir string, archivePath string, apps []string) error { continue } // Verify that we have enough space for uncompressed file. - if err := checkFreeSpace(tempDir, uint64(header.Size)); err != nil { + if err := checkFreeSpace(extractDir, uint64(header.Size)); err != nil { return trace.Wrap(err) } - dest := filepath.Join(toolsDir, strings.TrimPrefix(header.Name, "teleport/")) - t, err := renameio.TempFile(tempDir, dest) - if err != nil { - return trace.Wrap(err) - } - if err := os.Chmod(t.Name(), 0o755); err != nil { - _ = t.Cleanup() - return trace.Wrap(err) - } - if _, err := io.Copy(t, tarReader); err != nil { - _ = t.Cleanup() - return trace.Wrap(err) - } - if err := t.CloseAtomicallyReplace(); err != nil { - _ = t.Cleanup() - return trace.Wrap(err) - } - if err := t.Cleanup(); err != nil { + if err = func(header *tar.Header) error { + dest := filepath.Join(toolsDir, strings.TrimPrefix(header.Name, "teleport/")) + tempFile, err := renameio.TempFile(extractDir, dest) + if err != nil { + return trace.Wrap(err) + } + defer tempFile.Cleanup() + if err := os.Chmod(tempFile.Name(), 0o755); err != nil { + return trace.Wrap(err) + } + if _, err := io.Copy(tempFile, tarReader); err != nil { + return trace.Wrap(err) + } + if err := tempFile.CloseAtomicallyReplace(); err != nil { + return trace.Wrap(err) + } + return nil + }(header); err != nil { return trace.Wrap(err) } } @@ -105,7 +105,7 @@ func replaceTarGz(toolsDir string, archivePath string, apps []string) error { return trace.Wrap(gzipReader.Close()) } -func replacePkg(toolsDir string, archivePath string, hash string, apps []string) error { +func replacePkg(toolsDir string, archivePath string, extractDir string, apps []string) error { // Use "pkgutil" from the filesystem to expand the archive. In theory .pkg // files are xz archives, however it's still safer to use "pkgutil" in-case // Apple makes non-standard changes to the format. @@ -115,14 +115,13 @@ func replacePkg(toolsDir string, archivePath string, hash string, apps []string) if err != nil { return trace.Wrap(err) } - expandPath := filepath.Join(toolsDir, hash+"-pkg") - out, err := exec.Command(pkgutil, "--expand-full", archivePath, expandPath).Output() + out, err := exec.Command(pkgutil, "--expand-full", archivePath, extractDir).Output() if err != nil { slog.DebugContext(context.Background(), "failed to run pkgutil:", "output", out) return trace.Wrap(err) } - err = filepath.Walk(expandPath, func(path string, info os.FileInfo, err error) error { + err = filepath.Walk(extractDir, func(path string, info os.FileInfo, err error) error { if err != nil { return trace.Wrap(err) } diff --git a/lib/utils/packaging/unarchive_windows.go b/lib/utils/packaging/unarchive_windows.go index cd6654db8c793..602b8c473c009 100644 --- a/lib/utils/packaging/unarchive_windows.go +++ b/lib/utils/packaging/unarchive_windows.go @@ -20,7 +20,8 @@ package packaging -// ReplaceToolsBinaries un-archives package from tools directory and replaces defined apps by symlinks. -func ReplaceToolsBinaries(toolsDir string, archivePath string, hash string, apps []string) error { - return replaceZip(toolsDir, archivePath, hash, apps) +// ReplaceToolsBinaries extracts the package into `extractDir` and replaces the specified +// applications with symlinks in the tool's directory. +func ReplaceToolsBinaries(toolsDir string, archivePath string, extractPath string, apps []string) error { + return replaceZip(toolsDir, archivePath, extractPath, apps) } From deecf84c012787c4ee4a37522aa8acbb95b39cff Mon Sep 17 00:00:00 2001 From: Vadym Popov Date: Mon, 7 Oct 2024 21:18:08 -0400 Subject: [PATCH 08/13] CR changes --- integration/helpers/archive.go | 14 ++++++++------ lib/utils/packaging/unarchive.go | 12 ++++++++---- lib/utils/packaging/unarchive_test.go | 14 ++++++++++---- lib/utils/packaging/unarchive_unix.go | 20 +++++++++++--------- 4 files changed, 37 insertions(+), 23 deletions(-) diff --git a/integration/helpers/archive.go b/integration/helpers/archive.go index e89db12362f55..253fd835c9fc0 100644 --- a/integration/helpers/archive.go +++ b/integration/helpers/archive.go @@ -36,7 +36,7 @@ import ( // CompressDirToZipFile compresses a directory into `.zip` format, // preserving the relative file path structure of the source directory. -func CompressDirToZipFile(sourcePath, destPath string) (err error) { +func CompressDirToZipFile(ctx context.Context, sourcePath, destPath string) (err error) { archive, err := os.Create(destPath) if err != nil { return trace.Wrap(err) @@ -45,7 +45,7 @@ func CompressDirToZipFile(sourcePath, destPath string) (err error) { _ = archive.Close() if err != nil { if err := os.Remove(destPath); err != nil { - slog.ErrorContext(context.Background(), "failed to remove archive", "error", err) + slog.ErrorContext(ctx, "failed to remove archive", "error", err) } } }() @@ -84,7 +84,7 @@ func CompressDirToZipFile(sourcePath, destPath string) (err error) { // CompressDirToTarGzFile compresses a directory into .tar.gz format, // preserving the relative file path structure of the source directory. -func CompressDirToTarGzFile(sourcePath, destPath string) (err error) { +func CompressDirToTarGzFile(ctx context.Context, sourcePath, destPath string) (err error) { archive, err := os.Create(destPath) if err != nil { return trace.Wrap(err) @@ -93,7 +93,7 @@ func CompressDirToTarGzFile(sourcePath, destPath string) (err error) { _ = archive.Close() if err != nil { if err := os.Remove(destPath); err != nil { - slog.ErrorContext(context.Background(), "failed to remove archive", "error", err) + slog.ErrorContext(ctx, "failed to remove archive", "error", err) } } }() @@ -137,11 +137,13 @@ func CompressDirToTarGzFile(sourcePath, destPath string) (err error) { } // CompressDirToPkgFile runs for the macOS `pkgbuild` command to generate a .pkg file from the source. -func CompressDirToPkgFile(sourcePath, destPath, identifier string) error { +func CompressDirToPkgFile(ctx context.Context, sourcePath, destPath, identifier string) error { if runtime.GOOS != "darwin" { return trace.BadParameter("only darwin platform is supported for pkg file") } - cmd := exec.Command("pkgbuild", + cmd := exec.CommandContext( + ctx, + "pkgbuild", "--root", sourcePath, "--identifier", identifier, "--version", teleport.Version, diff --git a/lib/utils/packaging/unarchive.go b/lib/utils/packaging/unarchive.go index ddf273f8032cd..3b2ff0890b094 100644 --- a/lib/utils/packaging/unarchive.go +++ b/lib/utils/packaging/unarchive.go @@ -36,8 +36,8 @@ const ( ) // RemoveWithSuffix removes all files in dir that have the provided suffix, except for files named `skipName` -func RemoveWithSuffix(dir, suffix, skipName string) error { - err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { +func RemoveWithSuffix(extractDir, suffix, skipName string) error { + err := filepath.Walk(extractDir, func(path string, info os.FileInfo, err error) error { if err != nil { return trace.Wrap(err) } @@ -59,6 +59,10 @@ func RemoveWithSuffix(dir, suffix, skipName string) error { return trace.Wrap(err) } +// replaceZip un-archives the Teleport package in .zip format, iterates through +// the compressed content, and ignores everything not matching the app binaries specified +// in the apps argument. The data is extracted to extractDir, and symlinks are created +// in toolsDir pointing to the extractDir path with binaries. func replaceZip(toolsDir string, archivePath string, extractDir string, apps []string) error { f, err := os.Open(archivePath) if err != nil { @@ -84,7 +88,7 @@ func replaceZip(toolsDir string, archivePath string, extractDir string, apps []s } // Verify that we have enough space for uncompressed zipFile. if err := checkFreeSpace(extractDir, zipFile.UncompressedSize64); err != nil { - return trace.NewAggregate(err, f.Close()) + return trace.Wrap(err) } if err := func(zipFile *zip.File) error { @@ -111,7 +115,7 @@ func replaceZip(toolsDir string, archivePath string, extractDir string, apps []s if err := os.Symlink(dest, appPath); err != nil { return trace.Wrap(err) } - return nil + return trace.Wrap(destFile.Close()) }(zipFile); err != nil { return trace.Wrap(err) } diff --git a/lib/utils/packaging/unarchive_test.go b/lib/utils/packaging/unarchive_test.go index 852da20048e7d..e3be7346321f0 100644 --- a/lib/utils/packaging/unarchive_test.go +++ b/lib/utils/packaging/unarchive_test.go @@ -21,6 +21,7 @@ package packaging import ( + "context" "os" "path/filepath" "runtime" @@ -55,13 +56,18 @@ func TestPackaging(t *testing.T) { require.NoError(t, os.WriteFile(filepath.Join(sourceDir, "tsh"), []byte(script), 0o755)) require.NoError(t, os.WriteFile(filepath.Join(sourceDir, "tctl"), []byte(script), 0o755)) + ctx := context.Background() + t.Run("tar.gz", func(t *testing.T) { archivePath := filepath.Join(toolsDir, "tsh.tar.gz") - err = helpers.CompressDirToTarGzFile(sourceDir, archivePath) + err = helpers.CompressDirToTarGzFile(ctx, sourceDir, archivePath) require.NoError(t, err) require.FileExists(t, archivePath, "archive not created") - err = replaceTarGz(toolsDir, archivePath, extractDir, []string{"tsh", "tctl"}) + // For the .tar.gz format we extract app by app to check that content discard is not required. + err = replaceTarGz(toolsDir, archivePath, extractDir, []string{"tctl"}) + require.NoError(t, err) + err = replaceTarGz(toolsDir, archivePath, extractDir, []string{"tsh"}) require.NoError(t, err) assert.FileExists(t, filepath.Join(toolsDir, "tsh"), "script not created") assert.FileExists(t, filepath.Join(toolsDir, "tctl"), "script not created") @@ -76,7 +82,7 @@ func TestPackaging(t *testing.T) { t.Skip("unsupported platform") } archivePath := filepath.Join(toolsDir, "tsh.pkg") - err = helpers.CompressDirToPkgFile(sourceDir, archivePath, "com.example.pkgtest") + err = helpers.CompressDirToPkgFile(ctx, sourceDir, archivePath, "com.example.pkgtest") require.NoError(t, err) require.FileExists(t, archivePath, "archive not created") @@ -92,7 +98,7 @@ func TestPackaging(t *testing.T) { t.Run("zip", func(t *testing.T) { archivePath := filepath.Join(toolsDir, "tsh.zip") - err = helpers.CompressDirToZipFile(sourceDir, archivePath) + err = helpers.CompressDirToZipFile(ctx, sourceDir, archivePath) require.NoError(t, err) require.FileExists(t, archivePath, "archive not created") diff --git a/lib/utils/packaging/unarchive_unix.go b/lib/utils/packaging/unarchive_unix.go index c34038b4902ee..baada86afe872 100644 --- a/lib/utils/packaging/unarchive_unix.go +++ b/lib/utils/packaging/unarchive_unix.go @@ -23,10 +23,8 @@ package packaging import ( "archive/tar" "compress/gzip" - "context" "errors" "io" - "log/slog" "os" "os/exec" "path/filepath" @@ -49,6 +47,10 @@ func ReplaceToolsBinaries(toolsDir string, archivePath string, extractDir string } } +// replaceTarGz un-archives the Teleport package in .tar.gz format, iterates through +// the compressed content, and ignores everything not matching the app binaries specified +// in the apps argument. The data is extracted to extractDir, and symlinks are created +// in toolsDir pointing to the extractDir path with binaries. func replaceTarGz(toolsDir string, archivePath string, extractDir string, apps []string) error { f, err := os.Open(archivePath) if err != nil { @@ -70,9 +72,6 @@ func replaceTarGz(toolsDir string, archivePath string, extractDir string, apps [ if !slices.ContainsFunc(apps, func(s string) bool { return strings.HasSuffix(header.Name, s) }) { - if _, err := io.Copy(io.Discard, tarReader); err != nil { - slog.DebugContext(context.Background(), "failed to discard", "name", header.Name, "error", err) - } continue } // Verify that we have enough space for uncompressed file. @@ -96,7 +95,7 @@ func replaceTarGz(toolsDir string, archivePath string, extractDir string, apps [ if err := tempFile.CloseAtomicallyReplace(); err != nil { return trace.Wrap(err) } - return nil + return trace.Wrap(tempFile.Cleanup()) }(header); err != nil { return trace.Wrap(err) } @@ -105,6 +104,10 @@ func replaceTarGz(toolsDir string, archivePath string, extractDir string, apps [ return trace.Wrap(gzipReader.Close()) } +// replacePkg expands the Teleport package in .pkg format using the platform-dependent pkgutil utility. +// The data is extracted to extractDir, and symlinks are created in toolsDir pointing to the binaries +// in extractDir. Before creating the symlinks, each binary must be executed at least once to pass +// OS signature verification. func replacePkg(toolsDir string, archivePath string, extractDir string, apps []string) error { // Use "pkgutil" from the filesystem to expand the archive. In theory .pkg // files are xz archives, however it's still safer to use "pkgutil" in-case @@ -115,9 +118,8 @@ func replacePkg(toolsDir string, archivePath string, extractDir string, apps []s if err != nil { return trace.Wrap(err) } - out, err := exec.Command(pkgutil, "--expand-full", archivePath, extractDir).Output() - if err != nil { - slog.DebugContext(context.Background(), "failed to run pkgutil:", "output", out) + + if err = exec.Command(pkgutil, "--expand-full", archivePath, extractDir).Run(); err != nil { return trace.Wrap(err) } From 3a94b1dbe20f5a6bc294a3fc8aca84b1920d00ef Mon Sep 17 00:00:00 2001 From: Vadym Popov Date: Tue, 8 Oct 2024 15:58:11 -0400 Subject: [PATCH 09/13] Validate full size before un-archive Extract files to extractDir with ignore dir structure --- lib/utils/packaging/unarchive.go | 29 ++++++--- lib/utils/packaging/unarchive_unix.go | 78 ++++++++++++++++++------ lib/utils/packaging/unarchive_windows.go | 11 ++-- 3 files changed, 85 insertions(+), 33 deletions(-) diff --git a/lib/utils/packaging/unarchive.go b/lib/utils/packaging/unarchive.go index 3b2ff0890b094..bfe8f112b564b 100644 --- a/lib/utils/packaging/unarchive.go +++ b/lib/utils/packaging/unarchive.go @@ -60,10 +60,10 @@ func RemoveWithSuffix(extractDir, suffix, skipName string) error { } // replaceZip un-archives the Teleport package in .zip format, iterates through -// the compressed content, and ignores everything not matching the app binaries specified -// in the apps argument. The data is extracted to extractDir, and symlinks are created +// the compressed content, and ignores everything not matching the binaries specified +// in the execNames argument. The data is extracted to extractDir, and symlinks are created // in toolsDir pointing to the extractDir path with binaries. -func replaceZip(toolsDir string, archivePath string, extractDir string, apps []string) error { +func replaceZip(toolsDir string, archivePath string, extractDir string, execNames []string) error { f, err := os.Open(archivePath) if err != nil { return trace.Wrap(err) @@ -79,16 +79,27 @@ func replaceZip(toolsDir string, archivePath string, extractDir string, apps []s return trace.Wrap(err) } + var totalSize uint64 = 0 for _, zipFile := range zipReader.File { - // Skip over any files in the archive that are not defined apps. - if !slices.ContainsFunc(apps, func(s string) bool { - return strings.HasSuffix(zipFile.Name, s) + // Skip over any files in the archive that are not defined execNames. + if !slices.ContainsFunc(execNames, func(s string) bool { + return filepath.Base(zipFile.Name) == s }) { continue } - // Verify that we have enough space for uncompressed zipFile. - if err := checkFreeSpace(extractDir, zipFile.UncompressedSize64); err != nil { - return trace.Wrap(err) + totalSize += zipFile.UncompressedSize64 + } + // Verify that we have enough space for uncompressed zipFile. + if err := checkFreeSpace(extractDir, totalSize); err != nil { + return trace.Wrap(err) + } + + for _, zipFile := range zipReader.File { + // Skip over any files in the archive that are not defined execNames. + if !slices.ContainsFunc(execNames, func(s string) bool { + return filepath.Base(zipFile.Name) == s + }) { + continue } if err := func(zipFile *zip.File) error { diff --git a/lib/utils/packaging/unarchive_unix.go b/lib/utils/packaging/unarchive_unix.go index baada86afe872..50de040b1b115 100644 --- a/lib/utils/packaging/unarchive_unix.go +++ b/lib/utils/packaging/unarchive_unix.go @@ -30,20 +30,23 @@ import ( "path/filepath" "runtime" "slices" - "strings" "github.com/google/renameio/v2" "github.com/gravitational/trace" ) -// ReplaceToolsBinaries extracts the package into `extractDir` and replaces the specified -// applications with symlinks in the tool's directory. -func ReplaceToolsBinaries(toolsDir string, archivePath string, extractDir string, apps []string) error { +// ReplaceToolsBinaries extracts executables specified by execNames from archivePath into +// extractDir. After each executable is extracted, it is symlinked from extractDir/[name] to +// toolsDir/[name]. +// +// For Darwin, archivePath must be a .pkg file. +// For other POSIX, archivePath must be a gzipped tarball. +func ReplaceToolsBinaries(toolsDir string, archivePath string, extractDir string, execNames []string) error { switch runtime.GOOS { case "darwin": - return replacePkg(toolsDir, archivePath, extractDir, apps) + return replacePkg(toolsDir, archivePath, extractDir, execNames) default: - return replaceTarGz(toolsDir, archivePath, extractDir, apps) + return replaceTarGz(toolsDir, archivePath, extractDir, execNames) } } @@ -51,7 +54,10 @@ func ReplaceToolsBinaries(toolsDir string, archivePath string, extractDir string // the compressed content, and ignores everything not matching the app binaries specified // in the apps argument. The data is extracted to extractDir, and symlinks are created // in toolsDir pointing to the extractDir path with binaries. -func replaceTarGz(toolsDir string, archivePath string, extractDir string, apps []string) error { +func replaceTarGz(toolsDir string, archivePath string, extractDir string, execNames []string) error { + if err := validateFreeSpaceTarGz(archivePath, extractDir, execNames); err != nil { + return trace.Wrap(err) + } f, err := os.Open(archivePath) if err != nil { return trace.Wrap(err) @@ -68,20 +74,18 @@ func replaceTarGz(toolsDir string, archivePath string, extractDir string, apps [ if errors.Is(err, io.EOF) { break } - // Skip over any files in the archive that are not in apps. - if !slices.ContainsFunc(apps, func(s string) bool { - return strings.HasSuffix(header.Name, s) + if err != nil { + return trace.Wrap(err) + } + // Skip over any files in the archive that are not in execNames. + if !slices.ContainsFunc(execNames, func(s string) bool { + return filepath.Base(header.Name) == s }) { continue } - // Verify that we have enough space for uncompressed file. - if err := checkFreeSpace(extractDir, uint64(header.Size)); err != nil { - return trace.Wrap(err) - } if err = func(header *tar.Header) error { - dest := filepath.Join(toolsDir, strings.TrimPrefix(header.Name, "teleport/")) - tempFile, err := renameio.TempFile(extractDir, dest) + tempFile, err := renameio.TempFile(extractDir, filepath.Join(toolsDir, filepath.Base(header.Name))) if err != nil { return trace.Wrap(err) } @@ -104,11 +108,45 @@ func replaceTarGz(toolsDir string, archivePath string, extractDir string, apps [ return trace.Wrap(gzipReader.Close()) } +// validateFreeSpaceTarGz validates that extraction size match available disk space in `extractDir`. +func validateFreeSpaceTarGz(archivePath string, extractDir string, execNames []string) error { + f, err := os.Open(archivePath) + if err != nil { + return trace.Wrap(err) + } + defer f.Close() + + var totalSize uint64 + gzipReader, err := gzip.NewReader(f) + if err != nil { + return trace.Wrap(err) + } + tarReader := tar.NewReader(gzipReader) + for { + header, err := tarReader.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return trace.Wrap(err) + } + // Skip over any files in the archive that are not defined execNames. + if !slices.ContainsFunc(execNames, func(s string) bool { + return filepath.Base(header.Name) == s + }) { + continue + } + totalSize += uint64(header.Size) + } + + return trace.Wrap(checkFreeSpace(extractDir, totalSize)) +} + // replacePkg expands the Teleport package in .pkg format using the platform-dependent pkgutil utility. // The data is extracted to extractDir, and symlinks are created in toolsDir pointing to the binaries // in extractDir. Before creating the symlinks, each binary must be executed at least once to pass // OS signature verification. -func replacePkg(toolsDir string, archivePath string, extractDir string, apps []string) error { +func replacePkg(toolsDir string, archivePath string, extractDir string, execNames []string) error { // Use "pkgutil" from the filesystem to expand the archive. In theory .pkg // files are xz archives, however it's still safer to use "pkgutil" in-case // Apple makes non-standard changes to the format. @@ -130,9 +168,9 @@ func replacePkg(toolsDir string, archivePath string, extractDir string, apps []s if info.IsDir() { return nil } - // Skip over any files in the archive that are not in apps. - if !slices.ContainsFunc(apps, func(s string) bool { - return strings.HasSuffix(info.Name(), s) + // Skip over any files in the archive that are not in execNames. + if !slices.ContainsFunc(execNames, func(s string) bool { + return filepath.Base(info.Name()) == s }) { return nil } diff --git a/lib/utils/packaging/unarchive_windows.go b/lib/utils/packaging/unarchive_windows.go index 602b8c473c009..c07471adce83c 100644 --- a/lib/utils/packaging/unarchive_windows.go +++ b/lib/utils/packaging/unarchive_windows.go @@ -20,8 +20,11 @@ package packaging -// ReplaceToolsBinaries extracts the package into `extractDir` and replaces the specified -// applications with symlinks in the tool's directory. -func ReplaceToolsBinaries(toolsDir string, archivePath string, extractPath string, apps []string) error { - return replaceZip(toolsDir, archivePath, extractPath, apps) +// ReplaceToolsBinaries extracts executables specified by execNames from archivePath into +// extractDir. After each executable is extracted, it is symlinked from extractDir/[name] to +// toolsDir/[name]. +// +// For Windows, archivePath must be a .zip file. +func ReplaceToolsBinaries(toolsDir string, archivePath string, extractPath string, execNames []string) error { + return replaceZip(toolsDir, archivePath, extractPath, execNames) } From cece4a3f903400127e588bd44830898fe0698362 Mon Sep 17 00:00:00 2001 From: Vadym Popov Date: Tue, 8 Oct 2024 19:58:06 -0400 Subject: [PATCH 10/13] Change compressing with relative paths Add test for cleanup and fix skip logic --- integration/helpers/archive.go | 24 ++++++++++++--- lib/utils/packaging/unarchive.go | 38 +++++++++++++---------- lib/utils/packaging/unarchive_test.go | 44 +++++++++++++++++++++++++-- lib/utils/packaging/unarchive_unix.go | 12 +++----- 4 files changed, 87 insertions(+), 31 deletions(-) diff --git a/integration/helpers/archive.go b/integration/helpers/archive.go index 253fd835c9fc0..33843cd60b818 100644 --- a/integration/helpers/archive.go +++ b/integration/helpers/archive.go @@ -42,7 +42,10 @@ func CompressDirToZipFile(ctx context.Context, sourcePath, destPath string) (err return trace.Wrap(err) } defer func() { - _ = archive.Close() + if closeErr := archive.Close(); closeErr != nil { + err = closeErr + return + } if err != nil { if err := os.Remove(destPath); err != nil { slog.ErrorContext(ctx, "failed to remove archive", "error", err) @@ -63,7 +66,11 @@ func CompressDirToZipFile(ctx context.Context, sourcePath, destPath string) (err return trace.Wrap(err) } defer file.Close() - zipFileWriter, err := zipWriter.Create(filepath.Base(path)) + relPath, err := filepath.Rel(sourcePath, path) + if err != nil { + return trace.Wrap(err) + } + zipFileWriter, err := zipWriter.Create(relPath) if err != nil { return trace.Wrap(err) } @@ -79,7 +86,7 @@ func CompressDirToZipFile(ctx context.Context, sourcePath, destPath string) (err return trace.Wrap(err) } - return nil + return } // CompressDirToTarGzFile compresses a directory into .tar.gz format, @@ -90,7 +97,10 @@ func CompressDirToTarGzFile(ctx context.Context, sourcePath, destPath string) (e return trace.Wrap(err) } defer func() { - _ = archive.Close() + if closeErr := archive.Close(); closeErr != nil { + err = closeErr + return + } if err != nil { if err := os.Remove(destPath); err != nil { slog.ErrorContext(ctx, "failed to remove archive", "error", err) @@ -115,6 +125,10 @@ func CompressDirToTarGzFile(ctx context.Context, sourcePath, destPath string) (e if err != nil { return trace.Wrap(err) } + header.Name, err = filepath.Rel(sourcePath, path) + if err != nil { + return trace.Wrap(err) + } if err := tarWriter.WriteHeader(header); err != nil { return trace.Wrap(err) } @@ -133,7 +147,7 @@ func CompressDirToTarGzFile(ctx context.Context, sourcePath, destPath string) (e return trace.Wrap(err) } - return nil + return } // CompressDirToPkgFile runs for the macOS `pkgbuild` command to generate a .pkg file from the source. diff --git a/lib/utils/packaging/unarchive.go b/lib/utils/packaging/unarchive.go index bfe8f112b564b..f8115453d956d 100644 --- a/lib/utils/packaging/unarchive.go +++ b/lib/utils/packaging/unarchive.go @@ -35,28 +35,34 @@ const ( reservedFreeDisk = 10 * 1024 * 1024 ) -// RemoveWithSuffix removes all files in dir that have the provided suffix, except for files named `skipName` -func RemoveWithSuffix(extractDir, suffix, skipName string) error { - err := filepath.Walk(extractDir, func(path string, info os.FileInfo, err error) error { +// RemoveWithSuffix removes all that matches the provided suffix, except for file or directory with `skipName`. +func RemoveWithSuffix(dir, suffix, skipName string) error { + var removePaths []string + err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { if err != nil { return trace.Wrap(err) } - if !info.IsDir() { - return nil - } if skipName == info.Name() { return nil } if !strings.HasSuffix(info.Name(), suffix) { return nil } - // Found a stale expanded package. - if err := os.RemoveAll(path); err != nil { - return trace.Wrap(err) + removePaths = append(removePaths, path) + if info.IsDir() { + return filepath.SkipDir } return nil }) - return trace.Wrap(err) + if err != nil { + return trace.Wrap(err) + } + for _, path := range removePaths { + if err := os.RemoveAll(path); err != nil { + return trace.Wrap(err) + } + } + return nil } // replaceZip un-archives the Teleport package in .zip format, iterates through @@ -81,9 +87,10 @@ func replaceZip(toolsDir string, archivePath string, extractDir string, execName var totalSize uint64 = 0 for _, zipFile := range zipReader.File { + baseName := filepath.Base(zipFile.Name) // Skip over any files in the archive that are not defined execNames. if !slices.ContainsFunc(execNames, func(s string) bool { - return filepath.Base(zipFile.Name) == s + return baseName == s }) { continue } @@ -95,10 +102,9 @@ func replaceZip(toolsDir string, archivePath string, extractDir string, execName } for _, zipFile := range zipReader.File { + baseName := filepath.Base(zipFile.Name) // Skip over any files in the archive that are not defined execNames. - if !slices.ContainsFunc(execNames, func(s string) bool { - return filepath.Base(zipFile.Name) == s - }) { + if !slices.Contains(execNames, baseName) { continue } @@ -109,7 +115,7 @@ func replaceZip(toolsDir string, archivePath string, extractDir string, execName } defer file.Close() - dest := filepath.Join(extractDir, zipFile.Name) + dest := filepath.Join(extractDir, baseName) destFile, err := os.OpenFile(dest, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0755) if err != nil { return trace.Wrap(err) @@ -119,7 +125,7 @@ func replaceZip(toolsDir string, archivePath string, extractDir string, execName if _, err := io.Copy(destFile, file); err != nil { return trace.Wrap(err) } - appPath := filepath.Join(toolsDir, zipFile.Name) + appPath := filepath.Join(toolsDir, baseName) if err := os.Remove(appPath); err != nil && !os.IsNotExist(err) { return trace.Wrap(err) } diff --git a/lib/utils/packaging/unarchive_test.go b/lib/utils/packaging/unarchive_test.go index e3be7346321f0..9ad927b9b655c 100644 --- a/lib/utils/packaging/unarchive_test.go +++ b/lib/utils/packaging/unarchive_test.go @@ -52,9 +52,12 @@ func TestPackaging(t *testing.T) { require.NoError(t, os.RemoveAll(toolsDir)) }) - // Create test script for packaging. - require.NoError(t, os.WriteFile(filepath.Join(sourceDir, "tsh"), []byte(script), 0o755)) - require.NoError(t, os.WriteFile(filepath.Join(sourceDir, "tctl"), []byte(script), 0o755)) + // Create test script for packaging in relative path `teleport\bin` to ensure that + // binaries going to be identified and extracted flatten to `extractDir`. + binPath := filepath.Join(sourceDir, "teleport", "bin") + require.NoError(t, os.MkdirAll(binPath, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(binPath, "tsh"), []byte(script), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(binPath, "tctl"), []byte(script), 0o755)) ctx := context.Background() @@ -112,3 +115,38 @@ func TestPackaging(t *testing.T) { assert.Equal(t, script, string(data)) }) } + +// TestDirCleanUp verifies that helper for the cleanup removes directories +func TestDirCleanup(t *testing.T) { + testDir, err := os.MkdirTemp(os.TempDir(), "test") + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, os.RemoveAll(testDir)) + }) + + dirForRemove := "test-extract-pkg" + + // Creates directories `test/test-extract-pkg/test-extract-pkg` with exact names + // to ensure that only root one going to be removed recursively without any error. + path := filepath.Join(testDir, dirForRemove, dirForRemove) + require.NoError(t, os.MkdirAll(path, 0o755)) + // Also we create the directory that needs to be skipped, and it matches the remove + // pattern `test/skip-test-extract-pkg/test-extract-pkg`. + skipName := "skip-" + dirForRemove + skipPath := filepath.Join(testDir, skipName) + dirInSkipPath := filepath.Join(skipPath, dirForRemove) + require.NoError(t, os.MkdirAll(skipPath, 0o755)) + + err = RemoveWithSuffix(testDir, dirForRemove, skipName) + require.NoError(t, err) + + _, err = os.Stat(filepath.Join(testDir, dirForRemove)) + assert.True(t, os.IsNotExist(err)) + + filePath, err := os.Stat(skipPath) + require.NoError(t, err) + assert.True(t, filePath.IsDir()) + + _, err = os.Stat(dirInSkipPath) + assert.True(t, os.IsNotExist(err)) +} diff --git a/lib/utils/packaging/unarchive_unix.go b/lib/utils/packaging/unarchive_unix.go index 50de040b1b115..3be7d0c473ef9 100644 --- a/lib/utils/packaging/unarchive_unix.go +++ b/lib/utils/packaging/unarchive_unix.go @@ -77,15 +77,14 @@ func replaceTarGz(toolsDir string, archivePath string, extractDir string, execNa if err != nil { return trace.Wrap(err) } + baseName := filepath.Base(header.Name) // Skip over any files in the archive that are not in execNames. - if !slices.ContainsFunc(execNames, func(s string) bool { - return filepath.Base(header.Name) == s - }) { + if !slices.Contains(execNames, baseName) { continue } if err = func(header *tar.Header) error { - tempFile, err := renameio.TempFile(extractDir, filepath.Join(toolsDir, filepath.Base(header.Name))) + tempFile, err := renameio.TempFile(extractDir, filepath.Join(toolsDir, baseName)) if err != nil { return trace.Wrap(err) } @@ -130,10 +129,9 @@ func validateFreeSpaceTarGz(archivePath string, extractDir string, execNames []s if err != nil { return trace.Wrap(err) } + baseName := filepath.Base(header.Name) // Skip over any files in the archive that are not defined execNames. - if !slices.ContainsFunc(execNames, func(s string) bool { - return filepath.Base(header.Name) == s - }) { + if !slices.Contains(execNames, baseName) { continue } totalSize += uint64(header.Size) From f8bd0f9ca1ef86586cccd6ad159e7f66f959e3f3 Mon Sep 17 00:00:00 2001 From: Vadym Popov Date: Wed, 9 Oct 2024 12:51:57 -0400 Subject: [PATCH 11/13] CR changes --- integration/helpers/archive.go | 4 ++-- lib/utils/disk.go | 2 +- lib/utils/disk_windows.go | 2 +- lib/utils/packaging/unarchive.go | 4 +++- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/integration/helpers/archive.go b/integration/helpers/archive.go index 33843cd60b818..15efe4e740668 100644 --- a/integration/helpers/archive.go +++ b/integration/helpers/archive.go @@ -43,7 +43,7 @@ func CompressDirToZipFile(ctx context.Context, sourcePath, destPath string) (err } defer func() { if closeErr := archive.Close(); closeErr != nil { - err = closeErr + err = trace.Wrap(closeErr) return } if err != nil { @@ -98,7 +98,7 @@ func CompressDirToTarGzFile(ctx context.Context, sourcePath, destPath string) (e } defer func() { if closeErr := archive.Close(); closeErr != nil { - err = closeErr + err = trace.Wrap(closeErr) return } if err != nil { diff --git a/lib/utils/disk.go b/lib/utils/disk.go index 65172add5146c..9e2419527051a 100644 --- a/lib/utils/disk.go +++ b/lib/utils/disk.go @@ -46,7 +46,7 @@ func PercentUsed(path string) (float64, error) { return Round(ratio * 100), nil } -// FreeDiskWithReserve returns the available disk space on the disk at dir, minus `reservedFreeDisk`. +// FreeDiskWithReserve returns the available disk space (in bytes) on the disk at dir, minus `reservedFreeDisk`. func FreeDiskWithReserve(dir string, reservedFreeDisk uint64) (uint64, error) { var stat syscall.Statfs_t err := syscall.Statfs(dir, &stat) diff --git a/lib/utils/disk_windows.go b/lib/utils/disk_windows.go index 85b7a135cc3f8..8056849afa82b 100644 --- a/lib/utils/disk_windows.go +++ b/lib/utils/disk_windows.go @@ -31,7 +31,7 @@ func PercentUsed(path string) (float64, error) { return 0.0, trace.NotImplemented("disk usage not supported on Windows") } -// FreeDiskWithReserve returns the available disk space on the disk at dir, minus `reservedFreeDisk`. +// FreeDiskWithReserve returns the available disk space (in bytes) on the disk at dir, minus `reservedFreeDisk`. func FreeDiskWithReserve(dir string, reservedFreeDisk uint64) (uint64, error) { var avail uint64 err := windows.GetDiskFreeSpaceEx(windows.StringToUTF16Ptr(dir), &avail, nil, nil) diff --git a/lib/utils/packaging/unarchive.go b/lib/utils/packaging/unarchive.go index f8115453d956d..f1a197e095b1a 100644 --- a/lib/utils/packaging/unarchive.go +++ b/lib/utils/packaging/unarchive.go @@ -32,6 +32,8 @@ import ( ) const ( + // reservedFreeDisk is the predefined amount of free disk space (in bytes) required + // to remain available after extracting Teleport binaries. reservedFreeDisk = 10 * 1024 * 1024 ) @@ -141,7 +143,7 @@ func replaceZip(toolsDir string, archivePath string, extractDir string, execName return nil } -// checkFreeSpace verifies that we have enough requested space at specific directory. +// checkFreeSpace verifies that we have enough requested space (in bytes) at specific directory. func checkFreeSpace(path string, requested uint64) error { free, err := utils.FreeDiskWithReserve(path, reservedFreeDisk) if err != nil { From 98b1f0a0b69033ed71f7ea4cc56d267dd4692a11 Mon Sep 17 00:00:00 2001 From: Vadym Popov Date: Wed, 9 Oct 2024 15:43:34 -0400 Subject: [PATCH 12/13] CR changes --- integration/helpers/archive.go | 37 ++++++++++++++------------- lib/utils/packaging/unarchive_test.go | 13 +++------- 2 files changed, 23 insertions(+), 27 deletions(-) diff --git a/integration/helpers/archive.go b/integration/helpers/archive.go index 15efe4e740668..d2d07e28e82f7 100644 --- a/integration/helpers/archive.go +++ b/integration/helpers/archive.go @@ -34,27 +34,27 @@ import ( "github.com/gravitational/trace" ) -// CompressDirToZipFile compresses a directory into `.zip` format, +// CompressDirToZipFile compresses a source directory into `.zip` format and stores at `archivePath`, // preserving the relative file path structure of the source directory. -func CompressDirToZipFile(ctx context.Context, sourcePath, destPath string) (err error) { - archive, err := os.Create(destPath) +func CompressDirToZipFile(ctx context.Context, sourceDir, archivePath string) (err error) { + archive, err := os.Create(archivePath) if err != nil { return trace.Wrap(err) } defer func() { if closeErr := archive.Close(); closeErr != nil { - err = trace.Wrap(closeErr) + err = trace.NewAggregate(err, closeErr) return } if err != nil { - if err := os.Remove(destPath); err != nil { + if err := os.Remove(archivePath); err != nil { slog.ErrorContext(ctx, "failed to remove archive", "error", err) } } }() zipWriter := zip.NewWriter(archive) - err = filepath.Walk(sourcePath, func(path string, info os.FileInfo, err error) error { + err = filepath.Walk(sourceDir, func(path string, info os.FileInfo, err error) error { if err != nil { return trace.Wrap(err) } @@ -66,7 +66,7 @@ func CompressDirToZipFile(ctx context.Context, sourcePath, destPath string) (err return trace.Wrap(err) } defer file.Close() - relPath, err := filepath.Rel(sourcePath, path) + relPath, err := filepath.Rel(sourceDir, path) if err != nil { return trace.Wrap(err) } @@ -89,27 +89,27 @@ func CompressDirToZipFile(ctx context.Context, sourcePath, destPath string) (err return } -// CompressDirToTarGzFile compresses a directory into .tar.gz format, +// CompressDirToTarGzFile compresses a source directory into .tar.gz format and stores at `archivePath`, // preserving the relative file path structure of the source directory. -func CompressDirToTarGzFile(ctx context.Context, sourcePath, destPath string) (err error) { - archive, err := os.Create(destPath) +func CompressDirToTarGzFile(ctx context.Context, sourceDir, archivePath string) (err error) { + archive, err := os.Create(archivePath) if err != nil { return trace.Wrap(err) } defer func() { if closeErr := archive.Close(); closeErr != nil { - err = trace.Wrap(closeErr) + err = trace.NewAggregate(err, closeErr) return } if err != nil { - if err := os.Remove(destPath); err != nil { + if err := os.Remove(archivePath); err != nil { slog.ErrorContext(ctx, "failed to remove archive", "error", err) } } }() gzipWriter := gzip.NewWriter(archive) tarWriter := tar.NewWriter(gzipWriter) - err = filepath.Walk(sourcePath, func(path string, info os.FileInfo, err error) error { + err = filepath.Walk(sourceDir, func(path string, info os.FileInfo, err error) error { if err != nil { return err } @@ -125,7 +125,7 @@ func CompressDirToTarGzFile(ctx context.Context, sourcePath, destPath string) (e if err != nil { return trace.Wrap(err) } - header.Name, err = filepath.Rel(sourcePath, path) + header.Name, err = filepath.Rel(sourceDir, path) if err != nil { return trace.Wrap(err) } @@ -150,18 +150,19 @@ func CompressDirToTarGzFile(ctx context.Context, sourcePath, destPath string) (e return } -// CompressDirToPkgFile runs for the macOS `pkgbuild` command to generate a .pkg file from the source. -func CompressDirToPkgFile(ctx context.Context, sourcePath, destPath, identifier string) error { +// CompressDirToPkgFile runs for the macOS `pkgbuild` command to generate a .pkg +// archive file from the source directory. +func CompressDirToPkgFile(ctx context.Context, sourceDir, archivePath, identifier string) error { if runtime.GOOS != "darwin" { return trace.BadParameter("only darwin platform is supported for pkg file") } cmd := exec.CommandContext( ctx, "pkgbuild", - "--root", sourcePath, + "--root", sourceDir, "--identifier", identifier, "--version", teleport.Version, - destPath, + archivePath, ) return trace.Wrap(cmd.Run()) diff --git a/lib/utils/packaging/unarchive_test.go b/lib/utils/packaging/unarchive_test.go index 9ad927b9b655c..30933bbb75927 100644 --- a/lib/utils/packaging/unarchive_test.go +++ b/lib/utils/packaging/unarchive_test.go @@ -116,14 +116,9 @@ func TestPackaging(t *testing.T) { }) } -// TestDirCleanUp verifies that helper for the cleanup removes directories -func TestDirCleanup(t *testing.T) { - testDir, err := os.MkdirTemp(os.TempDir(), "test") - require.NoError(t, err) - t.Cleanup(func() { - require.NoError(t, os.RemoveAll(testDir)) - }) - +// TestRemoveWithSuffix verifies that helper for the cleanup removes directories +func TestRemoveWithSuffix(t *testing.T) { + testDir := t.TempDir() dirForRemove := "test-extract-pkg" // Creates directories `test/test-extract-pkg/test-extract-pkg` with exact names @@ -137,7 +132,7 @@ func TestDirCleanup(t *testing.T) { dirInSkipPath := filepath.Join(skipPath, dirForRemove) require.NoError(t, os.MkdirAll(skipPath, 0o755)) - err = RemoveWithSuffix(testDir, dirForRemove, skipName) + err := RemoveWithSuffix(testDir, dirForRemove, skipName) require.NoError(t, err) _, err = os.Stat(filepath.Join(testDir, dirForRemove)) From eee173cc0aad2edb15f345efc699fd75aaa69de3 Mon Sep 17 00:00:00 2001 From: Vadym Popov Date: Wed, 9 Oct 2024 18:49:21 -0400 Subject: [PATCH 13/13] Fix linter --- integration/helpers/archive.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/integration/helpers/archive.go b/integration/helpers/archive.go index d2d07e28e82f7..6e48108013d86 100644 --- a/integration/helpers/archive.go +++ b/integration/helpers/archive.go @@ -30,8 +30,9 @@ import ( "path/filepath" "runtime" - "github.com/gravitational/teleport" "github.com/gravitational/trace" + + "github.com/gravitational/teleport" ) // CompressDirToZipFile compresses a source directory into `.zip` format and stores at `archivePath`,