diff --git a/adr/0012-local-image-support-via-docker.md b/adr/0012-local-image-support-via-docker.md new file mode 100644 index 0000000000..8edd63bd0d --- /dev/null +++ b/adr/0012-local-image-support-via-docker.md @@ -0,0 +1,28 @@ +# 12. Local Image Support Via Docker + +Date: 2023-02-06 + +## Status + +Accepted + +## Context + +There has been a long-standing usabilty gap with Zarf when doing local development due to a lack of local image support. A solution was merged in, [#1173](https://github.com/defenseunicorns/zarf/pull/1173), and released in [v0.23.4](https://github.com/defenseunicorns/zarf/releases/tag/v0.23.4) to support this feature. Unfortuantely, we didn't realize there is a [glaring issue](https://github.com/defenseunicorns/zarf/issues/1214) with the implementation that causes Zarf to crash when trying to load large images into the local docker daemon. The docker daemon support in Crane is somewhat naive and can send a machine into an OOM condition due to how the tar stream is loaded into memory from the docker save action. Crane does have an option to avoid this issue, but at the cost of being much slower to load images from docker. + +We did extensive investigation into various strategies of loading docker images from the daemon including: crane, skopeo, the docker go client and executing the docker cli directly with varying levels of success. Unfortunately, some of the methods that did work well were up to 3 times slower than the current implementation, though they avoided the OOM issue. Lastly, the docker daemon save operations directly still ended up being slower than crane and docker produced a legacy format that would cause issues with [future package schema changes](https://github.com/defenseunicorns/zarf/issues/1319) we are planning for oci imports. + +| | **Docker** | **Crane** | +| -------------------------------------------------- | ---------- | --------- | +| Big Bang Core (cached) | 3m 1s | 1m 58s | +| Big Bang Core (cached + skip-sbom) | 1m 51s | 56s | +| 20 GB Single-Layer Image (local registry) | | 6m 14s | +| 20 GB Single-Layer Image (local registry + cached) | 5m 2s | 2m 10s | + +## Decision + +We had hoped to leverage docker caching and avoid crane caching moreso, but realized that caching was still occuring via Syft for SBOM. Additionaly, the extremely-large, local-only image is actually the edge case here and we created a recommended workaround in the FAQs as well as an inline alert when a large docker image is detected. This restores behavior to what it was before the docker daemon support was added, but with the added benefit of being able to load images from the docker daemon when they are available locally. + +## Consequences + +For most cases this will be a seamless transition back to the previous behavior while still supporting local-only images. While this will work for large images too, it will be slow and this is automatically communicated to the user via a warning/recommendation to use a local registry. diff --git a/docs/4-user-guide/1-the-zarf-cli/100-cli-commands/zarf_package_create.md b/docs/4-user-guide/1-the-zarf-cli/100-cli-commands/zarf_package_create.md index cbedae294b..1c641864f3 100644 --- a/docs/4-user-guide/1-the-zarf-cli/100-cli-commands/zarf_package_create.md +++ b/docs/4-user-guide/1-the-zarf-cli/100-cli-commands/zarf_package_create.md @@ -19,7 +19,6 @@ zarf package create [DIRECTORY] [flags] --confirm Confirm package creation without prompting -h, --help help for create -m, --max-package-size int Specify the maximum size of the package in megabytes, packages larger than this will be split into multiple parts. Use 0 to disable splitting. - --no-local-images Do not use local container images when creating this package -o, --output-directory string Specify the output directory for the created Zarf package -s, --sbom View SBOM contents after creating the package --sbom-out string Specify an output directory for the SBOMs from the created Zarf package diff --git a/docs/4-user-guide/3-zarf-schema.md b/docs/4-user-guide/3-zarf-schema.md index 74d4dded0f..99680906b0 100644 --- a/docs/4-user-guide/3-zarf-schema.md +++ b/docs/4-user-guide/3-zarf-schema.md @@ -343,12 +343,14 @@ Must be one of: | **Defined in** | #/definitions/ZarfComponent |
- name + name *  
+![Required](https://img.shields.io/badge/Required-red) + **Description:** The name of the component | | | @@ -1062,6 +1064,22 @@ Must be one of:
+
+ description + + +  +
+ +**Description:** Description of the action to be displayed during package execution instead of the command + +| | | +| -------- | -------- | +| **Type** | `string` | + +
+
+ diff --git a/docs/9-faq.md b/docs/9-faq.md index 73479946fa..a20df15a59 100644 --- a/docs/9-faq.md +++ b/docs/9-faq.md @@ -12,7 +12,6 @@ Zarf is statically compiled and written in [Go](https://golang.org/) and [Rust]( Zarf is under the [Apache License 2.0](https://github.com/defenseunicorns/zarf/blob/main/LICENSE). This is one of the most commonly used licenses for open source software. - ## What is the Zarf Agent? The Zarf Agent is a [Kubernetes Mutating Webhook](https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#mutatingadmissionwebhook) that is installed into the cluster during the `zarf init` operation. The Agent is responsible for modifying [Kubernetes PodSpec](https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#PodSpec) objects [Image](https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#Container.Image) fields to point to the Zarf Registry. This allows the cluster to pull images from the Zarf Registry instead of the internet without having to modify the original image references. The Agent also modifies [Flux GitRepository](https://fluxcd.io/docs/components/source/gitrepositories/) objects to point to the local Git Server. @@ -21,7 +20,7 @@ The Zarf Agent is a [Kubernetes Mutating Webhook](https://kubernetes.io/docs/ref During early discussions and [subsequent decision](../adr/0005-mutating-webhook.md) to use a Mutating Webhook, we decided to not have the Agent create any secrets in the cluster. This is to avoid the Agent having to have more privileges than it needs as well as avoid collisions with Helm. The Agent today simply responds to requests to patch PodSpec and GitRepository objects. -The Agent does not need to create any secrets in the cluster. Instead, during `zarf init` and `zarf package deploy`, secrets are automatically created as [Helm Postrender Hook](https://helm.sh/docs/topics/advanced/#post-rendering) for any namespaces Zarf sees. If you have resources managed by [Flux](https://fluxcd.io/) that are not in a namespace managed by Zarf, you can either create the secrets manually or include a manifest to create the namespace in your package and let Zarf create the secrets for you. +The Agent does not need to create any secrets in the cluster. Instead, during `zarf init` and `zarf package deploy`, secrets are automatically created as [Helm Postrender Hook](https://helm.sh/docs/topics/advanced/#post-rendering) for any namespaces Zarf sees. If you have resources managed by [Flux](https://fluxcd.io/) that are not in a namespace managed by Zarf, you can either create the secrets manually or include a manifest to create the namespace in your package and let Zarf create the secrets for you. ## How can a Kubernetes resource be excluded from the Zarf Agent? @@ -31,6 +30,37 @@ Resources can be excluded at the namespace or resources level by adding the `zar During the `zarf init` operation, the Zarf Agent will patch any existing namespaces with the `zarf.dev/agent: ignore` label to prevent the Agent from modifying any resources in that namespace. This is done because there is no way to guarantee the images used by pods in existing namespaces are available in the Zarf Registry. +## How can I improve the speed of loading larges images from Docker on `zarf package create`? + +Due to some limitations with how Docker provides access to local image layers, `zarf package create` has to rely on `docker save` under the hood which is [very slow overall](https://github.com/defenseunicorns/zarf/issues/1214) and also takes a long time to report progress. We experimented with many ways to improve this, but for now recommend leveraging a local docker registry to speed up the process. This can be done by running a local registry and pushing the images to it before running `zarf package create`. This will allow `zarf package create` to pull the images from the local registry instead of Docker. This can also be combined with [component actions](4-user-guide/5-component-actions.md) to make the process automatic. Given an example image of `my-giant-image:###ZARF_PKG_VAR_IMG###` you could do something like this: + +```sh +# Create a local registry +docker run -d -p 5000:5000 --restart=always --name registry registry:2 + +# Run the package create with a tag variable +zarf package create --set IMG=my-giant-image:v2 +``` + +```yaml +kind: ZarfPackageConfig +metadata: + name: giant-image-example + +components: + - name: main + actions: + # runs during "zarf package create" + onCreate: + # runs before the component is created + before: + - cmd: 'docker tag ###ZARF_PKG_VAR_IMG### localhost:5000/###ZARF_PKG_VAR_IMG###' + - cmd: 'docker push localhost:5000/###ZARF_PKG_VAR_IMG###' + + images: + - 'localhost:5000/###ZARF_PKG_VAR_IMG###' +``` + ## What is YOLO Mode and why would I use it? YOLO Mode is a special package metadata designation that be added to a package prior to `zarf package create` to allow the package to be installed without the need for a `zarf init` operation. In most cases this will not be used, but it can be useful for testing or for environments that manage their own registries and Git servers completely outside of Zarf. This can also be used as a way to transition slowly to using Zarf without having to do a full migration. diff --git a/examples/component-actions/zarf.yaml b/examples/component-actions/zarf.yaml index 317cb8deee..06462fbf82 100644 --- a/examples/component-actions/zarf.yaml +++ b/examples/component-actions/zarf.yaml @@ -39,6 +39,20 @@ components: # runs after the component is deployed after: - cmd: touch test-create-after.txt + - cmd: sleep 1 + - cmd: echo "I can print!" + - cmd: sleep 1 + - cmd: | + echo "multiline!" + sleep 1 + echo "updates!" + sleep 3 + echo "in!" + sleep 1 + echo "realtime!" + sleep 1 + description: multiline & description demo + - cmd: sleep 1 - name: on-deploy actions: diff --git a/go.mod b/go.mod index ecf803ba6c..006a81d657 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( github.com/goccy/go-yaml v1.9.8 github.com/google/go-containerregistry v0.13.0 github.com/mholt/archiver/v3 v3.5.1 + github.com/moby/moby v20.10.23+incompatible github.com/otiai10/copy v1.9.0 github.com/pkg/errors v0.9.1 github.com/pterm/pterm v0.12.54 diff --git a/go.sum b/go.sum index 20bd34a697..fe48ba5ef0 100644 --- a/go.sum +++ b/go.sum @@ -1266,6 +1266,8 @@ github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zx github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg= github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= +github.com/moby/moby v20.10.23+incompatible h1:5+Q6jGL7oH89tx+ms0fGsTYEXrQ3P4vuL3i7DayMUuM= +github.com/moby/moby v20.10.23+incompatible/go.mod h1:fDXVQ6+S340veQPv35CzDahGBmHsiclFwfEygB/TWMc= github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/moby/sys/mountinfo v0.6.0 h1:gUDhXQx58YNrpHlK4nSL+7y2pxFZkUcXqzFDKWdC0Oo= diff --git a/src/cmd/package.go b/src/cmd/package.go index c701ea509d..046c69df48 100644 --- a/src/cmd/package.go +++ b/src/cmd/package.go @@ -248,7 +248,6 @@ func bindCreateFlags() { v.SetDefault(V_PKG_CREATE_SBOM_OUTPUT, "") v.SetDefault(V_PKG_CREATE_SKIP_SBOM, false) v.SetDefault(V_PKG_CREATE_MAX_PACKAGE_SIZE, 0) - v.SetDefault(V_PKG_CREATE_NO_LOCAL_IMAGES, false) createFlags.StringToStringVar(&pkgConfig.CreateOpts.SetVariables, "set", v.GetStringMapString(V_PKG_CREATE_SET), "Specify package variables to set on the command line (KEY=value)") createFlags.StringVarP(&pkgConfig.CreateOpts.OutputDirectory, "output-directory", "o", v.GetString(V_PKG_CREATE_OUTPUT_DIR), "Specify the output directory for the created Zarf package") @@ -256,7 +255,6 @@ func bindCreateFlags() { createFlags.StringVar(&pkgConfig.CreateOpts.SBOMOutputDir, "sbom-out", v.GetString(V_PKG_CREATE_SBOM_OUTPUT), "Specify an output directory for the SBOMs from the created Zarf package") createFlags.BoolVar(&pkgConfig.CreateOpts.SkipSBOM, "skip-sbom", v.GetBool(V_PKG_CREATE_SKIP_SBOM), "Skip generating SBOM for this package") createFlags.IntVarP(&pkgConfig.CreateOpts.MaxPackageSizeMB, "max-package-size", "m", v.GetInt(V_PKG_CREATE_MAX_PACKAGE_SIZE), "Specify the maximum size of the package in megabytes, packages larger than this will be split into multiple parts. Use 0 to disable splitting.") - createFlags.BoolVar(&pkgConfig.CreateOpts.NoLocalImages, "no-local-images", v.GetBool(V_PKG_CREATE_NO_LOCAL_IMAGES), "Do not use local container images when creating this package") } func bindDeployFlags() { diff --git a/src/cmd/viper.go b/src/cmd/viper.go index 226db8150a..65add1400d 100644 --- a/src/cmd/viper.go +++ b/src/cmd/viper.go @@ -49,7 +49,6 @@ const ( V_PKG_CREATE_SBOM_OUTPUT = "package.create.sbom_output" V_PKG_CREATE_SKIP_SBOM = "package.create.skip_sbom" V_PKG_CREATE_MAX_PACKAGE_SIZE = "package.create.max_package_size" - V_PKG_CREATE_NO_LOCAL_IMAGES = "package.create.no_local_images" // Package deploy config keys V_PKG_DEPLOY_SET = "package.deploy.set" diff --git a/src/internal/packager/git/clone.go b/src/internal/packager/git/clone.go index 098d48dea2..db64b6fc2c 100644 --- a/src/internal/packager/git/clone.go +++ b/src/internal/packager/git/clone.go @@ -45,8 +45,8 @@ func (g *Git) clone(gitDirectory string, gitURL string, onlyFetchRef bool) (*git return repo, git.ErrRepositoryAlreadyExists } else if err != nil { - message.Debugf("Failed to clone repo: %s", err.Error()) - message.Infof("Falling back to host git for %s", gitURL) + message.Debugf("Failed to clone repo %s: %s", gitURL, err.Error()) + g.Spinner.Updatef("Falling back to host git for %s", gitURL) // If we can't clone with go-git, fallback to the host clone // Only support "all tags" due to the azure clone url format including a username @@ -56,10 +56,11 @@ func (g *Git) clone(gitDirectory string, gitURL string, onlyFetchRef bool) (*git cmdArgs = append(cmdArgs, "--no-tags") } - stdOut, stdErr, err := exec.CmdWithContext(context.TODO(), exec.Config{}, "git", cmdArgs...) - g.Spinner.Updatef(stdOut) - message.Debug(stdErr) - + execConfig := exec.Config{ + Stdout: g.Spinner, + Stderr: g.Spinner, + } + _, _, err := exec.CmdWithContext(context.TODO(), execConfig, "git", cmdArgs...) if err != nil { return nil, err } diff --git a/src/internal/packager/git/fetch.go b/src/internal/packager/git/fetch.go index 79bd4d465f..f876bea0a4 100644 --- a/src/internal/packager/git/fetch.go +++ b/src/internal/packager/git/fetch.go @@ -77,8 +77,8 @@ func (g *Git) fetch(gitDirectory string, fetchOptions *git.FetchOptions) error { if errors.Is(err, git.ErrTagExists) || errors.Is(err, git.NoErrAlreadyUpToDate) { message.Debug("Already fetched requested ref") } else if err != nil { - message.Debugf("Failed to fetch repo: %s", err) - message.Infof("Falling back to host git for %s", gitURL) + message.Debugf("Failed to fetch repo %s: %s", gitURL, err.Error()) + g.Spinner.Updatef("Falling back to host git for %s", gitURL) // If we can't fetch with go-git, fallback to the host fetch // Only support "all tags" due to the azure fetch url format including a username @@ -87,7 +87,9 @@ func (g *Git) fetch(gitDirectory string, fetchOptions *git.FetchOptions) error { cmdArgs = append(cmdArgs, refspec.String()) } execCfg := exec.Config{ - Dir: gitDirectory, + Dir: gitDirectory, + Stdout: g.Spinner, + Stderr: g.Spinner, } _, _, err := exec.CmdWithContext(context.TODO(), execCfg, "git", cmdArgs...) return err diff --git a/src/internal/packager/git/pull.go b/src/internal/packager/git/pull.go index 41ad149cf1..dad8af884c 100644 --- a/src/internal/packager/git/pull.go +++ b/src/internal/packager/git/pull.go @@ -18,17 +18,19 @@ import ( ) // DownloadRepoToTemp clones or updates a repo into a temp folder to perform ephemeral actions (i.e. process chart repos). -func (g *Git) DownloadRepoToTemp(gitURL string) string { - path, err := utils.MakeTempDir(config.CommonOptions.TempDirectory) - if err != nil { - message.Fatalf(err, "Unable to create tmpdir: %s", config.CommonOptions.TempDirectory) +func (g *Git) DownloadRepoToTemp(gitURL string) (path string, err error) { + if path, err = utils.MakeTempDir(config.CommonOptions.TempDirectory); err != nil { + return "", fmt.Errorf("unable to create tmpdir: %w", err) } + // If downloading to temp, grab all tags since the repo isn't being // packaged anyway, and it saves us from having to fetch the tags // later if we need them + if err = g.pull(gitURL, path, ""); err != nil { + return "", fmt.Errorf("unable to pull the git repo at %s: %w", gitURL, err) + } - err = g.pull(gitURL, path, "") - return path + return path, nil } // Pull clones or updates a git repository into the target folder. @@ -124,7 +126,7 @@ func (g *Git) pull(gitURL, targetFolder string, repoName string) error { _, err = g.removeLocalTagRefs() if err != nil { - return fmt.Errorf("Unable to remove unneeded local tag refs: %w", err) + return fmt.Errorf("unable to remove unneeded local tag refs: %w", err) } _, _ = g.removeLocalBranchRefs() _, _ = g.removeOnlineRemoteRefs() diff --git a/src/internal/packager/helm/repo.go b/src/internal/packager/helm/repo.go index d551a94363..ee18bfc682 100644 --- a/src/internal/packager/helm/repo.go +++ b/src/internal/packager/helm/repo.go @@ -54,23 +54,25 @@ func (h *Helm) DownloadChartFromGit(destination string) string { // Get the git repo gitCfg := git.NewWithSpinner(h.Cfg.State.GitServer, spinner) - tempPath := gitCfg.DownloadRepoToTemp(h.Chart.URL) + tempPath, err := gitCfg.DownloadRepoToTemp(h.Chart.URL) defer os.RemoveAll(tempPath) + if err != nil { + spinner.Fatalf(err, "Unable to download the git repo %s", h.Chart.URL) + } gitCfg.GitPath = tempPath // Switch to the correct tag gitCfg.CheckoutTag(h.Chart.Version) // Validate the chart - _, err := loader.LoadDir(filepath.Join(tempPath, h.Chart.GitPath)) - if err != nil { + chartPath := filepath.Join(tempPath, h.Chart.GitPath) + if _, err = loader.LoadDir(chartPath); err != nil { spinner.Fatalf(err, "Validation failed for chart %s (%s)", h.Chart.Name, err.Error()) } // Tell helm where to save the archive and create the package client.Destination = destination - name, err := client.Run(filepath.Join(tempPath, h.Chart.GitPath), nil) - + name, err := client.Run(chartPath, nil) if err != nil { spinner.Fatalf(err, "Helm is unable to save the archive and create the package %s", name) } diff --git a/src/internal/packager/images/common.go b/src/internal/packager/images/common.go index 631487ee93..4cc16ecf9e 100644 --- a/src/internal/packager/images/common.go +++ b/src/internal/packager/images/common.go @@ -17,6 +17,4 @@ type ImgConfig struct { NoChecksum bool Insecure bool - - NoLocalImages bool } diff --git a/src/internal/packager/images/pull.go b/src/internal/packager/images/pull.go index 5e847cb8da..11e45d9a2c 100644 --- a/src/internal/packager/images/pull.go +++ b/src/internal/packager/images/pull.go @@ -6,13 +6,9 @@ package images import ( "context" - "crypto/sha256" - "encoding/json" "errors" "fmt" "io" - "os" - "path" "path/filepath" "strings" @@ -26,13 +22,17 @@ import ( "github.com/google/go-containerregistry/pkg/v1/cache" "github.com/google/go-containerregistry/pkg/v1/daemon" "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/moby/moby/client" + "github.com/pterm/pterm" ) // PullAll pulls all of the images in the provided tag map. -func (i *ImgConfig) PullAll() (map[name.Tag]v1.Image, error) { +func (i *ImgConfig) PullAll() error { var ( - longer string - imgCount = len(i.ImgList) + longer string + imgCount = len(i.ImgList) + imageMap = map[string]v1.Image{} + tagToImage = map[name.Tag]v1.Image{} ) // Give some additional user feedback on larger image sets @@ -45,8 +45,6 @@ func (i *ImgConfig) PullAll() (map[name.Tag]v1.Image, error) { spinner := message.NewProgressSpinner("Loading metadata for %d images. %s", imgCount, longer) defer spinner.Stop() - imageMap := map[string]v1.Image{} - if message.GetLogLevel() >= message.DebugLevel { logs.Warn.SetOutput(spinner) logs.Progress.SetOutput(spinner) @@ -55,178 +53,118 @@ func (i *ImgConfig) PullAll() (map[name.Tag]v1.Image, error) { for idx, src := range i.ImgList { spinner.Updatef("Fetching image metadata (%d of %d): %s", idx+1, imgCount, src) - img, err := i.pullImage(src) + img, err := i.PullImage(src, spinner) if err != nil { - return nil, fmt.Errorf("failed to pull image %s: %w", src, err) + return fmt.Errorf("failed to pull image %s: %w", src, err) } imageMap[src] = img } - spinner.Updatef("Creating image tarball (this will take a while)") - - tagToImage := map[name.Tag]v1.Image{} - for src, img := range imageMap { - ref, err := name.ParseReference(src) + tag, err := name.NewTag(src, name.WeakValidation) if err != nil { - return nil, fmt.Errorf("failed to parse image reference %s: %w", src, err) - } - - tag, ok := ref.(name.Tag) - if !ok { - d, ok := ref.(name.Digest) - if !ok { - return nil, fmt.Errorf("image reference %s wasn't a tag or digest", src) - } - tag = d.Repository.Tag("digest-only") + return fmt.Errorf("failed to create tag for image %s: %w", src, err) } tagToImage[tag] = img } - spinner.Success() + spinner.Updatef("Preparing image sources and cache for image pulling") - progress := make(chan v1.Update, 200) + var ( + progress = make(chan v1.Update, 200) + progressBar *message.ProgressBar + title string + ) go func() { _ = tarball.MultiWriteToFile(i.TarballPath, tagToImage, tarball.WithProgress(progress)) }() - var progressBar *message.ProgressBar - var title string - for update := range progress { switch { case update.Error != nil && errors.Is(update.Error, io.EOF): progressBar.Success("Pulling %d images (%s)", len(imageMap), utils.ByteFormat(float64(update.Total), 2)) - return tagToImage, nil + return nil case update.Error != nil && strings.HasPrefix(update.Error.Error(), "archive/tar: missed writing "): // Handle potential image cache corruption with a more helpful error. See L#54 in libexec/src/archive/tar/writer.go message.Warnf("Potential image cache corruption: %s of %v bytes - try clearing cache with \"zarf tools clear-cache\"", update.Error.Error(), update.Total) - return nil, fmt.Errorf("failed to write image tarball: %w", update.Error) + return fmt.Errorf("failed to write image tarball: %w", update.Error) case update.Error != nil: - return nil, fmt.Errorf("failed to write image tarball: %w", update.Error) + return fmt.Errorf("failed to write image tarball: %w", update.Error) default: title = fmt.Sprintf("Pulling %d images (%s of %s)", len(imageMap), utils.ByteFormat(float64(update.Complete), 2), utils.ByteFormat(float64(update.Total), 2), ) if progressBar == nil { + spinner.Success() progressBar = message.NewProgressBar(update.Total, title) } progressBar.Update(update.Complete, title) } } - return tagToImage, nil + return nil } -// pullImage returns a v1.Image either by loading a local tarball, the pulling from the local daemon, or the wider internet -func (i *ImgConfig) pullImage(src string) (v1.Image, error) { - // Load image tarballs from the local filesystem +// PullImage returns a v1.Image either by loading a local tarball or the wider internet. +func (i *ImgConfig) PullImage(src string, spinner *message.Spinner) (img v1.Image, err error) { + // Load image tarballs from the local filesystem. if strings.HasSuffix(src, ".tar") || strings.HasSuffix(src, ".tar.gz") || strings.HasSuffix(src, ".tgz") { - message.Debugf("loading image tarball: %s", src) + spinner.Updatef("Reading image tarball: %s", src) return crane.Load(src, config.GetCraneOptions(true)...) } - // Unless disabled, attempt to pull the image from the local daemon - if !i.NoLocalImages { + // If crane is unable to pull the image, try to load it from the local docker daemon. + if _, err := crane.Manifest(src, config.GetCraneOptions(i.Insecure)...); err != nil { + message.Debugf("crane unable to pull image %s: %s", src, err) + spinner.Updatef("Falling back to docker for %s. This may take some time.", src) + + // Parse the image reference to get the image name. reference, err := name.ParseReference(src) if err != nil { - // log this error but don't return the error since we can still try pulling from the wider internet - message.Debugf("unable to parse the image reference, this might have impacts on pulling from the local daemon: %s", err.Error()) + return nil, fmt.Errorf("failed to parse image reference %s: %w", src, err) } - daemonOpts := daemon.WithContext(context.Background()) - if img, err := daemon.Image(reference, daemonOpts); err == nil { - message.Debugf("loading image from docker daemon: %s", src) - return img, err + // Attempt to connect to the local docker daemon. + ctx := context.TODO() + cli, err := client.NewClientWithOpts(client.FromEnv) + if err != nil { + return nil, fmt.Errorf("docker not available: %w", err) } - } + cli.NegotiateAPIVersion(ctx) - // We were unable to pull from the local daemon, so attempt to pull from the wider internet - img, err := crane.Pull(src, config.GetCraneOptions(i.Insecure)...) - if err != nil { - return nil, fmt.Errorf("failed to pull image %s: %w", src, err) - } + // Inspect the image to get the size. + rawImg, _, err := cli.ImageInspectWithRaw(ctx, src) + if err != nil { + return nil, fmt.Errorf("failed to inspect image %s via docker: %w", src, err) + } - message.Debugf("loading image with cache: %s", src) - imageCachePath := filepath.Join(config.GetAbsCachePath(), config.ZarfImageCacheDir) - img = cache.Image(img, cache.NewFilesystemCache(imageCachePath)) + // Warn the user if the image is large. + if rawImg.Size > 750*1000*1000 { + warn := pterm.DefaultParagraph.WithMaxWidth(80).Sprintf("%s is %s and may take a very long time to load via docker. "+ + "See https://docs.zarf.dev/docs/faq for suggestions on how to improve large local image loading operations.", + src, utils.ByteFormat(float64(rawImg.Size), 2)) + spinner.Warnf(warn) + } - return img, nil -} + // Use unbuffered opener to avoid OOM Kill issues https://github.com/defenseunicorns/zarf/issues/1214. + // This will also take for ever to load large images. + if img, err = daemon.Image(reference, daemon.WithUnbufferedOpener()); err != nil { + return nil, fmt.Errorf("failed to load image %s from docker daemon: %w", src, err) + } -// FormatCraneOCILayout ensures that all images are in the OCI format. -func FormatCraneOCILayout(ociPath string) error { - type IndexJSON struct { - SchemaVersion int `json:"schemaVersion"` - Manifests []struct { - MediaType string `json:"mediaType"` - Size int `json:"size"` - Digest string `json:"digest"` - } `json:"manifests"` + // The pull from the docker daemon was successful, return the image. + return img, err } - indexJSON, err := os.Open(path.Join(ociPath, "index.json")) - if err != nil { - message.Errorf(err, "Unable to open %s/index.json", ociPath) - return err - } - var index IndexJSON - byteValue, _ := io.ReadAll(indexJSON) - json.Unmarshal(byteValue, &index) - - digest := strings.TrimPrefix(index.Manifests[0].Digest, "sha256:") - b, err := os.ReadFile(path.Join(ociPath, "blobs", "sha256", digest)) - if err != nil { - message.Errorf(err, "Unable to open %s/blobs/sha256/%s", ociPath, digest) - return err - } - manifest := string(b) - // replace all docker media types w/ oci media types - manifest = strings.ReplaceAll(manifest, "application/vnd.docker.distribution.manifest.v2+json", "application/vnd.oci.image.manifest.v1+json") - manifest = strings.ReplaceAll(manifest, "application/vnd.docker.image.rootfs.diff.tar.gzip", "application/vnd.oci.image.layer.v1.tar+gzip") - - h := sha256.New() - h.Write([]byte(manifest)) - bs := h.Sum(nil) - - // Write the manifest to the blobs directory w/ the sha256 hash as the filename - manifestPath := path.Join(ociPath, "blobs", "sha256", fmt.Sprintf("%x", bs)) - manifestFile, err := os.Create(manifestPath) - if err != nil { - message.Errorf(err, "Unable to create %s/blobs/sha256/%x", ociPath, bs) - return err - } - defer manifestFile.Close() - _, err = manifestFile.WriteString(manifest) - if err != nil { - message.Errorf(err, "Unable to write to %s/blobs/sha256/%x", ociPath, bs) - return err + // Manifest was found, so use crane to pull the image. + if img, err = crane.Pull(src, config.GetCraneOptions(i.Insecure)...); err != nil { + return nil, fmt.Errorf("failed to pull image %s: %w", src, err) } - // Update the index.json to point to the new manifest - index.SchemaVersion = 2 - index.Manifests[0].Digest = fmt.Sprintf("sha256:%x", bs) - index.Manifests[0].Size = len(manifest) - index.Manifests[0].MediaType = "application/vnd.oci.image.manifest.v1+json" - indexJSON.Close() - _ = os.Remove(path.Join(ociPath, "index.json")) - indexJSON, err = os.Create(path.Join(ociPath, "index.json")) - if err != nil { - message.Errorf(err, "Unable to create %s/index.json", ociPath) - return err - } - indexJSONBytes, err := json.Marshal(index) - if err != nil { - message.Errorf(err, "Unable to marshal %s/index.json", ociPath) - return err - } - _, err = indexJSON.Write(indexJSONBytes) - if err != nil { - message.Errorf(err, "Unable to write to %s/index.json", ociPath) - return err - } - indexJSON.Close() + spinner.Updatef("Preparing image %s", src) + imageCachePath := filepath.Join(config.GetAbsCachePath(), config.ZarfImageCacheDir) + img = cache.Image(img, cache.NewFilesystemCache(imageCachePath)) - return nil + return img, nil } diff --git a/src/internal/packager/sbom/catalog.go b/src/internal/packager/sbom/catalog.go index 244b1a4df6..e2d3798f82 100644 --- a/src/internal/packager/sbom/catalog.go +++ b/src/internal/packager/sbom/catalog.go @@ -24,7 +24,6 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" "github.com/google/go-containerregistry/pkg/name" - v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/tarball" ) @@ -44,8 +43,8 @@ var transformRegex = regexp.MustCompile(`(?m)[^a-zA-Z0-9\.\-]`) var componentPrefix = "zarf-component-" // Catalog catalogs the given components and images to create an SBOM. -func Catalog(componentSBOMs map[string]*types.ComponentSBOM, tagToImage map[name.Tag]v1.Image, imagesPath, sbomPath string) { - imageCount := len(tagToImage) +func Catalog(componentSBOMs map[string]*types.ComponentSBOM, imgList []string, imagesPath, sbomPath string) { + imageCount := len(imgList) componentCount := len(componentSBOMs) builder := Builder{ spinner: message.NewProgressSpinner("Creating SBOMs for %d images and %d components with files.", imageCount, componentCount), @@ -59,7 +58,7 @@ func Catalog(componentSBOMs map[string]*types.ComponentSBOM, tagToImage map[name _ = utils.CreateDirectory(builder.sbomPath, 0700) // Generate a list of images and files for the sbom viewer - if json, err := builder.generateJSONList(componentSBOMs, tagToImage); err != nil { + if json, err := builder.generateJSONList(componentSBOMs, imgList); err != nil { builder.spinner.Fatalf(err, "Unable to generate the SBOM image list") } else { builder.jsonList = json @@ -68,7 +67,7 @@ func Catalog(componentSBOMs map[string]*types.ComponentSBOM, tagToImage map[name currImage := 1 // Generate SBOM for each image - for tag := range tagToImage { + for _, tag := range imgList { builder.spinner.Updatef("Creating image SBOMs (%d of %d): %s", currImage, imageCount, tag) jsonData, err := builder.createImageSBOM(tag) @@ -76,7 +75,7 @@ func Catalog(componentSBOMs map[string]*types.ComponentSBOM, tagToImage map[name builder.spinner.Fatalf(err, "Unable to create SBOM for image %s", tag) } - if err = builder.createSBOMViewerAsset(tag.String(), jsonData); err != nil { + if err = builder.createSBOMViewerAsset(tag, jsonData); err != nil { builder.spinner.Fatalf(err, "Unable to create SBOM viewer for image %s", tag) } @@ -107,7 +106,7 @@ func Catalog(componentSBOMs map[string]*types.ComponentSBOM, tagToImage map[name } // Include the compare tool if there are any image SBOMs OR component SBOMs - if len(componentSBOMs) > 0 || len(tagToImage) > 0 { + if len(componentSBOMs) > 0 || len(imgList) > 0 { if err := builder.createSBOMCompareAsset(); err != nil { builder.spinner.Fatalf(err, "Unable to create SBOM compare tool") } @@ -118,15 +117,27 @@ func Catalog(componentSBOMs map[string]*types.ComponentSBOM, tagToImage map[name // createImageSBOM uses syft to generate SBOM for an image, // some code/structure migrated from https://github.com/testifysec/go-witness/blob/v0.1.12/attestation/syft/syft.go. -func (b *Builder) createImageSBOM(tag name.Tag) ([]byte, error) { - // Get the image +func (b *Builder) createImageSBOM(src string) ([]byte, error) { + // Get the image reference. + tag, err := name.NewTag(src, name.WeakValidation) + if err != nil { + return nil, err + } + + // Load the image tarball. tarballImg, err := tarball.ImageFromPath(b.imagesPath, &tag) if err != nil { return nil, err } - // Create the sbom + // Create the sbom. imageCachePath := filepath.Join(b.cachePath, config.ZarfImageCacheDir) + + // Ensure the image cache directory exists. + if err := utils.CreateDirectory(imageCachePath, 0700); err != nil { + return nil, err + } + syftImage := image.NewImage(tarballImg, imageCachePath, image.WithTags(tag.String())) if err := syftImage.Read(); err != nil { return nil, err @@ -160,7 +171,7 @@ func (b *Builder) createImageSBOM(tag name.Tag) ([]byte, error) { } // Write the sbom to disk using the image tag as the filename - filename := fmt.Sprintf("%s.json", tag.String()) + filename := fmt.Sprintf("%s.json", tag) sbomFile, err := b.createSBOMFile(filename) if err != nil { return nil, err diff --git a/src/internal/packager/sbom/viewer.go b/src/internal/packager/sbom/viewer.go index 7a65064fb7..350b9d8b14 100644 --- a/src/internal/packager/sbom/viewer.go +++ b/src/internal/packager/sbom/viewer.go @@ -10,8 +10,6 @@ import ( "html/template" "github.com/defenseunicorns/zarf/src/types" - "github.com/google/go-containerregistry/pkg/name" - v1 "github.com/google/go-containerregistry/pkg/v1" ) func (b *Builder) createSBOMViewerAsset(identifier string, jsonData []byte) error { @@ -74,11 +72,11 @@ func (b *Builder) loadFileJS(name string) template.JS { } // This could be optimized, but loop over all the images and components to create a list of json files. -func (b *Builder) generateJSONList(componentToFiles map[string]*types.ComponentSBOM, tagToImage map[name.Tag]v1.Image) ([]byte, error) { +func (b *Builder) generateJSONList(componentToFiles map[string]*types.ComponentSBOM, imgList []string) ([]byte, error) { var jsonList []string - for tag := range tagToImage { - normalized := b.getNormalizedFileName(tag.String()) + for _, tag := range imgList { + normalized := b.getNormalizedFileName(tag) jsonList = append(jsonList, normalized) } diff --git a/src/pkg/message/spinner.go b/src/pkg/message/spinner.go index 041fd5cafa..2df82c00f1 100644 --- a/src/pkg/message/spinner.go +++ b/src/pkg/message/spinner.go @@ -5,7 +5,9 @@ package message import ( - "fmt" + "bufio" + "bytes" + "strings" "github.com/pterm/pterm" ) @@ -14,8 +16,10 @@ var activeSpinner *Spinner // Spinner is a wrapper around pterm.SpinnerPrinter. type Spinner struct { - spinner *pterm.SpinnerPrinter - startText string + spinner *pterm.SpinnerPrinter + startText string + termWidth int + preserveWrites bool } // NewProgressSpinner creates a new progress spinner. @@ -26,7 +30,7 @@ func NewProgressSpinner(format string, a ...any) *Spinner { } var spinner *pterm.SpinnerPrinter - text := fmt.Sprintf(format, a...) + text := pterm.Sprintf(format, a...) if NoProgress { Info(text) } else { @@ -40,19 +44,46 @@ func NewProgressSpinner(format string, a ...any) *Spinner { activeSpinner = &Spinner{ spinner: spinner, startText: text, + termWidth: pterm.GetTerminalWidth(), } return activeSpinner } +// EnablePreserveWrites enables preserving writes to the terminal. +func (p *Spinner) EnablePreserveWrites() { + p.preserveWrites = true +} + +// DisablePreserveWrites disables preserving writes to the terminal. +func (p *Spinner) DisablePreserveWrites() { + p.preserveWrites = false +} + // Write the given text to the spinner. -func (p *Spinner) Write(text []byte) (int, error) { - size := len(text) +func (p *Spinner) Write(raw []byte) (int, error) { + size := len(raw) if NoProgress { + pterm.Printfln(" %s", string(raw)) return size, nil } - Debug(string(text)) - return len(text), nil + + // Split the text into lines and update the spinner for each line. + scanner := bufio.NewScanner(bytes.NewReader(raw)) + scanner.Split(bufio.ScanLines) + for scanner.Scan() { + // Only be fancy if preserve writes is enabled. + if p.preserveWrites { + text := pterm.Sprintf(" %s", scanner.Text()) + pterm.Fprinto(p.spinner.Writer, strings.Repeat(" ", p.termWidth)) + pterm.Fprintln(p.spinner.Writer, text) + } else { + // Otherwise just update the spinner text. + p.spinner.UpdateText(scanner.Text()) + } + } + + return size, nil } // Updatef updates the spinner text. @@ -61,7 +92,7 @@ func (p *Spinner) Updatef(format string, a ...any) { return } - text := fmt.Sprintf(format, a...) + text := pterm.Sprintf(format, a...) p.spinner.UpdateText(text) } @@ -80,18 +111,18 @@ func (p *Spinner) Success() { // Successf prints a success message with the spinner and stops it. func (p *Spinner) Successf(format string, a ...any) { - text := fmt.Sprintf(format, a...) + text := pterm.Sprintf(format, a...) if p.spinner != nil { p.spinner.Success(text) - activeSpinner = nil } else { Info(text) } + p.Stop() } // Warnf prints a warning message with the spinner. func (p *Spinner) Warnf(format string, a ...any) { - text := fmt.Sprintf(format, a...) + text := pterm.Sprintf(format, a...) if p.spinner != nil { p.spinner.Warning(text) } else { diff --git a/src/pkg/packager/actions.go b/src/pkg/packager/actions.go index fffb2b9c37..d3532ebe74 100644 --- a/src/pkg/packager/actions.go +++ b/src/pkg/packager/actions.go @@ -30,8 +30,17 @@ func (p *Packager) runActions(defaultCfg types.ZarfComponentActionDefaults, acti // Run commands that a component has provided. func (p *Packager) runAction(defaultCfg types.ZarfComponentActionDefaults, action types.ZarfComponentAction, valueTemplate *template.Values) error { - spinner := message.NewProgressSpinner("Running command \"%s\"", action.Cmd) - defer spinner.Success() + var cmdEscaped string + + if action.Description != "" { + cmdEscaped = action.Description + } else { + cmdEscaped = escapeCmdForPrint(action.Cmd) + } + + spinner := message.NewProgressSpinner("Running command \"%s\"", cmdEscaped) + // Persist the spinner output so it doesn't get overwritten by the command output. + spinner.EnablePreserveWrites() var ( ctx context.Context @@ -52,7 +61,7 @@ func (p *Packager) runAction(defaultCfg types.ZarfComponentActionDefaults, actio cfg := actionGetCfg(defaultCfg, action, vars) if cmd, err = actionCmdMutation(action.Cmd); err != nil { - spinner.Errorf(err, "Error mutating command: %s", cmd) + spinner.Errorf(err, "Error mutating command: %s", cmdEscaped) } duration := time.Duration(cfg.MaxTotalSeconds) * time.Second @@ -61,56 +70,56 @@ func (p *Packager) runAction(defaultCfg types.ZarfComponentActionDefaults, actio // Keep trying until the max retries is reached. for remaining := cfg.MaxRetries + 1; remaining > 0; remaining-- { - // If no timeout is set, run the command and return or continue retrying. - if cfg.MaxTotalSeconds < 1 { - spinner.Updatef("Waiting for command \"%s\" (no timeout)", cmd) - + // Perform the action run. + tryCmd := func(ctx context.Context) error { // Try running the command and continue the retry loop if it fails. - if out, err = actionRun(context.TODO(), cfg, cmd); err != nil { - message.Debugf("command \"%s\" failed: %s", cmd, err.Error()) - continue + if out, err = actionRun(ctx, cfg, cmd, spinner); err != nil { + return err } + out = strings.TrimSpace(out) + // If an output variable is defined, set it. if action.SetVariable != "" { - p.setVariable(action.SetVariable, strings.TrimSpace(out)) + p.setVariable(action.SetVariable, out) } // If the command ran successfully, continue to the next action. + spinner.Successf("Completed command \"%s\"", cmdEscaped) + + return nil + } + + // If no timeout is set, run the command and return or continue retrying. + if cfg.MaxTotalSeconds < 1 { + spinner.Updatef("Waiting for \"%s\" (no timeout)", cmdEscaped) + if err := tryCmd(context.TODO()); err != nil { + continue + } + return nil } // Run the command on repeat until success or timeout. - spinner.Updatef("Waiting for command \"%s\" (timeout: %d seconds)", cmd, cfg.MaxTotalSeconds) + spinner.Updatef("Waiting for \"%s\" (timeout: %ds)", cmdEscaped, cfg.MaxTotalSeconds) select { // On timeout abort. case <-timeout: cancel() - return fmt.Errorf("command \"%s\" timed out", cmd) + return fmt.Errorf("command \"%s\" timed out", cmdEscaped) // Otherwise, try running the command. default: ctx, cancel = context.WithTimeout(context.Background(), duration) defer cancel() - - // Try running the command and continue the retry loop if it fails. - if out, err = actionRun(ctx, cfg, cmd); err != nil { - message.Debug(err) - continue - } - - // If an output variable is defined, set it. - if action.SetVariable != "" { - p.setVariable(action.SetVariable, strings.TrimSpace(out)) + if err := tryCmd(ctx); err == nil { + return nil } - - // If the command ran successfully, continue to the next action. - return nil } } // If we've reached this point, the retry limit has been reached. - return fmt.Errorf("command \"%s\" failed after %d retries", cmd, cfg.MaxRetries) + return fmt.Errorf("command \"%s\" failed after %d retries", cmdEscaped, cfg.MaxRetries) } // Perform some basic string mutations to make commands more useful. @@ -181,28 +190,42 @@ func actionGetCfg(cfg types.ZarfComponentActionDefaults, a types.ZarfComponentAc return cfg } -func actionRun(ctx context.Context, cfg types.ZarfComponentActionDefaults, cmd string) (string, error) { +func actionRun(ctx context.Context, cfg types.ZarfComponentActionDefaults, cmd string, spinner *message.Spinner) (string, error) { var shell string var shellArgs string if runtime.GOOS == "windows" { shell = "powershell" shellArgs = "-Command" + message.Debug("Running command in PowerShell: %s", cmd) } else { shell = "sh" shellArgs = "-c" + message.Debug("Running command in shell: %s", cmd) } execCfg := exec.Config{ - Print: !cfg.Mute, - Env: cfg.Env, - Dir: cfg.Dir, + Env: cfg.Env, + Dir: cfg.Dir, } - output, errOut, err := exec.CmdWithContext(ctx, execCfg, shell, shellArgs, cmd) - // Dump the command output in debug if output not already streamed. - if cfg.Mute { - message.Debug(output, errOut) + + if !cfg.Mute { + execCfg.Stdout = spinner + execCfg.Stderr = spinner } - return output, err + out, errOut, err := exec.CmdWithContext(ctx, execCfg, shell, shellArgs, cmd) + // Dump final complete output. + message.Debug(cmd, out, errOut) + + return out, err +} + +func escapeCmdForPrint(cmd string) string { + cmdEscaped := strings.ReplaceAll(cmd, "\n", "; ") + // Truncate the command if it is longer than 60 characters so it isn't too long. + if len(cmdEscaped) > 60 { + cmdEscaped = cmdEscaped[:57] + "..." + } + return cmdEscaped } diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index eaa4e86cad..d4641844f8 100644 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -27,8 +27,6 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" "github.com/google/go-containerregistry/pkg/crane" - "github.com/google/go-containerregistry/pkg/name" - v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/mholt/archiver/v3" ) @@ -62,6 +60,17 @@ func (p *Packager) Create(baseDir string) error { return fmt.Errorf("unable to fill variables in template: %s", err.Error()) } + seedImage := fmt.Sprintf("%s:%s", config.ZarfSeedImage, config.ZarfSeedTag) + + // Add the seed image to the registry component if this is an init config. + if p.cfg.IsInitConfig { + for idx, c := range p.cfg.Pkg.Components { + if c.Name == "zarf-registry" { + p.cfg.Pkg.Components[idx].Images = append(c.Images, seedImage) + } + } + } + // Save the transformed config if err := p.writeYaml(); err != nil { return fmt.Errorf("unable to write zarf.yaml: %w", err) @@ -76,23 +85,26 @@ func (p *Packager) Create(baseDir string) error { return fmt.Errorf("package creation canceled") } + // Save the seed image as an OCI image if this is an init config. if p.cfg.IsInitConfig { - // Load seed images into their own happy little tarball for ease of import on init - seedImage := fmt.Sprintf("%s:%s", config.ZarfSeedImage, config.ZarfSeedTag) - pulledImages, err := p.pullImages([]string{seedImage}, p.tmp.SeedImage) - if err != nil { - return fmt.Errorf("unable to pull the seed image after 3 attempts: %w", err) - } + spinner := message.NewProgressSpinner("Loading Zarf Registry Seed Image") + defer spinner.Stop() + ociPath := path.Join(p.tmp.Base, "seed-image") - for _, image := range pulledImages { - if err := crane.SaveOCI(image, ociPath); err != nil { - return fmt.Errorf("unable to save image %s as OCI: %w", image, err) - } + imgConfig := images.ImgConfig{ + Insecure: config.CommonOptions.Insecure, + } + + image, err := imgConfig.PullImage(seedImage, spinner) + if err != nil { + return fmt.Errorf("unable to pull seed image: %w", err) } - if err := images.FormatCraneOCILayout(ociPath); err != nil { - return fmt.Errorf("unable to format OCI layout: %w", err) + if err := crane.SaveOCI(image, ociPath); err != nil { + return fmt.Errorf("unable to save image %s as OCI: %w", image, err) } + + spinner.Success() } var combinedImageList []string @@ -124,13 +136,23 @@ func (p *Packager) Create(baseDir string) error { combinedImageList = append(combinedImageList, component.Images...) } - pulledImages := map[name.Tag]v1.Image{} + imgList := utils.Unique(combinedImageList) + // Images are handled separately from other component assets - if len(combinedImageList) > 0 { - uniqueList := utils.Unique(combinedImageList) + if len(imgList) > 0 { + message.HeaderInfof("📦 COMPONENT IMAGES") + + doPull := func() error { + imgConfig := images.ImgConfig{ + TarballPath: p.tmp.Images, + ImgList: imgList, + Insecure: config.CommonOptions.Insecure, + } - var err error - if pulledImages, err = p.pullImages(uniqueList, p.tmp.Images); err != nil { + return imgConfig.PullAll() + } + + if err := utils.Retry(doPull, 3, 5*time.Second); err != nil { return fmt.Errorf("unable to pull images after 3 attempts: %w", err) } } @@ -139,7 +161,7 @@ func (p *Packager) Create(baseDir string) error { if p.cfg.CreateOpts.SkipSBOM { message.Debug("Skipping image SBOM processing per --skip-sbom flag") } else { - sbom.Catalog(componentSBOMs, pulledImages, p.tmp.Images, p.tmp.Sboms) + sbom.Catalog(componentSBOMs, imgList, p.tmp.Images, p.tmp.Sboms) } // In case the directory was changed, reset to prevent breaking relative target paths @@ -216,24 +238,6 @@ func (p *Packager) Create(baseDir string) error { return nil } -func (p *Packager) pullImages(imgList []string, path string) (map[name.Tag]v1.Image, error) { - var pulledImages map[name.Tag]v1.Image - var err error - - return pulledImages, utils.Retry(func() error { - imgConfig := images.ImgConfig{ - TarballPath: path, - ImgList: imgList, - Insecure: config.CommonOptions.Insecure, - NoLocalImages: p.cfg.CreateOpts.NoLocalImages, - } - - pulledImages, err = imgConfig.PullAll() - - return err - }, 3, 5*time.Second) -} - func (p *Packager) addComponent(component types.ZarfComponent) (*types.ComponentSBOM, error) { message.HeaderInfof("📦 %s COMPONENT", strings.ToUpper(component.Name)) diff --git a/src/pkg/packager/deploy.go b/src/pkg/packager/deploy.go index 8e6859a9e9..b9edcb263a 100644 --- a/src/pkg/packager/deploy.go +++ b/src/pkg/packager/deploy.go @@ -164,24 +164,9 @@ func (p *Packager) deployInitComponent(component types.ZarfComponent) (charts [] // Do cleanup for when we inject the seed registry during initialization if isSeedRegistry { - err := p.cluster.StopInjectionMadness() - if err != nil { + if err := p.cluster.StopInjectionMadness(); err != nil { return charts, fmt.Errorf("unable to seed the Zarf Registry: %w", err) } - - seedImage := fmt.Sprintf("%s:%s", config.ZarfSeedImage, config.ZarfSeedTag) - imgConfig := images.ImgConfig{ - TarballPath: p.tmp.SeedImage, - ImgList: []string{seedImage}, - NoChecksum: true, - RegInfo: p.cfg.State.RegistryInfo, - Insecure: config.CommonOptions.Insecure, - } - - // Push the seed images into to Zarf registry - if err = imgConfig.PushToZarfRegistry(); err != nil { - return charts, fmt.Errorf("unable to push the seed images to the Zarf Registry: %w", err) - } } return charts, nil diff --git a/src/pkg/packager/variables.go b/src/pkg/packager/variables.go index af76f50a28..1b11b88156 100644 --- a/src/pkg/packager/variables.go +++ b/src/pkg/packager/variables.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" ) @@ -84,6 +85,7 @@ func (p *Packager) setActiveVariables() error { } func (p *Packager) setVariable(name, value string) { + message.Debugf("Setting variable '%s' to '%s'", name, value) p.cfg.SetVariableMap[name] = value } diff --git a/src/pkg/utils/exec/exec.go b/src/pkg/utils/exec/exec.go index bb1c3d368e..e59e327bb2 100644 --- a/src/pkg/utils/exec/exec.go +++ b/src/pkg/utils/exec/exec.go @@ -24,9 +24,11 @@ const colorWhite = "\x1b[37;1m" // Config is a struct for configuring the Cmd function. type Config struct { - Print bool - Dir string - Env []string + Print bool + Dir string + Env []string + Stdout io.Writer + Stderr io.Writer } // PrintCfg is a helper function for returning a Config struct with Print set to true. @@ -51,18 +53,6 @@ func CmdWithContext(ctx context.Context, config Config, command string, args ... return "", "", errors.New("command is required") } - // Print the command if requested. - if config.Print { - fmt.Println() - fmt.Printf(" %s", colorGreen) - fmt.Print(command + " ") - fmt.Printf("%s", colorCyan) - fmt.Printf("%v", args) - fmt.Printf("%s", colorWhite) - fmt.Printf("%s", colorReset) - fmt.Println("") - } - // Set up the command. cmd := exec.CommandContext(ctx, command, args...) cmd.Dir = config.Dir @@ -72,45 +62,75 @@ func CmdWithContext(ctx context.Context, config Config, command string, args ... cmdStdout, _ := cmd.StdoutPipe() cmdStderr, _ := cmd.StderrPipe() - var stdoutBuf, stderrBuf bytes.Buffer - stdout := io.MultiWriter(os.Stdout, &stdoutBuf) - stderr := io.MultiWriter(os.Stderr, &stderrBuf) + var ( + stdoutBuf, stderrBuf bytes.Buffer + errStdout, errStderr error + wg sync.WaitGroup + ) + + stdoutWriters := []io.Writer{ + &stdoutBuf, + } + + stdErrWriters := []io.Writer{ + &stderrBuf, + } + + // Add the writers if requested. + if config.Stdout != nil { + stdoutWriters = append(stdoutWriters, config.Stdout) + } + + if config.Stderr != nil { + stdErrWriters = append(stdErrWriters, config.Stderr) + } + + // Print to stdout if requested. + if config.Print { + stdoutWriters = append(stdoutWriters, os.Stdout) + stdErrWriters = append(stdErrWriters, os.Stderr) + } + + // Bind all the writers. + stdout := io.MultiWriter(stdoutWriters...) + stderr := io.MultiWriter(stdErrWriters...) + + // If we're printing, print the command. + if config.Print { + cmdString := fmt.Sprintf("%s%s %s%v%s%s", + colorGreen, command, colorCyan, args, colorWhite, colorReset) + fmt.Println(cmdString) + } // Start the command. if err := cmd.Start(); err != nil { return "", "", err } - // If printing live output, copy the command outputs to stdout/stderr. - if config.Print { - var errStdout, errStderr error - var wg sync.WaitGroup - - // Set the wait group to 2 so we wait for both stdout and stderr. - wg.Add(2) - - // Run a goroutine to capture the command's stdout live. - go func() { - _, errStdout = io.Copy(stdout, cmdStdout) - wg.Done() - }() - - // Run a goroutine to capture the command's stderr live. - go func() { - _, errStderr = io.Copy(stderr, cmdStderr) - wg.Done() - }() - - // Wait for the goroutines to finish. - wg.Wait() - - // Abort if there was an error capturing the command's outputs. - if errStdout != nil { - return "", "", fmt.Errorf("failed to capture the stdout command output: %w", errStdout) - } - if errStderr != nil { - return "", "", fmt.Errorf("failed to capture the stderr command output: %w", errStderr) - } + // Add to waitgroup for each goroutine. + wg.Add(2) + + // Run a goroutine to capture the command's stdout live. + go func() { + _, errStdout = io.Copy(stdout, cmdStdout) + wg.Done() + }() + + // Run a goroutine to capture the command's stderr live. + go func() { + _, errStderr = io.Copy(stderr, cmdStderr) + wg.Done() + }() + + // Wait for the goroutines to finish (if any). + wg.Wait() + + // Abort if there was an error capturing the command's outputs. + if errStdout != nil { + return "", "", fmt.Errorf("failed to capture the stdout command output: %w", errStdout) + } + if errStderr != nil { + return "", "", fmt.Errorf("failed to capture the stderr command output: %w", errStderr) } // Wait for the command to finish and return the buffered outputs, regardless of whether we printed them. diff --git a/src/test/e2e/02_component_actions_test.go b/src/test/e2e/02_component_actions_test.go index 29cead76a1..01728a1e83 100644 --- a/src/test/e2e/02_component_actions_test.go +++ b/src/test/e2e/02_component_actions_test.go @@ -38,6 +38,11 @@ func TestComponentActions(t *testing.T) { // Try creating the package to test the onCreate actions. stdOut, stdErr, err := e2e.execZarfCommand("package", "create", "examples/component-actions", "--confirm") require.NoError(t, err, stdOut, stdErr) + require.Contains(t, stdErr, "Completed command \"touch test-create-before.txt\"") + require.Contains(t, stdErr, "multiline!") + require.Contains(t, stdErr, "updates!") + require.Contains(t, stdErr, "realtime!") + require.Contains(t, stdErr, "Completed command \"multiline & description demo\"") // Test for package create prepare artifacts. for _, artifact := range createArtifacts { @@ -63,20 +68,20 @@ func TestComponentActions(t *testing.T) { // Deploy the simple action that should fail the timeout. stdOut, stdErr, err = e2e.execZarfCommand("package", "deploy", path, "--confirm", "--components=on-deploy-with-timeout") require.Error(t, err, stdOut, stdErr) - require.Contains(t, stdOut, "😭😭😭 this action failed because it took too long to run 😭😭😭") + require.Contains(t, stdErr, "😭😭😭 this action failed because it took too long to run 😭😭😭") // Test using a Zarf Variable within the action stdOut, stdErr, err = e2e.execZarfCommand("package", "deploy", path, "--confirm", "--components=on-deploy-with-variable", "-l=trace") require.NoError(t, err, stdOut, stdErr) - require.Contains(t, stdOut, "the dog says ruff") + require.Contains(t, stdErr, "the dog says ruff") // Test using dynamic and multiple-variables stdOut, stdErr, err = e2e.execZarfCommand("package", "deploy", path, "--confirm", "--components=on-deploy-with-dynamic-variable,on-deploy-with-multiple-variables", "-l=trace") require.NoError(t, err, stdOut, stdErr) - require.Contains(t, stdOut, "the cat says meow") - require.Contains(t, stdOut, "the dog says ruff") - require.Contains(t, stdOut, "the snake says hiss") - require.Contains(t, stdOut, "with a TF_VAR, the snake also says hiss") + require.Contains(t, stdErr, "the cat says meow") + require.Contains(t, stdErr, "the dog says ruff") + require.Contains(t, stdErr, "the snake says hiss") + require.Contains(t, stdErr, "with a TF_VAR, the snake also says hiss") // Test using environment variables stdOut, stdErr, err = e2e.execZarfCommand("package", "deploy", path, "--confirm", "--components=on-deploy-with-env-var") diff --git a/src/types/component.go b/src/types/component.go index 8db113f158..62a7329943 100644 --- a/src/types/component.go +++ b/src/types/component.go @@ -144,6 +144,7 @@ type ZarfComponentAction struct { Env []string `json:"env,omitempty" jsonschema:"description=Additional environment variables to set for the command"` Cmd string `json:"cmd,omitempty" jsonschema:"description=The command to run"` SetVariable string `json:"setVariable,omitempty" jsonschema:"description=The name of a variable to update with the output of the command. This variable will be available to all remaining actions and components in the package.,pattern=^[A-Z0-9_]+$"` + Description string `json:"description,omitempty" jsonschema:"description=Description of the action to be displayed during package execution instead of the command"` } // ZarfContainerTarget defines the destination info for a ZarfData target diff --git a/src/types/runtime.go b/src/types/runtime.go index d03b3e8c41..f0772d916a 100644 --- a/src/types/runtime.go +++ b/src/types/runtime.go @@ -42,7 +42,6 @@ type ZarfCreateOptions struct { SBOMOutputDir string `json:"sbomOutput" jsonschema:"description=Location to output an SBOM into after package creation"` SetVariables map[string]string `json:"setVariables" jsonschema:"description=Key-Value map of variable names and their corresponding values that will be used to template against the Zarf package being used"` MaxPackageSizeMB int `json:"maxPackageSizeMB" jsonschema:"description=Size of chunks to use when splitting a zarf package into multiple files in megabytes"` - NoLocalImages bool `json:"noLocalImages" jsonschema:"description=Disable the use of local container images during package creation"` } // ZarfPartialPackageData contains info about a partial package. diff --git a/src/ui/lib/api-types.ts b/src/ui/lib/api-types.ts index f1423173b7..beceae8218 100644 --- a/src/ui/lib/api-types.ts +++ b/src/ui/lib/api-types.ts @@ -308,6 +308,10 @@ export interface ZarfComponentAction { * The command to run */ cmd?: string; + /** + * Description of the action to be displayed during package execution instead of the command + */ + description?: string; /** * The working directory to run the command in (default is CWD) */ @@ -754,10 +758,6 @@ export interface ZarfCreateOptions { * Size of chunks to use when splitting a zarf package into multiple files in megabytes */ maxPackageSizeMB: number; - /** - * Disable the use of local container images during package creation - */ - noLocalImages: boolean; /** * Location where the finalized Zarf package will be placed */ @@ -1046,6 +1046,7 @@ const typeMap: any = { ], false), "ZarfComponentAction": o([ { json: "cmd", js: "cmd", typ: u(undefined, "") }, + { json: "description", js: "description", typ: u(undefined, "") }, { json: "dir", js: "dir", typ: u(undefined, "") }, { json: "env", js: "env", typ: u(undefined, a("")) }, { json: "maxRetries", js: "maxRetries", typ: u(undefined, 0) }, @@ -1185,7 +1186,6 @@ const typeMap: any = { ], false), "ZarfCreateOptions": o([ { json: "maxPackageSizeMB", js: "maxPackageSizeMB", typ: 0 }, - { json: "noLocalImages", js: "noLocalImages", typ: true }, { json: "outputDirectory", js: "outputDirectory", typ: "" }, { json: "sbom", js: "sbom", typ: true }, { json: "sbomOutput", js: "sbomOutput", typ: "" }, diff --git a/zarf.schema.json b/zarf.schema.json index ee8b753049..a9bd30c3a4 100644 --- a/zarf.schema.json +++ b/zarf.schema.json @@ -272,6 +272,10 @@ "pattern": "^[A-Z0-9_]+$", "type": "string", "description": "The name of a variable to update with the output of the command. This variable will be available to all remaining actions and components in the package." + }, + "description": { + "type": "string", + "description": "Description of the action to be displayed during package execution instead of the command" } }, "additionalProperties": false,