Add the ability to specify registry overrides on package create (#1683)

## Description

Allows registries to be overridden with registry mirrors on Zarf package
create.

## Related Issue

Fixes #1577 

- [x] Fix the faq for large local images - this can fix that use case.
#1679 (comment)

## Type of change

- [ ] Bug fix (non-breaking change which fixes an issue)
- [X] New feature (non-breaking change which adds functionality)
- [ ] Other (security config, docs update, etc)

## Checklist before merging

- [x] Test, docs, adr added or updated as needed
- [X] [Contributor Guide
Steps](https://github.com/defenseunicorns/zarf/blob/main/CONTRIBUTING.md#developer-workflow)
followed

---------

Co-authored-by: Jonathan Perry <[email protected]>

docs/2-the-zarf-cli/100-cli-commands/zarf_package_create.md

@@ -16,17 +16,18 @@ zarf package create [DIRECTORY] [flags]
 ## Options
 
 ```
-      --confirm                   Confirm package creation without prompting
-      --differential string       Build a package that only contains the differential changes from local resources and differing remote resources from the specified previously built package
-  -h, --help                      help for create
-  -k, --key string                Path to private key file for signing packages
-      --key-pass string           Password to the private key file used for signing packages
-  -m, --max-package-size int      Specify the maximum size of the package in megabytes, packages larger than this will be split into multiple parts. Use 0 to disable splitting.
-  -o, --output-directory string   Specify the output directory for the created Zarf package
-  -s, --sbom                      View SBOM contents after creating the package
-      --sbom-out string           Specify an output directory for the SBOMs from the created Zarf package
-      --set stringToString        Specify package variables to set on the command line (KEY=value) (default [])
-      --skip-sbom                 Skip generating SBOM for this package
+      --confirm                            Confirm package creation without prompting
+      --differential string                Build a package that only contains the differential changes from local resources and differing remote resources from the specified previously built package
+  -h, --help                               help for create
+  -k, --key string                         Path to private key file for signing packages
+      --key-pass string                    Password to the private key file used for signing packages
+  -m, --max-package-size int               Specify the maximum size of the package in megabytes, packages larger than this will be split into multiple parts. Use 0 to disable splitting.
+  -o, --output-directory string            Specify the output directory for the created Zarf package
+      --registry-override stringToString   Specify a map of domains to override on package create when pulling images (e.g. --registry-override docker.io=dockerio-reg.enterprise.intranet) (default [])
+  -s, --sbom                               View SBOM contents after creating the package
+      --sbom-out string                    Specify an output directory for the SBOMs from the created Zarf package
+      --set stringToString                 Specify package variables to set on the command line (KEY=value) (default [])
+      --skip-sbom                          Skip generating SBOM for this package
 ```
 
 ## Options inherited from parent commands

docs/3-create-a-zarf-package/4-zarf-schema.md

@@ -424,6 +424,46 @@ Must be one of:
 </blockquote>
 </details>
 
+<details open>
+<summary>
+<strong> <a name="build_registryOverrides"></a>registryOverrides *</strong>
+</summary>
+&nbsp;
+<blockquote>
+
+  ## build > registryOverrides
+![Required](https://img.shields.io/badge/Required-red)
+
+**Description:** Any registry domains that were overridden on package create when pulling images
+
+|                           |                                                                                                                                   |
+| ------------------------- | --------------------------------------------------------------------------------------------------------------------------------- |
+| **Type**                  | `object`                                                                                                                          |
+| **Additional properties** | [![Any type: allowed](https://img.shields.io/badge/Any%20type-allowed-green)](# "Additional Properties of any type are allowed.") |
+
+<details>
+<summary>
+<strong> <a name="build_registryOverrides_pattern1"></a>Pattern Property .*</strong>
+</summary>
+&nbsp;
+<blockquote>
+
+:::note
+All properties whose name matches the regular expression
+```.*``` ([Test](https://regex101.com/?regex=.%2A))
+must respect the following conditions
+:::
+
+|          |          |
+| -------- | -------- |
+| **Type** | `string` |
+
+</blockquote>
+</details>
+
+</blockquote>
+</details>
+
 </blockquote>
 </details>
 

docs/6-zarf-tutorials/0-creating-a-zarf-package.md

@@ -254,7 +254,7 @@ This will create a zarf package in the current directory with a package name tha
 
 :::tip
 
-You can learn more about what is going on behind the scenes of this process on the [package create lifecycle page](../3-create-a-zarf-package/5-package-create-lifecycle.md).
+You can learn more about what is going on behind the scenes of this process on the [package create lifecycle page](../3-create-a-zarf-package/5-package-create-lifecycle.md), and can view other useful command flags like `--differential` and `--registry-override` on the [package create command flags page](../2-the-zarf-cli/100-cli-commands/zarf_package_create.md).
 
 :::
 

docs/6-zarf-tutorials/8-custom-init-packages.md

@@ -74,6 +74,19 @@ As of v0.26.0 you can swap the `registry` and `agent` images by specifying diffe
 
 For other components, or older versions of Zarf, you can modify the manifests of the components you want to change in their individual packages under the `packages` folder of the Zarf repo.
 
+:::tip
+
+If your enterprise uses pull-through mirrors to host vetted images you can run the following command to create a Zarf 'init' package from those mirrors (where `<registry>.enterprise.corp` are your enterprise mirror(s)):
+
+```
+$ zarf package create . --set AGENT_IMAGE_TAG=vX.X.X \
+  --registry-override docker.io=dockerio.enterprise.corp \
+  --registry-override ghcr.io=ghcr.enterprise.corp \
+  --registry-override quay.io=quay.enterprise.corp
+```
+
+:::
+
 ### Removing Components
 
 You may not need or want all of the components in your 'init' package and may choose to slim down your package by removing them.  Because the [Zarf Package is composed](../3-create-a-zarf-package/2-zarf-components.md#composing-package-components) all you need to do is remove the component that imports the component you wish to exclude.

docs/8-faq.md

@@ -46,14 +46,16 @@ Additionally, when adopting resources, you should ensure that the namespaces you
 
 ## How can I improve the speed of loading large images from Docker on `zarf package create`?
 
-Due to some limitations with how Docker provides access to local image layers, `zarf package create` has to rely on `docker save` under the hood which is [very slow overall](https://github.com/defenseunicorns/zarf/issues/1214) and also takes a long time to report progress. We experimented with many ways to improve this, but for now recommend leveraging a local docker registry to speed up the process. This can be done by running a local registry and pushing the images to it before running `zarf package create`. This will allow `zarf package create` to pull the images from the local registry instead of Docker. This can also be combined with [component actions](3-create-a-zarf-package/6-component-actions.md) to make the process automatic. Given an example image of `my-giant-image:###ZARF_PKG_TMPL_IMG###` you could do something like this:
+Due to some limitations with how Docker provides access to local image layers, `zarf package create` has to rely on `docker save` under the hood which is [very slow overall](https://github.com/defenseunicorns/zarf/issues/1214) and also takes a long time to report progress. We experimented with many ways to improve this, but for now recommend leveraging a local docker registry to speed up the process.
+
+This can be done by running a local registry and pushing the images to it before running `zarf package create`. This will allow `zarf package create` to pull the images from the local registry instead of Docker. This can also be combined with [component actions](3-create-a-zarf-package/6-component-actions.md) and [`--registry-override`](./2-the-zarf-cli/100-cli-commands/zarf_package_create.md) to make the process automatic. Given an example image of `registry.enterprise.corp/my-giant-image:v2` you could do something like this:
 
 ```sh
 # Create a local registry
 docker run -d -p 5000:5000 --restart=always --name registry registry:2
 
 # Run the package create with a tag variable
-zarf package create --set IMG=my-giant-image:v2
+zarf package create --registry-override registry.enterprise.corp=localhost:5000 --set IMG=my-giant-image:v2
 ```
 
 ```yaml
@@ -68,11 +70,11 @@ components:
       onCreate:
         # runs before the component is created
         before:
-          - cmd: 'docker tag ###ZARF_PKG_TMPL_IMG### localhost:5000/###ZARF_PKG_TMPL_IMG###'
+          - cmd: 'docker tag registry.enterprise.corp/###ZARF_PKG_TMPL_IMG### localhost:5000/###ZARF_PKG_TMPL_IMG###'
           - cmd: 'docker push localhost:5000/###ZARF_PKG_TMPL_IMG###'
 
     images:
-      - 'localhost:5000/###ZARF_PKG_TMPL_IMG###'
+      - 'registry.enterprise.corp/###ZARF_PKG_TMPL_IMG###'
 ```
 
 ## Can I pull in more than http(s) git repos on `zarf package create`?

examples/helm-git-chart/zarf.yaml

@@ -1,6 +1,6 @@
 kind: ZarfPackageConfig
 metadata:
-  name: test-helm-git-chart
+  name: helm-git-chart
   description: "Deploys a helm chart from git"
 components:
   - name: demo-helm-git-chart

packages/distros/k3s/zarf.yaml

@@ -62,4 +62,4 @@ components:
         before:
           - cmd: if [ "$(arch)" != "arm64" ]; then echo "this package architecture is arm64, but the target system has a different architecture. These architectures must be the same" && exit 1; fi
             description: Check that the host architecture matches the package architecture
-            maxRetries: 0
+            maxRetries: 0

src/cmd/package.go

@@ -315,6 +315,7 @@ func bindCreateFlags() {
 	createFlags.IntVarP(&pkgConfig.CreateOpts.MaxPackageSizeMB, "max-package-size", "m", v.GetInt(V_PKG_CREATE_MAX_PACKAGE_SIZE), lang.CmdPackageCreateFlagMaxPackageSize)
 	createFlags.StringVarP(&pkgConfig.CreateOpts.SigningKeyPath, "key", "k", v.GetString(V_PKG_CREATE_SIGNING_KEY), lang.CmdPackageCreateFlagSigningKey)
 	createFlags.StringVar(&pkgConfig.CreateOpts.SigningKeyPassword, "key-pass", v.GetString(V_PKG_CREATE_SIGNING_KEY_PASSWORD), lang.CmdPackageCreateFlagSigningKeyPassword)
+	createFlags.StringToStringVar(&pkgConfig.CreateOpts.RegistryOverrides, "registry-override", v.GetStringMapString(V_PKG_CREATE_REGISTRY_OVERRIDE), lang.CmdPackageCreateFlagRegistryOverride)
 }
 
 func bindDeployFlags() {

src/cmd/viper.go

@@ -59,6 +59,7 @@ const (
 	V_PKG_CREATE_SIGNING_KEY          = "package.create.signing_key"
 	V_PKG_CREATE_SIGNING_KEY_PASSWORD = "package.create.signing_key_password"
 	V_PKG_CREATE_DIFFERENTIAL         = "package.create.differential"
+	V_PKG_CREATE_REGISTRY_OVERRIDE    = "package.create.registry_override"
 
 	// Package deploy config keys
 	V_PKG_DEPLOY_SET        = "package.deploy.set"

src/config/lang/english.go

@@ -217,6 +217,7 @@ zarf init --git-push-password={PASSWORD} --git-push-username={USERNAME} --git-ur
 	CmdPackageCreateFlagSigningKey         = "Path to private key file for signing packages"
 	CmdPackageCreateFlagSigningKeyPassword = "Password to the private key file used for signing packages"
 	CmdPackageCreateFlagDifferential       = "Build a package that only contains the differential changes from local resources and differing remote resources from the specified previously built package"
+	CmdPackageCreateFlagRegistryOverride   = "Specify a map of domains to override on package create when pulling images (e.g. --registry-override docker.io=dockerio-reg.enterprise.intranet)"
 
 	CmdPackageDeployFlagConfirm                = "Confirms package deployment without prompting. ONLY use with packages you trust. Skips prompts to review SBOM, configure variables, select optional components and review potential breaking changes."
 	CmdPackageDeployFlagAdoptExistingResources = "Adopts any pre-existing K8s resources into the Helm charts managed by Zarf. ONLY use when you have existing deployments you want Zarf to takeover."

src/internal/packager/git/clone.go

@@ -44,7 +44,7 @@ func (g *Git) clone(gitURL string, ref plumbing.ReferenceName) error {
 	// Clone the given repo.
 	repo, err := git.PlainClone(g.GitPath, false, cloneOptions)
 	if err != nil {
-		message.Debugf("Failed to clone repo %s: %s", gitURL, err.Error())
+		message.Warnf("Falling back to host 'git', failed to clone the repo with Zarf - %s: %s", gitURL, err.Error())
 		return g.gitCloneFallback(gitURL, ref)
 	}
 
@@ -66,8 +66,6 @@ func (g *Git) clone(gitURL string, ref plumbing.ReferenceName) error {
 
 // gitCloneFallback is a fallback if go-git fails to clone a repo.
 func (g *Git) gitCloneFallback(gitURL string, ref plumbing.ReferenceName) error {
-	g.Spinner.Updatef("Falling back to host git for %s", gitURL)
-
 	// If we can't clone with go-git, fallback to the host clone
 	// Only support "all tags" due to the azure clone url format including a username
 	cmdArgs := []string{"clone", "--origin", onlineRemoteName, gitURL, g.GitPath}

src/internal/packager/images/common.go

@@ -28,6 +28,8 @@ type ImgConfig struct {
 	Insecure bool
 
 	Architectures []string
+
+	RegistryOverrides map[string]string
 }
 
 // GetLegacyImgTarballPath returns the ImagesPath as if it were a path to a tarball instead of a directory.

src/internal/packager/images/pull.go

@@ -17,6 +17,7 @@ import (
 
 	"github.com/defenseunicorns/zarf/src/config"
 	"github.com/defenseunicorns/zarf/src/pkg/message"
+	"github.com/defenseunicorns/zarf/src/pkg/transform"
 	"github.com/defenseunicorns/zarf/src/pkg/utils"
 	"github.com/google/go-containerregistry/pkg/crane"
 	"github.com/google/go-containerregistry/pkg/logs"
@@ -55,9 +56,22 @@ func (i *ImgConfig) PullAll() error {
 	for idx, src := range i.ImgList {
 		spinner.Updatef("Fetching image metadata (%d of %d): %s", idx+1, imgCount, src)
 
-		img, err := i.PullImage(src, spinner)
+		srcParsed, err := transform.ParseImageRef(src)
 		if err != nil {
-			return fmt.Errorf("failed to pull image %s: %w", src, err)
+			return fmt.Errorf("failed to parse image ref %s: %w", src, err)
+		}
+
+		actualSrc := src
+		if overrideHost, present := i.RegistryOverrides[srcParsed.Host]; present {
+			actualSrc, err = transform.ImageTransformHostWithoutChecksum(overrideHost, src)
+			if err != nil {
+				return fmt.Errorf("failed to swap override host %s for %s: %w", overrideHost, src, err)
+			}
+		}
+
+		img, err := i.PullImage(actualSrc, spinner)
+		if err != nil {
+			return fmt.Errorf("failed to pull image %s: %w", actualSrc, err)
 		}
 		imageMap[src] = img
 	}

src/pkg/packager/create.go

@@ -150,10 +150,11 @@ func (p *Packager) Create(baseDir string) error {
 
 		doPull := func() error {
 			imgConfig := images.ImgConfig{
-				ImagesPath:    p.tmp.Images,
-				ImgList:       imgList,
-				Insecure:      config.CommonOptions.Insecure,
-				Architectures: []string{p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture},
+				ImagesPath:        p.tmp.Images,
+				ImgList:           imgList,
+				Insecure:          config.CommonOptions.Insecure,
+				Architectures:     []string{p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture},
+				RegistryOverrides: p.cfg.CreateOpts.RegistryOverrides,
 			}
 
 			return imgConfig.PullAll()

src/pkg/packager/yaml.go

@@ -77,5 +77,7 @@ func (p *Packager) writeYaml() error {
 		deprecated.PluralizeSetVariable,
 	}
 
+	p.cfg.Pkg.Build.RegistryOverrides = p.cfg.CreateOpts.RegistryOverrides
+
 	return utils.WriteYaml(p.tmp.ZarfYaml, p.cfg.Pkg, 0400)
 }