diff --git a/.github/workflows/test-e2e-shim.yml b/.github/workflows/test-e2e-shim.yml index 75e43f1ca1..4f04776759 100644 --- a/.github/workflows/test-e2e-shim.yml +++ b/.github/workflows/test-e2e-shim.yml @@ -11,6 +11,9 @@ on: - "docs/**" - "CODEOWNERS" +permissions: + contents: read + # This is here to act as a shim for branch protection rules to work correctly. # This is ugly but this seems to be the best way to do this since: # - Job names in a workflow must be unique diff --git a/src/cmd/tools/crane.go b/src/cmd/tools/crane.go index 779b3e5655..a887e8f80f 100644 --- a/src/cmd/tools/crane.go +++ b/src/cmd/tools/crane.go @@ -267,7 +267,7 @@ func pruneImages(_ *cobra.Command, _ []string) error { if err != nil { return err } - imageRefToDigest := map[string]string{} + referenceToDigest := map[string]string{} for _, image := range imageCatalog { imageRef := fmt.Sprintf("%s/%s", registryAddress, image) tags, err := crane.ListTags(imageRef, authOption) @@ -280,28 +280,28 @@ func pruneImages(_ *cobra.Command, _ []string) error { if err != nil { return err } - imageRefToDigest[taggedImageRef] = digest + referenceToDigest[taggedImageRef] = digest } } // Figure out which images are in the registry but not needed by packages imageDigestsToPrune := map[string]bool{} - for imageRef, digest := range imageRefToDigest { + for digestRef, digest := range referenceToDigest { if _, ok := pkgImages[digest]; !ok { - ref, err := transform.ParseImageRef(imageRef) + refInfo, err := transform.ParseImageRef(digestRef) if err != nil { return err } - imageRef = fmt.Sprintf("%s@%s", ref.Name, digest) - imageDigestsToPrune[imageRef] = true + digestRef = fmt.Sprintf("%s@%s", refInfo.Name, digest) + imageDigestsToPrune[digestRef] = true } } if len(imageDigestsToPrune) > 0 { message.Note(lang.CmdToolsRegistryPruneImageList) - for imageRef := range imageDigestsToPrune { - message.Info(imageRef) + for digestRef := range imageDigestsToPrune { + message.Info(digestRef) } confirm := config.CommonOptions.Confirm @@ -317,9 +317,9 @@ func pruneImages(_ *cobra.Command, _ []string) error { } } if confirm { - // Delete the image references that are to be pruned - for imageRef := range imageDigestsToPrune { - err = crane.Delete(imageRef, authOption) + // Delete the digest references that are to be pruned + for digestRef := range imageDigestsToPrune { + err = crane.Delete(digestRef, authOption) if err != nil { return err } diff --git a/src/internal/cluster/injector.go b/src/internal/cluster/injector.go index 2c0ca42afd..c9612827f9 100644 --- a/src/internal/cluster/injector.go +++ b/src/internal/cluster/injector.go @@ -29,7 +29,7 @@ import ( var payloadChunkSize = 1024 * 768 // StartInjectionMadness initializes a Zarf injection into the cluster. -func (c *Cluster) StartInjectionMadness(tempPath types.TempPaths, injectorSeedTags []string) { +func (c *Cluster) StartInjectionMadness(tempPath types.TempPaths, injectorSeedSrcs []string) { spinner := message.NewProgressSpinner("Attempting to bootstrap the seed image into the cluster") defer spinner.Stop() @@ -59,7 +59,7 @@ func (c *Cluster) StartInjectionMadness(tempPath types.TempPaths, injectorSeedTa } spinner.Updatef("Loading the seed image from the package") - if seedImages, err = c.loadSeedImages(tempPath, injectorSeedTags, spinner); err != nil { + if seedImages, err = c.loadSeedImages(tempPath, injectorSeedSrcs, spinner); err != nil { spinner.Fatalf(err, "Unable to load the injector seed image from the package") } @@ -129,26 +129,25 @@ func (c *Cluster) StopInjectionMadness() error { return c.DeleteService(ZarfNamespaceName, "zarf-injector") } -func (c *Cluster) loadSeedImages(tempPath types.TempPaths, injectorSeedTags []string, spinner *message.Spinner) ([]transform.Image, error) { +func (c *Cluster) loadSeedImages(tempPath types.TempPaths, injectorSeedSrcs []string, spinner *message.Spinner) ([]transform.Image, error) { seedImages := []transform.Image{} - tagToDigest := make(map[string]string) + localReferenceToDigest := make(map[string]string) // Load the injector-specific images and save them as seed-images - for _, src := range injectorSeedTags { + for _, src := range injectorSeedSrcs { spinner.Updatef("Loading the seed image '%s' from the package", src) - - img, err := utils.LoadOCIImage(tempPath.Images, src) + ref, err := transform.ParseImageRef(src) + if err != nil { + return seedImages, fmt.Errorf("failed to create ref for image %s: %w", src, err) + } + img, err := utils.LoadOCIImage(tempPath.Images, ref) if err != nil { return seedImages, err } crane.SaveOCI(img, tempPath.SeedImages) - imgRef, err := transform.ParseImageRef(src) - if err != nil { - return seedImages, err - } - seedImages = append(seedImages, imgRef) + seedImages = append(seedImages, ref) // Get the image digest so we can set an annotation in the image.json later imgDigest, err := img.Digest() @@ -156,10 +155,10 @@ func (c *Cluster) loadSeedImages(tempPath types.TempPaths, injectorSeedTags []st return seedImages, err } // This is done _without_ the domain (different from pull.go) since the injector only handles local images - tagToDigest[imgRef.Path+imgRef.TagOrDigest] = imgDigest.String() + localReferenceToDigest[ref.Path+ref.TagOrDigest] = imgDigest.String() } - if err := utils.AddImageNameAnnotation(tempPath.SeedImages, tagToDigest); err != nil { + if err := utils.AddImageNameAnnotation(tempPath.SeedImages, localReferenceToDigest); err != nil { return seedImages, fmt.Errorf("unable to format OCI layout: %w", err) } diff --git a/src/internal/packager/images/common.go b/src/internal/packager/images/common.go index d6e9663fb9..e4284d20a7 100644 --- a/src/internal/packager/images/common.go +++ b/src/internal/packager/images/common.go @@ -9,17 +9,18 @@ import ( "os" "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" "github.com/google/go-containerregistry/pkg/crane" v1 "github.com/google/go-containerregistry/pkg/v1" ) -// ImgConfig is the main struct for managing container images. -type ImgConfig struct { +// ImageConfig is the main struct for managing container images. +type ImageConfig struct { ImagesPath string - ImgList []string + ImageList []transform.Image RegInfo types.RegistryInfo @@ -33,17 +34,17 @@ type ImgConfig struct { } // GetLegacyImgTarballPath returns the ImagesPath as if it were a path to a tarball instead of a directory. -func (i *ImgConfig) GetLegacyImgTarballPath() string { +func (i *ImageConfig) GetLegacyImgTarballPath() string { return fmt.Sprintf("%s.tar", i.ImagesPath) } -// LoadImageFromPackage returns a v1.Image from the image tag specified, or an error if the image cannot be found. -func (i ImgConfig) LoadImageFromPackage(imgTag string) (v1.Image, error) { - // If the package still has a images.tar that contains all of the images, use crane to load the specific tag we want +// LoadImageFromPackage returns a v1.Image from the specified image, or an error if the image cannot be found. +func (i ImageConfig) LoadImageFromPackage(refInfo transform.Image) (v1.Image, error) { + // If the package still has a images.tar that contains all of the images, use crane to load the specific reference (crane tag) we want if _, statErr := os.Stat(i.GetLegacyImgTarballPath()); statErr == nil { - return crane.LoadTag(i.GetLegacyImgTarballPath(), imgTag, config.GetCraneOptions(i.Insecure, i.Architectures...)...) + return crane.LoadTag(i.GetLegacyImgTarballPath(), refInfo.Reference, config.GetCraneOptions(i.Insecure, i.Architectures...)...) } // Load the image from the OCI formatted images directory - return utils.LoadOCIImage(i.ImagesPath, imgTag) + return utils.LoadOCIImage(i.ImagesPath, refInfo) } diff --git a/src/internal/packager/images/pull.go b/src/internal/packager/images/pull.go index ce13f134f0..51be578214 100644 --- a/src/internal/packager/images/pull.go +++ b/src/internal/packager/images/pull.go @@ -32,63 +32,62 @@ import ( "github.com/pterm/pterm" ) -// PullAll pulls all of the images in the provided tag map. -func (i *ImgConfig) PullAll() error { +// PullAll pulls all of the images in the provided ref map. +func (i *ImageConfig) PullAll() error { var ( - longer string - imgCount = len(i.ImgList) - imageMap = map[string]v1.Image{} - tagToImage = map[name.Tag]v1.Image{} - tagToDigest = make(map[string]string) + longer string + imageCount = len(i.ImageList) + refInfoToImage = map[transform.Image]v1.Image{} + referenceToDigest = make(map[string]string) ) + type imgInfo struct { + refInfo transform.Image + img v1.Image + } + + type digestInfo struct { + refInfo transform.Image + digest string + } + // Give some additional user feedback on larger image sets - if imgCount > 15 { + if imageCount > 15 { longer = "This step may take a couple of minutes to complete." - } else if imgCount > 5 { + } else if imageCount > 5 { longer = "This step may take several seconds to complete." } - spinner := message.NewProgressSpinner("Loading metadata for %d images. %s", imgCount, longer) + spinner := message.NewProgressSpinner("Loading metadata for %d images. %s", imageCount, longer) defer spinner.Stop() logs.Warn.SetOutput(&message.DebugWriter{}) logs.Progress.SetOutput(&message.DebugWriter{}) - type srcAndImg struct { - src string - img v1.Image - } - - metadataImageConcurrency := utils.NewConcurrencyTools[srcAndImg, error](len(i.ImgList)) + metadataImageConcurrency := utils.NewConcurrencyTools[imgInfo, error](len(i.ImageList)) defer metadataImageConcurrency.Cancel() - spinner.Updatef("Fetching image metadata (0 of %d)", len(i.ImgList)) + spinner.Updatef("Fetching image metadata (0 of %d)", len(i.ImageList)) // Spawn a goroutine for each image to load its metadata - for _, src := range i.ImgList { + for _, refInfo := range i.ImageList { // Create a closure so that we can pass the src into the goroutine - src := src + refInfo := refInfo go func() { // Make sure to call Done() on the WaitGroup when the goroutine finishes defer metadataImageConcurrency.WaitGroupDone() - srcParsed, err := transform.ParseImageRef(src) - if err != nil { - metadataImageConcurrency.ErrorChan <- fmt.Errorf("failed to parse image ref %s: %w", src, err) - return - } - if metadataImageConcurrency.IsDone() { return } - actualSrc := src - if overrideHost, present := i.RegistryOverrides[srcParsed.Host]; present { - actualSrc, err = transform.ImageTransformHostWithoutChecksum(overrideHost, src) + actualSrc := refInfo.Reference + if overrideHost, present := i.RegistryOverrides[refInfo.Host]; present { + var err error + actualSrc, err = transform.ImageTransformHostWithoutChecksum(overrideHost, refInfo.Reference) if err != nil { - metadataImageConcurrency.ErrorChan <- fmt.Errorf("failed to swap override host %s for %s: %w", overrideHost, src, err) + metadataImageConcurrency.ErrorChan <- fmt.Errorf("failed to swap override host %s for %s: %w", overrideHost, refInfo.Reference, err) return } } @@ -107,17 +106,17 @@ func (i *ImgConfig) PullAll() error { return } - metadataImageConcurrency.ProgressChan <- srcAndImg{src: src, img: img} + metadataImageConcurrency.ProgressChan <- imgInfo{refInfo: refInfo, img: img} }() } - onMetadataProgress := func(finishedImage srcAndImg, iteration int) { - spinner.Updatef("Fetching image metadata (%d of %d): %s", iteration+1, len(i.ImgList), finishedImage.src) - imageMap[finishedImage.src] = finishedImage.img + onMetadataProgress := func(finishedImage imgInfo, iteration int) { + spinner.Updatef("Fetching image metadata (%d of %d): %s", iteration+1, len(i.ImageList), finishedImage.refInfo.Reference) + refInfoToImage[finishedImage.refInfo] = finishedImage.img } onMetadataError := func(err error) error { - return fmt.Errorf("Failed to load metadata for all images. This may be due to a network error or an invalid image reference: %w", err) + return fmt.Errorf("failed to load metadata for all images. This may be due to a network error or an invalid image reference: %w", err) } if err := metadataImageConcurrency.WaitWithProgress(onMetadataProgress, onMetadataError); err != nil { @@ -132,16 +131,11 @@ func (i *ImgConfig) PullAll() error { totalBytes := int64(0) processedLayers := make(map[string]v1.Layer) - for src, img := range imageMap { - tag, err := name.NewTag(src, name.WeakValidation) - if err != nil { - return fmt.Errorf("failed to create tag for image %s: %w", src, err) - } - tagToImage[tag] = img + for refInfo, img := range refInfoToImage { // Get the byte size for this image layers, err := img.Layers() if err != nil { - return fmt.Errorf("unable to get layers for image %s: %w", src, err) + return fmt.Errorf("unable to get layers for image %s: %w", refInfo.Reference, err) } for _, layer := range layers { layerDigest, err := layer.Digest() @@ -163,11 +157,6 @@ func (i *ImgConfig) PullAll() error { } spinner.Updatef("Preparing image sources and cache for image pulling") - type digestAndTag struct { - digest string - tag string - } - // Create special sauce crane Path object // If it already exists use it cranePath, err := layout.FromPath(i.ImagesPath) @@ -180,12 +169,12 @@ func (i *ImgConfig) PullAll() error { } } - for tag, img := range tagToImage { + for refInfo, img := range refInfoToImage { imgDigest, err := img.Digest() if err != nil { - return fmt.Errorf("unable to get digest for image %s: %w", tag, err) + return fmt.Errorf("unable to get digest for image %s: %w", refInfo.Reference, err) } - tagToDigest[tag.String()] = imgDigest.String() + referenceToDigest[refInfo.Reference] = imgDigest.String() } spinner.Success() @@ -194,7 +183,7 @@ func (i *ImgConfig) PullAll() error { doneSaving := make(chan int) var progressBarWaitGroup sync.WaitGroup progressBarWaitGroup.Add(1) - go utils.RenderProgressBarForLocalDirWrite(i.ImagesPath, totalBytes, &progressBarWaitGroup, doneSaving, fmt.Sprintf("Pulling %d images", imgCount)) + go utils.RenderProgressBarForLocalDirWrite(i.ImagesPath, totalBytes, &progressBarWaitGroup, doneSaving, fmt.Sprintf("Pulling %d images", imageCount)) // Spawn a goroutine for each layer to write it to disk using crane @@ -343,15 +332,15 @@ func (i *ImgConfig) PullAll() error { return err } - imageSavingConcurrency := utils.NewConcurrencyTools[digestAndTag, error](len(tagToImage)) + imageSavingConcurrency := utils.NewConcurrencyTools[digestInfo, error](len(refInfoToImage)) defer imageSavingConcurrency.Cancel() // Spawn a goroutine for each image to write it's config and manifest to disk using crane // All layers should already be in place so this should be extremely fast - for tag, img := range tagToImage { - // Create a closure so that we can pass the tag and img into the goroutine - tag, img := tag, img + for refInfo, img := range refInfoToImage { + // Create a closure so that we can pass the refInfo and img into the goroutine + refInfo, img := refInfo, img go func() { // Make sure to call Done() on the WaitGroup when the goroutine finishes defer imageSavingConcurrency.WaitGroupDone() @@ -368,7 +357,7 @@ func (i *ImgConfig) PullAll() error { if strings.HasPrefix(err.Error(), "error writing layer: expected blob size") { message.Warnf("Potential image cache corruption: %s - try clearing cache with \"zarf tools clear-cache\"", err.Error()) } - imageSavingConcurrency.ErrorChan <- fmt.Errorf("error when trying to save the img (%s): %w", tag.Name(), err) + imageSavingConcurrency.ErrorChan <- fmt.Errorf("error when trying to save the img (%s): %w", refInfo.Reference, err) return } @@ -387,12 +376,12 @@ func (i *ImgConfig) PullAll() error { return } - imageSavingConcurrency.ProgressChan <- digestAndTag{digest: imgDigest.String(), tag: tag.String()} + imageSavingConcurrency.ProgressChan <- digestInfo{digest: imgDigest.String(), refInfo: refInfo} }() } - onImageSavingProgress := func(finishedImage digestAndTag, iteration int) { - tagToDigest[finishedImage.tag] = finishedImage.digest + onImageSavingProgress := func(finishedImage digestInfo, iteration int) { + referenceToDigest[finishedImage.refInfo.Reference] = finishedImage.digest } onImageSavingError := func(err error) error { @@ -409,7 +398,7 @@ func (i *ImgConfig) PullAll() error { // for every image sequentially append OCI descriptor - for tag, img := range tagToImage { + for refInfo, img := range refInfoToImage { desc, err := partial.Descriptor(img) if err != nil { return err @@ -425,10 +414,10 @@ func (i *ImgConfig) PullAll() error { return err } - tagToDigest[tag.String()] = imgDigest.String() + referenceToDigest[refInfo.Reference] = imgDigest.String() } - if err := utils.AddImageNameAnnotation(i.ImagesPath, tagToDigest); err != nil { + if err := utils.AddImageNameAnnotation(i.ImagesPath, referenceToDigest); err != nil { return fmt.Errorf("unable to format OCI layout: %w", err) } @@ -440,7 +429,7 @@ func (i *ImgConfig) PullAll() error { } // PullImage returns a v1.Image either by loading a local tarball or the wider internet. -func (i *ImgConfig) PullImage(src string, spinner *message.Spinner) (img v1.Image, err error) { +func (i *ImageConfig) PullImage(src string, spinner *message.Spinner) (img v1.Image, err error) { // Load image tarballs from the local filesystem. if strings.HasSuffix(src, ".tar") || strings.HasSuffix(src, ".tar.gz") || strings.HasSuffix(src, ".tgz") { spinner.Updatef("Reading image tarball: %s", src) diff --git a/src/internal/packager/images/push.go b/src/internal/packager/images/push.go index af2d56192e..7008a69a6e 100644 --- a/src/internal/packager/images/push.go +++ b/src/internal/packager/images/push.go @@ -20,21 +20,21 @@ import ( // PushToZarfRegistry pushes a provided image into the configured Zarf registry // This function will optionally shorten the image name while appending a checksum of the original image name. -func (i *ImgConfig) PushToZarfRegistry() error { +func (i *ImageConfig) PushToZarfRegistry() error { message.Debug("images.PushToZarfRegistry()") logs.Warn.SetOutput(&message.DebugWriter{}) logs.Progress.SetOutput(&message.DebugWriter{}) - imageMap := map[string]v1.Image{} + refInfoToImage := map[transform.Image]v1.Image{} var totalSize int64 // Build an image list from the references - for _, src := range i.ImgList { - img, err := i.LoadImageFromPackage(src) + for _, refInfo := range i.ImageList { + img, err := i.LoadImageFromPackage(refInfo) if err != nil { return err } - imageMap[src] = img + refInfoToImage[refInfo] = img imgSize, err := calcImgSize(img) if err != nil { return err @@ -49,7 +49,7 @@ func (i *ImgConfig) PushToZarfRegistry() error { httpTransport := http.DefaultTransport.(*http.Transport).Clone() httpTransport.TLSClientConfig.InsecureSkipVerify = i.Insecure - progressBar := message.NewProgressBar(totalSize, fmt.Sprintf("Pushing %d images to the zarf registry", len(i.ImgList))) + progressBar := message.NewProgressBar(totalSize, fmt.Sprintf("Pushing %d images to the zarf registry", len(i.ImageList))) craneTransport := utils.NewTransport(httpTransport, progressBar) pushOptions := config.GetCraneOptions(i.Insecure, i.Architectures...) @@ -91,18 +91,18 @@ func (i *ImgConfig) PushToZarfRegistry() error { registryURL = i.RegInfo.Address } - for src, img := range imageMap { - srcTruncated := message.Truncate(src, 55, true) - progressBar.UpdateTitle(fmt.Sprintf("Pushing %s", srcTruncated)) + for refInfo, img := range refInfoToImage { + refTruncated := message.Truncate(refInfo.Reference, 55, true) + progressBar.UpdateTitle(fmt.Sprintf("Pushing %s", refTruncated)) // If this is not a no checksum image push it for use with the Zarf agent if !i.NoChecksum { - offlineNameCRC, err := transform.ImageTransformHost(registryURL, src) + offlineNameCRC, err := transform.ImageTransformHost(registryURL, refInfo.Reference) if err != nil { return err } - message.Debugf("crane.Push() %s:%s -> %s)", i.ImagesPath, src, offlineNameCRC) + message.Debugf("crane.Push() %s:%s -> %s)", i.ImagesPath, refInfo.Reference, offlineNameCRC) if err = crane.Push(img, offlineNameCRC, pushOptions...); err != nil { return err @@ -111,19 +111,19 @@ func (i *ImgConfig) PushToZarfRegistry() error { // To allow for other non-zarf workloads to easily see the images upload a non-checksum version // (this may result in collisions but this is acceptable for this use case) - offlineName, err := transform.ImageTransformHostWithoutChecksum(registryURL, src) + offlineName, err := transform.ImageTransformHostWithoutChecksum(registryURL, refInfo.Reference) if err != nil { return err } - message.Debugf("crane.Push() %s:%s -> %s)", i.ImagesPath, src, offlineName) + message.Debugf("crane.Push() %s:%s -> %s)", i.ImagesPath, refInfo.Reference, offlineName) if err = crane.Push(img, offlineName, pushOptions...); err != nil { return err } } - progressBar.Successf("Pushed %d images to the zarf registry", len(i.ImgList)) + progressBar.Successf("Pushed %d images to the zarf registry", len(i.ImageList)) return nil } diff --git a/src/internal/packager/sbom/catalog.go b/src/internal/packager/sbom/catalog.go index e0fee676f6..265235fae6 100755 --- a/src/internal/packager/sbom/catalog.go +++ b/src/internal/packager/sbom/catalog.go @@ -22,9 +22,9 @@ import ( "github.com/anchore/syft/syft/source" "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" - "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/mholt/archiver/v3" ) @@ -46,9 +46,8 @@ var transformRegex = regexp.MustCompile(`(?m)[^a-zA-Z0-9\.\-]`) var componentPrefix = "zarf-component-" // Catalog catalogs the given components and images to create an SBOM. -// func Catalog(componentSBOMs map[string]*types.ComponentSBOM, imgList []string, imagesPath, sbomPath string) error { -func Catalog(componentSBOMs map[string]*types.ComponentSBOM, imgList []string, tmpPaths types.TempPaths) error { - imageCount := len(imgList) +func Catalog(componentSBOMs map[string]*types.ComponentSBOM, imageList []transform.Image, tmpPaths types.TempPaths) error { + imageCount := len(imageList) componentCount := len(componentSBOMs) builder := Builder{ spinner: message.NewProgressSpinner("Creating SBOMs for %d images and %d components with files.", imageCount, componentCount), @@ -63,7 +62,7 @@ func Catalog(componentSBOMs map[string]*types.ComponentSBOM, imgList []string, t _ = utils.CreateDirectory(builder.tmpSBOMPath, 0700) // Generate a list of images and files for the sbom viewer - json, err := builder.generateJSONList(componentSBOMs, imgList) + json, err := builder.generateJSONList(componentSBOMs, imageList) if err != nil { builder.spinner.Errorf(err, "Unable to generate the SBOM image list") return err @@ -72,24 +71,24 @@ func Catalog(componentSBOMs map[string]*types.ComponentSBOM, imgList []string, t // Generate SBOM for each image currImage := 1 - for _, tag := range imgList { - builder.spinner.Updatef("Creating image SBOMs (%d of %d): %s", currImage, imageCount, tag) + for _, refInfo := range imageList { + builder.spinner.Updatef("Creating image SBOMs (%d of %d): %s", currImage, imageCount, refInfo.Reference) // Get the image that we are creating an SBOM for - img, err := utils.LoadOCIImage(tmpPaths.Images, tag) + img, err := utils.LoadOCIImage(tmpPaths.Images, refInfo) if err != nil { builder.spinner.Errorf(err, "Unable to load the image to generate an SBOM") return err } - jsonData, err := builder.createImageSBOM(img, tag) + jsonData, err := builder.createImageSBOM(img, refInfo.Reference) if err != nil { - builder.spinner.Errorf(err, "Unable to create SBOM for image %s", tag) + builder.spinner.Errorf(err, "Unable to create SBOM for image %s", refInfo.Reference) return err } - if err = builder.createSBOMViewerAsset(tag, jsonData); err != nil { - builder.spinner.Errorf(err, "Unable to create SBOM viewer for image %s", tag) + if err = builder.createSBOMViewerAsset(refInfo.Reference, jsonData); err != nil { + builder.spinner.Errorf(err, "Unable to create SBOM viewer for image %s", refInfo.Reference) return err } @@ -122,7 +121,7 @@ func Catalog(componentSBOMs map[string]*types.ComponentSBOM, imgList []string, t } // Include the compare tool if there are any image SBOMs OR component SBOMs - if len(componentSBOMs) > 0 || len(imgList) > 0 { + if len(componentSBOMs) > 0 || len(imageList) > 0 { if err := builder.createSBOMCompareAsset(); err != nil { builder.spinner.Errorf(err, "Unable to create SBOM compare tool") return err @@ -152,11 +151,11 @@ func Catalog(componentSBOMs map[string]*types.ComponentSBOM, imgList []string, t // createImageSBOM uses syft to generate SBOM for an image, // some code/structure migrated from https://github.com/testifysec/go-witness/blob/v0.1.12/attestation/syft/syft.go. -func (b *Builder) createImageSBOM(img v1.Image, tagStr string) ([]byte, error) { +func (b *Builder) createImageSBOM(img v1.Image, src string) ([]byte, error) { // Get the image reference. - tag, err := name.NewTag(tagStr, name.WeakValidation) + refInfo, err := transform.ParseImageRef(src) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to create ref for image %s: %w", src, err) } // Create the sbom. @@ -167,7 +166,7 @@ func (b *Builder) createImageSBOM(img v1.Image, tagStr string) ([]byte, error) { return nil, err } - syftImage := image.NewImage(img, file.NewTempDirGenerator("zarf"), imageCachePath, image.WithTags(tag.String())) + syftImage := image.NewImage(img, file.NewTempDirGenerator("zarf"), imageCachePath, image.WithTags(refInfo.Reference)) if err := syftImage.Read(); err != nil { return nil, err } @@ -199,8 +198,8 @@ func (b *Builder) createImageSBOM(img v1.Image, tagStr string) ([]byte, error) { return nil, err } - // Write the sbom to disk using the image tag as the filename - filename := fmt.Sprintf("%s.json", tag) + // Write the sbom to disk using the image ref as the filename + filename := fmt.Sprintf("%s.json", refInfo.Reference) sbomFile, err := b.createSBOMFile(filename) if err != nil { return nil, err diff --git a/src/internal/packager/sbom/viewer.go b/src/internal/packager/sbom/viewer.go index 350b9d8b14..55e564bca1 100644 --- a/src/internal/packager/sbom/viewer.go +++ b/src/internal/packager/sbom/viewer.go @@ -9,6 +9,7 @@ import ( "fmt" "html/template" + "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/defenseunicorns/zarf/src/types" ) @@ -72,11 +73,11 @@ func (b *Builder) loadFileJS(name string) template.JS { } // This could be optimized, but loop over all the images and components to create a list of json files. -func (b *Builder) generateJSONList(componentToFiles map[string]*types.ComponentSBOM, imgList []string) ([]byte, error) { +func (b *Builder) generateJSONList(componentToFiles map[string]*types.ComponentSBOM, imageList []transform.Image) ([]byte, error) { var jsonList []string - for _, tag := range imgList { - normalized := b.getNormalizedFileName(tag) + for _, refInfo := range imageList { + normalized := b.getNormalizedFileName(refInfo.Reference) jsonList = append(jsonList, normalized) } diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index 4ecaf938b1..9f850bb9ea 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -116,7 +116,7 @@ func (p *Packager) Create(baseDir string) error { return fmt.Errorf("package creation canceled") } - var combinedImageList []string + var combinedImageList []transform.Image componentSBOMs := map[string]*types.ComponentSBOM{} for idx, component := range p.cfg.Pkg.Components { onCreate := component.Actions.OnCreate @@ -147,7 +147,13 @@ func (p *Packager) Create(baseDir string) error { } // Combine all component images into a single entry for efficient layer reuse. - combinedImageList = append(combinedImageList, component.Images...) + for _, src := range component.Images { + refInfo, err := transform.ParseImageRef(src) + if err != nil { + return fmt.Errorf("failed to create ref for image %s: %w", src, err) + } + combinedImageList = append(combinedImageList, refInfo) + } // Remove the temp directory for this component before archiving. err = os.RemoveAll(filepath.Join(p.tmp.Components, component.Name, types.TempFolder)) @@ -156,16 +162,16 @@ func (p *Packager) Create(baseDir string) error { } } - imgList := helpers.Unique(combinedImageList) + imageList := helpers.Unique(combinedImageList) // Images are handled separately from other component assets. - if len(imgList) > 0 { + if len(imageList) > 0 { message.HeaderInfof("📦 PACKAGE IMAGES") doPull := func() error { - imgConfig := images.ImgConfig{ + imgConfig := images.ImageConfig{ ImagesPath: p.tmp.Images, - ImgList: imgList, + ImageList: imageList, Insecure: config.CommonOptions.Insecure, Architectures: []string{p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture}, RegistryOverrides: p.cfg.CreateOpts.RegistryOverrides, @@ -183,7 +189,7 @@ func (p *Packager) Create(baseDir string) error { if p.cfg.CreateOpts.SkipSBOM { message.Debug("Skipping image SBOM processing per --skip-sbom flag") } else { - if err := sbom.Catalog(componentSBOMs, imgList, p.tmp); err != nil { + if err := sbom.Catalog(componentSBOMs, imageList, p.tmp); err != nil { return fmt.Errorf("unable to create an SBOM catalog for the package: %w", err) } } @@ -722,7 +728,7 @@ func (p *Packager) removeCopiesFromDifferentialPackage() error { newRepoList := []string{} // Generate a list of all unique images for this component for _, img := range component.Images { - // If a image doesn't have a tag (or is a commonly reused tag), we will include this image in the differential package + // If a image doesn't have a ref (or is a commonly reused ref), we will include this image in the differential package imgRef, err := transform.ParseImageRef(img) if err != nil { return fmt.Errorf("unable to parse image ref %s: %s", img, err.Error()) diff --git a/src/pkg/packager/deploy.go b/src/pkg/packager/deploy.go index 00eea92d9e..2b48c65b04 100644 --- a/src/pkg/packager/deploy.go +++ b/src/pkg/packager/deploy.go @@ -23,6 +23,7 @@ import ( "github.com/defenseunicorns/zarf/src/internal/packager/images" "github.com/defenseunicorns/zarf/src/internal/packager/template" "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" @@ -432,9 +433,20 @@ func (p *Packager) pushImagesToRegistry(componentImages []string, noImgChecksum return nil } - imgConfig := images.ImgConfig{ + var combinedImageList []transform.Image + for _, src := range componentImages { + ref, err := transform.ParseImageRef(src) + if err != nil { + return fmt.Errorf("failed to create ref for image %s: %w", src, err) + } + combinedImageList = append(combinedImageList, ref) + } + + imageList := helpers.Unique(combinedImageList) + + imgConfig := images.ImageConfig{ ImagesPath: p.tmp.Images, - ImgList: componentImages, + ImageList: imageList, NoChecksum: noImgChecksum, RegInfo: p.cfg.State.RegistryInfo, Insecure: config.CommonOptions.Insecure, diff --git a/src/pkg/utils/image.go b/src/pkg/utils/image.go index b007df3fd3..bd6b4d2db7 100644 --- a/src/pkg/utils/image.go +++ b/src/pkg/utils/image.go @@ -10,13 +10,14 @@ import ( "os" "path/filepath" + "github.com/defenseunicorns/zarf/src/pkg/transform" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/layout" ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) -// LoadOCIImage returns a v1.Image with the image tag specified from a location provided, or an error if the image cannot be found. -func LoadOCIImage(imgPath, imgTag string) (v1.Image, error) { +// LoadOCIImage returns a v1.Image with the image ref specified from a location provided, or an error if the image cannot be found. +func LoadOCIImage(imgPath string, refInfo transform.Image) (v1.Image, error) { // Use the manifest within the index.json to load the specific image we want layoutPath := layout.Path(imgPath) imgIdx, err := layoutPath.ImageIndex() @@ -28,19 +29,22 @@ func LoadOCIImage(imgPath, imgTag string) (v1.Image, error) { return nil, err } - // Search through all the manifests within this package until we find the annotation that matches our tag + // Search through all the manifests within this package until we find the annotation that matches our ref for _, manifest := range idxManifest.Manifests { - if manifest.Annotations[ocispec.AnnotationBaseImageName] == imgTag { + if manifest.Annotations[ocispec.AnnotationBaseImageName] == refInfo.Reference || + // A backwards compatibility shim for older Zarf versions that would leave docker.io off of image annotations + (manifest.Annotations[ocispec.AnnotationBaseImageName] == refInfo.Path+refInfo.TagOrDigest && refInfo.Host == "docker.io") { + // This is the image we are looking for, load it and then return return layoutPath.Image(manifest.Digest) } } - return nil, fmt.Errorf("unable to find image (%s) at the path (%s)", imgTag, imgPath) + return nil, fmt.Errorf("unable to find image (%s) at the path (%s)", refInfo.Reference, imgPath) } -// AddImageNameAnnotation adds an annotation to the index.json file so that the deploying code can figure out what the image tag <-> digest shasum will be. -func AddImageNameAnnotation(ociPath string, tagToDigest map[string]string) error { +// AddImageNameAnnotation adds an annotation to the index.json file so that the deploying code can figure out what the image reference <-> digest shasum will be. +func AddImageNameAnnotation(ociPath string, referenceToDigest map[string]string) error { indexPath := filepath.Join(ociPath, "index.json") // Read the file contents and turn it into a usable struct that we can manipulate @@ -61,16 +65,16 @@ func AddImageNameAnnotation(ociPath string, tagToDigest map[string]string) error var baseImageName string - for tag, digest := range tagToDigest { + for reference, digest := range referenceToDigest { if digest == manifest.Digest.String() { - baseImageName = tag + baseImageName = reference } } if baseImageName != "" { manifest.Annotations[ocispec.AnnotationBaseImageName] = baseImageName index.Manifests[idx] = manifest - delete(tagToDigest, baseImageName) + delete(referenceToDigest, baseImageName) } } diff --git a/src/test/e2e/06_create_sbom_test.go b/src/test/e2e/06_create_sbom_test.go index 322db362ce..1af4adf739 100644 --- a/src/test/e2e/06_create_sbom_test.go +++ b/src/test/e2e/06_create_sbom_test.go @@ -25,9 +25,9 @@ func TestCreateSBOM(t *testing.T) { require.NoError(t, err, stdOut, stdErr) require.Contains(t, stdErr, "Creating SBOMs for 1 images and 0 components with files.") // Test that the game package generates the SBOMs we expect (images only) - require.FileExists(t, filepath.Join(sbomPath, "dos-games", "sbom-viewer-defenseunicorns_zarf-game_multi-tile-dark.html")) + require.FileExists(t, filepath.Join(sbomPath, "dos-games", "sbom-viewer-docker.io_defenseunicorns_zarf-game_multi-tile-dark.html")) require.FileExists(t, filepath.Join(sbomPath, "dos-games", "compare.html")) - require.FileExists(t, filepath.Join(sbomPath, "dos-games", "defenseunicorns_zarf-game_multi-tile-dark.json")) + require.FileExists(t, filepath.Join(sbomPath, "dos-games", "docker.io_defenseunicorns_zarf-game_multi-tile-dark.json")) // Clean the SBOM path so it is force to be recreated e2e.CleanFiles(sbomPath) @@ -35,11 +35,11 @@ func TestCreateSBOM(t *testing.T) { stdOut, stdErr, err = e2e.Zarf("package", "inspect", pkgName, "--sbom-out", sbomPath) require.NoError(t, err, stdOut, stdErr) // Test that the game package generates the SBOMs we expect (images only) - _, err = os.ReadFile(filepath.Join(sbomPath, "dos-games", "sbom-viewer-defenseunicorns_zarf-game_multi-tile-dark.html")) + _, err = os.ReadFile(filepath.Join(sbomPath, "dos-games", "sbom-viewer-docker.io_defenseunicorns_zarf-game_multi-tile-dark.html")) require.NoError(t, err) _, err = os.ReadFile(filepath.Join(sbomPath, "dos-games", "compare.html")) require.NoError(t, err) - _, err = os.ReadFile(filepath.Join(sbomPath, "dos-games", "defenseunicorns_zarf-game_multi-tile-dark.json")) + _, err = os.ReadFile(filepath.Join(sbomPath, "dos-games", "docker.io_defenseunicorns_zarf-game_multi-tile-dark.json")) require.NoError(t, err) // Pull the current zarf binary version to find the corresponding init package @@ -51,12 +51,12 @@ func TestCreateSBOM(t *testing.T) { stdOut, stdErr, err = e2e.Zarf("package", "inspect", initName, "--sbom-out", sbomPath) require.NoError(t, err, stdOut, stdErr) // Test that we preserve the filepath - _, err = os.ReadFile(filepath.Join(sbomPath, "dos-games", "sbom-viewer-defenseunicorns_zarf-game_multi-tile-dark.html")) + _, err = os.ReadFile(filepath.Join(sbomPath, "dos-games", "sbom-viewer-docker.io_defenseunicorns_zarf-game_multi-tile-dark.html")) require.NoError(t, err) // Test that the init package generates the SBOMs we expect (images + component files) - _, err = os.ReadFile(filepath.Join(sbomPath, "init", "sbom-viewer-gitea_gitea_1.19.3-rootless.html")) + _, err = os.ReadFile(filepath.Join(sbomPath, "init", "sbom-viewer-docker.io_gitea_gitea_1.19.3-rootless.html")) require.NoError(t, err) - _, err = os.ReadFile(filepath.Join(sbomPath, "init", "gitea_gitea_1.19.3-rootless.json")) + _, err = os.ReadFile(filepath.Join(sbomPath, "init", "docker.io_gitea_gitea_1.19.3-rootless.json")) require.NoError(t, err) _, err = os.ReadFile(filepath.Join(sbomPath, "init", "sbom-viewer-zarf-component-k3s.html")) require.NoError(t, err) diff --git a/src/test/e2e/26_simple_packages_test.go b/src/test/e2e/26_simple_packages_test.go index 34c5f7eed3..fba578ab4e 100644 --- a/src/test/e2e/26_simple_packages_test.go +++ b/src/test/e2e/26_simple_packages_test.go @@ -7,6 +7,7 @@ package test import ( "fmt" "net/http" + "path/filepath" "testing" "github.com/defenseunicorns/zarf/src/internal/cluster" @@ -17,7 +18,7 @@ func TestDosGames(t *testing.T) { t.Log("E2E: Dos games") e2e.SetupWithCluster(t) - path := fmt.Sprintf("build/zarf-package-dos-games-%s-1.0.0.tar.zst", e2e.Arch) + path := filepath.Join("build", fmt.Sprintf("zarf-package-dos-games-%s-1.0.0.tar.zst", e2e.Arch)) // Deploy the game stdOut, stdErr, err := e2e.Zarf("package", "deploy", path, "--confirm") @@ -36,13 +37,24 @@ func TestDosGames(t *testing.T) { stdOut, stdErr, err = e2e.Zarf("package", "remove", "dos-games", "--confirm") require.NoError(t, err, stdOut, stdErr) + + testCreate := filepath.Join("src", "test", "packages", "26-image-dos-games") + testDeploy := filepath.Join("build", fmt.Sprintf("zarf-package-dos-games-images-%s.tar.zst", e2e.Arch)) + + // Create the game image test package + stdOut, stdErr, err = e2e.Zarf("package", "create", testCreate, "-o", "build", "--confirm") + require.NoError(t, err, stdOut, stdErr) + + // Deploy the game image test package + stdOut, stdErr, err = e2e.Zarf("package", "deploy", testDeploy, "--confirm") + require.NoError(t, err, stdOut, stdErr) } func TestManifests(t *testing.T) { t.Log("E2E: Local, Remote, and Kustomize Manifests") e2e.SetupWithCluster(t) - path := fmt.Sprintf("build/zarf-package-manifests-%s-0.0.1.tar.zst", e2e.Arch) + path := filepath.Join("build", fmt.Sprintf("zarf-package-manifests-%s-0.0.1.tar.zst", e2e.Arch)) // Deploy the package stdOut, stdErr, err := e2e.Zarf("package", "deploy", path, "--confirm") diff --git a/src/test/packages/26-image-dos-games/zarf.yaml b/src/test/packages/26-image-dos-games/zarf.yaml new file mode 100644 index 0000000000..06131de09b --- /dev/null +++ b/src/test/packages/26-image-dos-games/zarf.yaml @@ -0,0 +1,12 @@ +kind: ZarfPackageConfig +metadata: + name: dos-games-images + description: Simple example to test the various ways you can specify image references. + +components: + - name: baseline + required: true + images: + - defenseunicorns/zarf-game:multi-tile-dark + - defenseunicorns/zarf-game:multi-tile-dark@sha256:f78e442f0f3eb3e9459b5ae6b1a8fda62f8dfe818112e7d130a4e8ae72b3cbff + - defenseunicorns/zarf-game@sha256:f78e442f0f3eb3e9459b5ae6b1a8fda62f8dfe818112e7d130a4e8ae72b3cbff