Skip to content

Commit

Permalink
ref -> refInfo
Browse files Browse the repository at this point in the history
  • Loading branch information
@Racer159
Racer159 committed Sep 28, 2023
1 parent 59f8cdb commit 120991a
Show file tree
Hide file tree
Showing 9 changed files with 95 additions and 95 deletions.
22 changes: 11 additions & 11 deletions src/cmd/tools/crane.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -267,7 +267,7 @@ func pruneImages(_ *cobra.Command, _ []string) error {
if err != nil {
return err
}
imageRefToDigest := map[string]string{}
referenceToDigest := map[string]string{}
for _, image := range imageCatalog {
imageRef := fmt.Sprintf("%s/%s", registryAddress, image)
tags, err := crane.ListTags(imageRef, authOption)
Expand All @@ -280,28 +280,28 @@ func pruneImages(_ *cobra.Command, _ []string) error {
if err != nil {
return err
}
imageRefToDigest[taggedImageRef] = digest
referenceToDigest[taggedImageRef] = digest
}
}

// Figure out which images are in the registry but not needed by packages
imageDigestsToPrune := map[string]bool{}
for imageRef, digest := range imageRefToDigest {
for digestRef, digest := range referenceToDigest {
if _, ok := pkgImages[digest]; !ok {
ref, err := transform.ParseImageRef(imageRef)
refInfo, err := transform.ParseImageRef(digestRef)
if err != nil {
return err
}
imageRef = fmt.Sprintf("%s@%s", ref.Name, digest)
imageDigestsToPrune[imageRef] = true
digestRef = fmt.Sprintf("%s@%s", refInfo.Name, digest)
imageDigestsToPrune[digestRef] = true
}
}

if len(imageDigestsToPrune) > 0 {
message.Note(lang.CmdToolsRegistryPruneImageList)

for imageRef := range imageDigestsToPrune {
message.Info(imageRef)
for digestRef := range imageDigestsToPrune {
message.Info(digestRef)
}

confirm := config.CommonOptions.Confirm
Expand All @@ -317,9 +317,9 @@ func pruneImages(_ *cobra.Command, _ []string) error {
}
}
if confirm {
// Delete the image references that are to be pruned
for imageRef := range imageDigestsToPrune {
err = crane.Delete(imageRef, authOption)
// Delete the digest references that are to be pruned
for digestRef := range imageDigestsToPrune {
err = crane.Delete(digestRef, authOption)
if err != nil {
return err
}
Expand Down
18 changes: 9 additions & 9 deletions src/internal/packager/images/common.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,11 @@ import (
v1 "github.com/google/go-containerregistry/pkg/v1"
)

// ImgConfig is the main struct for managing container images.
type ImgConfig struct {
// ImageConfig is the main struct for managing container images.
type ImageConfig struct {
ImagesPath string

ImgList []transform.Image
ImageList []transform.Image

RegInfo types.RegistryInfo

Expand All @@ -34,17 +34,17 @@ type ImgConfig struct {
}

// GetLegacyImgTarballPath returns the ImagesPath as if it were a path to a tarball instead of a directory.
func (i *ImgConfig) GetLegacyImgTarballPath() string {
func (i *ImageConfig) GetLegacyImgTarballPath() string {
return fmt.Sprintf("%s.tar", i.ImagesPath)
}

// LoadImageFromPackage returns a v1.Image from the image ref specified, or an error if the image cannot be found.
func (i ImgConfig) LoadImageFromPackage(ref transform.Image) (v1.Image, error) {
// If the package still has a images.tar that contains all of the images, use crane to load the specific ref (tag) we want
// LoadImageFromPackage returns a v1.Image from the specified image, or an error if the image cannot be found.
func (i ImageConfig) LoadImageFromPackage(refInfo transform.Image) (v1.Image, error) {
// If the package still has a images.tar that contains all of the images, use crane to load the specific reference (crane tag) we want
if _, statErr := os.Stat(i.GetLegacyImgTarballPath()); statErr == nil {
return crane.LoadTag(i.GetLegacyImgTarballPath(), ref.Reference, config.GetCraneOptions(i.Insecure, i.Architectures...)...)
return crane.LoadTag(i.GetLegacyImgTarballPath(), refInfo.Reference, config.GetCraneOptions(i.Insecure, i.Architectures...)...)
}

// Load the image from the OCI formatted images directory
return utils.LoadOCIImage(i.ImagesPath, ref)
return utils.LoadOCIImage(i.ImagesPath, refInfo)
}
86 changes: 43 additions & 43 deletions src/internal/packager/images/pull.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,42 +33,42 @@ import (
)

// PullAll pulls all of the images in the provided ref map.
func (i *ImgConfig) PullAll() error {
func (i *ImageConfig) PullAll() error {
var (
longer string
imgCount = len(i.ImgList)
refToImage = map[transform.Image]v1.Image{}
refReferenceToDigest = make(map[string]string)
longer string
imageCount = len(i.ImageList)
refInfoToImage = map[transform.Image]v1.Image{}
referenceToDigest = make(map[string]string)
)

// Give some additional user feedback on larger image sets
if imgCount > 15 {
if imageCount > 15 {
longer = "This step may take a couple of minutes to complete."
} else if imgCount > 5 {
} else if imageCount > 5 {
longer = "This step may take several seconds to complete."
}

spinner := message.NewProgressSpinner("Loading metadata for %d images. %s", imgCount, longer)
spinner := message.NewProgressSpinner("Loading metadata for %d images. %s", imageCount, longer)
defer spinner.Stop()

logs.Warn.SetOutput(&message.DebugWriter{})
logs.Progress.SetOutput(&message.DebugWriter{})

type refAndImg struct {
ref transform.Image
img v1.Image
refInfo transform.Image
img v1.Image
}

metadataImageConcurrency := utils.NewConcurrencyTools[refAndImg, error](len(i.ImgList))
metadataImageConcurrency := utils.NewConcurrencyTools[refAndImg, error](len(i.ImageList))

defer metadataImageConcurrency.Cancel()

spinner.Updatef("Fetching image metadata (0 of %d)", len(i.ImgList))
spinner.Updatef("Fetching image metadata (0 of %d)", len(i.ImageList))

// Spawn a goroutine for each image to load its metadata
for _, ref := range i.ImgList {
for _, refInfo := range i.ImageList {
// Create a closure so that we can pass the src into the goroutine
ref := ref
refInfo := refInfo
go func() {
// Make sure to call Done() on the WaitGroup when the goroutine finishes
defer metadataImageConcurrency.WaitGroupDone()
Expand All @@ -77,12 +77,12 @@ func (i *ImgConfig) PullAll() error {
return
}

actualSrc := ref.Reference
if overrideHost, present := i.RegistryOverrides[ref.Host]; present {
actualSrc := refInfo.Reference
if overrideHost, present := i.RegistryOverrides[refInfo.Host]; present {
var err error
actualSrc, err = transform.ImageTransformHostWithoutChecksum(overrideHost, ref.Reference)
actualSrc, err = transform.ImageTransformHostWithoutChecksum(overrideHost, refInfo.Reference)
if err != nil {
metadataImageConcurrency.ErrorChan <- fmt.Errorf("failed to swap override host %s for %s: %w", overrideHost, ref.Reference, err)
metadataImageConcurrency.ErrorChan <- fmt.Errorf("failed to swap override host %s for %s: %w", overrideHost, refInfo.Reference, err)
return
}
}
Expand All @@ -101,17 +101,17 @@ func (i *ImgConfig) PullAll() error {
return
}

metadataImageConcurrency.ProgressChan <- refAndImg{ref: ref, img: img}
metadataImageConcurrency.ProgressChan <- refAndImg{refInfo: refInfo, img: img}
}()
}

onMetadataProgress := func(finishedImage refAndImg, iteration int) {
spinner.Updatef("Fetching image metadata (%d of %d): %s", iteration+1, len(i.ImgList), finishedImage.ref.Reference)
refToImage[finishedImage.ref] = finishedImage.img
spinner.Updatef("Fetching image metadata (%d of %d): %s", iteration+1, len(i.ImageList), finishedImage.refInfo.Reference)
refInfoToImage[finishedImage.refInfo] = finishedImage.img
}

onMetadataError := func(err error) error {
return fmt.Errorf("Failed to load metadata for all images. This may be due to a network error or an invalid image reference: %w", err)
return fmt.Errorf("failed to load metadata for all images. This may be due to a network error or an invalid image reference: %w", err)
}

if err := metadataImageConcurrency.WaitWithProgress(onMetadataProgress, onMetadataError); err != nil {
Expand All @@ -126,11 +126,11 @@ func (i *ImgConfig) PullAll() error {

totalBytes := int64(0)
processedLayers := make(map[string]v1.Layer)
for ref, img := range refToImage {
for refInfo, img := range refInfoToImage {
// Get the byte size for this image
layers, err := img.Layers()
if err != nil {
return fmt.Errorf("unable to get layers for image %s: %w", ref.Reference, err)
return fmt.Errorf("unable to get layers for image %s: %w", refInfo.Reference, err)
}
for _, layer := range layers {
layerDigest, err := layer.Digest()
Expand All @@ -152,9 +152,9 @@ func (i *ImgConfig) PullAll() error {
}
spinner.Updatef("Preparing image sources and cache for image pulling")

type refAndDigest struct {
digest string
ref string
type refInfoAndDigest struct {
refInfo transform.Image
digest string
}

// Create special sauce crane Path object
Expand All @@ -169,12 +169,12 @@ func (i *ImgConfig) PullAll() error {
}
}

for ref, img := range refToImage {
for refInfo, img := range refInfoToImage {
imgDigest, err := img.Digest()
if err != nil {
return fmt.Errorf("unable to get digest for image %s: %w", ref.Reference, err)
return fmt.Errorf("unable to get digest for image %s: %w", refInfo.Reference, err)
}
refReferenceToDigest[ref.Reference] = imgDigest.String()
referenceToDigest[refInfo.Reference] = imgDigest.String()
}

spinner.Success()
Expand All @@ -183,7 +183,7 @@ func (i *ImgConfig) PullAll() error {
doneSaving := make(chan int)
var progressBarWaitGroup sync.WaitGroup
progressBarWaitGroup.Add(1)
go utils.RenderProgressBarForLocalDirWrite(i.ImagesPath, totalBytes, &progressBarWaitGroup, doneSaving, fmt.Sprintf("Pulling %d images", imgCount))
go utils.RenderProgressBarForLocalDirWrite(i.ImagesPath, totalBytes, &progressBarWaitGroup, doneSaving, fmt.Sprintf("Pulling %d images", imageCount))

// Spawn a goroutine for each layer to write it to disk using crane

Expand Down Expand Up @@ -332,15 +332,15 @@ func (i *ImgConfig) PullAll() error {
return err
}

imageSavingConcurrency := utils.NewConcurrencyTools[refAndDigest, error](len(refToImage))
imageSavingConcurrency := utils.NewConcurrencyTools[refInfoAndDigest, error](len(refInfoToImage))

defer imageSavingConcurrency.Cancel()

// Spawn a goroutine for each image to write it's config and manifest to disk using crane
// All layers should already be in place so this should be extremely fast
for ref, img := range refToImage {
// Create a closure so that we can pass the ref and img into the goroutine
ref, img := ref, img
for refInfo, img := range refInfoToImage {
// Create a closure so that we can pass the refInfo and img into the goroutine
refInfo, img := refInfo, img
go func() {
// Make sure to call Done() on the WaitGroup when the goroutine finishes
defer imageSavingConcurrency.WaitGroupDone()
Expand All @@ -357,7 +357,7 @@ func (i *ImgConfig) PullAll() error {
if strings.HasPrefix(err.Error(), "error writing layer: expected blob size") {
message.Warnf("Potential image cache corruption: %s - try clearing cache with \"zarf tools clear-cache\"", err.Error())
}
imageSavingConcurrency.ErrorChan <- fmt.Errorf("error when trying to save the img (%s): %w", ref.Reference, err)
imageSavingConcurrency.ErrorChan <- fmt.Errorf("error when trying to save the img (%s): %w", refInfo.Reference, err)
return
}

Expand All @@ -376,12 +376,12 @@ func (i *ImgConfig) PullAll() error {
return
}

imageSavingConcurrency.ProgressChan <- refAndDigest{digest: imgDigest.String(), ref: ref.Reference}
imageSavingConcurrency.ProgressChan <- refInfoAndDigest{digest: imgDigest.String(), refInfo: refInfo}
}()
}

onImageSavingProgress := func(finishedImage refAndDigest, iteration int) {
refReferenceToDigest[finishedImage.ref] = finishedImage.digest
onImageSavingProgress := func(finishedImage refInfoAndDigest, iteration int) {
referenceToDigest[finishedImage.refInfo.Reference] = finishedImage.digest
}

onImageSavingError := func(err error) error {
Expand All @@ -398,7 +398,7 @@ func (i *ImgConfig) PullAll() error {

// for every image sequentially append OCI descriptor

for ref, img := range refToImage {
for refInfo, img := range refInfoToImage {
desc, err := partial.Descriptor(img)
if err != nil {
return err
Expand All @@ -414,10 +414,10 @@ func (i *ImgConfig) PullAll() error {
return err
}

refReferenceToDigest[ref.Reference] = imgDigest.String()
referenceToDigest[refInfo.Reference] = imgDigest.String()
}

if err := utils.AddImageNameAnnotation(i.ImagesPath, refReferenceToDigest); err != nil {
if err := utils.AddImageNameAnnotation(i.ImagesPath, referenceToDigest); err != nil {
return fmt.Errorf("unable to format OCI layout: %w", err)
}

Expand All @@ -429,7 +429,7 @@ func (i *ImgConfig) PullAll() error {
}

// PullImage returns a v1.Image either by loading a local tarball or the wider internet.
func (i *ImgConfig) PullImage(src string, spinner *message.Spinner) (img v1.Image, err error) {
func (i *ImageConfig) PullImage(src string, spinner *message.Spinner) (img v1.Image, err error) {
// Load image tarballs from the local filesystem.
if strings.HasSuffix(src, ".tar") || strings.HasSuffix(src, ".tar.gz") || strings.HasSuffix(src, ".tgz") {
spinner.Updatef("Reading image tarball: %s", src)
Expand Down
8 changes: 4 additions & 4 deletions src/internal/packager/images/push.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ import (

// PushToZarfRegistry pushes a provided image into the configured Zarf registry
// This function will optionally shorten the image name while appending a checksum of the original image name.
func (i *ImgConfig) PushToZarfRegistry() error {
func (i *ImageConfig) PushToZarfRegistry() error {
message.Debug("images.PushToZarfRegistry()")

logs.Warn.SetOutput(&message.DebugWriter{})
Expand All @@ -29,7 +29,7 @@ func (i *ImgConfig) PushToZarfRegistry() error {
imageMap := map[transform.Image]v1.Image{}
var totalSize int64
// Build an image list from the references
for _, ref := range i.ImgList {
for _, ref := range i.ImageList {
img, err := i.LoadImageFromPackage(ref)
if err != nil {
return err
Expand All @@ -49,7 +49,7 @@ func (i *ImgConfig) PushToZarfRegistry() error {

httpTransport := http.DefaultTransport.(*http.Transport).Clone()
httpTransport.TLSClientConfig.InsecureSkipVerify = i.Insecure
progressBar := message.NewProgressBar(totalSize, fmt.Sprintf("Pushing %d images to the zarf registry", len(i.ImgList)))
progressBar := message.NewProgressBar(totalSize, fmt.Sprintf("Pushing %d images to the zarf registry", len(i.ImageList)))
craneTransport := utils.NewTransport(httpTransport, progressBar)

pushOptions := config.GetCraneOptions(i.Insecure, i.Architectures...)
Expand Down Expand Up @@ -123,7 +123,7 @@ func (i *ImgConfig) PushToZarfRegistry() error {
}
}

progressBar.Successf("Pushed %d images to the zarf registry", len(i.ImgList))
progressBar.Successf("Pushed %d images to the zarf registry", len(i.ImageList))

return nil
}
Expand Down
Loading

0 comments on commit 120991a

Please sign in to comment.