diff --git a/copy/copy.go b/copy/copy.go index 30d8a44641..3368c30ed8 100644 --- a/copy/copy.go +++ b/copy/copy.go @@ -15,6 +15,7 @@ import ( "github.com/containers/image/v4/docker/reference" "github.com/containers/image/v4/image" + "github.com/containers/image/v4/internal" "github.com/containers/image/v4/manifest" "github.com/containers/image/v4/pkg/blobinfocache" "github.com/containers/image/v4/pkg/compression" @@ -22,6 +23,7 @@ import ( "github.com/containers/image/v4/transports" "github.com/containers/image/v4/types" digest "github.com/opencontainers/go-digest" + imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/vbauerster/mpb" @@ -110,6 +112,32 @@ type imageCopier struct { canSubstituteBlobs bool } +const ( + // CopyOnlyCurrentRuntimeImage is a value which, when set in + // Options.MultipleImages, indicates that the caller expects only one + // image to be copied, so if the source reference refers to a list of + // images, one that matches the current system will be selected. + CopyOnlyCurrentRuntimeImage MultipleImagesOption = iota + // CopyAllImages is a value which, when set in Options.MultipleImages, + // indicates that the caller expects to copy multiple images, and if the + // source reference refers to a list of images, but the target reference + // can only accept one image, an error should be returned. + CopyAllImages + // CopyOnlyInstances is a value which, when set in + // Options.MultipleImages, indicates that the caller expects the source + // reference to be a list of images, and wants only specific instances + // from it copied (or none of them, if the list of instances to copy is + // empty), along with the list itself. If the target reference can + // only accept one image (i.e., it cannot accept lists), an error + // should be returned. + CopyOnlyInstances +) + +// MultipleImagesOption is either CopyOnlyCurrentRuntimeImage or CopyAllImages, to control +// whether copy.Image() copies only an image which matches the current runtime environment, or +// all images which match the supplied reference. +type MultipleImagesOption int + // Options allows supplying non-default configuration modifying the behavior of CopyImage. type Options struct { RemoveSignatures bool // Remove any pre-existing signatures. SignBy will still add a new signature. @@ -121,12 +149,14 @@ type Options struct { Progress chan types.ProgressProperties // Reported to when ProgressInterval has arrived for a single artifact+offset. // manifest MIME type of image set by user. "" is default and means use the autodetection to the the manifest MIME type ForceManifestMIMEType string + MultipleImages MultipleImagesOption // set to either CopyOnlyCurrentRuntimeImage, CopyAllImages, or CopyOnlyInstances + Instances []digest.Digest // if MultipleImages is CopyOnlyInstances, copy only these instances and the list itself } // Image copies image from srcRef to destRef, using policyContext to validate // source image admissibility. It returns the manifest which was written to // the new copy of the image. -func Image(ctx context.Context, policyContext *signature.PolicyContext, destRef, srcRef types.ImageReference, options *Options) (manifest []byte, retErr error) { +func Image(ctx context.Context, policyContext *signature.PolicyContext, destRef, srcRef types.ImageReference, options *Options) (copiedManifest []byte, retErr error) { // NOTE this function uses an output parameter for the error return value. // Setting this and returning is the ideal way to return an error. // @@ -136,6 +166,9 @@ func Image(ctx context.Context, policyContext *signature.PolicyContext, destRef, options = &Options{} } + if options.MultipleImages != CopyAllImages && options.MultipleImages != CopyOnlyCurrentRuntimeImage && options.MultipleImages != CopyOnlyInstances { + return nil, errors.Errorf("Invalid value for options.MultipleImages: %d", options.MultipleImages) + } reportWriter := ioutil.Discard if options.ReportWriter != nil { @@ -205,23 +238,52 @@ func Image(ctx context.Context, policyContext *signature.PolicyContext, destRef, return nil, errors.Wrapf(err, "Error determining manifest MIME type for %s", transports.ImageName(srcRef)) } + // Give Commit() a pointer to the unparsed source image in its original form. This may not exactly fit the intended + // use for WithValue(), though, so we may need to switch to passing it directly to dest.Commit() and c.copyOneImage(). + ctx = context.WithValue(ctx, internal.CopyUnparsedTopImageKey, unparsedToplevel) + if !multiImage { - // The simple case: Just copy a single image. - if manifest, err = c.copyOneImage(ctx, policyContext, options, unparsedToplevel); err != nil { + // The simple case: just copy a single image. + if copiedManifest, _, _, err = c.copyOneImage(ctx, policyContext, options, unparsedToplevel, nil); err != nil { return nil, err } } else { - // This is a manifest list. Choose a single image and copy it. - // FIXME: Copy to destinations which support manifest lists, one image at a time. - instanceDigest, err := image.ChooseManifestInstanceFromManifestList(ctx, options.SourceCtx, unparsedToplevel) - if err != nil { - return nil, errors.Wrapf(err, "Error choosing an image from manifest list %s", transports.ImageName(srcRef)) - } - logrus.Debugf("Source is a manifest list; copying (only) instance %s", instanceDigest) - unparsedInstance := image.UnparsedInstance(rawSource, &instanceDigest) + if options.MultipleImages == CopyOnlyCurrentRuntimeImage { + // This is a manifest list, and we weren't asked to copy multiple images. Choose a single image to copy, + // and copy it. + mfest, manifestType, err := unparsedToplevel.Manifest(ctx) + if err != nil { + return nil, errors.Wrapf(err, "Error reading manifest for %s", transports.ImageName(srcRef)) + } + manifestList, err := manifest.ListFromBlob(mfest, manifestType) + if err != nil { + return nil, errors.Wrapf(err, "Error parsing primary manifest as list for %s", transports.ImageName(srcRef)) + } + instanceDigest, err := manifestList.ChooseInstance(options.SourceCtx) + if err != nil { + return nil, errors.Wrapf(err, "Error choosing an image from manifest list %s", transports.ImageName(srcRef)) + } + logrus.Debugf("Source is a manifest list; copying (only) instance %s", instanceDigest) + unparsedInstance := image.UnparsedInstance(rawSource, &instanceDigest) - if manifest, err = c.copyOneImage(ctx, policyContext, options, unparsedInstance); err != nil { - return nil, err + if copiedManifest, _, _, err = c.copyOneImage(ctx, policyContext, options, unparsedInstance, nil); err != nil { + return nil, err + } + } else { /* options.MultipleImages == CopyAllImages or options.MultipleImages == CopyOnlyInstances, */ + // If we were asked to copy multiple images and can't, that's an error. + if !supportsMultipleImages(c.dest) { + return nil, errors.Errorf("Error copying multiple images: destination reference %q does not support multiple images", transports.ImageName(destRef)) + } + // Copy some or all of the images. + switch options.MultipleImages { + case CopyAllImages: + logrus.Debugf("Source is a manifest list; copying all instances") + case CopyOnlyInstances: + logrus.Debugf("Source is a manifest list; copying some instances") + } + if copiedManifest, _, err = c.copyMultipleImages(ctx, policyContext, options, unparsedToplevel); err != nil { + return nil, err + } } } @@ -229,56 +291,222 @@ func Image(ctx context.Context, policyContext *signature.PolicyContext, destRef, return nil, errors.Wrap(err, "Error committing the finished image") } - return manifest, nil + return copiedManifest, nil } -// Image copies a single (on-manifest-list) image unparsedImage, using policyContext to validate +// Checks if the destination supports accepting multiple images by checking if it can support +// manifest types that are lists of other manifests. +func supportsMultipleImages(dest types.ImageDestination) bool { + mtypes := dest.SupportedManifestMIMETypes() + if len(mtypes) == 0 { + // Anything goes! + return true + } + for _, mtype := range mtypes { + if manifest.MIMETypeIsMultiImage(mtype) { + return true + } + } + return false +} + +// copyMultipleImages copies all of an image's instances, using policyContext to validate // source image admissibility. -func (c *copier) copyOneImage(ctx context.Context, policyContext *signature.PolicyContext, options *Options, unparsedImage *image.UnparsedImage) (manifestBytes []byte, retErr error) { +func (c *copier) copyMultipleImages(ctx context.Context, policyContext *signature.PolicyContext, options *Options, unparsedToplevel *image.UnparsedImage) (copiedManifest []byte, copiedManifestType string, retErr error) { + // Parse the list and get a copy of the original value after it's re-encoded. + manifestList, manifestType, err := unparsedToplevel.Manifest(ctx) + if err != nil { + return nil, "", errors.Wrapf(err, "Error reading manifest list") + } + list, err := manifest.ListFromBlob(manifestList, manifestType) + if err != nil { + return nil, "", errors.Wrapf(err, "Error parsing manifest list %q", string(manifestList)) + } + originalList := list.Clone() + + // Read and/or clear the set of signatures for this list. + var sigs [][]byte + if options.RemoveSignatures { + sigs = [][]byte{} + } else { + c.Printf("Getting image list signatures\n") + s, err := c.rawSource.GetSignatures(ctx, nil) + if err != nil { + return nil, "", errors.Wrap(err, "Error reading signatures") + } + sigs = s + } + if len(sigs) != 0 { + c.Printf("Checking if image list destination supports signatures\n") + if err := c.dest.SupportsSignatures(ctx); err != nil { + return nil, "", errors.Wrap(err, "Can not copy signatures") + } + } + + // Copy each image, or just the ones we want to copy, in turn. + instanceDigests := list.Instances() + updates := make([]manifest.ListUpdate, len(instanceDigests)) + for i, instanceDigest := range instanceDigests { + if options.MultipleImages == CopyOnlyInstances { + skip := true + for _, instance := range options.Instances { + if instance == instanceDigest { + skip = false + break + } + } + if skip { + update, err := list.Instance(instanceDigest) + if err != nil { + return nil, "", err + } + logrus.Debugf("Skipping instance %s (%d/%d)", instanceDigest, i+1, len(instanceDigests)) + updates[i] = update + continue + } + } + logrus.Debugf("Copying instance %s (%d/%d)", instanceDigest, i+1, len(instanceDigests)) + unparsedInstance := image.UnparsedInstance(c.rawSource, &instanceDigest) + updatedManifest, updatedManifestType, updatedManifestDigest, err := c.copyOneImage(ctx, policyContext, options, unparsedInstance, &instanceDigest) + if err != nil { + return nil, "", err + } + // Record the result of a possible conversion here. + update := manifest.ListUpdate{ + Digest: updatedManifestDigest, + Size: int64(len(updatedManifest)), + MediaType: updatedManifestType, + } + updates[i] = update + } + + // Now apply the updates. + if err = list.UpdateInstances(updates); err != nil { + return nil, "", errors.Wrapf(err, "Error updating manifest list") + } + + // Check if the updates meaningfully changed the list of images. + listIsModified := false + if !reflect.DeepEqual(list.Instances(), originalList.Instances()) { + listIsModified = true + } + + // Determine if we need to convert the manifest to a different format. + forceListMIMEType := options.ForceManifestMIMEType + switch forceListMIMEType { + case manifest.DockerV2Schema1MediaType, manifest.DockerV2Schema1SignedMediaType, manifest.DockerV2Schema2MediaType: + forceListMIMEType = manifest.DockerV2ListMediaType + case imgspecv1.MediaTypeImageManifest: + forceListMIMEType = imgspecv1.MediaTypeImageIndex + } + selectedType, err := c.determineListConversion(manifestType, c.dest.SupportedManifestMIMETypes(), forceListMIMEType) + if err != nil { + return nil, "", errors.Wrapf(err, "Error determining manifest list type to write to destination") + } + if selectedType != list.MIMEType() { + list, err = list.ConvertToMIMEType(selectedType) + if err != nil { + return nil, "", errors.Wrapf(err, "Error converting manifest list to list with MIME type %q", selectedType) + } + } + + // If we can't use the original value, but we have to change it, flag an error. + if listIsModified { + canModifyManifest := (len(sigs) == 0) + if !canModifyManifest { + return nil, "", errors.Errorf("Internal error: copyMultipleImages() needs to use an updated manifest but that was known to be forbidden") + } + manifestList, err = list.Serialize() + if err != nil { + return nil, "", errors.Wrapf(err, "Error encoding updated manifest list (%q: %#v)", list.MIMEType(), list.Instances()) + } + logrus.Debugf("Manifest list has been updated") + } + + // Save the manifest list. + if err = c.dest.PutManifest(ctx, manifestList, nil); err != nil { + return nil, "", errors.Wrapf(err, "Error writing manifest list %q", string(manifestList)) + } + + // Sign the manifest list. + if options.SignBy != "" { + newSig, err := c.createSignature(manifestList, options.SignBy) + if err != nil { + return nil, "", err + } + sigs = append(sigs, newSig) + } + + c.Printf("Storing list signatures\n") + if err := c.dest.PutSignatures(ctx, sigs, nil); err != nil { + return nil, "", errors.Wrap(err, "Error writing signatures") + } + + return manifestList, selectedType, nil +} + +// copyOneImage copies a single (non-manifest-list) image unparsedImage, using policyContext to validate +// source image admissibility. +func (c *copier) copyOneImage(ctx context.Context, policyContext *signature.PolicyContext, options *Options, unparsedImage *image.UnparsedImage, targetInstance *digest.Digest) (retManifest []byte, retManifestType string, retManifestDigest digest.Digest, retErr error) { // The caller is handling manifest lists; this could happen only if a manifest list contains a manifest list. // Make sure we fail cleanly in such cases. multiImage, err := isMultiImage(ctx, unparsedImage) if err != nil { // FIXME FIXME: How to name a reference for the sub-image? - return nil, errors.Wrapf(err, "Error determining manifest MIME type for %s", transports.ImageName(unparsedImage.Reference())) + return nil, "", "", errors.Wrapf(err, "Error determining manifest MIME type for %s", transports.ImageName(unparsedImage.Reference())) } if multiImage { - return nil, fmt.Errorf("Unexpectedly received a manifest list instead of a manifest for a single image") + return nil, "", "", fmt.Errorf("Unexpectedly received a manifest list instead of a manifest for a single image") } // Please keep this policy check BEFORE reading any other information about the image. - // (the multiImage check above only matches the MIME type, which we have received anyway. + // (The multiImage check above only matches the MIME type, which we have received anyway. // Actual parsing of anything should be deferred.) if allowed, err := policyContext.IsRunningImageAllowed(ctx, unparsedImage); !allowed || err != nil { // Be paranoid and fail if either return value indicates so. - return nil, errors.Wrap(err, "Source image rejected") + return nil, "", "", errors.Wrap(err, "Source image rejected") } src, err := image.FromUnparsedImage(ctx, options.SourceCtx, unparsedImage) if err != nil { - return nil, errors.Wrapf(err, "Error initializing image from source %s", transports.ImageName(c.rawSource.Reference())) + return nil, "", "", errors.Wrapf(err, "Error initializing image from source %s", transports.ImageName(c.rawSource.Reference())) } // If the destination is a digested reference, make a note of that, determine what digest value we're - // expecting, and check that the source manifest matches it. + // expecting, and check that the source manifest matches it. If the source manifest doesn't, but it's + // one item from a manifest list that matches it, accept that as a match. destIsDigestedReference := false if named := c.dest.Reference().DockerReference(); named != nil { if digested, ok := named.(reference.Digested); ok { destIsDigestedReference = true sourceManifest, _, err := src.Manifest(ctx) if err != nil { - return nil, errors.Wrapf(err, "Error reading manifest from source image") + return nil, "", "", errors.Wrapf(err, "Error reading manifest from source image") } matches, err := manifest.MatchesDigest(sourceManifest, digested.Digest()) if err != nil { - return nil, errors.Wrapf(err, "Error computing digest of source image's manifest") + return nil, "", "", errors.Wrapf(err, "Error computing digest of source image's manifest") + } + if !matches { + if value := ctx.Value(internal.CopyUnparsedTopImageKey); value != nil { + if unparsedToplevel, ok := value.(*image.UnparsedImage); ok { + manifestList, _, err := unparsedToplevel.Manifest(ctx) + if err != nil { + return nil, "", "", errors.Wrapf(err, "Error reading manifest from source image") + } + matches, err = manifest.MatchesDigest(manifestList, digested.Digest()) + if err != nil { + return nil, "", "", errors.Wrapf(err, "Error computing digest of source image's manifest") + } + } + } } if !matches { - return nil, errors.New("Digest of source image's manifest would not match destination reference") + return nil, "", "", errors.New("Digest of source image's manifest would not match destination reference") } } } if err := checkImageDestinationForCurrentRuntimeOS(ctx, options.DestinationCtx, src, c.dest); err != nil { - return nil, err + return nil, "", "", err } var sigs [][]byte @@ -288,14 +516,14 @@ func (c *copier) copyOneImage(ctx context.Context, policyContext *signature.Poli c.Printf("Getting image source signatures\n") s, err := src.Signatures(ctx) if err != nil { - return nil, errors.Wrap(err, "Error reading signatures") + return nil, "", "", errors.Wrap(err, "Error reading signatures") } sigs = s } if len(sigs) != 0 { c.Printf("Checking if image destination supports signatures\n") if err := c.dest.SupportsSignatures(ctx); err != nil { - return nil, errors.Wrap(err, "Can not copy signatures") + return nil, "", "", errors.Wrap(err, "Can not copy signatures") } } @@ -315,28 +543,29 @@ func (c *copier) copyOneImage(ctx context.Context, policyContext *signature.Poli ic.canSubstituteBlobs = ic.canModifyManifest && options.SignBy == "" if err := ic.updateEmbeddedDockerReference(); err != nil { - return nil, err + return nil, "", "", err } // We compute preferredManifestMIMEType only to show it in error messages. // Without having to add this context in an error message, we would be happy enough to know only that no conversion is needed. preferredManifestMIMEType, otherManifestMIMETypeCandidates, err := ic.determineManifestConversion(ctx, c.dest.SupportedManifestMIMETypes(), options.ForceManifestMIMEType) if err != nil { - return nil, err + return nil, "", "", err } // If src.UpdatedImageNeedsLayerDiffIDs(ic.manifestUpdates) will be true, it needs to be true by the time we get here. ic.diffIDsAreNeeded = src.UpdatedImageNeedsLayerDiffIDs(*ic.manifestUpdates) if err := ic.copyLayers(ctx); err != nil { - return nil, err + return nil, "", "", err } // With docker/distribution registries we do not know whether the registry accepts schema2 or schema1 only; // and at least with the OpenShift registry "acceptschema2" option, there is no way to detect the support // without actually trying to upload something and getting a types.ManifestTypeRejectedError. // So, try the preferred manifest MIME type. If the process succeeds, fine… - manifestBytes, err = ic.copyUpdatedConfigAndManifest(ctx) + manifestBytes, retManifestDigest, err := ic.copyUpdatedConfigAndManifest(ctx, targetInstance) + retManifestType = preferredManifestMIMEType if err != nil { logrus.Debugf("Writing manifest using preferred type %s failed: %v", preferredManifestMIMEType, err) // … if it fails, _and_ the failure is because the manifest is rejected, we may have other options. @@ -344,14 +573,14 @@ func (c *copier) copyOneImage(ctx context.Context, policyContext *signature.Poli // We don’t have other options. // In principle the code below would handle this as well, but the resulting error message is fairly ugly. // Don’t bother the user with MIME types if we have no choice. - return nil, err + return nil, "", "", err } // If the original MIME type is acceptable, determineManifestConversion always uses it as preferredManifestMIMEType. // So if we are here, we will definitely be trying to convert the manifest. // With !ic.canModifyManifest, that would just be a string of repeated failures for the same reason, // so let’s bail out early and with a better error message. if !ic.canModifyManifest { - return nil, errors.Wrap(err, "Writing manifest failed (and converting it is not possible)") + return nil, "", "", errors.Wrap(err, "Writing manifest failed (and converting it is not possible)") } // errs is a list of errors when trying various manifest types. Also serves as an "upload succeeded" flag when set to nil. @@ -359,7 +588,7 @@ func (c *copier) copyOneImage(ctx context.Context, policyContext *signature.Poli for _, manifestMIMEType := range otherManifestMIMETypeCandidates { logrus.Debugf("Trying to use manifest type %s…", manifestMIMEType) ic.manifestUpdates.ManifestMIMEType = manifestMIMEType - attemptedManifest, err := ic.copyUpdatedConfigAndManifest(ctx) + attemptedManifest, attemptedManifestDigest, err := ic.copyUpdatedConfigAndManifest(ctx, targetInstance) if err != nil { logrus.Debugf("Upload of manifest type %s failed: %v", manifestMIMEType, err) errs = append(errs, fmt.Sprintf("%s(%v)", manifestMIMEType, err)) @@ -368,28 +597,30 @@ func (c *copier) copyOneImage(ctx context.Context, policyContext *signature.Poli // We have successfully uploaded a manifest. manifestBytes = attemptedManifest + retManifestDigest = attemptedManifestDigest + retManifestType = manifestMIMEType errs = nil // Mark this as a success so that we don't abort below. break } if errs != nil { - return nil, fmt.Errorf("Uploading manifest failed, attempted the following formats: %s", strings.Join(errs, ", ")) + return nil, "", "", fmt.Errorf("Uploading manifest failed, attempted the following formats: %s", strings.Join(errs, ", ")) } } if options.SignBy != "" { newSig, err := c.createSignature(manifestBytes, options.SignBy) if err != nil { - return nil, err + return nil, "", "", err } sigs = append(sigs, newSig) } c.Printf("Storing signatures\n") - if err := c.dest.PutSignatures(ctx, sigs); err != nil { - return nil, errors.Wrap(err, "Error writing signatures") + if err := c.dest.PutSignatures(ctx, sigs, targetInstance); err != nil { + return nil, "", "", errors.Wrap(err, "Error writing signatures") } - return manifestBytes, nil + return manifestBytes, retManifestType, retManifestDigest, nil } // Printf writes a formatted string to c.reportWriter. @@ -554,12 +785,13 @@ func layerDigestsDiffer(a, b []types.BlobInfo) bool { } // copyUpdatedConfigAndManifest updates the image per ic.manifestUpdates, if necessary, -// stores the resulting config and manifest to the destination, and returns the stored manifest. -func (ic *imageCopier) copyUpdatedConfigAndManifest(ctx context.Context) ([]byte, error) { +// stores the resulting config and manifest to the destination, and returns the stored manifest +// and its digest. +func (ic *imageCopier) copyUpdatedConfigAndManifest(ctx context.Context, instanceDigest *digest.Digest) ([]byte, digest.Digest, error) { pendingImage := ic.src if !reflect.DeepEqual(*ic.manifestUpdates, types.ManifestUpdateOptions{InformationOnly: ic.manifestUpdates.InformationOnly}) { if !ic.canModifyManifest { - return nil, errors.Errorf("Internal error: copy needs an updated manifest but that was known to be forbidden") + return nil, "", errors.Errorf("Internal error: copy needs an updated manifest but that was known to be forbidden") } if !ic.diffIDsAreNeeded && ic.src.UpdatedImageNeedsLayerDiffIDs(*ic.manifestUpdates) { // We have set ic.diffIDsAreNeeded based on the preferred MIME type returned by determineManifestConversion. @@ -568,28 +800,35 @@ func (ic *imageCopier) copyUpdatedConfigAndManifest(ctx context.Context) ([]byte // when ic.c.dest.SupportedManifestMIMETypes() includes both s1 and s2, the upload using s1 failed, and we are now trying s2. // Supposedly s2-only registries do not exist or are extremely rare, so failing with this error message is good enough for now. // If handling such registries turns out to be necessary, we could compute ic.diffIDsAreNeeded based on the full list of manifest MIME type candidates. - return nil, errors.Errorf("Can not convert image to %s, preparing DiffIDs for this case is not supported", ic.manifestUpdates.ManifestMIMEType) + return nil, "", errors.Errorf("Can not convert image to %s, preparing DiffIDs for this case is not supported", ic.manifestUpdates.ManifestMIMEType) } pi, err := ic.src.UpdatedImage(ctx, *ic.manifestUpdates) if err != nil { - return nil, errors.Wrap(err, "Error creating an updated image manifest") + return nil, "", errors.Wrap(err, "Error creating an updated image manifest") } pendingImage = pi } - manifest, _, err := pendingImage.Manifest(ctx) + man, _, err := pendingImage.Manifest(ctx) if err != nil { - return nil, errors.Wrap(err, "Error reading manifest") + return nil, "", errors.Wrap(err, "Error reading manifest") } if err := ic.c.copyConfig(ctx, pendingImage); err != nil { - return nil, err + return nil, "", err } ic.c.Printf("Writing manifest to image destination\n") - if err := ic.c.dest.PutManifest(ctx, manifest); err != nil { - return nil, errors.Wrap(err, "Error writing manifest") + manifestDigest, err := manifest.Digest(man) + if err != nil { + return nil, "", err + } + if instanceDigest != nil { + instanceDigest = &manifestDigest + } + if err := ic.c.dest.PutManifest(ctx, man, instanceDigest); err != nil { + return nil, "", errors.Wrap(err, "Error writing manifest") } - return manifest, nil + return man, manifestDigest, nil } // newProgressPool creates a *mpb.Progress and a cleanup function. diff --git a/copy/manifest.go b/copy/manifest.go index 7c981fcad2..9f2168f59a 100644 --- a/copy/manifest.go +++ b/copy/manifest.go @@ -119,3 +119,38 @@ func isMultiImage(ctx context.Context, img types.UnparsedImage) (bool, error) { } return manifest.MIMETypeIsMultiImage(mt), nil } + +// determineListConversion returns the MIME type to which we should convert a list of manifests. +// Returns the preferred manifest MIME type (whether we are converting to it or using it unmodified), +// and a list of other possible alternatives, in order. +func (c *copier) determineListConversion(currentListMIMEType string, destSupportedListMIMETypes []string, forcedListMIMEType string) (string, error) { + // If there's no list of supported types, then anything we support is expected to be supported. + if len(destSupportedListMIMETypes) == 0 { + destSupportedListMIMETypes = manifest.SupportedListMIMETypes + } + // The lowest priority is the first member of the list of acceptable types that is a list. + selectedType := destSupportedListMIMETypes[0] + if !manifest.MIMETypeIsMultiImage(selectedType) { + for i := range destSupportedListMIMETypes { + if manifest.MIMETypeIsMultiImage(destSupportedListMIMETypes[i]) { + selectedType = destSupportedListMIMETypes[i] + break + } + } + } + if forcedListMIMEType != "" { + // If we're forcing it, we prefer the forced value over everything else. + selectedType = forcedListMIMEType + } else { + // If the current type is in the list of acceptable types, we prefer that, + // to avoid having to do a format conversion. + for _, allowedType := range destSupportedListMIMETypes { + if allowedType == currentListMIMEType { + selectedType = allowedType + break + } + } + } + // Done. + return selectedType, nil +} diff --git a/copy/manifest_test.go b/copy/manifest_test.go index c8d25b91f5..a9ff787b6a 100644 --- a/copy/manifest_test.go +++ b/copy/manifest_test.go @@ -203,6 +203,8 @@ func TestIsMultiImage(t *testing.T) { }{ {manifest.DockerV2ListMediaType, true}, {manifest.DockerV2Schema2MediaType, false}, + {v1.MediaTypeImageManifest, false}, + {v1.MediaTypeImageIndex, true}, } { src := fakeImageSource(c.mt) res, err := isMultiImage(context.Background(), src) diff --git a/directory/directory_dest.go b/directory/directory_dest.go index 18f7dde70d..71c3e49055 100644 --- a/directory/directory_dest.go +++ b/directory/directory_dest.go @@ -199,16 +199,23 @@ func (d *dirImageDestination) TryReusingBlob(ctx context.Context, info types.Blo } // PutManifest writes manifest to the destination. +// If instanceDigest is not nil, it contains a digest of the specific manifest instance to write the manifest for (when +// the primary manifest is a manifest list); this should always be nil if the primary manifest is not a manifest list. +// It is expected but not enforced that the instanceDigest, when specified, matches the digest of `manifest` as generated +// by `manifest.Digest()`. // FIXME? This should also receive a MIME type if known, to differentiate between schema versions. // If the destination is in principle available, refuses this manifest type (e.g. it does not recognize the schema), // but may accept a different manifest type, the returned error must be an ManifestTypeRejectedError. -func (d *dirImageDestination) PutManifest(ctx context.Context, manifest []byte) error { - return ioutil.WriteFile(d.ref.manifestPath(), manifest, 0644) +func (d *dirImageDestination) PutManifest(ctx context.Context, manifest []byte, instanceDigest *digest.Digest) error { + return ioutil.WriteFile(d.ref.manifestPath(instanceDigest), manifest, 0644) } -func (d *dirImageDestination) PutSignatures(ctx context.Context, signatures [][]byte) error { +// PutSignatures writes a set of signatures to the destination. +// If instanceDigest is not nil, it contains a digest of the specific manifest instance to write or overwrite the signatures for +// (when the primary manifest is a manifest list); this should always be nil if the primary manifest is not a manifest list. +func (d *dirImageDestination) PutSignatures(ctx context.Context, signatures [][]byte, instanceDigest *digest.Digest) error { for i, sig := range signatures { - if err := ioutil.WriteFile(d.ref.signaturePath(i), sig, 0644); err != nil { + if err := ioutil.WriteFile(d.ref.signaturePath(i, instanceDigest), sig, 0644); err != nil { return err } } diff --git a/directory/directory_src.go b/directory/directory_src.go index 921c1941cc..d97b68d99c 100644 --- a/directory/directory_src.go +++ b/directory/directory_src.go @@ -9,7 +9,6 @@ import ( "github.com/containers/image/v4/manifest" "github.com/containers/image/v4/types" "github.com/opencontainers/go-digest" - "github.com/pkg/errors" ) type dirImageSource struct { @@ -38,10 +37,7 @@ func (s *dirImageSource) Close() error { // If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve (when the primary manifest is a manifest list); // this never happens if the primary manifest is not a manifest list (e.g. if the source never returns manifest lists). func (s *dirImageSource) GetManifest(ctx context.Context, instanceDigest *digest.Digest) ([]byte, string, error) { - if instanceDigest != nil { - return nil, "", errors.Errorf(`Getting target manifest not supported by "dir:"`) - } - m, err := ioutil.ReadFile(s.ref.manifestPath()) + m, err := ioutil.ReadFile(s.ref.manifestPath(instanceDigest)) if err != nil { return nil, "", err } @@ -73,12 +69,9 @@ func (s *dirImageSource) GetBlob(ctx context.Context, info types.BlobInfo, cache // (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list // (e.g. if the source never returns manifest lists). func (s *dirImageSource) GetSignatures(ctx context.Context, instanceDigest *digest.Digest) ([][]byte, error) { - if instanceDigest != nil { - return nil, errors.Errorf(`Manifests lists are not supported by "dir:"`) - } signatures := [][]byte{} for i := 0; ; i++ { - signature, err := ioutil.ReadFile(s.ref.signaturePath(i)) + signature, err := ioutil.ReadFile(s.ref.signaturePath(i, instanceDigest)) if err != nil { if os.IsNotExist(err) { break @@ -90,7 +83,14 @@ func (s *dirImageSource) GetSignatures(ctx context.Context, instanceDigest *dige return signatures, nil } -// LayerInfosForCopy() returns updated layer info that should be used when copying, in preference to values in the manifest, if specified. -func (s *dirImageSource) LayerInfosForCopy(ctx context.Context) ([]types.BlobInfo, error) { +// LayerInfosForCopy returns either nil (meaning the values in the manifest are fine), or updated values for the layer +// blobsums that are listed in the image's manifest. If values are returned, they should be used when using GetBlob() +// to read the image's layers. +// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve BlobInfos for +// (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list +// (e.g. if the source never returns manifest lists). +// The Digest field is guaranteed to be provided; Size may be -1. +// WARNING: The list may contain duplicates, and they are semantically relevant. +func (s *dirImageSource) LayerInfosForCopy(context.Context, *digest.Digest) ([]types.BlobInfo, error) { return nil, nil } diff --git a/directory/directory_test.go b/directory/directory_test.go index 65f3302608..26cdec2abe 100644 --- a/directory/directory_test.go +++ b/directory/directory_test.go @@ -32,10 +32,15 @@ func TestGetPutManifest(t *testing.T) { defer os.RemoveAll(tmpDir) man := []byte("test-manifest") + list := []byte("test-manifest-list") + md, err := manifest.Digest(man) + require.NoError(t, err) dest, err := ref.NewImageDestination(context.Background(), nil) require.NoError(t, err) defer dest.Close() - err = dest.PutManifest(context.Background(), man) + err = dest.PutManifest(context.Background(), man, &md) + assert.NoError(t, err) + err = dest.PutManifest(context.Background(), list, nil) assert.NoError(t, err) err = dest.Commit(context.Background()) assert.NoError(t, err) @@ -45,14 +50,13 @@ func TestGetPutManifest(t *testing.T) { defer src.Close() m, mt, err := src.GetManifest(context.Background(), nil) assert.NoError(t, err) - assert.Equal(t, man, m) + assert.Equal(t, list, m) assert.Equal(t, "", mt) - // Non-default instances are not supported - md, err := manifest.Digest(man) - require.NoError(t, err) - _, _, err = src.GetManifest(context.Background(), &md) - assert.Error(t, err) + m, mt, err = src.GetManifest(context.Background(), &md) + assert.NoError(t, err) + assert.Equal(t, man, m) + assert.Equal(t, "", mt) } func TestGetPutBlob(t *testing.T) { @@ -148,10 +152,27 @@ func TestGetPutSignatures(t *testing.T) { } err = dest.SupportsSignatures(context.Background()) assert.NoError(t, err) - err = dest.PutManifest(context.Background(), man) + err = dest.PutManifest(context.Background(), man, nil) + require.NoError(t, err) + + err = dest.PutSignatures(context.Background(), signatures, nil) + listSignatures := [][]byte{ + []byte("sig3"), + []byte("sig4"), + } + md, err := manifest.Digest(man) + require.NoError(t, err) + + err = dest.SupportsSignatures(context.Background()) + assert.NoError(t, err) + err = dest.PutManifest(context.Background(), man, nil) + require.NoError(t, err) + err = dest.PutManifest(context.Background(), man, &md) require.NoError(t, err) - err = dest.PutSignatures(context.Background(), signatures) + err = dest.PutSignatures(context.Background(), listSignatures, nil) + assert.NoError(t, err) + err = dest.PutSignatures(context.Background(), signatures, &md) assert.NoError(t, err) err = dest.Commit(context.Background()) assert.NoError(t, err) @@ -161,13 +182,11 @@ func TestGetPutSignatures(t *testing.T) { defer src.Close() sigs, err := src.GetSignatures(context.Background(), nil) assert.NoError(t, err) - assert.Equal(t, signatures, sigs) + assert.Equal(t, listSignatures, sigs) - // Non-default instances are not supported - md, err := manifest.Digest(man) - require.NoError(t, err) - _, err = src.GetSignatures(context.Background(), &md) - assert.Error(t, err) + sigs, err = src.GetSignatures(context.Background(), &md) + assert.NoError(t, err) + assert.Equal(t, signatures, sigs) } func TestSourceReference(t *testing.T) { diff --git a/directory/directory_transport.go b/directory/directory_transport.go index 29ac7115f3..1229e5a9e8 100644 --- a/directory/directory_transport.go +++ b/directory/directory_transport.go @@ -166,18 +166,24 @@ func (ref dirReference) DeleteImage(ctx context.Context, sys *types.SystemContex } // manifestPath returns a path for the manifest within a directory using our conventions. -func (ref dirReference) manifestPath() string { +func (ref dirReference) manifestPath(instanceDigest *digest.Digest) string { + if instanceDigest != nil { + return filepath.Join(ref.path, instanceDigest.Encoded()+".manifest.json") + } return filepath.Join(ref.path, "manifest.json") } // layerPath returns a path for a layer tarball within a directory using our conventions. func (ref dirReference) layerPath(digest digest.Digest) string { // FIXME: Should we keep the digest identification? - return filepath.Join(ref.path, digest.Hex()) + return filepath.Join(ref.path, digest.Encoded()) } // signaturePath returns a path for a signature within a directory using our conventions. -func (ref dirReference) signaturePath(index int) string { +func (ref dirReference) signaturePath(index int, instanceDigest *digest.Digest) string { + if instanceDigest != nil { + return filepath.Join(ref.path, fmt.Sprintf(instanceDigest.Encoded()+".signature-%d", index+1)) + } return filepath.Join(ref.path, fmt.Sprintf("signature-%d", index+1)) } diff --git a/directory/directory_transport_test.go b/directory/directory_transport_test.go index 1fbe6ca832..c2b7440395 100644 --- a/directory/directory_transport_test.go +++ b/directory/directory_transport_test.go @@ -9,6 +9,7 @@ import ( _ "github.com/containers/image/v4/internal/testing/explicitfilepath-tmpdir" "github.com/containers/image/v4/types" + digest "github.com/opencontainers/go-digest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -157,7 +158,7 @@ func TestReferenceNewImage(t *testing.T) { defer dest.Close() mFixture, err := ioutil.ReadFile("../manifest/fixtures/v2s1.manifest.json") require.NoError(t, err) - err = dest.PutManifest(context.Background(), mFixture) + err = dest.PutManifest(context.Background(), mFixture, nil) assert.NoError(t, err) err = dest.Commit(context.Background()) assert.NoError(t, err) @@ -174,7 +175,7 @@ func TestReferenceNewImageNoValidManifest(t *testing.T) { dest, err := ref.NewImageDestination(context.Background(), nil) require.NoError(t, err) defer dest.Close() - err = dest.PutManifest(context.Background(), []byte(`{"schemaVersion":1}`)) + err = dest.PutManifest(context.Background(), []byte(`{"schemaVersion":1}`), nil) assert.NoError(t, err) err = dest.Commit(context.Background()) assert.NoError(t, err) @@ -207,11 +208,14 @@ func TestReferenceDeleteImage(t *testing.T) { } func TestReferenceManifestPath(t *testing.T) { + dhex := digest.Digest("sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef") + ref, tmpDir := refToTempDir(t) defer os.RemoveAll(tmpDir) dirRef, ok := ref.(dirReference) require.True(t, ok) - assert.Equal(t, tmpDir+"/manifest.json", dirRef.manifestPath()) + assert.Equal(t, tmpDir+"/manifest.json", dirRef.manifestPath(nil)) + assert.Equal(t, tmpDir+"/"+dhex.Encoded()+".manifest.json", dirRef.manifestPath(&dhex)) } func TestReferenceLayerPath(t *testing.T) { @@ -225,12 +229,16 @@ func TestReferenceLayerPath(t *testing.T) { } func TestReferenceSignaturePath(t *testing.T) { + dhex := digest.Digest("sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef") + ref, tmpDir := refToTempDir(t) defer os.RemoveAll(tmpDir) dirRef, ok := ref.(dirReference) require.True(t, ok) - assert.Equal(t, tmpDir+"/signature-1", dirRef.signaturePath(0)) - assert.Equal(t, tmpDir+"/signature-10", dirRef.signaturePath(9)) + assert.Equal(t, tmpDir+"/signature-1", dirRef.signaturePath(0, nil)) + assert.Equal(t, tmpDir+"/signature-10", dirRef.signaturePath(9, nil)) + assert.Equal(t, tmpDir+"/"+dhex.Encoded()+".signature-1", dirRef.signaturePath(0, &dhex)) + assert.Equal(t, tmpDir+"/"+dhex.Encoded()+".signature-10", dirRef.signaturePath(9, &dhex)) } func TestReferenceVersionPath(t *testing.T) { diff --git a/docker/archive/src.go b/docker/archive/src.go index feea0decd5..749d621751 100644 --- a/docker/archive/src.go +++ b/docker/archive/src.go @@ -33,8 +33,3 @@ func newImageSource(ctx context.Context, ref archiveReference) (types.ImageSourc func (s *archiveImageSource) Reference() types.ImageReference { return s.ref } - -// LayerInfosForCopy() returns updated layer info that should be used when reading, in preference to values in the manifest, if specified. -func (s *archiveImageSource) LayerInfosForCopy(ctx context.Context) ([]types.BlobInfo, error) { - return nil, nil -} diff --git a/docker/daemon/daemon_src.go b/docker/daemon/daemon_src.go index f6f60aaf91..e1a01fd7dd 100644 --- a/docker/daemon/daemon_src.go +++ b/docker/daemon/daemon_src.go @@ -55,8 +55,3 @@ func newImageSource(ctx context.Context, sys *types.SystemContext, ref daemonRef func (s *daemonImageSource) Reference() types.ImageReference { return s.ref } - -// LayerInfosForCopy() returns updated layer info that should be used when reading, in preference to values in the manifest, if specified. -func (s *daemonImageSource) LayerInfosForCopy(ctx context.Context) ([]types.BlobInfo, error) { - return nil, nil -} diff --git a/docker/docker_image_dest.go b/docker/docker_image_dest.go index 0f351ab594..355db4d948 100644 --- a/docker/docker_image_dest.go +++ b/docker/docker_image_dest.go @@ -61,6 +61,8 @@ func (d *dockerImageDestination) SupportedManifestMIMETypes() []string { return []string{ imgspecv1.MediaTypeImageManifest, manifest.DockerV2Schema2MediaType, + imgspecv1.MediaTypeImageIndex, + manifest.DockerV2ListMediaType, manifest.DockerV2Schema1SignedMediaType, manifest.DockerV2Schema1MediaType, } @@ -343,20 +345,36 @@ func (d *dockerImageDestination) TryReusingBlob(ctx context.Context, info types. } // PutManifest writes manifest to the destination. +// If instanceDigest is not nil, it contains a digest of the specific manifest instance to overwrite the manifest for (when +// the primary manifest is a manifest list); this should always be nil if the primary manifest is not a manifest list. +// It is expected but not enforced that the instanceDigest, when specified, matches the digest of `manifest` as generated +// by `manifest.Digest()`. // FIXME? This should also receive a MIME type if known, to differentiate between schema versions. // If the destination is in principle available, refuses this manifest type (e.g. it does not recognize the schema), // but may accept a different manifest type, the returned error must be an ManifestTypeRejectedError. -func (d *dockerImageDestination) PutManifest(ctx context.Context, m []byte) error { - digest, err := manifest.Digest(m) - if err != nil { - return err +func (d *dockerImageDestination) PutManifest(ctx context.Context, m []byte, instanceDigest *digest.Digest) error { + refTail := "" + if instanceDigest != nil { + // If the instanceDigest is provided, then use it as the reftail, as the reference, + // whether it includes a tag or a digest, refers to the list as a whole, and not this + // particular instance. + refTail = instanceDigest.String() + } else { + // Compute the digest of the main manifest, or the list if it's a list, so that we + // have a digest value to use if we're asked to save a signature for the manifest. + digest, err := manifest.Digest(m) + if err != nil { + return err + } + d.manifestDigest = digest + // The refTail should be either a digest (which we expect to match the value we just + // computed) or a tag name. + refTail, err = d.ref.tagOrDigest() + if err != nil { + return err + } } - d.manifestDigest = digest - refTail, err := d.ref.tagOrDigest() - if err != nil { - return err - } path := fmt.Sprintf(manifestPath, reference.Path(d.ref.ref), refTail) headers := map[string][]string{} @@ -416,19 +434,30 @@ func isManifestInvalidError(err error) bool { } } -func (d *dockerImageDestination) PutSignatures(ctx context.Context, signatures [][]byte) error { +// PutSignatures uploads a set of signatures to the relevant lookaside or API extension point. +// If instanceDigest is not nil, it contains a digest of the specific manifest instance to upload the signatures for (when +// the primary manifest is a manifest list); this should always be nil if the primary manifest is not a manifest list. +func (d *dockerImageDestination) PutSignatures(ctx context.Context, signatures [][]byte, instanceDigest *digest.Digest) error { // Do not fail if we don’t really need to support signatures. if len(signatures) == 0 { return nil } + if instanceDigest == nil { + if d.manifestDigest.String() == "" { + // This shouldn’t happen, ImageDestination users are required to call PutManifest before PutSignatures + return errors.Errorf("Unknown manifest digest, can't add signatures") + } + instanceDigest = &d.manifestDigest + } + if err := d.c.detectProperties(ctx); err != nil { return err } switch { case d.c.signatureBase != nil: - return d.putSignaturesToLookaside(signatures) + return d.putSignaturesToLookaside(signatures, instanceDigest) case d.c.supportsSignatures: - return d.putSignaturesToAPIExtension(ctx, signatures) + return d.putSignaturesToAPIExtension(ctx, signatures, instanceDigest) default: return errors.Errorf("X-Registry-Supports-Signatures extension not supported, and lookaside is not configured") } @@ -436,7 +465,7 @@ func (d *dockerImageDestination) PutSignatures(ctx context.Context, signatures [ // putSignaturesToLookaside implements PutSignatures() from the lookaside location configured in s.c.signatureBase, // which is not nil. -func (d *dockerImageDestination) putSignaturesToLookaside(signatures [][]byte) error { +func (d *dockerImageDestination) putSignaturesToLookaside(signatures [][]byte, instanceDigest *digest.Digest) error { // FIXME? This overwrites files one at a time, definitely not atomic. // A failure when updating signatures with a reordered copy could lose some of them. @@ -445,14 +474,9 @@ func (d *dockerImageDestination) putSignaturesToLookaside(signatures [][]byte) e return nil } - if d.manifestDigest.String() == "" { - // This shouldn’t happen, ImageDestination users are required to call PutManifest before PutSignatures - return errors.Errorf("Unknown manifest digest, can't add signatures") - } - // NOTE: Keep this in sync with docs/signature-protocols.md! for i, signature := range signatures { - url := signatureStorageURL(d.c.signatureBase, d.manifestDigest, i) + url := signatureStorageURL(d.c.signatureBase, *instanceDigest, i) if url == nil { return errors.Errorf("Internal error: signatureStorageURL with non-nil base returned nil") } @@ -467,7 +491,7 @@ func (d *dockerImageDestination) putSignaturesToLookaside(signatures [][]byte) e // is enough for dockerImageSource to stop looking for other signatures, so that // is sufficient. for i := len(signatures); ; i++ { - url := signatureStorageURL(d.c.signatureBase, d.manifestDigest, i) + url := signatureStorageURL(d.c.signatureBase, *instanceDigest, i) if url == nil { return errors.Errorf("Internal error: signatureStorageURL with non-nil base returned nil") } @@ -527,22 +551,17 @@ func (c *dockerClient) deleteOneSignature(url *url.URL) (missing bool, err error } // putSignaturesToAPIExtension implements PutSignatures() using the X-Registry-Supports-Signatures API extension. -func (d *dockerImageDestination) putSignaturesToAPIExtension(ctx context.Context, signatures [][]byte) error { +func (d *dockerImageDestination) putSignaturesToAPIExtension(ctx context.Context, signatures [][]byte, instanceDigest *digest.Digest) error { // Skip dealing with the manifest digest, or reading the old state, if not necessary. if len(signatures) == 0 { return nil } - if d.manifestDigest.String() == "" { - // This shouldn’t happen, ImageDestination users are required to call PutManifest before PutSignatures - return errors.Errorf("Unknown manifest digest, can't add signatures") - } - // Because image signatures are a shared resource in Atomic Registry, the default upload // always adds signatures. Eventually we should also allow removing signatures, // but the X-Registry-Supports-Signatures API extension does not support that yet. - existingSignatures, err := d.c.getExtensionsSignatures(ctx, d.ref, d.manifestDigest) + existingSignatures, err := d.c.getExtensionsSignatures(ctx, d.ref, *instanceDigest) if err != nil { return err } @@ -567,7 +586,7 @@ sigExists: if err != nil || n != 16 { return errors.Wrapf(err, "Error generating random signature len %d", n) } - signatureName = fmt.Sprintf("%s@%032x", d.manifestDigest.String(), randBytes) + signatureName = fmt.Sprintf("%s@%032x", instanceDigest.String(), randBytes) if _, ok := existingSigNames[signatureName]; !ok { break } diff --git a/docker/docker_image_src.go b/docker/docker_image_src.go index 353b1a6c59..23df287609 100644 --- a/docker/docker_image_src.go +++ b/docker/docker_image_src.go @@ -103,8 +103,15 @@ func (s *dockerImageSource) Close() error { return nil } -// LayerInfosForCopy() returns updated layer info that should be used when reading, in preference to values in the manifest, if specified. -func (s *dockerImageSource) LayerInfosForCopy(ctx context.Context) ([]types.BlobInfo, error) { +// LayerInfosForCopy returns either nil (meaning the values in the manifest are fine), or updated values for the layer +// blobsums that are listed in the image's manifest. If values are returned, they should be used when using GetBlob() +// to read the image's layers. +// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve BlobInfos for +// (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list +// (e.g. if the source never returns manifest lists). +// The Digest field is guaranteed to be provided; Size may be -1. +// WARNING: The list may contain duplicates, and they are semantically relevant. +func (s *dockerImageSource) LayerInfosForCopy(context.Context, *digest.Digest) ([]types.BlobInfo, error) { return nil, nil } diff --git a/docker/tarfile/dest.go b/docker/tarfile/dest.go index aec8404b67..eb10ed788c 100644 --- a/docker/tarfile/dest.go +++ b/docker/tarfile/dest.go @@ -195,10 +195,15 @@ func (d *Destination) createRepositoriesFile(rootLayerID string) error { } // PutManifest writes manifest to the destination. +// The instanceDigest value is expected to always be nil, because this transport does not support manifest lists, so +// there can be no secondary manifests. // FIXME? This should also receive a MIME type if known, to differentiate between schema versions. // If the destination is in principle available, refuses this manifest type (e.g. it does not recognize the schema), // but may accept a different manifest type, the returned error must be an ManifestTypeRejectedError. -func (d *Destination) PutManifest(ctx context.Context, m []byte) error { +func (d *Destination) PutManifest(ctx context.Context, m []byte, instanceDigest *digest.Digest) error { + if instanceDigest != nil { + return errors.New(`Manifest lists are not supported for docker tar files`) + } // We do not bother with types.ManifestTypeRejectedError; our .SupportedManifestMIMETypes() above is already providing only one alternative, // so the caller trying a different manifest kind would be pointless. var man manifest.Schema2 @@ -390,10 +395,13 @@ func (d *Destination) sendFile(path string, expectedSize int64, stream io.Reader return nil } -// PutSignatures adds the given signatures to the docker tarfile (currently not -// supported). MUST be called after PutManifest (signatures reference manifest -// contents) -func (d *Destination) PutSignatures(ctx context.Context, signatures [][]byte) error { +// PutSignatures would add the given signatures to the docker tarfile (currently not supported). +// The instanceDigest value is expected to always be nil, because this transport does not support manifest lists, so +// there can be no secondary manifests. MUST be called after PutManifest (signatures reference manifest contents). +func (d *Destination) PutSignatures(ctx context.Context, signatures [][]byte, instanceDigest *digest.Digest) error { + if instanceDigest != nil { + return errors.Errorf(`Manifest lists are not supported for docker tar files`) + } if len(signatures) != 0 { return errors.Errorf("Storing signatures for docker tar files is not supported") } diff --git a/docker/tarfile/src.go b/docker/tarfile/src.go index 78e4d6f656..b84afd6aa8 100644 --- a/docker/tarfile/src.go +++ b/docker/tarfile/src.go @@ -349,10 +349,12 @@ func (s *Source) prepareLayerData(tarManifest *ManifestItem, parsedConfig *manif // It may use a remote (= slow) service. // If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve (when the primary manifest is a manifest list); // this never happens if the primary manifest is not a manifest list (e.g. if the source never returns manifest lists). +// This source implementation does not support manifest lists, so the passed-in instanceDigest should always be nil, +// as the primary manifest can not be a list, so there can be no secondary instances. func (s *Source) GetManifest(ctx context.Context, instanceDigest *digest.Digest) ([]byte, string, error) { if instanceDigest != nil { // How did we even get here? GetManifest(ctx, nil) has returned a manifest.DockerV2Schema2MediaType. - return nil, "", errors.Errorf(`Manifest lists are not supported by "docker-daemon:"`) + return nil, "", errors.New(`Manifest lists are not supported by "docker-daemon:"`) } if s.generatedManifest == nil { if err := s.ensureCachedDataIsPresent(); err != nil { @@ -466,9 +468,8 @@ func (s *Source) GetBlob(ctx context.Context, info types.BlobInfo, cache types.B } // GetSignatures returns the image's signatures. It may use a remote (= slow) service. -// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve signatures for -// (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list -// (e.g. if the source never returns manifest lists). +// This source implementation does not support manifest lists, so the passed-in instanceDigest should always be nil, +// as there can be no secondary manifests. func (s *Source) GetSignatures(ctx context.Context, instanceDigest *digest.Digest) ([][]byte, error) { if instanceDigest != nil { // How did we even get here? GetManifest(ctx, nil) has returned a manifest.DockerV2Schema2MediaType. @@ -476,3 +477,14 @@ func (s *Source) GetSignatures(ctx context.Context, instanceDigest *digest.Diges } return [][]byte{}, nil } + +// LayerInfosForCopy returns either nil (meaning the values in the manifest are fine), or updated values for the layer +// blobsums that are listed in the image's manifest. If values are returned, they should be used when using GetBlob() +// to read the image's layers. +// This source implementation does not support manifest lists, so the passed-in instanceDigest should always be nil, +// as the primary manifest can not be a list, so there can be no secondary manifests. +// The Digest field is guaranteed to be provided; Size may be -1. +// WARNING: The list may contain duplicates, and they are semantically relevant. +func (s *Source) LayerInfosForCopy(context.Context, *digest.Digest) ([]types.BlobInfo, error) { + return nil, nil +} diff --git a/image/docker_list.go b/image/docker_list.go index a11cd06b99..7e22489840 100644 --- a/image/docker_list.go +++ b/image/docker_list.go @@ -2,69 +2,24 @@ package image import ( "context" - "encoding/json" - "fmt" - "runtime" "github.com/containers/image/v4/manifest" "github.com/containers/image/v4/types" - "github.com/opencontainers/go-digest" "github.com/pkg/errors" ) -type platformSpec struct { - Architecture string `json:"architecture"` - OS string `json:"os"` - OSVersion string `json:"os.version,omitempty"` - OSFeatures []string `json:"os.features,omitempty"` - Variant string `json:"variant,omitempty"` - Features []string `json:"features,omitempty"` // removed in OCI -} - -// A manifestDescriptor references a platform-specific manifest. -type manifestDescriptor struct { - manifest.Schema2Descriptor - Platform platformSpec `json:"platform"` -} - -type manifestList struct { - SchemaVersion int `json:"schemaVersion"` - MediaType string `json:"mediaType"` - Manifests []manifestDescriptor `json:"manifests"` -} - -// chooseDigestFromManifestList parses blob as a schema2 manifest list, -// and returns the digest of the image appropriate for the current environment. -func chooseDigestFromManifestList(sys *types.SystemContext, blob []byte) (digest.Digest, error) { - wantedArch := runtime.GOARCH - if sys != nil && sys.ArchitectureChoice != "" { - wantedArch = sys.ArchitectureChoice - } - wantedOS := runtime.GOOS - if sys != nil && sys.OSChoice != "" { - wantedOS = sys.OSChoice - } - - list := manifestList{} - if err := json.Unmarshal(blob, &list); err != nil { - return "", err - } - for _, d := range list.Manifests { - if d.Platform.Architecture == wantedArch && d.Platform.OS == wantedOS { - return d.Digest, nil - } - } - return "", fmt.Errorf("no image found in manifest list for architecture %s, OS %s", wantedArch, wantedOS) -} - func manifestSchema2FromManifestList(ctx context.Context, sys *types.SystemContext, src types.ImageSource, manblob []byte) (genericManifest, error) { - targetManifestDigest, err := chooseDigestFromManifestList(sys, manblob) + list, err := manifest.Schema2ListFromManifest(manblob) + if err != nil { + return nil, errors.Wrapf(err, "Error parsing schema2 manifest list") + } + targetManifestDigest, err := list.ChooseInstance(sys) if err != nil { - return nil, err + return nil, errors.Wrapf(err, "Error choosing image instance") } manblob, mt, err := src.GetManifest(ctx, &targetManifestDigest) if err != nil { - return nil, err + return nil, errors.Wrapf(err, "Error loading manifest for target platform") } matches, err := manifest.MatchesDigest(manblob, targetManifestDigest) @@ -72,23 +27,8 @@ func manifestSchema2FromManifestList(ctx context.Context, sys *types.SystemConte return nil, errors.Wrap(err, "Error computing manifest digest") } if !matches { - return nil, errors.Errorf("Manifest image does not match selected manifest digest %s", targetManifestDigest) + return nil, errors.Errorf("Image manifest does not match selected manifest digest %s", targetManifestDigest) } return manifestInstanceFromBlob(ctx, sys, src, manblob, mt) } - -// ChooseManifestInstanceFromManifestList returns a digest of a manifest appropriate -// for the current system from the manifest available from src. -func ChooseManifestInstanceFromManifestList(ctx context.Context, sys *types.SystemContext, src types.UnparsedImage) (digest.Digest, error) { - // For now this only handles manifest.DockerV2ListMediaType; we can generalize it later, - // probably along with manifest list editing. - blob, mt, err := src.Manifest(ctx) - if err != nil { - return "", err - } - if mt != manifest.DockerV2ListMediaType { - return "", fmt.Errorf("Internal error: Trying to select an image from a non-manifest-list manifest type %s", mt) - } - return chooseDigestFromManifestList(sys, blob) -} diff --git a/image/docker_list_test.go b/image/docker_list_test.go deleted file mode 100644 index 6bfbae7151..0000000000 --- a/image/docker_list_test.go +++ /dev/null @@ -1,44 +0,0 @@ -package image - -import ( - "bytes" - "io/ioutil" - "path/filepath" - "testing" - - "github.com/containers/image/v4/types" - "github.com/opencontainers/go-digest" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestChooseDigestFromManifestList(t *testing.T) { - manifest, err := ioutil.ReadFile(filepath.Join("fixtures", "schema2list.json")) - require.NoError(t, err) - - // Match found - for arch, expected := range map[string]digest.Digest{ - "amd64": "sha256:030fcb92e1487b18c974784dcc110a93147c9fc402188370fbfd17efabffc6af", - "s390x": "sha256:e5aa1b0a24620228b75382997a0977f609b3ca3a95533dafdef84c74cc8df642", - // There are several "arm" images with different variants; - // the current code returns the first match. NOTE: This is NOT an API promise. - "arm": "sha256:9142d97ef280a7953cf1a85716de49a24cc1dd62776352afad67e635331ff77a", - } { - digest, err := chooseDigestFromManifestList(&types.SystemContext{ - ArchitectureChoice: arch, - OSChoice: "linux", - }, manifest) - require.NoError(t, err, arch) - assert.Equal(t, expected, digest) - } - - // Invalid manifest list - _, err = chooseDigestFromManifestList(&types.SystemContext{ - ArchitectureChoice: "amd64", OSChoice: "linux", - }, bytes.Join([][]byte{manifest, []byte("!INVALID")}, nil)) - assert.Error(t, err) - - // Not found - _, err = chooseDigestFromManifestList(&types.SystemContext{OSChoice: "Unmatched"}, manifest) - assert.Error(t, err) -} diff --git a/image/docker_schema2_test.go b/image/docker_schema2_test.go index 8a901cbd5f..2266d6be93 100644 --- a/image/docker_schema2_test.go +++ b/image/docker_schema2_test.go @@ -41,7 +41,7 @@ func (f unusedImageSource) GetBlob(context.Context, types.BlobInfo, types.BlobIn func (f unusedImageSource) GetSignatures(context.Context, *digest.Digest) ([][]byte, error) { panic("Unexpected call to a mock function") } -func (f unusedImageSource) LayerInfosForCopy(ctx context.Context) ([]types.BlobInfo, error) { +func (f unusedImageSource) LayerInfosForCopy(context.Context, *digest.Digest) ([]types.BlobInfo, error) { panic("Unexpected call to a mock function") } @@ -430,10 +430,10 @@ func (d *memoryImageDest) PutBlob(ctx context.Context, stream io.Reader, inputIn func (d *memoryImageDest) TryReusingBlob(context.Context, types.BlobInfo, types.BlobInfoCache, bool) (bool, types.BlobInfo, error) { panic("Unexpected call to a mock function") } -func (d *memoryImageDest) PutManifest(ctx context.Context, m []byte) error { +func (d *memoryImageDest) PutManifest(context.Context, []byte, *digest.Digest) error { panic("Unexpected call to a mock function") } -func (d *memoryImageDest) PutSignatures(ctx context.Context, signatures [][]byte) error { +func (d *memoryImageDest) PutSignatures(ctx context.Context, signatures [][]byte, instanceDigest *digest.Digest) error { panic("Unexpected call to a mock function") } func (d *memoryImageDest) Commit(ctx context.Context) error { diff --git a/image/fixtures/oci1index.json b/image/fixtures/oci1index.json new file mode 100644 index 0000000000..066f058db1 --- /dev/null +++ b/image/fixtures/oci1index.json @@ -0,0 +1,30 @@ +{ + "schemaVersion": 2, + "manifests": [ + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "size": 7143, + "digest": "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f", + "platform": { + "architecture": "ppc64le", + "os": "linux" + } + }, + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "size": 7682, + "digest": "sha256:5b0bcabd1ed22e9fb1310cf6c2dec7cdef19f0ad69efa1f392e94a4333501270", + "platform": { + "architecture": "amd64", + "os": "linux", + "os.features": [ + "sse4" + ] + } + } + ], + "annotations": { + "com.example.key1": "value1", + "com.example.key2": "value2" + } +} diff --git a/image/manifest.go b/image/manifest.go index f384d2fb8a..fa5fd6f4dd 100644 --- a/image/manifest.go +++ b/image/manifest.go @@ -58,6 +58,8 @@ func manifestInstanceFromBlob(ctx context.Context, sys *types.SystemContext, src return manifestSchema2FromManifest(src, manblob) case manifest.DockerV2ListMediaType: return manifestSchema2FromManifestList(ctx, sys, src, manblob) + case imgspecv1.MediaTypeImageIndex: + return manifestOCI1FromImageIndex(ctx, sys, src, manblob) default: // Note that this may not be reachable, manifest.NormalizedMIMEType has a default for unknown values. return nil, fmt.Errorf("Unimplemented manifest MIME type %s", mt) } diff --git a/image/oci_index.go b/image/oci_index.go new file mode 100644 index 0000000000..32a3b75051 --- /dev/null +++ b/image/oci_index.go @@ -0,0 +1,34 @@ +package image + +import ( + "context" + + "github.com/containers/image/v4/manifest" + "github.com/containers/image/v4/types" + "github.com/pkg/errors" +) + +func manifestOCI1FromImageIndex(ctx context.Context, sys *types.SystemContext, src types.ImageSource, manblob []byte) (genericManifest, error) { + index, err := manifest.OCI1IndexFromManifest(manblob) + if err != nil { + return nil, errors.Wrapf(err, "Error parsing OCI1 index") + } + targetManifestDigest, err := index.ChooseInstance(sys) + if err != nil { + return nil, errors.Wrapf(err, "Error choosing image instance") + } + manblob, mt, err := src.GetManifest(ctx, &targetManifestDigest) + if err != nil { + return nil, errors.Wrapf(err, "Error loading manifest for target platform") + } + + matches, err := manifest.MatchesDigest(manblob, targetManifestDigest) + if err != nil { + return nil, errors.Wrap(err, "Error computing manifest digest") + } + if !matches { + return nil, errors.Errorf("Image manifest does not match selected manifest digest %s", targetManifestDigest) + } + + return manifestInstanceFromBlob(ctx, sys, src, manblob, mt) +} diff --git a/image/sourced.go b/image/sourced.go index d2a3e2ee62..6d59b6df02 100644 --- a/image/sourced.go +++ b/image/sourced.go @@ -100,5 +100,5 @@ func (i *sourcedImage) Manifest(ctx context.Context) ([]byte, string, error) { } func (i *sourcedImage) LayerInfosForCopy(ctx context.Context) ([]types.BlobInfo, error) { - return i.UnparsedImage.src.LayerInfosForCopy(ctx) + return i.UnparsedImage.src.LayerInfosForCopy(ctx, i.UnparsedImage.instanceDigest) } diff --git a/internal/magic.go b/internal/magic.go new file mode 100644 index 0000000000..fdbe5c8281 --- /dev/null +++ b/internal/magic.go @@ -0,0 +1,7 @@ +package internal + +type copyUnparsedTopImageKeyType int + +// CopyUnparsedTopImageKey is used to store a *image.UnparsedImage in a context +// object in copy.Image(). +const CopyUnparsedTopImageKey = copyUnparsedTopImageKeyType(1) diff --git a/manifest/docker_list.go b/manifest/docker_list.go new file mode 100644 index 0000000000..e16f8bd786 --- /dev/null +++ b/manifest/docker_list.go @@ -0,0 +1,212 @@ +package manifest + +import ( + "encoding/json" + "fmt" + "runtime" + + "github.com/containers/image/v4/types" + "github.com/opencontainers/go-digest" + imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" +) + +// Schema2PlatformSpec describes the platform which a particular manifest is +// specialized for. +type Schema2PlatformSpec struct { + Architecture string `json:"architecture"` + OS string `json:"os"` + OSVersion string `json:"os.version,omitempty"` + OSFeatures []string `json:"os.features,omitempty"` + Variant string `json:"variant,omitempty"` + Features []string `json:"features,omitempty"` // removed in OCI +} + +// Schema2ManifestDescriptor references a platform-specific manifest. +type Schema2ManifestDescriptor struct { + Schema2Descriptor + Platform Schema2PlatformSpec `json:"platform"` +} + +// Schema2List is a list of platform-specific manifests. +type Schema2List struct { + SchemaVersion int `json:"schemaVersion"` + MediaType string `json:"mediaType"` + Manifests []Schema2ManifestDescriptor `json:"manifests"` +} + +// MIMEType returns the MIME type of this particular manifest list. +func (list *Schema2List) MIMEType() string { + return list.MediaType +} + +// Instances returns a list of the manifests that this list knows of. +func (list *Schema2List) Instances() []digest.Digest { + results := make([]digest.Digest, len(list.Manifests)) + for i, m := range list.Manifests { + results[i] = m.Digest + } + return results +} + +// Instance returns the size and MIME type of a particular instance in the list. +func (list *Schema2List) Instance(instanceDigest digest.Digest) (ListUpdate, error) { + for _, manifest := range list.Manifests { + if manifest.Digest == instanceDigest { + return ListUpdate{ + Digest: manifest.Digest, + Size: manifest.Size, + MediaType: manifest.MediaType, + }, nil + } + } + return ListUpdate{}, errors.Errorf("unable to find instance %s passed to Schema2List.Instances", instanceDigest) +} + +// UpdateInstances updates the sizes, digests, and media types of the manifests +// which the list catalogs. +func (list *Schema2List) UpdateInstances(updates []ListUpdate) error { + if len(updates) != len(list.Manifests) { + return errors.Errorf("incorrect number of update entries passed to Schema2List.UpdateInstances: expected %d, got %d", len(list.Manifests), len(updates)) + } + for i := range updates { + if err := updates[i].Digest.Validate(); err != nil { + return errors.Wrapf(err, "update %d of %d passed to Schema2List.UpdateInstances contained an invalid digest", i+1, len(updates)) + } + list.Manifests[i].Digest = updates[i].Digest + if updates[i].Size < 0 { + return errors.Errorf("update %d of %d passed to Schema2List.UpdateInstances had an invalid size (%d)", i+1, len(updates), updates[i].Size) + } + list.Manifests[i].Size = updates[i].Size + if updates[i].MediaType == "" { + return errors.Errorf("update %d of %d passed to Schema2List.UpdateInstances had no media type", i+1, len(updates)) + } + list.Manifests[i].MediaType = updates[i].MediaType + } + return nil +} + +// ChooseInstance parses blob as a schema2 manifest list, and returns the digest +// of the image which is appropriate for the current environment. +func (list *Schema2List) ChooseInstance(ctx *types.SystemContext) (digest.Digest, error) { + wantedArch := runtime.GOARCH + if ctx != nil && ctx.ArchitectureChoice != "" { + wantedArch = ctx.ArchitectureChoice + } + wantedOS := runtime.GOOS + if ctx != nil && ctx.OSChoice != "" { + wantedOS = ctx.OSChoice + } + + for _, d := range list.Manifests { + if d.Platform.Architecture == wantedArch && d.Platform.OS == wantedOS { + return d.Digest, nil + } + } + return "", fmt.Errorf("no image found in manifest list for architecture %s, OS %s", wantedArch, wantedOS) +} + +// Serialize returns the list in a blob format. +// NOTE: Serialize() does not in general reproduce the original blob if this object was loaded from one, even if no modifications were made! +func (list *Schema2List) Serialize() ([]byte, error) { + buf, err := json.Marshal(list) + if err != nil { + return nil, errors.Wrapf(err, "error marshaling Schema2List %#v", list) + } + return buf, nil +} + +// Schema2ListFromComponents creates a Schema2 manifest list instance from the +// supplied data. +func Schema2ListFromComponents(components []Schema2ManifestDescriptor) *Schema2List { + list := Schema2List{ + SchemaVersion: 2, + MediaType: DockerV2ListMediaType, + Manifests: make([]Schema2ManifestDescriptor, len(components)), + } + for i, component := range components { + m := Schema2ManifestDescriptor{ + Schema2Descriptor{ + MediaType: component.MediaType, + Size: component.Size, + Digest: component.Digest, + URLs: dupStringSlice(component.URLs), + }, + Schema2PlatformSpec{ + Architecture: component.Platform.Architecture, + OS: component.Platform.OS, + OSVersion: component.Platform.OSVersion, + OSFeatures: dupStringSlice(component.Platform.OSFeatures), + Variant: component.Platform.Variant, + Features: dupStringSlice(component.Platform.Features), + }, + } + list.Manifests[i] = m + } + return &list +} + +// Schema2ListClone creates a deep copy of the passed-in list. +func Schema2ListClone(list *Schema2List) *Schema2List { + return Schema2ListFromComponents(list.Manifests) +} + +// ToOCI1Index returns the list encoded as an OCI1 index. +func (list *Schema2List) ToOCI1Index() (*OCI1Index, error) { + components := make([]imgspecv1.Descriptor, 0, len(list.Manifests)) + for _, manifest := range list.Manifests { + converted := imgspecv1.Descriptor{ + MediaType: manifest.MediaType, + Size: manifest.Size, + Digest: manifest.Digest, + URLs: dupStringSlice(manifest.URLs), + Platform: &imgspecv1.Platform{ + OS: manifest.Platform.OS, + Architecture: manifest.Platform.Architecture, + OSFeatures: dupStringSlice(manifest.Platform.OSFeatures), + OSVersion: manifest.Platform.OSVersion, + Variant: manifest.Platform.Variant, + }, + } + components = append(components, converted) + } + oci := OCI1IndexFromComponents(components, nil) + return oci, nil +} + +// ToSchema2List returns the list encoded as a Schema2 list. +func (list *Schema2List) ToSchema2List() (*Schema2List, error) { + return Schema2ListClone(list), nil +} + +// Schema2ListFromManifest creates a Schema2 manifest list instance from marshalled +// JSON, presumably generated by encoding a Schema2 manifest list. +func Schema2ListFromManifest(manifest []byte) (*Schema2List, error) { + list := Schema2List{ + Manifests: []Schema2ManifestDescriptor{}, + } + if err := json.Unmarshal(manifest, &list); err != nil { + return nil, errors.Wrapf(err, "error unmarshaling Schema2List %q", string(manifest)) + } + return &list, nil +} + +// Clone returns a deep copy of this list and its contents. +func (list *Schema2List) Clone() List { + return Schema2ListClone(list) +} + +// ConvertToMIMEType converts the passed-in manifest list to a manifest +// list of the specified type. +func (list *Schema2List) ConvertToMIMEType(manifestMIMEType string) (List, error) { + switch normalized := NormalizedMIMEType(manifestMIMEType); normalized { + case DockerV2ListMediaType: + return list.Clone(), nil + case imgspecv1.MediaTypeImageIndex: + return list.ToOCI1Index() + case DockerV2Schema1MediaType, DockerV2Schema1SignedMediaType, imgspecv1.MediaTypeImageManifest, DockerV2Schema2MediaType: + return nil, fmt.Errorf("Can not convert manifest list to MIME type %q, which is not a list type", manifestMIMEType) + } + // Note that this may not be reachable, NormalizedMIMEType has a default for unknown values. + return nil, fmt.Errorf("Unimplemented manifest MIME type %s", manifestMIMEType) +} diff --git a/manifest/list.go b/manifest/list.go new file mode 100644 index 0000000000..2db5584645 --- /dev/null +++ b/manifest/list.go @@ -0,0 +1,106 @@ +package manifest + +import ( + "fmt" + + "github.com/containers/image/v4/types" + digest "github.com/opencontainers/go-digest" + imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" +) + +var ( + // SupportedListMIMETypes is a list of the manifest list types that we know how to + // read/manipulate/write. + SupportedListMIMETypes = []string{ + DockerV2ListMediaType, + imgspecv1.MediaTypeImageIndex, + } +) + +// List is an interface for parsing, modifying lists of image manifests. +// Callers can either use this abstract interface without understanding the details of the formats, +// or instantiate a specific implementation (e.g. manifest.OCI1Index) and access the public members +// directly. +type List interface { + // MIMEType returns the MIME type of this particular manifest list. + MIMEType() string + + // Instances returns a list of the manifests that this list knows of, other than its own. + Instances() []digest.Digest + + // Update information about the list's instances. The length of the passed-in slice must + // match the length of the list of instances which the list already contains, and every field + // must be specified. + UpdateInstances([]ListUpdate) error + + // Instance returns the size and MIME type of a particular instance in the list. + Instance(digest.Digest) (ListUpdate, error) + + // ChooseInstance selects which manifest is most appropriate for the platform described by the + // SystemContext, or for the current platform if the SystemContext doesn't specify any details. + ChooseInstance(ctx *types.SystemContext) (digest.Digest, error) + + // Serialize returns the list in a blob format. + // NOTE: Serialize() does not in general reproduce the original blob if this object was loaded + // from, even if no modifications were made! + Serialize() ([]byte, error) + + // ConvertToMIMEType returns the list rebuilt to the specified MIME type, or an error. + ConvertToMIMEType(mimeType string) (List, error) + + // Clone returns a deep copy of this list and its contents. + Clone() List +} + +// ListUpdate includes the fields which a List's UpdateInstances() method will modify. +type ListUpdate struct { + Digest digest.Digest + Size int64 + MediaType string +} + +// dupStringSlice returns a deep copy of a slice of strings, or nil if the +// source slice is empty. +func dupStringSlice(list []string) []string { + if len(list) == 0 { + return nil + } + dup := make([]string, len(list)) + for i := range list { + dup[i] = list[i] + } + return dup +} + +// dupStringStringMap returns a deep copy of a map[string]string, or nil if the +// passed-in map is nil or has no keys. +func dupStringStringMap(m map[string]string) map[string]string { + if len(m) == 0 { + return nil + } + result := make(map[string]string) + for k, v := range m { + result[k] = v + } + return result +} + +// ListFromBlob parses a list of manifests. +func ListFromBlob(manifest []byte, manifestMIMEType string) (List, error) { + normalized := NormalizedMIMEType(manifestMIMEType) + switch normalized { + case DockerV2ListMediaType: + return Schema2ListFromManifest(manifest) + case imgspecv1.MediaTypeImageIndex: + return OCI1IndexFromManifest(manifest) + case DockerV2Schema1MediaType, DockerV2Schema1SignedMediaType, imgspecv1.MediaTypeImageManifest, DockerV2Schema2MediaType: + return nil, fmt.Errorf("Treating single images as manifest lists is not implemented") + } + return nil, fmt.Errorf("Unimplemented manifest list MIME type %s (normalized as %s)", manifestMIMEType, normalized) +} + +// ConvertListToMIMEType converts the passed-in manifest list to a manifest +// list of the specified type. +func ConvertListToMIMEType(list List, manifestMIMEType string) (List, error) { + return list.ConvertToMIMEType(manifestMIMEType) +} diff --git a/manifest/list_test.go b/manifest/list_test.go new file mode 100644 index 0000000000..7135c31fa0 --- /dev/null +++ b/manifest/list_test.go @@ -0,0 +1,149 @@ +package manifest + +import ( + "io/ioutil" + "path/filepath" + "testing" + + "github.com/containers/image/v4/types" + digest "github.com/opencontainers/go-digest" + imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func isOCI1Index(i interface{}) bool { + switch i.(type) { + case *OCI1Index: + return true + } + return false +} + +func isSchema2List(i interface{}) bool { + switch i.(type) { + case *Schema2List: + return true + } + return false +} + +func cloneOCI1Index(i interface{}) List { + if impl, ok := i.(*OCI1Index); ok { + return OCI1IndexClone(impl) + } + return nil +} + +func cloneSchema2List(i interface{}) List { + if impl, ok := i.(*Schema2List); ok { + return Schema2ListClone(impl) + } + return nil +} + +func pare(m List) { + if impl, ok := m.(*OCI1Index); ok { + impl.Annotations = nil + } + if impl, ok := m.(*Schema2List); ok { + for i := range impl.Manifests { + impl.Manifests[i].Platform.Features = nil + } + } + return +} + +func TestParseLists(t *testing.T) { + cases := []struct { + path string + mimeType string + }{ + {"ociv1.image.index.json", imgspecv1.MediaTypeImageIndex}, + {"v2list.manifest.json", DockerV2ListMediaType}, + } + for _, c := range cases { + manifest, err := ioutil.ReadFile(filepath.Join("fixtures", c.path)) + require.NoError(t, err, "error reading file %q", filepath.Join("fixtures", c.path)) + assert.Equal(t, GuessMIMEType(manifest), c.mimeType) + + _, err = FromBlob(manifest, c.mimeType) + require.Error(t, err, "manifest list %q should not parse as single images", c.path) + + m, err := ListFromBlob(manifest, c.mimeType) + require.NoError(t, err, "manifest list %q should parse as list types", c.path) + assert.Equal(t, m.MIMEType(), c.mimeType, "manifest %q is not of the expected MIME type", c.path) + + clone := m.Clone() + assert.Equal(t, clone, m, "manifest %q is missing some fields after being cloned", c.path) + + pare(m) + + index, err := m.ConvertToMIMEType(imgspecv1.MediaTypeImageIndex) + require.NoError(t, err, "error converting %q to an OCI1Index", c.path) + + list, err := m.ConvertToMIMEType(DockerV2ListMediaType) + require.NoError(t, err, "error converting %q to an Schema2List", c.path) + + index2, err := list.ConvertToMIMEType(imgspecv1.MediaTypeImageIndex) + assert.Equal(t, index, index2, "index %q lost data in conversion", c.path) + + list2, err := index.ConvertToMIMEType(DockerV2ListMediaType) + assert.Equal(t, list, list2, "list %q lost data in conversion", c.path) + } +} + +func TestChooseInstance(t *testing.T) { + for _, manifestList := range []struct { + listFile string + matchedInstances map[string]digest.Digest + unmatchedInstances []string + }{ + { + listFile: "schema2list.json", + matchedInstances: map[string]digest.Digest{ + "amd64": "sha256:030fcb92e1487b18c974784dcc110a93147c9fc402188370fbfd17efabffc6af", + "s390x": "sha256:e5aa1b0a24620228b75382997a0977f609b3ca3a95533dafdef84c74cc8df642", + // There are several "arm" images with different variants; + // the current code returns the first match. NOTE: This is NOT an API promise. + "arm": "sha256:9142d97ef280a7953cf1a85716de49a24cc1dd62776352afad67e635331ff77a", + }, + unmatchedInstances: []string{ + "unmatched", + }, + }, + { + listFile: "oci1index.json", + matchedInstances: map[string]digest.Digest{ + "amd64": "sha256:5b0bcabd1ed22e9fb1310cf6c2dec7cdef19f0ad69efa1f392e94a4333501270", + "ppc64le": "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f", + }, + unmatchedInstances: []string{ + "unmatched", + }, + }, + } { + man, err := ioutil.ReadFile(filepath.Join("..", "image", "fixtures", manifestList.listFile)) + require.NoError(t, err) + rawManifest := man + list, err := ListFromBlob(rawManifest, GuessMIMEType(rawManifest)) + require.NoError(t, err) + // Match found + for arch, expected := range manifestList.matchedInstances { + digest, err := list.ChooseInstance(&types.SystemContext{ + ArchitectureChoice: arch, + OSChoice: "linux", + }) + require.NoError(t, err, arch) + assert.Equal(t, expected, digest) + } + // Not found + for _, arch := range manifestList.unmatchedInstances { + _, err := list.ChooseInstance(&types.SystemContext{ + ArchitectureChoice: arch, + OSChoice: "linux", + }) + assert.Error(t, err) + } + } +} diff --git a/manifest/manifest.go b/manifest/manifest.go index 32af97ea81..a1f98e40c2 100644 --- a/manifest/manifest.go +++ b/manifest/manifest.go @@ -52,6 +52,7 @@ var DefaultRequestedManifestMIMETypes = []string{ DockerV2Schema1SignedMediaType, DockerV2Schema1MediaType, DockerV2ListMediaType, + imgspecv1.MediaTypeImageIndex, } // Manifest is an interface for parsing, modifying image manifests in isolation. @@ -140,8 +141,11 @@ func GuessMIMEType(manifest []byte) string { if err := json.Unmarshal(manifest, &ociIndex); err != nil { return "" } - if len(ociIndex.Manifests) != 0 && ociIndex.Manifests[0].MediaType == imgspecv1.MediaTypeImageManifest { - return imgspecv1.MediaTypeImageIndex + if len(ociIndex.Manifests) != 0 { + if ociMan.Config.MediaType == "" { + return imgspecv1.MediaTypeImageIndex + } + return ociMan.Config.MediaType } return DockerV2Schema2MediaType } @@ -199,7 +203,7 @@ func AddDummyV2S1Signature(manifest []byte) ([]byte, error) { // MIMETypeIsMultiImage returns true if mimeType is a list of images func MIMETypeIsMultiImage(mimeType string) bool { - return mimeType == DockerV2ListMediaType + return mimeType == DockerV2ListMediaType || mimeType == imgspecv1.MediaTypeImageIndex } // NormalizedMIMEType returns the effective MIME type of a manifest MIME type returned by a server, @@ -213,6 +217,7 @@ func NormalizedMIMEType(input string) string { return DockerV2Schema1SignedMediaType case DockerV2Schema1MediaType, DockerV2Schema1SignedMediaType, imgspecv1.MediaTypeImageManifest, + imgspecv1.MediaTypeImageIndex, DockerV2Schema2MediaType, DockerV2ListMediaType: return input @@ -232,7 +237,8 @@ func NormalizedMIMEType(input string) string { // FromBlob returns a Manifest instance for the specified manifest blob and the corresponding MIME type func FromBlob(manblob []byte, mt string) (Manifest, error) { - switch NormalizedMIMEType(mt) { + nmt := NormalizedMIMEType(mt) + switch nmt { case DockerV2Schema1MediaType, DockerV2Schema1SignedMediaType: return Schema1FromManifest(manblob) case imgspecv1.MediaTypeImageManifest: @@ -241,9 +247,10 @@ func FromBlob(manblob []byte, mt string) (Manifest, error) { return Schema2FromManifest(manblob) case DockerV2ListMediaType: return nil, fmt.Errorf("Treating manifest lists as individual manifests is not implemented") - default: // Note that this may not be reachable, NormalizedMIMEType has a default for unknown values. - return nil, fmt.Errorf("Unimplemented manifest MIME type %s", mt) + case imgspecv1.MediaTypeImageIndex: + return nil, fmt.Errorf("Treating image indices as individual manifests is not implemented") } + return nil, fmt.Errorf("Unimplemented manifest MIME type %s (normalized as %s)", mt, nmt) } // layerInfosToStrings converts a list of layer infos, presumably obtained from a Manifest.LayerInfos() diff --git a/manifest/manifest_test.go b/manifest/manifest_test.go index 7ed9e042c6..739a6f5af1 100644 --- a/manifest/manifest_test.go +++ b/manifest/manifest_test.go @@ -134,6 +134,8 @@ func TestMIMETypeIsMultiImage(t *testing.T) { {DockerV2Schema1MediaType, false}, {DockerV2Schema1SignedMediaType, false}, {DockerV2Schema2MediaType, false}, + {imgspecv1.MediaTypeImageIndex, true}, + {imgspecv1.MediaTypeImageManifest, false}, } { res := MIMETypeIsMultiImage(c.mt) assert.Equal(t, c.expected, res, c.mt) diff --git a/manifest/oci_index.go b/manifest/oci_index.go new file mode 100644 index 0000000000..f1db0cea32 --- /dev/null +++ b/manifest/oci_index.go @@ -0,0 +1,217 @@ +package manifest + +import ( + "encoding/json" + "fmt" + "runtime" + + "github.com/containers/image/v4/types" + "github.com/opencontainers/go-digest" + imgspec "github.com/opencontainers/image-spec/specs-go" + imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" +) + +// OCI1Index is just an alias for the OCI index type, but one which we can +// provide methods for. +type OCI1Index struct { + imgspecv1.Index +} + +// MIMEType returns the MIME type of this particular manifest index. +func (index *OCI1Index) MIMEType() string { + return imgspecv1.MediaTypeImageIndex +} + +// Instances returns a index of the manifests that this index knows of. +func (index *OCI1Index) Instances() []digest.Digest { + results := make([]digest.Digest, len(index.Manifests)) + for i, m := range index.Manifests { + results[i] = m.Digest + } + return results +} + +// Instance returns the size and MIME type of a particular instance in the index. +func (index *OCI1Index) Instance(instanceDigest digest.Digest) (ListUpdate, error) { + for _, manifest := range index.Manifests { + if manifest.Digest == instanceDigest { + return ListUpdate{ + Digest: manifest.Digest, + Size: manifest.Size, + MediaType: manifest.MediaType, + }, nil + } + } + return ListUpdate{}, errors.Errorf("unable to find instance %s in OCI1Index", instanceDigest) +} + +// UpdateInstances updates the sizes, digests, and media types of the manifests +// which the list catalogs. +func (index *OCI1Index) UpdateInstances(updates []ListUpdate) error { + if len(updates) != len(index.Manifests) { + return errors.Errorf("incorrect number of update entries passed to OCI1Index.UpdateInstances: expected %d, got %d", len(index.Manifests), len(updates)) + } + for i := range updates { + if err := updates[i].Digest.Validate(); err != nil { + return errors.Wrapf(err, "update %d of %d passed to OCI1Index.UpdateInstances contained an invalid digest", i+1, len(updates)) + } + index.Manifests[i].Digest = updates[i].Digest + if updates[i].Size < 0 { + return errors.Errorf("update %d of %d passed to OCI1Index.UpdateInstances had an invalid size (%d)", i+1, len(updates), updates[i].Size) + } + index.Manifests[i].Size = updates[i].Size + if updates[i].MediaType == "" { + return errors.Errorf("update %d of %d passed to OCI1Index.UpdateInstances had no media type", i+1, len(updates)) + } + index.Manifests[i].MediaType = updates[i].MediaType + } + return nil +} + +// ChooseInstance parses blob as an oci v1 manifest index, and returns the digest +// of the image which is appropriate for the current environment. +func (index *OCI1Index) ChooseInstance(ctx *types.SystemContext) (digest.Digest, error) { + wantedArch := runtime.GOARCH + if ctx != nil && ctx.ArchitectureChoice != "" { + wantedArch = ctx.ArchitectureChoice + } + wantedOS := runtime.GOOS + if ctx != nil && ctx.OSChoice != "" { + wantedOS = ctx.OSChoice + } + + for _, d := range index.Manifests { + if d.Platform != nil && d.Platform.Architecture == wantedArch && d.Platform.OS == wantedOS { + return d.Digest, nil + } + } + for _, d := range index.Manifests { + if d.Platform == nil { + return d.Digest, nil + } + } + return "", fmt.Errorf("no image found in image index for architecture %s, OS %s", wantedArch, wantedOS) +} + +// Serialize returns the index in a blob format. +// NOTE: Serialize() does not in general reproduce the original blob if this object was loaded from one, even if no modifications were made! +func (index *OCI1Index) Serialize() ([]byte, error) { + buf, err := json.Marshal(index) + if err != nil { + return nil, errors.Wrapf(err, "error marshaling OCI1Index %#v", index) + } + return buf, nil +} + +// OCI1IndexFromComponents creates an OCI1 image index instance from the +// supplied data. +func OCI1IndexFromComponents(components []imgspecv1.Descriptor, annotations map[string]string) *OCI1Index { + index := OCI1Index{ + imgspecv1.Index{ + Versioned: imgspec.Versioned{SchemaVersion: 2}, + Manifests: make([]imgspecv1.Descriptor, len(components)), + Annotations: dupStringStringMap(annotations), + }, + } + for i, component := range components { + var platform *imgspecv1.Platform + if component.Platform != nil { + platform = &imgspecv1.Platform{ + Architecture: component.Platform.Architecture, + OS: component.Platform.OS, + OSVersion: component.Platform.OSVersion, + OSFeatures: dupStringSlice(component.Platform.OSFeatures), + Variant: component.Platform.Variant, + } + } + m := imgspecv1.Descriptor{ + MediaType: component.MediaType, + Size: component.Size, + Digest: component.Digest, + URLs: dupStringSlice(component.URLs), + Annotations: dupStringStringMap(component.Annotations), + Platform: platform, + } + index.Manifests[i] = m + } + return &index +} + +// OCI1IndexClone creates a deep copy of the passed-in index. +func OCI1IndexClone(index *OCI1Index) *OCI1Index { + return OCI1IndexFromComponents(index.Manifests, index.Annotations) +} + +// ToOCI1Index returns the index encoded as an OCI1 index. +func (index *OCI1Index) ToOCI1Index() (*OCI1Index, error) { + return OCI1IndexClone(index), nil +} + +// ToSchema2List returns the index encoded as a Schema2 list. +func (index *OCI1Index) ToSchema2List() (*Schema2List, error) { + components := make([]Schema2ManifestDescriptor, 0, len(index.Manifests)) + for _, manifest := range index.Manifests { + platform := manifest.Platform + if platform == nil { + platform = &imgspecv1.Platform{ + OS: runtime.GOOS, + Architecture: runtime.GOARCH, + } + } + converted := Schema2ManifestDescriptor{ + Schema2Descriptor{ + MediaType: manifest.MediaType, + Size: manifest.Size, + Digest: manifest.Digest, + URLs: dupStringSlice(manifest.URLs), + }, + Schema2PlatformSpec{ + OS: platform.OS, + Architecture: platform.Architecture, + OSFeatures: dupStringSlice(platform.OSFeatures), + OSVersion: platform.OSVersion, + Variant: platform.Variant, + }, + } + components = append(components, converted) + } + s2 := Schema2ListFromComponents(components) + return s2, nil +} + +// OCI1IndexFromManifest creates an OCI1 manifest index instance from marshalled +// JSON, presumably generated by encoding a OCI1 manifest index. +func OCI1IndexFromManifest(manifest []byte) (*OCI1Index, error) { + index := OCI1Index{ + Index: imgspecv1.Index{ + Versioned: imgspec.Versioned{SchemaVersion: 2}, + Manifests: []imgspecv1.Descriptor{}, + Annotations: make(map[string]string), + }, + } + if err := json.Unmarshal(manifest, &index); err != nil { + return nil, errors.Wrapf(err, "error unmarshaling OCI1Index %q", string(manifest)) + } + return &index, nil +} + +// Clone returns a deep copy of this list and its contents. +func (index *OCI1Index) Clone() List { + return OCI1IndexClone(index) +} + +// ConvertToMIMEType converts the passed-in image index to a manifest list of +// the specified type. +func (index *OCI1Index) ConvertToMIMEType(manifestMIMEType string) (List, error) { + switch normalized := NormalizedMIMEType(manifestMIMEType); normalized { + case DockerV2ListMediaType: + return index.ToSchema2List() + case imgspecv1.MediaTypeImageIndex: + return index.Clone(), nil + case DockerV2Schema1MediaType, DockerV2Schema1SignedMediaType, imgspecv1.MediaTypeImageManifest, DockerV2Schema2MediaType: + return nil, fmt.Errorf("Can not convert image index to MIME type %q, which is not a list type", manifestMIMEType) + } + // Note that this may not be reachable, NormalizedMIMEType has a default for unknown values. + return nil, fmt.Errorf("Unimplemented manifest MIME type %s", manifestMIMEType) +} diff --git a/oci/archive/oci_dest.go b/oci/archive/oci_dest.go index 2455ed575a..3c2d8a3356 100644 --- a/oci/archive/oci_dest.go +++ b/oci/archive/oci_dest.go @@ -7,6 +7,7 @@ import ( "github.com/containers/image/v4/types" "github.com/containers/storage/pkg/archive" + digest "github.com/opencontainers/go-digest" "github.com/pkg/errors" ) @@ -105,13 +106,20 @@ func (d *ociArchiveImageDestination) TryReusingBlob(ctx context.Context, info ty return d.unpackedDest.TryReusingBlob(ctx, info, cache, canSubstitute) } -// PutManifest writes manifest to the destination -func (d *ociArchiveImageDestination) PutManifest(ctx context.Context, m []byte) error { - return d.unpackedDest.PutManifest(ctx, m) +// PutManifest writes the manifest to the destination. +// If instanceDigest is not nil, it contains a digest of the specific manifest instance to overwrite the manifest for (when +// the primary manifest is a manifest list); this should always be nil if the primary manifest is not a manifest list. +// It is expected but not enforced that the instanceDigest, when specified, matches the digest of `manifest` as generated +// by `manifest.Digest()`. +func (d *ociArchiveImageDestination) PutManifest(ctx context.Context, m []byte, instanceDigest *digest.Digest) error { + return d.unpackedDest.PutManifest(ctx, m, instanceDigest) } -func (d *ociArchiveImageDestination) PutSignatures(ctx context.Context, signatures [][]byte) error { - return d.unpackedDest.PutSignatures(ctx, signatures) +// PutSignatures writes a set of signatures to the destination. +// If instanceDigest is not nil, it contains a digest of the specific manifest instance to write or overwrite the signatures for +// (when the primary manifest is a manifest list); this should always be nil if the primary manifest is not a manifest list. +func (d *ociArchiveImageDestination) PutSignatures(ctx context.Context, signatures [][]byte, instanceDigest *digest.Digest) error { + return d.unpackedDest.PutSignatures(ctx, signatures, instanceDigest) } // Commit marks the process of storing the image as successful and asks for the image to be persisted diff --git a/oci/archive/oci_src.go b/oci/archive/oci_src.go index 8a479883fa..e54b423e41 100644 --- a/oci/archive/oci_src.go +++ b/oci/archive/oci_src.go @@ -96,7 +96,14 @@ func (s *ociArchiveImageSource) GetSignatures(ctx context.Context, instanceDiges return s.unpackedSrc.GetSignatures(ctx, instanceDigest) } -// LayerInfosForCopy() returns updated layer info that should be used when reading, in preference to values in the manifest, if specified. -func (s *ociArchiveImageSource) LayerInfosForCopy(ctx context.Context) ([]types.BlobInfo, error) { - return nil, nil +// LayerInfosForCopy returns either nil (meaning the values in the manifest are fine), or updated values for the layer +// blobsums that are listed in the image's manifest. If values are returned, they should be used when using GetBlob() +// to read the image's layers. +// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve BlobInfos for +// (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list +// (e.g. if the source never returns manifest lists). +// The Digest field is guaranteed to be provided; Size may be -1. +// WARNING: The list may contain duplicates, and they are semantically relevant. +func (s *ociArchiveImageSource) LayerInfosForCopy(ctx context.Context, instanceDigest *digest.Digest) ([]types.BlobInfo, error) { + return s.unpackedSrc.LayerInfosForCopy(ctx, instanceDigest) } diff --git a/oci/layout/oci_dest.go b/oci/layout/oci_dest.go index 20925d3dca..098c01ed25 100644 --- a/oci/layout/oci_dest.go +++ b/oci/layout/oci_dest.go @@ -38,6 +38,7 @@ func newImageDestination(sys *types.SystemContext, ref ociReference) (types.Imag Versioned: imgspec.Versioned{ SchemaVersion: 2, }, + Annotations: make(map[string]string), } } @@ -73,6 +74,7 @@ func (d *ociImageDestination) Close() error { func (d *ociImageDestination) SupportedManifestMIMETypes() []string { return []string{ imgspecv1.MediaTypeImageManifest, + imgspecv1.MediaTypeImageIndex, } } @@ -205,20 +207,27 @@ func (d *ociImageDestination) TryReusingBlob(ctx context.Context, info types.Blo return true, types.BlobInfo{Digest: info.Digest, Size: finfo.Size()}, nil } -// PutManifest writes manifest to the destination. +// PutManifest writes a manifest to the destination. Per our list of supported manifest MIME types, +// this should be either an OCI manifest (possibly converted to this format by the caller) or index, +// neither of which we'll need to modify further. +// If instanceDigest is not nil, it contains a digest of the specific manifest instance to overwrite the manifest for (when +// the primary manifest is a manifest list); this should always be nil if the primary manifest is not a manifest list. +// It is expected but not enforced that the instanceDigest, when specified, matches the digest of `manifest` as generated +// by `manifest.Digest()`. // FIXME? This should also receive a MIME type if known, to differentiate between schema versions. // If the destination is in principle available, refuses this manifest type (e.g. it does not recognize the schema), // but may accept a different manifest type, the returned error must be an ManifestTypeRejectedError. -func (d *ociImageDestination) PutManifest(ctx context.Context, m []byte) error { - digest, err := manifest.Digest(m) - if err != nil { - return err +func (d *ociImageDestination) PutManifest(ctx context.Context, m []byte, instanceDigest *digest.Digest) error { + var digest digest.Digest + var err error + if instanceDigest != nil { + digest = *instanceDigest + } else { + digest, err = manifest.Digest(m) + if err != nil { + return err + } } - desc := imgspecv1.Descriptor{} - desc.Digest = digest - // TODO(runcom): beaware and add support for OCI manifest list - desc.MediaType = imgspecv1.MediaTypeImageManifest - desc.Size = int64(len(m)) blobPath, err := d.ref.blobPath(digest, d.sharedBlobDir) if err != nil { @@ -231,32 +240,59 @@ func (d *ociImageDestination) PutManifest(ctx context.Context, m []byte) error { return err } - if d.ref.image != "" { - annotations := make(map[string]string) - annotations["org.opencontainers.image.ref.name"] = d.ref.image - desc.Annotations = annotations + if instanceDigest != nil { + return nil } - desc.Platform = &imgspecv1.Platform{ - Architecture: runtime.GOARCH, - OS: runtime.GOOS, + + // If we had platform information, we'd build an imgspecv1.Platform structure here. + + // Start filling out the descriptor for this entry + desc := imgspecv1.Descriptor{} + desc.Digest = digest + desc.Size = int64(len(m)) + if d.ref.image != "" { + desc.Annotations = make(map[string]string) + desc.Annotations[imgspecv1.AnnotationRefName] = d.ref.image } + + // If we knew the MIME type, we wouldn't have to guess here. + desc.MediaType = manifest.GuessMIMEType(m) + d.addManifest(&desc) return nil } func (d *ociImageDestination) addManifest(desc *imgspecv1.Descriptor) { + // If the new entry has a name, remove any conflicting names which we already have. + if desc.Annotations != nil && desc.Annotations[imgspecv1.AnnotationRefName] != "" { + // The name is being set on a new entry, so remove any older ones that had the same name. + // We might be storing an index and all of its component images, and we'll want to attach + // the name to the last one, which is the index. + for i, manifest := range d.index.Manifests { + if manifest.Annotations[imgspecv1.AnnotationRefName] == desc.Annotations[imgspecv1.AnnotationRefName] { + delete(d.index.Manifests[i].Annotations, imgspecv1.AnnotationRefName) + break + } + } + } + // If it has the same digest as another entry in the index, we already overwrote the file, + // so just pick up the other information. for i, manifest := range d.index.Manifests { - if manifest.Annotations["org.opencontainers.image.ref.name"] == desc.Annotations["org.opencontainers.image.ref.name"] { - // TODO Should there first be a cleanup based on the descriptor we are going to replace? + if manifest.Digest == desc.Digest { + // Replace it completely. d.index.Manifests[i] = *desc return } } + // It's a new entry to be added to the index. d.index.Manifests = append(d.index.Manifests, *desc) } -func (d *ociImageDestination) PutSignatures(ctx context.Context, signatures [][]byte) error { +// PutSignatures would add the given signatures to the oci layout (currently not supported). +// If instanceDigest is not nil, it contains a digest of the specific manifest instance to write or overwrite the signatures for +// (when the primary manifest is a manifest list); this should always be nil if the primary manifest is not a manifest list. +func (d *ociImageDestination) PutSignatures(ctx context.Context, signatures [][]byte, instanceDigest *digest.Digest) error { if len(signatures) != 0 { return errors.Errorf("Pushing signatures for OCI images is not supported") } diff --git a/oci/layout/oci_dest_test.go b/oci/layout/oci_dest_test.go index edc510b3ba..fa9e0f9234 100644 --- a/oci/layout/oci_dest_test.go +++ b/oci/layout/oci_dest_test.go @@ -1,13 +1,16 @@ package layout import ( + "bytes" "context" + "io/ioutil" "os" "path/filepath" "testing" "github.com/containers/image/v4/pkg/blobinfocache/memory" "github.com/containers/image/v4/types" + digest "github.com/opencontainers/go-digest" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -96,20 +99,46 @@ func TestPutManifestTwice(t *testing.T) { ociRef, ok := ref.(ociReference) require.True(t, ok) + putTestConfig(t, ociRef, tmpDir) putTestManifest(t, ociRef, tmpDir) putTestManifest(t, ociRef, tmpDir) index, err := ociRef.getIndex() assert.NoError(t, err) - assert.Equal(t, 1, len(index.Manifests), "Unexpected number of manifests") + assert.Equal(t, 2, len(index.Manifests), "Unexpected number of manifests") +} + +func putTestConfig(t *testing.T, ociRef ociReference, tmpDir string) { + data, err := ioutil.ReadFile("../../image/fixtures/oci1-config.json") + assert.NoError(t, err) + imageDest, err := newImageDestination(nil, ociRef) + assert.NoError(t, err) + + cache := memory.New() + + _, err = imageDest.PutBlob(context.Background(), bytes.NewReader(data), types.BlobInfo{Size: int64(len(data)), Digest: digest.FromBytes(data)}, cache, true) + assert.NoError(t, err) + + err = imageDest.Commit(context.Background()) + assert.NoError(t, err) + + paths := []string{} + filepath.Walk(tmpDir, func(path string, info os.FileInfo, err error) error { + paths = append(paths, path) + return nil + }) + + digest := digest.FromBytes(data).Encoded() + assert.Contains(t, paths, filepath.Join(tmpDir, "blobs", "sha256", digest), "The OCI directory does not contain the new config data") } func putTestManifest(t *testing.T, ociRef ociReference, tmpDir string) { + data, err := ioutil.ReadFile("../../image/fixtures/oci1.json") + assert.NoError(t, err) imageDest, err := newImageDestination(nil, ociRef) assert.NoError(t, err) - data := []byte("abc") - err = imageDest.PutManifest(context.Background(), data) + err = imageDest.PutManifest(context.Background(), data, nil) assert.NoError(t, err) err = imageDest.Commit(context.Background()) @@ -121,6 +150,6 @@ func putTestManifest(t *testing.T, ociRef ociReference, tmpDir string) { return nil }) - digest := "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + digest := digest.FromBytes(data).Encoded() assert.Contains(t, paths, filepath.Join(tmpDir, "blobs", "sha256", digest), "The OCI directory does not contain the new manifest data") } diff --git a/oci/layout/oci_src.go b/oci/layout/oci_src.go index dd6c6c4a66..f9977f8136 100644 --- a/oci/layout/oci_src.go +++ b/oci/layout/oci_src.go @@ -8,6 +8,7 @@ import ( "os" "strconv" + "github.com/containers/image/v4/manifest" "github.com/containers/image/v4/pkg/tlsclientconfig" "github.com/containers/image/v4/types" "github.com/docker/go-connections/tlsconfig" @@ -18,6 +19,7 @@ import ( type ociImageSource struct { ref ociReference + index *imgspecv1.Index descriptor imgspecv1.Descriptor client *http.Client sharedBlobDir string @@ -41,7 +43,11 @@ func newImageSource(sys *types.SystemContext, ref ociReference) (types.ImageSour if err != nil { return nil, err } - d := &ociImageSource{ref: ref, descriptor: descriptor, client: client} + index, err := ref.getIndex() + if err != nil { + return nil, err + } + d := &ociImageSource{ref: ref, index: index, descriptor: descriptor, client: client} if sys != nil { // TODO(jonboulle): check dir existence? d.sharedBlobDir = sys.OCISharedBlobDirPath @@ -66,28 +72,33 @@ func (s *ociImageSource) Close() error { func (s *ociImageSource) GetManifest(ctx context.Context, instanceDigest *digest.Digest) ([]byte, string, error) { var dig digest.Digest var mimeType string + var err error + if instanceDigest == nil { dig = digest.Digest(s.descriptor.Digest) mimeType = s.descriptor.MediaType } else { dig = *instanceDigest - // XXX: instanceDigest means that we don't immediately have the context of what - // mediaType the manifest has. In OCI this means that we don't know - // what reference it came from, so we just *assume* that its - // MediaTypeImageManifest. - // FIXME: We should actually be able to look up the manifest in the index, - // and see the MIME type there. - mimeType = imgspecv1.MediaTypeImageManifest + for _, md := range s.index.Manifests { + if md.Digest == dig { + mimeType = md.MediaType + break + } + } } manifestPath, err := s.ref.blobPath(dig, s.sharedBlobDir) if err != nil { return nil, "", err } + m, err := ioutil.ReadFile(manifestPath) if err != nil { return nil, "", err } + if mimeType == "" { + mimeType = manifest.GuessMIMEType(m) + } return m, mimeType, nil } @@ -157,8 +168,15 @@ func (s *ociImageSource) getExternalBlob(ctx context.Context, urls []string) (io return nil, 0, errWrap } -// LayerInfosForCopy() returns updated layer info that should be used when reading, in preference to values in the manifest, if specified. -func (s *ociImageSource) LayerInfosForCopy(ctx context.Context) ([]types.BlobInfo, error) { +// LayerInfosForCopy returns either nil (meaning the values in the manifest are fine), or updated values for the layer +// blobsums that are listed in the image's manifest. If values are returned, they should be used when using GetBlob() +// to read the image's layers. +// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve BlobInfos for +// (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list +// (e.g. if the source never returns manifest lists). +// The Digest field is guaranteed to be provided; Size may be -1. +// WARNING: The list may contain duplicates, and they are semantically relevant. +func (s *ociImageSource) LayerInfosForCopy(context.Context, *digest.Digest) ([]types.BlobInfo, error) { return nil, nil } diff --git a/oci/layout/oci_transport.go b/oci/layout/oci_transport.go index 259852b4dc..8b6be8ead8 100644 --- a/oci/layout/oci_transport.go +++ b/oci/layout/oci_transport.go @@ -195,10 +195,10 @@ func (ref ociReference) getManifestDescriptor() (imgspecv1.Descriptor, error) { } else { // if image specified, look through all manifests for a match for _, md := range index.Manifests { - if md.MediaType != imgspecv1.MediaTypeImageManifest { + if md.MediaType != imgspecv1.MediaTypeImageManifest && md.MediaType != imgspecv1.MediaTypeImageIndex { continue } - refName, ok := md.Annotations["org.opencontainers.image.ref.name"] + refName, ok := md.Annotations[imgspecv1.AnnotationRefName] if !ok { continue } diff --git a/openshift/openshift.go b/openshift/openshift.go index 51fff6269c..e795e14b04 100644 --- a/openshift/openshift.go +++ b/openshift/openshift.go @@ -231,16 +231,16 @@ func (s *openshiftImageSource) GetBlob(ctx context.Context, info types.BlobInfo, // (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list // (e.g. if the source never returns manifest lists). func (s *openshiftImageSource) GetSignatures(ctx context.Context, instanceDigest *digest.Digest) ([][]byte, error) { - var imageName string + var imageStreamImageName string if instanceDigest == nil { if err := s.ensureImageIsResolved(ctx); err != nil { return nil, err } - imageName = s.imageStreamImageName + imageStreamImageName = s.imageStreamImageName } else { - imageName = instanceDigest.String() + imageStreamImageName = instanceDigest.String() } - image, err := s.client.getImage(ctx, imageName) + image, err := s.client.getImage(ctx, imageStreamImageName) if err != nil { return nil, err } @@ -253,8 +253,15 @@ func (s *openshiftImageSource) GetSignatures(ctx context.Context, instanceDigest return sigs, nil } -// LayerInfosForCopy() returns updated layer info that should be used when reading, in preference to values in the manifest, if specified. -func (s *openshiftImageSource) LayerInfosForCopy(ctx context.Context) ([]types.BlobInfo, error) { +// LayerInfosForCopy returns either nil (meaning the values in the manifest are fine), or updated values for the layer +// blobsums that are listed in the image's manifest. If values are returned, they should be used when using GetBlob() +// to read the image's layers. +// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve BlobInfos for +// (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list +// (e.g. if the source never returns manifest lists). +// The Digest field is guaranteed to be provided; Size may be -1. +// WARNING: The list may contain duplicates, and they are semantically relevant. +func (s *openshiftImageSource) LayerInfosForCopy(ctx context.Context, instanceDigest *digest.Digest) ([]types.BlobInfo, error) { return nil, nil } @@ -414,20 +421,28 @@ func (d *openshiftImageDestination) TryReusingBlob(ctx context.Context, info typ // FIXME? This should also receive a MIME type if known, to differentiate between schema versions. // If the destination is in principle available, refuses this manifest type (e.g. it does not recognize the schema), // but may accept a different manifest type, the returned error must be an ManifestTypeRejectedError. -func (d *openshiftImageDestination) PutManifest(ctx context.Context, m []byte) error { - manifestDigest, err := manifest.Digest(m) - if err != nil { - return err +func (d *openshiftImageDestination) PutManifest(ctx context.Context, m []byte, instanceDigest *digest.Digest) error { + if instanceDigest == nil { + manifestDigest, err := manifest.Digest(m) + if err != nil { + return err + } + d.imageStreamImageName = manifestDigest.String() } - d.imageStreamImageName = manifestDigest.String() - - return d.docker.PutManifest(ctx, m) + return d.docker.PutManifest(ctx, m, instanceDigest) } -func (d *openshiftImageDestination) PutSignatures(ctx context.Context, signatures [][]byte) error { - if d.imageStreamImageName == "" { - return errors.Errorf("Internal error: Unknown manifest digest, can't add signatures") +func (d *openshiftImageDestination) PutSignatures(ctx context.Context, signatures [][]byte, instanceDigest *digest.Digest) error { + var imageStreamName string + if instanceDigest == nil { + if d.imageStreamImageName == "" { + return errors.Errorf("Internal error: Unknown manifest digest, can't add signatures") + } + imageStreamName = d.imageStreamImageName + } else { + imageStreamName = instanceDigest.String() } + // Because image signatures are a shared resource in Atomic Registry, the default upload // always adds signatures. Eventually we should also allow removing signatures. @@ -435,7 +450,7 @@ func (d *openshiftImageDestination) PutSignatures(ctx context.Context, signature return nil // No need to even read the old state. } - image, err := d.client.getImage(ctx, d.imageStreamImageName) + image, err := d.client.getImage(ctx, imageStreamName) if err != nil { return err } @@ -460,7 +475,7 @@ sigExists: if err != nil || n != 16 { return errors.Wrapf(err, "Error generating random signature len %d", n) } - signatureName = fmt.Sprintf("%s@%032x", d.imageStreamImageName, randBytes) + signatureName = fmt.Sprintf("%s@%032x", imageStreamName, randBytes) if _, ok := existingSigNames[signatureName]; !ok { break } diff --git a/ostree/ostree_dest.go b/ostree/ostree_dest.go index 9e1436e29e..0b345028fa 100644 --- a/ostree/ostree_dest.go +++ b/ostree/ostree_dest.go @@ -376,10 +376,16 @@ func (d *ostreeImageDestination) TryReusingBlob(ctx context.Context, info types. } // PutManifest writes manifest to the destination. +// The instanceDigest value is expected to always be nil, because this transport does not support manifest lists, so +// there can be no secondary manifests. // FIXME? This should also receive a MIME type if known, to differentiate between schema versions. // If the destination is in principle available, refuses this manifest type (e.g. it does not recognize the schema), // but may accept a different manifest type, the returned error must be an ManifestTypeRejectedError. -func (d *ostreeImageDestination) PutManifest(ctx context.Context, manifestBlob []byte) error { +func (d *ostreeImageDestination) PutManifest(ctx context.Context, manifestBlob []byte, instanceDigest *digest.Digest) error { + if instanceDigest != nil { + return errors.New(`Manifest lists are not supported by "ostree:"`) + } + d.manifest = string(manifestBlob) if err := json.Unmarshal(manifestBlob, &d.schema); err != nil { @@ -400,7 +406,14 @@ func (d *ostreeImageDestination) PutManifest(ctx context.Context, manifestBlob [ return ioutil.WriteFile(manifestPath, manifestBlob, 0644) } -func (d *ostreeImageDestination) PutSignatures(ctx context.Context, signatures [][]byte) error { +// PutSignatures writes signatures to the destination. +// The instanceDigest value is expected to always be nil, because this transport does not support manifest lists, so +// there can be no secondary manifests. +func (d *ostreeImageDestination) PutSignatures(ctx context.Context, signatures [][]byte, instanceDigest *digest.Digest) error { + if instanceDigest != nil { + return errors.New(`Manifest lists are not supported by "ostree:"`) + } + path := filepath.Join(d.tmpDirPath, d.ref.signaturePath(0)) if err := ensureParentDirectoryExists(path); err != nil { return err diff --git a/ostree/ostree_src.go b/ostree/ostree_src.go index ecb6e3f84a..62182b1bc0 100644 --- a/ostree/ostree_src.go +++ b/ostree/ostree_src.go @@ -98,9 +98,11 @@ func (s *ostreeImageSource) getTarSplitData(blob string) ([]byte, error) { // GetManifest returns the image's manifest along with its MIME type (which may be empty when it can't be determined but the manifest is available). // It may use a remote (= slow) service. +// This source implementation does not support manifest lists, so the passed-in instanceDigest should always be nil, +// as the primary manifest can not be a list, so there can be non-default instances. func (s *ostreeImageSource) GetManifest(ctx context.Context, instanceDigest *digest.Digest) ([]byte, string, error) { if instanceDigest != nil { - return nil, "", errors.Errorf(`Manifest lists are not supported by "ostree:"`) + return nil, "", errors.New(`Manifest lists are not supported by "ostree:"`) } if s.repo == nil { repo, err := openRepo(s.ref.repo) @@ -275,7 +277,7 @@ func (s *ostreeImageSource) GetBlob(ctx context.Context, info types.BlobInfo, ca // Ensure s.compressed is initialized. It is build by LayerInfosForCopy. if s.compressed == nil { - _, err := s.LayerInfosForCopy(ctx) + _, err := s.LayerInfosForCopy(ctx, nil) if err != nil { return nil, -1, err } @@ -337,9 +339,12 @@ func (s *ostreeImageSource) GetBlob(ctx context.Context, info types.BlobInfo, ca return rc, layerSize, nil } +// GetSignatures returns the image's signatures. It may use a remote (= slow) service. +// This source implementation does not support manifest lists, so the passed-in instanceDigest should always be nil, +// as there can be no secondary manifests. func (s *ostreeImageSource) GetSignatures(ctx context.Context, instanceDigest *digest.Digest) ([][]byte, error) { if instanceDigest != nil { - return nil, errors.New("manifest lists are not supported by this transport") + return nil, errors.New(`Manifest lists are not supported by "ostree:"`) } lenSignatures, err := s.getLenSignatures() if err != nil { @@ -372,9 +377,18 @@ func (s *ostreeImageSource) GetSignatures(ctx context.Context, instanceDigest *d return signatures, nil } -// LayerInfosForCopy() returns the list of layer blobs that make up the root filesystem of -// the image, after they've been decompressed. -func (s *ostreeImageSource) LayerInfosForCopy(ctx context.Context) ([]types.BlobInfo, error) { +// LayerInfosForCopy returns either nil (meaning the values in the manifest are fine), or updated values for the layer +// blobsums that are listed in the image's manifest. If values are returned, they should be used when using GetBlob() +// to read the image's layers. +// This source implementation does not support manifest lists, so the passed-in instanceDigest should always be nil, +// as the primary manifest can not be a list, so there can be secondary manifests. +// The Digest field is guaranteed to be provided; Size may be -1. +// WARNING: The list may contain duplicates, and they are semantically relevant. +func (s *ostreeImageSource) LayerInfosForCopy(ctx context.Context, instanceDigest *digest.Digest) ([]types.BlobInfo, error) { + if instanceDigest != nil { + return nil, errors.New(`Manifest lists are not supported by "ostree:"`) + } + updatedBlobInfos := []types.BlobInfo{} manifestBlob, manifestType, err := s.GetManifest(ctx, nil) if err != nil { diff --git a/storage/storage_image.go b/storage/storage_image.go index 4e913b84c0..cca4cf014a 100644 --- a/storage/storage_image.go +++ b/storage/storage_image.go @@ -6,6 +6,7 @@ import ( "bytes" "context" "encoding/json" + stderrors "errors" "fmt" "io" "io/ioutil" @@ -16,6 +17,7 @@ import ( "github.com/containers/image/v4/docker/reference" "github.com/containers/image/v4/image" + "github.com/containers/image/v4/internal" "github.com/containers/image/v4/internal/tmpdir" "github.com/containers/image/v4/manifest" "github.com/containers/image/v4/pkg/blobinfocache/none" @@ -32,38 +34,39 @@ import ( var ( // ErrBlobDigestMismatch is returned when PutBlob() is given a blob // with a digest-based name that doesn't match its contents. - ErrBlobDigestMismatch = errors.New("blob digest mismatch") + ErrBlobDigestMismatch = stderrors.New("blob digest mismatch") // ErrBlobSizeMismatch is returned when PutBlob() is given a blob // with an expected size that doesn't match the reader. - ErrBlobSizeMismatch = errors.New("blob size mismatch") - // ErrNoManifestLists is returned when GetManifest() is called. - // with a non-nil instanceDigest. - ErrNoManifestLists = errors.New("manifest lists are not supported by this transport") + ErrBlobSizeMismatch = stderrors.New("blob size mismatch") // ErrNoSuchImage is returned when we attempt to access an image which // doesn't exist in the storage area. ErrNoSuchImage = storage.ErrNotAnImage ) type storageImageSource struct { - imageRef storageReference - image *storage.Image - layerPosition map[digest.Digest]int // Where we are in reading a blob's layers - cachedManifest []byte // A cached copy of the manifest, if already known, or nil - getBlobMutex sync.Mutex // Mutex to sync state for parallel GetBlob executions - SignatureSizes []int `json:"signature-sizes,omitempty"` // List of sizes of each signature slice + imageRef storageReference + image *storage.Image + layerPosition map[digest.Digest]int // Where we are in reading a blob's layers + cachedManifest []byte // A cached copy of the manifest, if already known, or nil + getBlobMutex sync.Mutex // Mutex to sync state for parallel GetBlob executions + SignatureSizes []int `json:"signature-sizes,omitempty"` // List of sizes of each signature slice + SignaturesSizes map[digest.Digest][]int `json:"signatures-sizes,omitempty"` // List of sizes of each signature slice } type storageImageDestination struct { - imageRef storageReference - directory string // Temporary directory where we store blobs until Commit() time - nextTempFileID int32 // A counter that we use for computing filenames to assign to blobs - manifest []byte // Manifest contents, temporary - signatures []byte // Signature contents, temporary - putBlobMutex sync.Mutex // Mutex to sync state for parallel PutBlob executions - blobDiffIDs map[digest.Digest]digest.Digest // Mapping from layer blobsums to their corresponding DiffIDs - fileSizes map[digest.Digest]int64 // Mapping from layer blobsums to their sizes - filenames map[digest.Digest]string // Mapping from layer blobsums to names of files we used to hold them - SignatureSizes []int `json:"signature-sizes,omitempty"` // List of sizes of each signature slice + imageRef storageReference + directory string // Temporary directory where we store blobs until Commit() time + nextTempFileID int32 // A counter that we use for computing filenames to assign to blobs + manifest []byte // Manifest contents, temporary + manifests [][]byte // Additional manifest contents, temporary + signatures []byte // Signature contents, temporary + signatureses map[digest.Digest][]byte // Instance signature contents, temporary + putBlobMutex sync.Mutex // Mutex to sync state for parallel PutBlob executions + blobDiffIDs map[digest.Digest]digest.Digest // Mapping from layer blobsums to their corresponding DiffIDs + fileSizes map[digest.Digest]int64 // Mapping from layer blobsums to their sizes + filenames map[digest.Digest]string // Mapping from layer blobsums to names of files we used to hold them + SignatureSizes []int `json:"signature-sizes,omitempty"` // List of sizes of each signature slice + SignaturesSizes map[digest.Digest][]int `json:"signatures-sizes,omitempty"` // Sizes of each manifest's signature slice } type storageImageCloser struct { @@ -72,26 +75,33 @@ type storageImageCloser struct { } // manifestBigDataKey returns a key suitable for recording a manifest with the specified digest using storage.Store.ImageBigData and related functions. -// If a specific manifest digest is explicitly requested by the user, the key retruned function should be used preferably; +// If a specific manifest digest is explicitly requested by the user, the key returned by this function should be used preferably; // for compatibility, if a manifest is not available under this key, check also storage.ImageDigestBigDataKey func manifestBigDataKey(digest digest.Digest) string { return storage.ImageDigestManifestBigDataNamePrefix + "-" + digest.String() } +// signatureBigDataKey returns a key suitable for recording the signatures associated with the manifest with the specified digest using storage.Store.ImageBigData and related functions. +// If a specific manifest digest is explicitly requested by the user, the key returned by this function should be used preferably; +func signatureBigDataKey(digest digest.Digest) string { + return "signature-" + digest.Encoded() +} + // newImageSource sets up an image for reading. -func newImageSource(imageRef storageReference) (*storageImageSource, error) { +func newImageSource(ctx context.Context, sys *types.SystemContext, imageRef storageReference) (*storageImageSource, error) { // First, locate the image. - img, err := imageRef.resolveImage() + img, err := imageRef.resolveImage(sys) if err != nil { return nil, err } // Build the reader object. image := &storageImageSource{ - imageRef: imageRef, - image: img, - layerPosition: make(map[digest.Digest]int), - SignatureSizes: []int{}, + imageRef: imageRef, + image: img, + layerPosition: make(map[digest.Digest]int), + SignatureSizes: []int{}, + SignaturesSizes: make(map[digest.Digest][]int), } if img.Metadata != "" { if err := json.Unmarshal([]byte(img.Metadata), image); err != nil { @@ -182,7 +192,12 @@ func (s *storageImageSource) getBlobAndLayerID(info types.BlobInfo) (rc io.ReadC // GetManifest() reads the image's manifest. func (s *storageImageSource) GetManifest(ctx context.Context, instanceDigest *digest.Digest) (manifestBlob []byte, MIMEType string, err error) { if instanceDigest != nil { - return nil, "", ErrNoManifestLists + key := manifestBigDataKey(*instanceDigest) + blob, err := s.imageRef.transport.store.ImageBigData(s.image.ID, key) + if err != nil { + return nil, "", errors.Wrapf(err, "error reading manifest for image instance %q", *instanceDigest) + } + return blob, manifest.GuessMIMEType(blob), err } if len(s.cachedManifest) == 0 { // The manifest is stored as a big data item. @@ -214,11 +229,14 @@ func (s *storageImageSource) GetManifest(ctx context.Context, instanceDigest *di // LayerInfosForCopy() returns the list of layer blobs that make up the root filesystem of // the image, after they've been decompressed. -func (s *storageImageSource) LayerInfosForCopy(ctx context.Context) ([]types.BlobInfo, error) { - manifestBlob, manifestType, err := s.GetManifest(ctx, nil) +func (s *storageImageSource) LayerInfosForCopy(ctx context.Context, instanceDigest *digest.Digest) ([]types.BlobInfo, error) { + manifestBlob, manifestType, err := s.GetManifest(ctx, instanceDigest) if err != nil { return nil, errors.Wrapf(err, "error reading image manifest for %q", s.image.ID) } + if manifest.MIMETypeIsMultiImage(manifestType) { + return nil, errors.Errorf("can't copy layers for a manifest list (shouldn't be attempted)") + } man, err := manifest.FromBlob(manifestBlob, manifestType) if err != nil { return nil, errors.Wrapf(err, "error parsing image manifest for %q", s.image.ID) @@ -292,25 +310,34 @@ func buildLayerInfosForCopy(manifestInfos []manifest.LayerInfo, physicalInfos [] // GetSignatures() parses the image's signatures blob into a slice of byte slices. func (s *storageImageSource) GetSignatures(ctx context.Context, instanceDigest *digest.Digest) (signatures [][]byte, err error) { - if instanceDigest != nil { - return nil, ErrNoManifestLists - } var offset int sigslice := [][]byte{} signature := []byte{} - if len(s.SignatureSizes) > 0 { - signatureBlob, err := s.imageRef.transport.store.ImageBigData(s.image.ID, "signatures") + signatureSizes := s.SignatureSizes + key := "signatures" + instance := "default instance" + if instanceDigest != nil { + signatureSizes = s.SignaturesSizes[*instanceDigest] + key = signatureBigDataKey(*instanceDigest) + instance = instanceDigest.Encoded() + } else { + } + if len(signatureSizes) > 0 { + signatureBlob, err := s.imageRef.transport.store.ImageBigData(s.image.ID, key) if err != nil { - return nil, errors.Wrapf(err, "error looking up signatures data for image %q", s.image.ID) + return nil, errors.Wrapf(err, "error looking up signatures data for image %q (%s)", s.image.ID, instance) } signature = signatureBlob } - for _, length := range s.SignatureSizes { + for _, length := range signatureSizes { + if offset+length > len(signature) { + return nil, errors.Wrapf(err, "error looking up signatures data for image %q (%s): expected at least %d bytes, only found %d", s.image.ID, instance, len(signature), offset+length) + } sigslice = append(sigslice, signature[offset:offset+length]) offset += length } if offset != len(signature) { - return nil, errors.Errorf("signatures data contained %d extra bytes", len(signatures)-offset) + return nil, errors.Errorf("signatures data (%s) contained %d extra bytes", instance, len(signatures)-offset) } return sigslice, nil } @@ -323,12 +350,14 @@ func newImageDestination(imageRef storageReference) (*storageImageDestination, e return nil, errors.Wrapf(err, "error creating a temporary directory") } image := &storageImageDestination{ - imageRef: imageRef, - directory: directory, - blobDiffIDs: make(map[digest.Digest]digest.Digest), - fileSizes: make(map[digest.Digest]int64), - filenames: make(map[digest.Digest]string), - SignatureSizes: []int{}, + imageRef: imageRef, + directory: directory, + signatureses: make(map[digest.Digest][]byte), + blobDiffIDs: make(map[digest.Digest]digest.Digest), + fileSizes: make(map[digest.Digest]int64), + filenames: make(map[digest.Digest]string), + SignatureSizes: []int{}, + SignaturesSizes: make(map[digest.Digest][]int), } return image, nil } @@ -404,10 +433,10 @@ func (s *storageImageDestination) PutBlob(ctx context.Context, stream io.Reader, } // Ensure that any information that we were given about the blob is correct. if blobinfo.Digest.Validate() == nil && blobinfo.Digest != hasher.Digest() { - return errorBlobInfo, ErrBlobDigestMismatch + return errorBlobInfo, errors.WithStack(ErrBlobDigestMismatch) } if blobinfo.Size >= 0 && blobinfo.Size != counter.Count { - return errorBlobInfo, ErrBlobSizeMismatch + return errorBlobInfo, errors.WithStack(ErrBlobSizeMismatch) } // Record information about the blob. s.putBlobMutex.Lock() @@ -579,6 +608,15 @@ func (s *storageImageDestination) getConfigBlob(info types.BlobInfo) ([]byte, er return nil, errors.New("blob not found") } +func (s *storageImageDestination) unparsedTopLevelManifest(ctx context.Context) ([]byte, string, error) { + if value := ctx.Value(internal.CopyUnparsedTopImageKey); value != nil { + if unparsedTopLevel, ok := value.(*image.UnparsedImage); ok { + return unparsedTopLevel.Manifest(ctx) + } + } + return nil, "", nil +} + func (s *storageImageDestination) Commit(ctx context.Context) error { // Find the list of layer blobs. if len(s.manifest) == 0 { @@ -747,7 +785,32 @@ func (s *storageImageDestination) Commit(ctx context.Context) error { return errors.Wrapf(err, "error saving big data %q for image %q", blob.String(), img.ID) } } - // Set the reference's name on the image. + // If this is one image from a list, add its manifest to the "additional manifest" list. + topLevelManifest, _, err := s.unparsedTopLevelManifest(ctx) + if err != nil { + return errors.Wrapf(err, "error checking for top level manifest") + } + manifests := s.manifests + if topLevelManifest != nil { + manifests = append(manifests, topLevelManifest) + } + // Save the additional/alternate manifests. + for _, additionalManifest := range manifests { + additionalDigest, err := manifest.Digest(additionalManifest) + if err != nil { + return errors.Wrapf(err, "error digesting additional manifest") + } + key := manifestBigDataKey(additionalDigest) + if err := s.imageRef.transport.store.SetImageBigData(img.ID, key, additionalManifest, manifest.Digest); err != nil { + if _, err2 := s.imageRef.transport.store.DeleteImage(img.ID, true); err2 != nil { + logrus.Debugf("error deleting incomplete image %q: %v", img.ID, err2) + } + logrus.Debugf("error saving additional manifest for image %q: %v", img.ID, err) + return err + } + } + // Set the reference's name on the image. We don't need to worry about avoiding duplicate + // values because SetNames() will deduplicate the list that we pass to it. if name := s.imageRef.DockerReference(); len(oldNames) > 0 || name != nil { names := []string{} if name != nil { @@ -772,14 +835,16 @@ func (s *storageImageDestination) Commit(ctx context.Context) error { if err != nil { return errors.Wrapf(err, "error computing manifest digest") } - if err := s.imageRef.transport.store.SetImageBigData(img.ID, manifestBigDataKey(manifestDigest), s.manifest, manifest.Digest); err != nil { + key := manifestBigDataKey(manifestDigest) + if err := s.imageRef.transport.store.SetImageBigData(img.ID, key, s.manifest, manifest.Digest); err != nil { if _, err2 := s.imageRef.transport.store.DeleteImage(img.ID, true); err2 != nil { logrus.Debugf("error deleting incomplete image %q: %v", img.ID, err2) } logrus.Debugf("error saving manifest for image %q: %v", img.ID, err) return err } - if err := s.imageRef.transport.store.SetImageBigData(img.ID, storage.ImageDigestBigDataKey, s.manifest, manifest.Digest); err != nil { + key = storage.ImageDigestBigDataKey + if err := s.imageRef.transport.store.SetImageBigData(img.ID, key, s.manifest, manifest.Digest); err != nil { if _, err2 := s.imageRef.transport.store.DeleteImage(img.ID, true); err2 != nil { logrus.Debugf("error deleting incomplete image %q: %v", img.ID, err2) } @@ -796,6 +861,16 @@ func (s *storageImageDestination) Commit(ctx context.Context) error { return err } } + for instanceDigest, signatures := range s.signatureses { + key := signatureBigDataKey(instanceDigest) + if err := s.imageRef.transport.store.SetImageBigData(img.ID, key, signatures, manifest.Digest); err != nil { + if _, err2 := s.imageRef.transport.store.DeleteImage(img.ID, true); err2 != nil { + logrus.Debugf("error deleting incomplete image %q: %v", img.ID, err2) + } + logrus.Debugf("error saving signatures for image %q: %v", img.ID, err) + return err + } + } // Save our metadata. metadata, err := json.Marshal(s) if err != nil { @@ -830,21 +905,33 @@ func (s *storageImageDestination) SupportedManifestMIMETypes() []string { } // PutManifest writes the manifest to the destination. -func (s *storageImageDestination) PutManifest(ctx context.Context, manifestBlob []byte) error { +func (s *storageImageDestination) PutManifest(ctx context.Context, manifestBlob []byte, instanceDigest *digest.Digest) error { if s.imageRef.named != nil { if digested, ok := s.imageRef.named.(reference.Digested); ok { matches, err := manifest.MatchesDigest(manifestBlob, digested.Digest()) if err != nil { return err } + if !matches { + if topLevelManifest, _, err := s.unparsedTopLevelManifest(ctx); topLevelManifest != nil && err == nil { + matches, err = manifest.MatchesDigest(topLevelManifest, digested.Digest()) + if err != nil { + return err + } + } + } if !matches { return fmt.Errorf("Manifest does not match expected digest %s", digested.Digest()) } } } - - s.manifest = make([]byte, len(manifestBlob)) - copy(s.manifest, manifestBlob) + newBlob := make([]byte, len(manifestBlob)) + copy(newBlob, manifestBlob) + if instanceDigest == nil { + s.manifest = newBlob + } else { + s.manifests = append(s.manifests, newBlob) + } return nil } @@ -873,7 +960,7 @@ func (s *storageImageDestination) IgnoresEmbeddedDockerReference() bool { } // PutSignatures records the image's signatures for committing as a single data blob. -func (s *storageImageDestination) PutSignatures(ctx context.Context, signatures [][]byte) error { +func (s *storageImageDestination) PutSignatures(ctx context.Context, signatures [][]byte, instanceDigest *digest.Digest) error { sizes := []int{} sigblob := []byte{} for _, sig := range signatures { @@ -883,8 +970,21 @@ func (s *storageImageDestination) PutSignatures(ctx context.Context, signatures copy(newblob[len(sigblob):], sig) sigblob = newblob } - s.signatures = sigblob - s.SignatureSizes = sizes + if instanceDigest == nil { + s.signatures = sigblob + s.SignatureSizes = sizes + } + if instanceDigest == nil && len(s.manifest) > 0 { + manifestDigest, err := manifest.Digest(s.manifest) + if err != nil { + return err + } + instanceDigest = &manifestDigest + } + if instanceDigest != nil { + s.signatureses[*instanceDigest] = sigblob + s.SignaturesSizes[*instanceDigest] = sizes + } return nil } @@ -940,7 +1040,7 @@ func (s *storageImageCloser) Size() (int64, error) { // newImage creates an image that also knows its size func newImage(ctx context.Context, sys *types.SystemContext, s storageReference) (types.ImageCloser, error) { - src, err := newImageSource(s) + src, err := newImageSource(ctx, sys, s) if err != nil { return nil, err } diff --git a/storage/storage_reference.go b/storage/storage_reference.go index 7ad20817b8..e27bf16770 100644 --- a/storage/storage_reference.go +++ b/storage/storage_reference.go @@ -7,8 +7,11 @@ import ( "strings" "github.com/containers/image/v4/docker/reference" + "github.com/containers/image/v4/manifest" "github.com/containers/image/v4/types" "github.com/containers/storage" + digest "github.com/opencontainers/go-digest" + imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -51,9 +54,58 @@ func imageMatchesRepo(image *storage.Image, ref reference.Named) bool { return false } +func imageMatchesSystemContext(store storage.Store, img *storage.Image, manifestDigest digest.Digest, sys *types.SystemContext) bool { + key := manifestBigDataKey(manifestDigest) + manifestBytes, err := store.ImageBigData(img.ID, key) + if err != nil { + return false + } + manifestType := manifest.GuessMIMEType(manifestBytes) + if manifest.MIMETypeIsMultiImage(manifestType) { + list, err := manifest.ListFromBlob(manifestBytes, manifestType) + if err != nil { + return false + } + manifestDigest, err = list.ChooseInstance(sys) + if err != nil { + return false + } + key = manifestBigDataKey(manifestDigest) + manifestBytes, err = store.ImageBigData(img.ID, key) + if err != nil { + return false + } + manifestType = manifest.GuessMIMEType(manifestBytes) + } + m, err := manifest.FromBlob(manifestBytes, manifestType) + getConfig := func(blobInfo types.BlobInfo) ([]byte, error) { + return store.ImageBigData(img.ID, blobInfo.Digest.String()) + } + ii, err := m.Inspect(getConfig) + if err != nil { + return false + } + index := manifest.OCI1IndexFromComponents([]imgspecv1.Descriptor{{ + MediaType: imgspecv1.MediaTypeImageManifest, + Digest: manifestDigest, + Size: int64(len(manifestBytes)), + Platform: &imgspecv1.Platform{ + OS: ii.Os, + Architecture: ii.Architecture, + }, + }}, nil) + instanceDigest, err := index.ChooseInstance(sys) + if err != nil { + return false + } + key = manifestBigDataKey(instanceDigest) + _, err = store.ImageBigData(img.ID, key) + return err == nil +} + // Resolve the reference's name to an image ID in the store, if there's already // one present with the same name or ID, and return the image. -func (s *storageReference) resolveImage() (*storage.Image, error) { +func (s *storageReference) resolveImage(sys *types.SystemContext) (*storage.Image, error) { var loadedImage *storage.Image if s.id == "" && s.named != nil { // Look for an image that has the expanded reference name as an explicit Name value. @@ -72,9 +124,10 @@ func (s *storageReference) resolveImage() (*storage.Image, error) { if err == nil && len(images) > 0 { for _, image := range images { if imageMatchesRepo(image, s.named) { - loadedImage = image - s.id = image.ID - break + if loadedImage == nil || imageMatchesSystemContext(s.transport.store, image, digested.Digest(), sys) { + loadedImage = image + s.id = image.ID + } } } } @@ -202,7 +255,7 @@ func (s storageReference) NewImage(ctx context.Context, sys *types.SystemContext } func (s storageReference) DeleteImage(ctx context.Context, sys *types.SystemContext) error { - img, err := s.resolveImage() + img, err := s.resolveImage(sys) if err != nil { return err } @@ -217,7 +270,7 @@ func (s storageReference) DeleteImage(ctx context.Context, sys *types.SystemCont } func (s storageReference) NewImageSource(ctx context.Context, sys *types.SystemContext) (types.ImageSource, error) { - return newImageSource(s) + return newImageSource(ctx, sys, s) } func (s storageReference) NewImageDestination(ctx context.Context, sys *types.SystemContext) (types.ImageDestination, error) { diff --git a/storage/storage_test.go b/storage/storage_test.go index 0cf4b51680..037c488de5 100644 --- a/storage/storage_test.go +++ b/storage/storage_test.go @@ -14,10 +14,12 @@ import ( "io/ioutil" "os" "path/filepath" + "reflect" "strings" "testing" "time" + imanifest "github.com/containers/image/v4/manifest" "github.com/containers/image/v4/pkg/blobinfocache/memory" "github.com/containers/image/v4/types" "github.com/containers/storage" @@ -271,6 +273,10 @@ func makeLayer(t *testing.T, compression archive.Compression) (ddigest.Digest, i if n != len(buf) { t.Fatalf("Short write writing tar header: %d < %d", n, len(buf)) } + err = twriter.Flush() + if err != nil { + t.Fatalf("Error flushing output to tar archive: %v", err) + } }() _, err = io.Copy(&tbuffer, preader) if err != nil { @@ -394,10 +400,10 @@ func TestWriteRead(t *testing.T) { manifest = strings.Replace(manifest, "%li", li, -1) manifest = strings.Replace(manifest, "%ci", sum.Hex(), -1) t.Logf("this manifest is %q", manifest) - if err := dest.PutManifest(context.Background(), []byte(manifest)); err != nil { + if err := dest.PutManifest(context.Background(), []byte(manifest), nil); err != nil { t.Fatalf("Error saving manifest to destination: %v", err) } - if err := dest.PutSignatures(context.Background(), signatures); err != nil { + if err := dest.PutSignatures(context.Background(), signatures, nil); err != nil { t.Fatalf("Error saving signatures to destination: %v", err) } if err := dest.Commit(context.Background()); err != nil { @@ -454,10 +460,16 @@ func TestWriteRead(t *testing.T) { t.Fatalf("GetManifest(%q) returned error %v", ref.StringWithinTransport(), err) } t.Logf("this manifest's type appears to be %q", manifestType) - sum = ddigest.SHA256.FromBytes([]byte(manifest)) - _, _, err = src.GetManifest(context.Background(), &sum) - if err == nil { - t.Fatalf("GetManifest(%q) with an instanceDigest is supposed to fail", ref.StringWithinTransport()) + sum, err = imanifest.Digest([]byte(manifest)) + if err != nil { + t.Fatalf("manifest.Digest() returned error %v", err) + } + retrieved, _, err := src.GetManifest(context.Background(), &sum) + if err != nil { + t.Fatalf("GetManifest(%q) with an instanceDigest is supposed to succeed", ref.StringWithinTransport()) + } + if string(retrieved) != string(manifest) { + t.Fatalf("GetManifest(%q) with an instanceDigest retrieved a different manifest", ref.StringWithinTransport()) } sigs, err := src.GetSignatures(context.Background(), nil) if err != nil { @@ -474,9 +486,12 @@ func TestWriteRead(t *testing.T) { t.Fatalf("Signature %d was corrupted", i) } } - _, err = src.GetSignatures(context.Background(), &sum) - if err == nil { - t.Fatalf("GetSignatures(%q) with instanceDigest is supposed to fail", ref.StringWithinTransport()) + sigs2, err := src.GetSignatures(context.Background(), &sum) + if err != nil { + t.Fatalf("GetSignatures(%q) with instance %s returned error %v", ref.StringWithinTransport(), sum.String(), err) + } + if !reflect.DeepEqual(sigs, sigs2) { + t.Fatalf("GetSignatures(%q) with instance %s returned a different result", ref.StringWithinTransport(), sum.String()) } for _, layerInfo := range layerInfos { buf := bytes.Buffer{} @@ -493,6 +508,7 @@ func TestWriteRead(t *testing.T) { t.Fatalf("Error decompressing layer %q from %q", layerInfo.Digest, ref.StringWithinTransport()) } n, err := io.Copy(&buf, decompressed) + layer.Close() if layerInfo.Size >= 0 && compressed.Count != layerInfo.Size { t.Fatalf("Blob size is different than expected: %d != %d, read %d", compressed.Count, layerInfo.Size, n) } @@ -556,7 +572,7 @@ func TestDuplicateName(t *testing.T) { ] } `, digest, size) - if err := dest.PutManifest(context.Background(), []byte(manifest)); err != nil { + if err := dest.PutManifest(context.Background(), []byte(manifest), nil); err != nil { t.Fatalf("Error storing manifest to destination: %v", err) } if err := dest.Commit(context.Background()); err != nil { @@ -591,7 +607,7 @@ func TestDuplicateName(t *testing.T) { ] } `, digest, size) - if err := dest.PutManifest(context.Background(), []byte(manifest)); err != nil { + if err := dest.PutManifest(context.Background(), []byte(manifest), nil); err != nil { t.Fatalf("Error storing manifest to destination: %v", err) } if err := dest.Commit(context.Background()); err != nil { @@ -643,7 +659,7 @@ func TestDuplicateID(t *testing.T) { ] } `, digest, size) - if err := dest.PutManifest(context.Background(), []byte(manifest)); err != nil { + if err := dest.PutManifest(context.Background(), []byte(manifest), nil); err != nil { t.Fatalf("Error storing manifest to destination: %v", err) } if err := dest.Commit(context.Background()); err != nil { @@ -678,7 +694,7 @@ func TestDuplicateID(t *testing.T) { ] } `, digest, size) - if err := dest.PutManifest(context.Background(), []byte(manifest)); err != nil { + if err := dest.PutManifest(context.Background(), []byte(manifest), nil); err != nil { t.Fatalf("Error storing manifest to destination: %v", err) } if err := dest.Commit(context.Background()); errors.Cause(err) != storage.ErrDuplicateID { @@ -733,7 +749,7 @@ func TestDuplicateNameID(t *testing.T) { ] } `, digest, size) - if err := dest.PutManifest(context.Background(), []byte(manifest)); err != nil { + if err := dest.PutManifest(context.Background(), []byte(manifest), nil); err != nil { t.Fatalf("Error storing manifest to destination: %v", err) } if err := dest.Commit(context.Background()); err != nil { @@ -768,7 +784,7 @@ func TestDuplicateNameID(t *testing.T) { ] } `, digest, size) - if err := dest.PutManifest(context.Background(), []byte(manifest)); err != nil { + if err := dest.PutManifest(context.Background(), []byte(manifest), nil); err != nil { t.Fatalf("Error storing manifest to destination: %v", err) } if err := dest.Commit(context.Background()); errors.Cause(err) != storage.ErrDuplicateID { @@ -889,7 +905,7 @@ func TestSize(t *testing.T) { ] } `, configInfo.Size, configInfo.Digest, digest1, size1, digest2, size2) - if err := dest.PutManifest(context.Background(), []byte(manifest)); err != nil { + if err := dest.PutManifest(context.Background(), []byte(manifest), nil); err != nil { t.Fatalf("Error storing manifest to destination: %v", err) } if err := dest.Commit(context.Background()); err != nil { @@ -905,8 +921,8 @@ func TestSize(t *testing.T) { if usize == -1 || err != nil { t.Fatalf("Error calculating image size: %v", err) } - if int(usize) != len(config)+int(usize1)+int(usize2)+len(manifest) { - t.Fatalf("Unexpected image size: %d != %d + %d + %d + %d", usize, len(config), usize1, usize2, len(manifest)) + if int(usize) != len(config)+int(usize1)+int(usize2)+2*len(manifest) { + t.Fatalf("Unexpected image size: %d != %d + %d + %d + %d (%d)", usize, len(config), usize1, usize2, len(manifest), len(config)+int(usize1)+int(usize2)+2*len(manifest)) } img.Close() } @@ -1000,7 +1016,7 @@ func TestDuplicateBlob(t *testing.T) { ] } `, configInfo.Size, configInfo.Digest, digest1, size1, digest2, size2, digest1, size1, digest2, size2) - if err := dest.PutManifest(context.Background(), []byte(manifest)); err != nil { + if err := dest.PutManifest(context.Background(), []byte(manifest), nil); err != nil { t.Fatalf("Error storing manifest to destination: %v", err) } if err := dest.Commit(context.Background()); err != nil { diff --git a/storage/storage_transport.go b/storage/storage_transport.go index 48b909c035..3f87d0c275 100644 --- a/storage/storage_transport.go +++ b/storage/storage_transport.go @@ -288,7 +288,7 @@ func (s storageTransport) GetStoreImage(store storage.Store, ref types.ImageRefe } if sref, ok := ref.(*storageReference); ok { tmpRef := *sref - if img, err := tmpRef.resolveImage(); err == nil { + if img, err := tmpRef.resolveImage(&types.SystemContext{}); err == nil { return img, nil } } diff --git a/tarball/tarball_src.go b/tarball/tarball_src.go index ead1a50bd3..f5c3e958ab 100644 --- a/tarball/tarball_src.go +++ b/tarball/tarball_src.go @@ -248,9 +248,8 @@ func (is *tarballImageSource) GetManifest(ctx context.Context, instanceDigest *d } // GetSignatures returns the image's signatures. It may use a remote (= slow) service. -// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve signatures for -// (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list -// (e.g. if the source never returns manifest lists). +// This source implementation does not support manifest lists, so the passed-in instanceDigest should always be nil, +// as there can be no secondary manifests. func (*tarballImageSource) GetSignatures(ctx context.Context, instanceDigest *digest.Digest) ([][]byte, error) { if instanceDigest != nil { return nil, fmt.Errorf("manifest lists are not supported by the %q transport", transportName) @@ -262,7 +261,14 @@ func (is *tarballImageSource) Reference() types.ImageReference { return &is.reference } -// LayerInfosForCopy() returns updated layer info that should be used when reading, in preference to values in the manifest, if specified. -func (*tarballImageSource) LayerInfosForCopy(ctx context.Context) ([]types.BlobInfo, error) { +// LayerInfosForCopy returns either nil (meaning the values in the manifest are fine), or updated values for the layer +// blobsums that are listed in the image's manifest. If values are returned, they should be used when using GetBlob() +// to read the image's layers. +// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve BlobInfos for +// (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list +// (e.g. if the source never returns manifest lists). +// The Digest field is guaranteed to be provided; Size may be -1. +// WARNING: The list may contain duplicates, and they are semantically relevant. +func (*tarballImageSource) LayerInfosForCopy(context.Context, *digest.Digest) ([]types.BlobInfo, error) { return nil, nil } diff --git a/types/types.go b/types/types.go index edb1cf66df..5b1a6b1387 100644 --- a/types/types.go +++ b/types/types.go @@ -227,10 +227,15 @@ type ImageSource interface { // (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list // (e.g. if the source never returns manifest lists). GetSignatures(ctx context.Context, instanceDigest *digest.Digest) ([][]byte, error) - // LayerInfosForCopy returns either nil (meaning the values in the manifest are fine), or updated values for the layer blobsums that are listed in the image's manifest. + // LayerInfosForCopy returns either nil (meaning the values in the manifest are fine), or updated values for the layer + // blobsums that are listed in the image's manifest. If values are returned, they should be used when using GetBlob() + // to read the image's layers. + // If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve BlobInfos for + // (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list + // (e.g. if the source never returns manifest lists). // The Digest field is guaranteed to be provided; Size may be -1. // WARNING: The list may contain duplicates, and they are semantically relevant. - LayerInfosForCopy(ctx context.Context) ([]BlobInfo, error) + LayerInfosForCopy(ctx context.Context, instanceDigest *digest.Digest) ([]BlobInfo, error) } // ImageDestination is a service, possibly remote (= slow), to store components of a single image. @@ -286,11 +291,19 @@ type ImageDestination interface { // May use and/or update cache. TryReusingBlob(ctx context.Context, info BlobInfo, cache BlobInfoCache, canSubstitute bool) (bool, BlobInfo, error) // PutManifest writes manifest to the destination. + // If instanceDigest is not nil, it contains a digest of the specific manifest instance to write the manifest for + // (when the primary manifest is a manifest list); this should always be nil if the primary manifest is not a manifest list. + // It is expected but not enforced that the instanceDigest, when specified, matches the digest of `manifest` as generated + // by `manifest.Digest()`. // FIXME? This should also receive a MIME type if known, to differentiate between schema versions. // If the destination is in principle available, refuses this manifest type (e.g. it does not recognize the schema), // but may accept a different manifest type, the returned error must be an ManifestTypeRejectedError. - PutManifest(ctx context.Context, manifest []byte) error - PutSignatures(ctx context.Context, signatures [][]byte) error + PutManifest(ctx context.Context, manifest []byte, instanceDigest *digest.Digest) error + // PutSignatures writes a set of signatures to the destination. + // If instanceDigest is not nil, it contains a digest of the specific manifest instance to write or overwrite the signatures for + // (when the primary manifest is a manifest list); this should always be nil if the primary manifest is not a manifest list. + // MUST be called after PutManifest (signatures may reference manifest contents). + PutSignatures(ctx context.Context, signatures [][]byte, instanceDigest *digest.Digest) error // Commit marks the process of storing the image as successful and asks for the image to be persisted. // WARNING: This does not have any transactional semantics: // - Uploaded data MAY be visible to others before Commit() is called