diff --git a/copy/copy.go b/copy/copy.go index 30d8a44641..bdb727d97d 100644 --- a/copy/copy.go +++ b/copy/copy.go @@ -22,6 +22,7 @@ import ( "github.com/containers/image/v4/transports" "github.com/containers/image/v4/types" digest "github.com/opencontainers/go-digest" + imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/vbauerster/mpb" @@ -110,6 +111,37 @@ type imageCopier struct { canSubstituteBlobs bool } +const ( + // CopySystemImage is the default value which, when set in + // Options.ImageListSelection, indicates that the caller expects only one + // image to be copied, so if the source reference refers to a list of + // images, one that matches the current system will be selected. + CopySystemImage ImageListSelection = iota + // CopyAllImages is a value which, when set in Options.ImageListSelection, + // indicates that the caller expects to copy multiple images, and if + // the source reference refers to a list, that the list and every image + // to which it refers will be copied. If the source reference refers + // to a list, the target reference can not accept lists, an error + // should be returned. + CopyAllImages + // CopySpecificImages is a value which, when set in + // Options.ImageListSelection, indicates that the caller expects the + // source reference to be either a single image or a list of images, + // and if the source reference is a list, wants only specific instances + // from it copied (or none of them, if the list of instances to copy is + // empty), along with the list itself. If the target reference can + // only accept one image (i.e., it cannot accept lists), an error + // should be returned. + CopySpecificImages +) + +// ImageListSelection is one of CopySystemImage, CopyAllImages, or +// CopySpecificImages, to control whether, when the source reference is a list, +// copy.Image() copies only an image which matches the current runtime +// environment, or all images which match the supplied reference, or only +// specific images from the source reference. +type ImageListSelection int + // Options allows supplying non-default configuration modifying the behavior of CopyImage. type Options struct { RemoveSignatures bool // Remove any pre-existing signatures. SignBy will still add a new signature. @@ -121,12 +153,24 @@ type Options struct { Progress chan types.ProgressProperties // Reported to when ProgressInterval has arrived for a single artifact+offset. // manifest MIME type of image set by user. "" is default and means use the autodetection to the the manifest MIME type ForceManifestMIMEType string + ImageListSelection ImageListSelection // set to either CopySystemImage (the default), CopyAllImages, or CopySpecificImages to control which instances we copy when the source reference is a list; ignored if the source reference is not a list + Instances []digest.Digest // if ImageListSelection is CopySpecificImages, copy only these instances and the list itself +} + +// validateImageListSelection returns an error if the passed-in value is not one that we recognize as a valid ImageListSelection value +func validateImageListSelection(selection ImageListSelection) error { + switch selection { + default: + return errors.Errorf("Invalid value for options.ImageListSelection: %d", selection) + case CopySystemImage, CopyAllImages, CopySpecificImages: + } + return nil } // Image copies image from srcRef to destRef, using policyContext to validate // source image admissibility. It returns the manifest which was written to // the new copy of the image. -func Image(ctx context.Context, policyContext *signature.PolicyContext, destRef, srcRef types.ImageReference, options *Options) (manifest []byte, retErr error) { +func Image(ctx context.Context, policyContext *signature.PolicyContext, destRef, srcRef types.ImageReference, options *Options) (copiedManifest []byte, retErr error) { // NOTE this function uses an output parameter for the error return value. // Setting this and returning is the ideal way to return an error. // @@ -136,6 +180,10 @@ func Image(ctx context.Context, policyContext *signature.PolicyContext, destRef, options = &Options{} } + if err := validateImageListSelection(options.ImageListSelection); err != nil { + return nil, err + } + reportWriter := ioutil.Discard if options.ReportWriter != nil { @@ -206,79 +254,269 @@ func Image(ctx context.Context, policyContext *signature.PolicyContext, destRef, } if !multiImage { - // The simple case: Just copy a single image. - if manifest, err = c.copyOneImage(ctx, policyContext, options, unparsedToplevel); err != nil { + // The simple case: just copy a single image. + if copiedManifest, _, _, err = c.copyOneImage(ctx, policyContext, options, unparsedToplevel, unparsedToplevel, nil); err != nil { return nil, err } - } else { - // This is a manifest list. Choose a single image and copy it. - // FIXME: Copy to destinations which support manifest lists, one image at a time. - instanceDigest, err := image.ChooseManifestInstanceFromManifestList(ctx, options.SourceCtx, unparsedToplevel) + } else if options.ImageListSelection == CopySystemImage { + // This is a manifest list, and we weren't asked to copy multiple images. Choose a single image that + // matches the current system to copy, and copy it. + mfest, manifestType, err := unparsedToplevel.Manifest(ctx) + if err != nil { + return nil, errors.Wrapf(err, "Error reading manifest for %s", transports.ImageName(srcRef)) + } + manifestList, err := manifest.ListFromBlob(mfest, manifestType) + if err != nil { + return nil, errors.Wrapf(err, "Error parsing primary manifest as list for %s", transports.ImageName(srcRef)) + } + instanceDigest, err := manifestList.ChooseInstance(options.DestinationCtx) // try to pick one that matches options.DestinationCtx if err != nil { return nil, errors.Wrapf(err, "Error choosing an image from manifest list %s", transports.ImageName(srcRef)) } - logrus.Debugf("Source is a manifest list; copying (only) instance %s", instanceDigest) + logrus.Debugf("Source is a manifest list; copying (only) instance %s for current system", instanceDigest) unparsedInstance := image.UnparsedInstance(rawSource, &instanceDigest) - if manifest, err = c.copyOneImage(ctx, policyContext, options, unparsedInstance); err != nil { + if copiedManifest, _, _, err = c.copyOneImage(ctx, policyContext, options, unparsedToplevel, unparsedInstance, nil); err != nil { + return nil, err + } + } else { /* options.ImageListSelection == CopyAllImages or options.ImageListSelection == CopySpecificImages, */ + // If we were asked to copy multiple images and can't, that's an error. + if !supportsMultipleImages(c.dest) { + return nil, errors.Errorf("Error copying multiple images: destination transport %q does not support copying multiple images as a group", destRef.Transport().Name()) + } + // Copy some or all of the images. + switch options.ImageListSelection { + case CopyAllImages: + logrus.Debugf("Source is a manifest list; copying all instances") + case CopySpecificImages: + logrus.Debugf("Source is a manifest list; copying some instances") + } + if copiedManifest, _, err = c.copyMultipleImages(ctx, policyContext, options, unparsedToplevel); err != nil { return nil, err } } - if err := c.dest.Commit(ctx); err != nil { + if err := c.dest.Commit(ctx, unparsedToplevel); err != nil { return nil, errors.Wrap(err, "Error committing the finished image") } - return manifest, nil + return copiedManifest, nil +} + +// Checks if the destination supports accepting multiple images by checking if it can support +// manifest types that are lists of other manifests. +func supportsMultipleImages(dest types.ImageDestination) bool { + mtypes := dest.SupportedManifestMIMETypes() + if len(mtypes) == 0 { + // Anything goes! + return true + } + for _, mtype := range mtypes { + if manifest.MIMETypeIsMultiImage(mtype) { + return true + } + } + return false +} + +// copyMultipleImages copies some or all of an image list's instances, using +// policyContext to validate source image admissibility. +func (c *copier) copyMultipleImages(ctx context.Context, policyContext *signature.PolicyContext, options *Options, unparsedToplevel *image.UnparsedImage) (copiedManifest []byte, copiedManifestType string, retErr error) { + // Parse the list and get a copy of the original value after it's re-encoded. + manifestList, manifestType, err := unparsedToplevel.Manifest(ctx) + if err != nil { + return nil, "", errors.Wrapf(err, "Error reading manifest list") + } + list, err := manifest.ListFromBlob(manifestList, manifestType) + if err != nil { + return nil, "", errors.Wrapf(err, "Error parsing manifest list %q", string(manifestList)) + } + originalList := list.Clone() + + // Read and/or clear the set of signatures for this list. + var sigs [][]byte + if options.RemoveSignatures { + sigs = [][]byte{} + } else { + c.Printf("Getting image list signatures\n") + s, err := c.rawSource.GetSignatures(ctx, nil) + if err != nil { + return nil, "", errors.Wrap(err, "Error reading signatures") + } + sigs = s + } + if len(sigs) != 0 { + c.Printf("Checking if image list destination supports signatures\n") + if err := c.dest.SupportsSignatures(ctx); err != nil { + return nil, "", errors.Wrap(err, "Can not copy signatures") + } + } + + // Determine if we'll need to convert the manifest list to a different format. + forceListMIMEType := options.ForceManifestMIMEType + switch forceListMIMEType { + case manifest.DockerV2Schema1MediaType, manifest.DockerV2Schema1SignedMediaType, manifest.DockerV2Schema2MediaType: + forceListMIMEType = manifest.DockerV2ListMediaType + case imgspecv1.MediaTypeImageManifest: + forceListMIMEType = imgspecv1.MediaTypeImageIndex + } + selectedListType, err := c.determineListConversion(manifestType, c.dest.SupportedManifestMIMETypes(), forceListMIMEType) + if err != nil { + return nil, "", errors.Wrapf(err, "Error determining manifest list type to write to destination") + } + if selectedListType != list.MIMEType() { + canModifyManifestList := (len(sigs) == 0) + if !canModifyManifestList { + return nil, "", errors.Errorf("Error: manifest list must be converted to type %q to be written to destination, but that would invalidate signatures", selectedListType) + } + } + + // Copy each image, or just the ones we want to copy, in turn. + instanceDigests := list.Instances() + updates := make([]manifest.ListUpdate, len(instanceDigests)) + for i, instanceDigest := range instanceDigests { + if options.ImageListSelection == CopySpecificImages { + skip := true + for _, instance := range options.Instances { + if instance == instanceDigest { + skip = false + break + } + } + if skip { + update, err := list.Instance(instanceDigest) + if err != nil { + return nil, "", err + } + logrus.Debugf("Skipping instance %s (%d/%d)", instanceDigest, i+1, len(instanceDigests)) + // Record the digest/size/type of the manifest that we didn't copy. + updates[i] = update + continue + } + } + logrus.Debugf("Copying instance %s (%d/%d)", instanceDigest, i+1, len(instanceDigests)) + unparsedInstance := image.UnparsedInstance(c.rawSource, &instanceDigest) + updatedManifest, updatedManifestType, updatedManifestDigest, err := c.copyOneImage(ctx, policyContext, options, unparsedToplevel, unparsedInstance, &instanceDigest) + if err != nil { + return nil, "", err + } + // Record the result of a possible conversion here. + update := manifest.ListUpdate{ + Digest: updatedManifestDigest, + Size: int64(len(updatedManifest)), + MediaType: updatedManifestType, + } + updates[i] = update + } + + // Now reset the digest/size/types of the manifests in the list to account for any conversions that we made. + if err = list.UpdateInstances(updates); err != nil { + return nil, "", errors.Wrapf(err, "Error updating manifest list") + } + + // Check if the updates meaningfully changed the list of images. + listIsModified := false + if !reflect.DeepEqual(list.Instances(), originalList.Instances()) { + listIsModified = true + } + + // Perform the list conversion. + if selectedListType != list.MIMEType() { + list, err = list.ConvertToMIMEType(selectedListType) + if err != nil { + return nil, "", errors.Wrapf(err, "Error converting manifest list to list with MIME type %q", selectedListType) + } + } + + // If we can't use the original value, but we have to change it, flag an error. + if listIsModified { + manifestList, err = list.Serialize() + if err != nil { + return nil, "", errors.Wrapf(err, "Error encoding updated manifest list (%q: %#v)", list.MIMEType(), list.Instances()) + } + logrus.Debugf("Manifest list has been updated") + } + + // Save the manifest list. + if err = c.dest.PutManifest(ctx, manifestList, nil); err != nil { + return nil, "", errors.Wrapf(err, "Error writing manifest list %q", string(manifestList)) + } + + // Sign the manifest list. + if options.SignBy != "" { + newSig, err := c.createSignature(manifestList, options.SignBy) + if err != nil { + return nil, "", err + } + sigs = append(sigs, newSig) + } + + c.Printf("Storing list signatures\n") + if err := c.dest.PutSignatures(ctx, sigs, nil); err != nil { + return nil, "", errors.Wrap(err, "Error writing signatures") + } + + return manifestList, selectedListType, nil } -// Image copies a single (on-manifest-list) image unparsedImage, using policyContext to validate +// copyOneImage copies a single (non-manifest-list) image unparsedImage, using policyContext to validate // source image admissibility. -func (c *copier) copyOneImage(ctx context.Context, policyContext *signature.PolicyContext, options *Options, unparsedImage *image.UnparsedImage) (manifestBytes []byte, retErr error) { +func (c *copier) copyOneImage(ctx context.Context, policyContext *signature.PolicyContext, options *Options, unparsedToplevel, unparsedImage *image.UnparsedImage, targetInstance *digest.Digest) (retManifest []byte, retManifestType string, retManifestDigest digest.Digest, retErr error) { // The caller is handling manifest lists; this could happen only if a manifest list contains a manifest list. // Make sure we fail cleanly in such cases. multiImage, err := isMultiImage(ctx, unparsedImage) if err != nil { // FIXME FIXME: How to name a reference for the sub-image? - return nil, errors.Wrapf(err, "Error determining manifest MIME type for %s", transports.ImageName(unparsedImage.Reference())) + return nil, "", "", errors.Wrapf(err, "Error determining manifest MIME type for %s", transports.ImageName(unparsedImage.Reference())) } if multiImage { - return nil, fmt.Errorf("Unexpectedly received a manifest list instead of a manifest for a single image") + return nil, "", "", fmt.Errorf("Unexpectedly received a manifest list instead of a manifest for a single image") } // Please keep this policy check BEFORE reading any other information about the image. - // (the multiImage check above only matches the MIME type, which we have received anyway. + // (The multiImage check above only matches the MIME type, which we have received anyway. // Actual parsing of anything should be deferred.) if allowed, err := policyContext.IsRunningImageAllowed(ctx, unparsedImage); !allowed || err != nil { // Be paranoid and fail if either return value indicates so. - return nil, errors.Wrap(err, "Source image rejected") + return nil, "", "", errors.Wrap(err, "Source image rejected") } src, err := image.FromUnparsedImage(ctx, options.SourceCtx, unparsedImage) if err != nil { - return nil, errors.Wrapf(err, "Error initializing image from source %s", transports.ImageName(c.rawSource.Reference())) + return nil, "", "", errors.Wrapf(err, "Error initializing image from source %s", transports.ImageName(c.rawSource.Reference())) } // If the destination is a digested reference, make a note of that, determine what digest value we're - // expecting, and check that the source manifest matches it. + // expecting, and check that the source manifest matches it. If the source manifest doesn't, but it's + // one item from a manifest list that matches it, accept that as a match. destIsDigestedReference := false if named := c.dest.Reference().DockerReference(); named != nil { if digested, ok := named.(reference.Digested); ok { destIsDigestedReference = true sourceManifest, _, err := src.Manifest(ctx) if err != nil { - return nil, errors.Wrapf(err, "Error reading manifest from source image") + return nil, "", "", errors.Wrapf(err, "Error reading manifest from source image") } matches, err := manifest.MatchesDigest(sourceManifest, digested.Digest()) if err != nil { - return nil, errors.Wrapf(err, "Error computing digest of source image's manifest") + return nil, "", "", errors.Wrapf(err, "Error computing digest of source image's manifest") } if !matches { - return nil, errors.New("Digest of source image's manifest would not match destination reference") + manifestList, _, err := unparsedToplevel.Manifest(ctx) + if err != nil { + return nil, "", "", errors.Wrapf(err, "Error reading manifest from source image") + } + matches, err = manifest.MatchesDigest(manifestList, digested.Digest()) + if err != nil { + return nil, "", "", errors.Wrapf(err, "Error computing digest of source image's manifest") + } + if !matches { + return nil, "", "", errors.New("Digest of source image's manifest would not match destination reference") + } } } } if err := checkImageDestinationForCurrentRuntimeOS(ctx, options.DestinationCtx, src, c.dest); err != nil { - return nil, err + return nil, "", "", err } var sigs [][]byte @@ -288,14 +526,14 @@ func (c *copier) copyOneImage(ctx context.Context, policyContext *signature.Poli c.Printf("Getting image source signatures\n") s, err := src.Signatures(ctx) if err != nil { - return nil, errors.Wrap(err, "Error reading signatures") + return nil, "", "", errors.Wrap(err, "Error reading signatures") } sigs = s } if len(sigs) != 0 { c.Printf("Checking if image destination supports signatures\n") if err := c.dest.SupportsSignatures(ctx); err != nil { - return nil, errors.Wrap(err, "Can not copy signatures") + return nil, "", "", errors.Wrap(err, "Can not copy signatures") } } @@ -315,28 +553,29 @@ func (c *copier) copyOneImage(ctx context.Context, policyContext *signature.Poli ic.canSubstituteBlobs = ic.canModifyManifest && options.SignBy == "" if err := ic.updateEmbeddedDockerReference(); err != nil { - return nil, err + return nil, "", "", err } // We compute preferredManifestMIMEType only to show it in error messages. // Without having to add this context in an error message, we would be happy enough to know only that no conversion is needed. preferredManifestMIMEType, otherManifestMIMETypeCandidates, err := ic.determineManifestConversion(ctx, c.dest.SupportedManifestMIMETypes(), options.ForceManifestMIMEType) if err != nil { - return nil, err + return nil, "", "", err } // If src.UpdatedImageNeedsLayerDiffIDs(ic.manifestUpdates) will be true, it needs to be true by the time we get here. ic.diffIDsAreNeeded = src.UpdatedImageNeedsLayerDiffIDs(*ic.manifestUpdates) if err := ic.copyLayers(ctx); err != nil { - return nil, err + return nil, "", "", err } // With docker/distribution registries we do not know whether the registry accepts schema2 or schema1 only; // and at least with the OpenShift registry "acceptschema2" option, there is no way to detect the support // without actually trying to upload something and getting a types.ManifestTypeRejectedError. // So, try the preferred manifest MIME type. If the process succeeds, fine… - manifestBytes, err = ic.copyUpdatedConfigAndManifest(ctx) + manifestBytes, retManifestDigest, err := ic.copyUpdatedConfigAndManifest(ctx, targetInstance) + retManifestType = preferredManifestMIMEType if err != nil { logrus.Debugf("Writing manifest using preferred type %s failed: %v", preferredManifestMIMEType, err) // … if it fails, _and_ the failure is because the manifest is rejected, we may have other options. @@ -344,14 +583,14 @@ func (c *copier) copyOneImage(ctx context.Context, policyContext *signature.Poli // We don’t have other options. // In principle the code below would handle this as well, but the resulting error message is fairly ugly. // Don’t bother the user with MIME types if we have no choice. - return nil, err + return nil, "", "", err } // If the original MIME type is acceptable, determineManifestConversion always uses it as preferredManifestMIMEType. // So if we are here, we will definitely be trying to convert the manifest. // With !ic.canModifyManifest, that would just be a string of repeated failures for the same reason, // so let’s bail out early and with a better error message. if !ic.canModifyManifest { - return nil, errors.Wrap(err, "Writing manifest failed (and converting it is not possible)") + return nil, "", "", errors.Wrap(err, "Writing manifest failed (and converting it is not possible)") } // errs is a list of errors when trying various manifest types. Also serves as an "upload succeeded" flag when set to nil. @@ -359,7 +598,7 @@ func (c *copier) copyOneImage(ctx context.Context, policyContext *signature.Poli for _, manifestMIMEType := range otherManifestMIMETypeCandidates { logrus.Debugf("Trying to use manifest type %s…", manifestMIMEType) ic.manifestUpdates.ManifestMIMEType = manifestMIMEType - attemptedManifest, err := ic.copyUpdatedConfigAndManifest(ctx) + attemptedManifest, attemptedManifestDigest, err := ic.copyUpdatedConfigAndManifest(ctx, targetInstance) if err != nil { logrus.Debugf("Upload of manifest type %s failed: %v", manifestMIMEType, err) errs = append(errs, fmt.Sprintf("%s(%v)", manifestMIMEType, err)) @@ -368,28 +607,30 @@ func (c *copier) copyOneImage(ctx context.Context, policyContext *signature.Poli // We have successfully uploaded a manifest. manifestBytes = attemptedManifest + retManifestDigest = attemptedManifestDigest + retManifestType = manifestMIMEType errs = nil // Mark this as a success so that we don't abort below. break } if errs != nil { - return nil, fmt.Errorf("Uploading manifest failed, attempted the following formats: %s", strings.Join(errs, ", ")) + return nil, "", "", fmt.Errorf("Uploading manifest failed, attempted the following formats: %s", strings.Join(errs, ", ")) } } if options.SignBy != "" { newSig, err := c.createSignature(manifestBytes, options.SignBy) if err != nil { - return nil, err + return nil, "", "", err } sigs = append(sigs, newSig) } c.Printf("Storing signatures\n") - if err := c.dest.PutSignatures(ctx, sigs); err != nil { - return nil, errors.Wrap(err, "Error writing signatures") + if err := c.dest.PutSignatures(ctx, sigs, targetInstance); err != nil { + return nil, "", "", errors.Wrap(err, "Error writing signatures") } - return manifestBytes, nil + return manifestBytes, retManifestType, retManifestDigest, nil } // Printf writes a formatted string to c.reportWriter. @@ -554,12 +795,13 @@ func layerDigestsDiffer(a, b []types.BlobInfo) bool { } // copyUpdatedConfigAndManifest updates the image per ic.manifestUpdates, if necessary, -// stores the resulting config and manifest to the destination, and returns the stored manifest. -func (ic *imageCopier) copyUpdatedConfigAndManifest(ctx context.Context) ([]byte, error) { +// stores the resulting config and manifest to the destination, and returns the stored manifest +// and its digest. +func (ic *imageCopier) copyUpdatedConfigAndManifest(ctx context.Context, instanceDigest *digest.Digest) ([]byte, digest.Digest, error) { pendingImage := ic.src if !reflect.DeepEqual(*ic.manifestUpdates, types.ManifestUpdateOptions{InformationOnly: ic.manifestUpdates.InformationOnly}) { if !ic.canModifyManifest { - return nil, errors.Errorf("Internal error: copy needs an updated manifest but that was known to be forbidden") + return nil, "", errors.Errorf("Internal error: copy needs an updated manifest but that was known to be forbidden") } if !ic.diffIDsAreNeeded && ic.src.UpdatedImageNeedsLayerDiffIDs(*ic.manifestUpdates) { // We have set ic.diffIDsAreNeeded based on the preferred MIME type returned by determineManifestConversion. @@ -568,28 +810,35 @@ func (ic *imageCopier) copyUpdatedConfigAndManifest(ctx context.Context) ([]byte // when ic.c.dest.SupportedManifestMIMETypes() includes both s1 and s2, the upload using s1 failed, and we are now trying s2. // Supposedly s2-only registries do not exist or are extremely rare, so failing with this error message is good enough for now. // If handling such registries turns out to be necessary, we could compute ic.diffIDsAreNeeded based on the full list of manifest MIME type candidates. - return nil, errors.Errorf("Can not convert image to %s, preparing DiffIDs for this case is not supported", ic.manifestUpdates.ManifestMIMEType) + return nil, "", errors.Errorf("Can not convert image to %s, preparing DiffIDs for this case is not supported", ic.manifestUpdates.ManifestMIMEType) } pi, err := ic.src.UpdatedImage(ctx, *ic.manifestUpdates) if err != nil { - return nil, errors.Wrap(err, "Error creating an updated image manifest") + return nil, "", errors.Wrap(err, "Error creating an updated image manifest") } pendingImage = pi } - manifest, _, err := pendingImage.Manifest(ctx) + man, _, err := pendingImage.Manifest(ctx) if err != nil { - return nil, errors.Wrap(err, "Error reading manifest") + return nil, "", errors.Wrap(err, "Error reading manifest") } if err := ic.c.copyConfig(ctx, pendingImage); err != nil { - return nil, err + return nil, "", err } ic.c.Printf("Writing manifest to image destination\n") - if err := ic.c.dest.PutManifest(ctx, manifest); err != nil { - return nil, errors.Wrap(err, "Error writing manifest") + manifestDigest, err := manifest.Digest(man) + if err != nil { + return nil, "", err + } + if instanceDigest != nil { + instanceDigest = &manifestDigest + } + if err := ic.c.dest.PutManifest(ctx, man, instanceDigest); err != nil { + return nil, "", errors.Wrap(err, "Error writing manifest") } - return manifest, nil + return man, manifestDigest, nil } // newProgressPool creates a *mpb.Progress and a cleanup function. diff --git a/copy/manifest.go b/copy/manifest.go index 7c981fcad2..35ae1c5f88 100644 --- a/copy/manifest.go +++ b/copy/manifest.go @@ -119,3 +119,36 @@ func isMultiImage(ctx context.Context, img types.UnparsedImage) (bool, error) { } return manifest.MIMETypeIsMultiImage(mt), nil } + +// determineListConversion takes the current MIME type of a list of manifests, +// the list of MIME types supported for a given destination, and a possible +// forced value, and returns the MIME type to which we should convert the list +// of manifests, whether we are converting to it or using it unmodified. +func (c *copier) determineListConversion(currentListMIMEType string, destSupportedMIMETypes []string, forcedListMIMEType string) (string, error) { + // If there's no list of supported types, then anything we support is expected to be supported. + if len(destSupportedMIMETypes) == 0 { + destSupportedMIMETypes = manifest.SupportedListMIMETypes + } + var selectedType string + for i := range destSupportedMIMETypes { + // The second priority is the first member of the list of acceptable types that is a list, + // but keep going in case current type occurs later in the list. + if selectedType == "" && manifest.MIMETypeIsMultiImage(destSupportedMIMETypes[i]) { + selectedType = destSupportedMIMETypes[i] + } + // The first priority is the current type, if it's in the list, since that lets us avoid a + // conversion that isn't strictly necessary. + if destSupportedMIMETypes[i] == currentListMIMEType { + selectedType = destSupportedMIMETypes[i] + } + } + if forcedListMIMEType != "" { + // If we're forcing it, we prefer the forced value over everything else. + selectedType = forcedListMIMEType + } + if selectedType == "" { + return "", errors.Errorf("destination does not support any supported manifest list types (%v)", manifest.SupportedListMIMETypes) + } + // Done. + return selectedType, nil +} diff --git a/copy/manifest_test.go b/copy/manifest_test.go index c8d25b91f5..a9ff787b6a 100644 --- a/copy/manifest_test.go +++ b/copy/manifest_test.go @@ -203,6 +203,8 @@ func TestIsMultiImage(t *testing.T) { }{ {manifest.DockerV2ListMediaType, true}, {manifest.DockerV2Schema2MediaType, false}, + {v1.MediaTypeImageManifest, false}, + {v1.MediaTypeImageIndex, true}, } { src := fakeImageSource(c.mt) res, err := isMultiImage(context.Background(), src) diff --git a/directory/directory_dest.go b/directory/directory_dest.go index 18f7dde70d..c23d1ce167 100644 --- a/directory/directory_dest.go +++ b/directory/directory_dest.go @@ -199,16 +199,23 @@ func (d *dirImageDestination) TryReusingBlob(ctx context.Context, info types.Blo } // PutManifest writes manifest to the destination. +// If instanceDigest is not nil, it contains a digest of the specific manifest instance to write the manifest for (when +// the primary manifest is a manifest list); this should always be nil if the primary manifest is not a manifest list. +// It is expected but not enforced that the instanceDigest, when specified, matches the digest of `manifest` as generated +// by `manifest.Digest()`. // FIXME? This should also receive a MIME type if known, to differentiate between schema versions. // If the destination is in principle available, refuses this manifest type (e.g. it does not recognize the schema), // but may accept a different manifest type, the returned error must be an ManifestTypeRejectedError. -func (d *dirImageDestination) PutManifest(ctx context.Context, manifest []byte) error { - return ioutil.WriteFile(d.ref.manifestPath(), manifest, 0644) +func (d *dirImageDestination) PutManifest(ctx context.Context, manifest []byte, instanceDigest *digest.Digest) error { + return ioutil.WriteFile(d.ref.manifestPath(instanceDigest), manifest, 0644) } -func (d *dirImageDestination) PutSignatures(ctx context.Context, signatures [][]byte) error { +// PutSignatures writes a set of signatures to the destination. +// If instanceDigest is not nil, it contains a digest of the specific manifest instance to write or overwrite the signatures for +// (when the primary manifest is a manifest list); this should always be nil if the primary manifest is not a manifest list. +func (d *dirImageDestination) PutSignatures(ctx context.Context, signatures [][]byte, instanceDigest *digest.Digest) error { for i, sig := range signatures { - if err := ioutil.WriteFile(d.ref.signaturePath(i), sig, 0644); err != nil { + if err := ioutil.WriteFile(d.ref.signaturePath(i, instanceDigest), sig, 0644); err != nil { return err } } @@ -219,7 +226,7 @@ func (d *dirImageDestination) PutSignatures(ctx context.Context, signatures [][] // WARNING: This does not have any transactional semantics: // - Uploaded data MAY be visible to others before Commit() is called // - Uploaded data MAY be removed or MAY remain around if Close() is called without Commit() (i.e. rollback is allowed but not guaranteed) -func (d *dirImageDestination) Commit(ctx context.Context) error { +func (d *dirImageDestination) Commit(context.Context, types.UnparsedImage) error { return nil } diff --git a/directory/directory_src.go b/directory/directory_src.go index 921c1941cc..d97b68d99c 100644 --- a/directory/directory_src.go +++ b/directory/directory_src.go @@ -9,7 +9,6 @@ import ( "github.com/containers/image/v4/manifest" "github.com/containers/image/v4/types" "github.com/opencontainers/go-digest" - "github.com/pkg/errors" ) type dirImageSource struct { @@ -38,10 +37,7 @@ func (s *dirImageSource) Close() error { // If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve (when the primary manifest is a manifest list); // this never happens if the primary manifest is not a manifest list (e.g. if the source never returns manifest lists). func (s *dirImageSource) GetManifest(ctx context.Context, instanceDigest *digest.Digest) ([]byte, string, error) { - if instanceDigest != nil { - return nil, "", errors.Errorf(`Getting target manifest not supported by "dir:"`) - } - m, err := ioutil.ReadFile(s.ref.manifestPath()) + m, err := ioutil.ReadFile(s.ref.manifestPath(instanceDigest)) if err != nil { return nil, "", err } @@ -73,12 +69,9 @@ func (s *dirImageSource) GetBlob(ctx context.Context, info types.BlobInfo, cache // (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list // (e.g. if the source never returns manifest lists). func (s *dirImageSource) GetSignatures(ctx context.Context, instanceDigest *digest.Digest) ([][]byte, error) { - if instanceDigest != nil { - return nil, errors.Errorf(`Manifests lists are not supported by "dir:"`) - } signatures := [][]byte{} for i := 0; ; i++ { - signature, err := ioutil.ReadFile(s.ref.signaturePath(i)) + signature, err := ioutil.ReadFile(s.ref.signaturePath(i, instanceDigest)) if err != nil { if os.IsNotExist(err) { break @@ -90,7 +83,14 @@ func (s *dirImageSource) GetSignatures(ctx context.Context, instanceDigest *dige return signatures, nil } -// LayerInfosForCopy() returns updated layer info that should be used when copying, in preference to values in the manifest, if specified. -func (s *dirImageSource) LayerInfosForCopy(ctx context.Context) ([]types.BlobInfo, error) { +// LayerInfosForCopy returns either nil (meaning the values in the manifest are fine), or updated values for the layer +// blobsums that are listed in the image's manifest. If values are returned, they should be used when using GetBlob() +// to read the image's layers. +// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve BlobInfos for +// (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list +// (e.g. if the source never returns manifest lists). +// The Digest field is guaranteed to be provided; Size may be -1. +// WARNING: The list may contain duplicates, and they are semantically relevant. +func (s *dirImageSource) LayerInfosForCopy(context.Context, *digest.Digest) ([]types.BlobInfo, error) { return nil, nil } diff --git a/directory/directory_test.go b/directory/directory_test.go index 65f3302608..e09b800710 100644 --- a/directory/directory_test.go +++ b/directory/directory_test.go @@ -32,12 +32,17 @@ func TestGetPutManifest(t *testing.T) { defer os.RemoveAll(tmpDir) man := []byte("test-manifest") + list := []byte("test-manifest-list") + md, err := manifest.Digest(man) + require.NoError(t, err) dest, err := ref.NewImageDestination(context.Background(), nil) require.NoError(t, err) defer dest.Close() - err = dest.PutManifest(context.Background(), man) + err = dest.PutManifest(context.Background(), man, &md) assert.NoError(t, err) - err = dest.Commit(context.Background()) + err = dest.PutManifest(context.Background(), list, nil) + assert.NoError(t, err) + err = dest.Commit(context.Background(), nil) assert.NoError(t, err) src, err := ref.NewImageSource(context.Background(), nil) @@ -45,14 +50,13 @@ func TestGetPutManifest(t *testing.T) { defer src.Close() m, mt, err := src.GetManifest(context.Background(), nil) assert.NoError(t, err) - assert.Equal(t, man, m) + assert.Equal(t, list, m) assert.Equal(t, "", mt) - // Non-default instances are not supported - md, err := manifest.Digest(man) - require.NoError(t, err) - _, _, err = src.GetManifest(context.Background(), &md) - assert.Error(t, err) + m, mt, err = src.GetManifest(context.Background(), &md) + assert.NoError(t, err) + assert.Equal(t, man, m) + assert.Equal(t, "", mt) } func TestGetPutBlob(t *testing.T) { @@ -67,7 +71,7 @@ func TestGetPutBlob(t *testing.T) { assert.Equal(t, types.PreserveOriginal, dest.DesiredLayerCompression()) info, err := dest.PutBlob(context.Background(), bytes.NewReader(blob), types.BlobInfo{Digest: digest.Digest("sha256:digest-test"), Size: int64(9)}, cache, false) assert.NoError(t, err) - err = dest.Commit(context.Background()) + err = dest.Commit(context.Background(), nil) assert.NoError(t, err) assert.Equal(t, int64(9), info.Size) assert.Equal(t, digest.FromBytes(blob), info.Digest) @@ -126,7 +130,7 @@ func TestPutBlobDigestFailure(t *testing.T) { _, err = dest.PutBlob(context.Background(), reader, types.BlobInfo{Digest: blobDigest, Size: -1}, cache, false) assert.Error(t, err) assert.Contains(t, digestErrorString, err.Error()) - err = dest.Commit(context.Background()) + err = dest.Commit(context.Background(), nil) assert.NoError(t, err) _, err = os.Lstat(blobPath) @@ -148,12 +152,29 @@ func TestGetPutSignatures(t *testing.T) { } err = dest.SupportsSignatures(context.Background()) assert.NoError(t, err) - err = dest.PutManifest(context.Background(), man) + err = dest.PutManifest(context.Background(), man, nil) + require.NoError(t, err) + + err = dest.PutSignatures(context.Background(), signatures, nil) + listSignatures := [][]byte{ + []byte("sig3"), + []byte("sig4"), + } + md, err := manifest.Digest(man) + require.NoError(t, err) + + err = dest.SupportsSignatures(context.Background()) + assert.NoError(t, err) + err = dest.PutManifest(context.Background(), man, nil) + require.NoError(t, err) + err = dest.PutManifest(context.Background(), man, &md) require.NoError(t, err) - err = dest.PutSignatures(context.Background(), signatures) + err = dest.PutSignatures(context.Background(), listSignatures, nil) + assert.NoError(t, err) + err = dest.PutSignatures(context.Background(), signatures, &md) assert.NoError(t, err) - err = dest.Commit(context.Background()) + err = dest.Commit(context.Background(), nil) assert.NoError(t, err) src, err := ref.NewImageSource(context.Background(), nil) @@ -161,13 +182,11 @@ func TestGetPutSignatures(t *testing.T) { defer src.Close() sigs, err := src.GetSignatures(context.Background(), nil) assert.NoError(t, err) - assert.Equal(t, signatures, sigs) + assert.Equal(t, listSignatures, sigs) - // Non-default instances are not supported - md, err := manifest.Digest(man) - require.NoError(t, err) - _, err = src.GetSignatures(context.Background(), &md) - assert.Error(t, err) + sigs, err = src.GetSignatures(context.Background(), &md) + assert.NoError(t, err) + assert.Equal(t, signatures, sigs) } func TestSourceReference(t *testing.T) { diff --git a/directory/directory_transport.go b/directory/directory_transport.go index 29ac7115f3..1229e5a9e8 100644 --- a/directory/directory_transport.go +++ b/directory/directory_transport.go @@ -166,18 +166,24 @@ func (ref dirReference) DeleteImage(ctx context.Context, sys *types.SystemContex } // manifestPath returns a path for the manifest within a directory using our conventions. -func (ref dirReference) manifestPath() string { +func (ref dirReference) manifestPath(instanceDigest *digest.Digest) string { + if instanceDigest != nil { + return filepath.Join(ref.path, instanceDigest.Encoded()+".manifest.json") + } return filepath.Join(ref.path, "manifest.json") } // layerPath returns a path for a layer tarball within a directory using our conventions. func (ref dirReference) layerPath(digest digest.Digest) string { // FIXME: Should we keep the digest identification? - return filepath.Join(ref.path, digest.Hex()) + return filepath.Join(ref.path, digest.Encoded()) } // signaturePath returns a path for a signature within a directory using our conventions. -func (ref dirReference) signaturePath(index int) string { +func (ref dirReference) signaturePath(index int, instanceDigest *digest.Digest) string { + if instanceDigest != nil { + return filepath.Join(ref.path, fmt.Sprintf(instanceDigest.Encoded()+".signature-%d", index+1)) + } return filepath.Join(ref.path, fmt.Sprintf("signature-%d", index+1)) } diff --git a/directory/directory_transport_test.go b/directory/directory_transport_test.go index 1fbe6ca832..b1fed5eceb 100644 --- a/directory/directory_transport_test.go +++ b/directory/directory_transport_test.go @@ -9,6 +9,7 @@ import ( _ "github.com/containers/image/v4/internal/testing/explicitfilepath-tmpdir" "github.com/containers/image/v4/types" + digest "github.com/opencontainers/go-digest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -157,9 +158,9 @@ func TestReferenceNewImage(t *testing.T) { defer dest.Close() mFixture, err := ioutil.ReadFile("../manifest/fixtures/v2s1.manifest.json") require.NoError(t, err) - err = dest.PutManifest(context.Background(), mFixture) + err = dest.PutManifest(context.Background(), mFixture, nil) assert.NoError(t, err) - err = dest.Commit(context.Background()) + err = dest.Commit(context.Background(), nil) assert.NoError(t, err) img, err := ref.NewImage(context.Background(), nil) @@ -174,9 +175,9 @@ func TestReferenceNewImageNoValidManifest(t *testing.T) { dest, err := ref.NewImageDestination(context.Background(), nil) require.NoError(t, err) defer dest.Close() - err = dest.PutManifest(context.Background(), []byte(`{"schemaVersion":1}`)) + err = dest.PutManifest(context.Background(), []byte(`{"schemaVersion":1}`), nil) assert.NoError(t, err) - err = dest.Commit(context.Background()) + err = dest.Commit(context.Background(), nil) assert.NoError(t, err) _, err = ref.NewImage(context.Background(), nil) @@ -207,11 +208,14 @@ func TestReferenceDeleteImage(t *testing.T) { } func TestReferenceManifestPath(t *testing.T) { + dhex := digest.Digest("sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef") + ref, tmpDir := refToTempDir(t) defer os.RemoveAll(tmpDir) dirRef, ok := ref.(dirReference) require.True(t, ok) - assert.Equal(t, tmpDir+"/manifest.json", dirRef.manifestPath()) + assert.Equal(t, tmpDir+"/manifest.json", dirRef.manifestPath(nil)) + assert.Equal(t, tmpDir+"/"+dhex.Encoded()+".manifest.json", dirRef.manifestPath(&dhex)) } func TestReferenceLayerPath(t *testing.T) { @@ -225,12 +229,16 @@ func TestReferenceLayerPath(t *testing.T) { } func TestReferenceSignaturePath(t *testing.T) { + dhex := digest.Digest("sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef") + ref, tmpDir := refToTempDir(t) defer os.RemoveAll(tmpDir) dirRef, ok := ref.(dirReference) require.True(t, ok) - assert.Equal(t, tmpDir+"/signature-1", dirRef.signaturePath(0)) - assert.Equal(t, tmpDir+"/signature-10", dirRef.signaturePath(9)) + assert.Equal(t, tmpDir+"/signature-1", dirRef.signaturePath(0, nil)) + assert.Equal(t, tmpDir+"/signature-10", dirRef.signaturePath(9, nil)) + assert.Equal(t, tmpDir+"/"+dhex.Encoded()+".signature-1", dirRef.signaturePath(0, &dhex)) + assert.Equal(t, tmpDir+"/"+dhex.Encoded()+".signature-10", dirRef.signaturePath(9, &dhex)) } func TestReferenceVersionPath(t *testing.T) { diff --git a/docker/archive/dest.go b/docker/archive/dest.go index 9e06e7c96a..1126ed276c 100644 --- a/docker/archive/dest.go +++ b/docker/archive/dest.go @@ -67,6 +67,6 @@ func (d *archiveImageDestination) Close() error { // WARNING: This does not have any transactional semantics: // - Uploaded data MAY be visible to others before Commit() is called // - Uploaded data MAY be removed or MAY remain around if Close() is called without Commit() (i.e. rollback is allowed but not guaranteed) -func (d *archiveImageDestination) Commit(ctx context.Context) error { +func (d *archiveImageDestination) Commit(ctx context.Context, unparsedToplevel types.UnparsedImage) error { return d.Destination.Commit(ctx) } diff --git a/docker/archive/src.go b/docker/archive/src.go index feea0decd5..749d621751 100644 --- a/docker/archive/src.go +++ b/docker/archive/src.go @@ -33,8 +33,3 @@ func newImageSource(ctx context.Context, ref archiveReference) (types.ImageSourc func (s *archiveImageSource) Reference() types.ImageReference { return s.ref } - -// LayerInfosForCopy() returns updated layer info that should be used when reading, in preference to values in the manifest, if specified. -func (s *archiveImageSource) LayerInfosForCopy(ctx context.Context) ([]types.BlobInfo, error) { - return nil, nil -} diff --git a/docker/daemon/daemon_dest.go b/docker/daemon/daemon_dest.go index 2c56ab9344..7738e9ae0d 100644 --- a/docker/daemon/daemon_dest.go +++ b/docker/daemon/daemon_dest.go @@ -124,7 +124,7 @@ func (d *daemonImageDestination) Reference() types.ImageReference { // WARNING: This does not have any transactional semantics: // - Uploaded data MAY be visible to others before Commit() is called // - Uploaded data MAY be removed or MAY remain around if Close() is called without Commit() (i.e. rollback is allowed but not guaranteed) -func (d *daemonImageDestination) Commit(ctx context.Context) error { +func (d *daemonImageDestination) Commit(ctx context.Context, unparsedToplevel types.UnparsedImage) error { logrus.Debugf("docker-daemon: Closing tar stream") if err := d.Destination.Commit(ctx); err != nil { return err diff --git a/docker/daemon/daemon_src.go b/docker/daemon/daemon_src.go index f6f60aaf91..e1a01fd7dd 100644 --- a/docker/daemon/daemon_src.go +++ b/docker/daemon/daemon_src.go @@ -55,8 +55,3 @@ func newImageSource(ctx context.Context, sys *types.SystemContext, ref daemonRef func (s *daemonImageSource) Reference() types.ImageReference { return s.ref } - -// LayerInfosForCopy() returns updated layer info that should be used when reading, in preference to values in the manifest, if specified. -func (s *daemonImageSource) LayerInfosForCopy(ctx context.Context) ([]types.BlobInfo, error) { - return nil, nil -} diff --git a/docker/docker_image_dest.go b/docker/docker_image_dest.go index 0f351ab594..769a705078 100644 --- a/docker/docker_image_dest.go +++ b/docker/docker_image_dest.go @@ -61,6 +61,8 @@ func (d *dockerImageDestination) SupportedManifestMIMETypes() []string { return []string{ imgspecv1.MediaTypeImageManifest, manifest.DockerV2Schema2MediaType, + imgspecv1.MediaTypeImageIndex, + manifest.DockerV2ListMediaType, manifest.DockerV2Schema1SignedMediaType, manifest.DockerV2Schema1MediaType, } @@ -343,20 +345,43 @@ func (d *dockerImageDestination) TryReusingBlob(ctx context.Context, info types. } // PutManifest writes manifest to the destination. +// When the primary manifest is a manifest list, if instanceDigest is nil, we're saving the list +// itself, else instanceDigest contains a digest of the specific manifest instance to overwrite the +// manifest for; when the primary manifest is not a manifest list, instanceDigest should always be nil. // FIXME? This should also receive a MIME type if known, to differentiate between schema versions. // If the destination is in principle available, refuses this manifest type (e.g. it does not recognize the schema), // but may accept a different manifest type, the returned error must be an ManifestTypeRejectedError. -func (d *dockerImageDestination) PutManifest(ctx context.Context, m []byte) error { - digest, err := manifest.Digest(m) - if err != nil { - return err +func (d *dockerImageDestination) PutManifest(ctx context.Context, m []byte, instanceDigest *digest.Digest) error { + refTail := "" + if instanceDigest != nil { + // If the instanceDigest is provided, then use it as the refTail, because the reference, + // whether it includes a tag or a digest, refers to the list as a whole, and not this + // particular instance. + refTail = instanceDigest.String() + // Double-check that the manifest we've been given matches the digest we've been given. + matches, err := manifest.MatchesDigest(m, *instanceDigest) + if err != nil { + return errors.Wrapf(err, "error digesting manifest in PutManifest") + } + if !matches { + return errors.New("Attempted to PutManifest using an explicitly specified digest that didn't match the manifest") + } + } else { + // Compute the digest of the main manifest, or the list if it's a list, so that we + // have a digest value to use if we're asked to save a signature for the manifest. + digest, err := manifest.Digest(m) + if err != nil { + return err + } + d.manifestDigest = digest + // The refTail should be either a digest (which we expect to match the value we just + // computed) or a tag name. + refTail, err = d.ref.tagOrDigest() + if err != nil { + return err + } } - d.manifestDigest = digest - refTail, err := d.ref.tagOrDigest() - if err != nil { - return err - } path := fmt.Sprintf(manifestPath, reference.Path(d.ref.ref), refTail) headers := map[string][]string{} @@ -416,19 +441,30 @@ func isManifestInvalidError(err error) bool { } } -func (d *dockerImageDestination) PutSignatures(ctx context.Context, signatures [][]byte) error { +// PutSignatures uploads a set of signatures to the relevant lookaside or API extension point. +// If instanceDigest is not nil, it contains a digest of the specific manifest instance to upload the signatures for (when +// the primary manifest is a manifest list); this should always be nil if the primary manifest is not a manifest list. +func (d *dockerImageDestination) PutSignatures(ctx context.Context, signatures [][]byte, instanceDigest *digest.Digest) error { // Do not fail if we don’t really need to support signatures. if len(signatures) == 0 { return nil } + if instanceDigest == nil { + if d.manifestDigest.String() == "" { + // This shouldn’t happen, ImageDestination users are required to call PutManifest before PutSignatures + return errors.Errorf("Unknown manifest digest, can't add signatures") + } + instanceDigest = &d.manifestDigest + } + if err := d.c.detectProperties(ctx); err != nil { return err } switch { case d.c.signatureBase != nil: - return d.putSignaturesToLookaside(signatures) + return d.putSignaturesToLookaside(signatures, instanceDigest) case d.c.supportsSignatures: - return d.putSignaturesToAPIExtension(ctx, signatures) + return d.putSignaturesToAPIExtension(ctx, signatures, instanceDigest) default: return errors.Errorf("X-Registry-Supports-Signatures extension not supported, and lookaside is not configured") } @@ -436,7 +472,7 @@ func (d *dockerImageDestination) PutSignatures(ctx context.Context, signatures [ // putSignaturesToLookaside implements PutSignatures() from the lookaside location configured in s.c.signatureBase, // which is not nil. -func (d *dockerImageDestination) putSignaturesToLookaside(signatures [][]byte) error { +func (d *dockerImageDestination) putSignaturesToLookaside(signatures [][]byte, instanceDigest *digest.Digest) error { // FIXME? This overwrites files one at a time, definitely not atomic. // A failure when updating signatures with a reordered copy could lose some of them. @@ -445,14 +481,9 @@ func (d *dockerImageDestination) putSignaturesToLookaside(signatures [][]byte) e return nil } - if d.manifestDigest.String() == "" { - // This shouldn’t happen, ImageDestination users are required to call PutManifest before PutSignatures - return errors.Errorf("Unknown manifest digest, can't add signatures") - } - // NOTE: Keep this in sync with docs/signature-protocols.md! for i, signature := range signatures { - url := signatureStorageURL(d.c.signatureBase, d.manifestDigest, i) + url := signatureStorageURL(d.c.signatureBase, *instanceDigest, i) if url == nil { return errors.Errorf("Internal error: signatureStorageURL with non-nil base returned nil") } @@ -467,7 +498,7 @@ func (d *dockerImageDestination) putSignaturesToLookaside(signatures [][]byte) e // is enough for dockerImageSource to stop looking for other signatures, so that // is sufficient. for i := len(signatures); ; i++ { - url := signatureStorageURL(d.c.signatureBase, d.manifestDigest, i) + url := signatureStorageURL(d.c.signatureBase, *instanceDigest, i) if url == nil { return errors.Errorf("Internal error: signatureStorageURL with non-nil base returned nil") } @@ -527,22 +558,17 @@ func (c *dockerClient) deleteOneSignature(url *url.URL) (missing bool, err error } // putSignaturesToAPIExtension implements PutSignatures() using the X-Registry-Supports-Signatures API extension. -func (d *dockerImageDestination) putSignaturesToAPIExtension(ctx context.Context, signatures [][]byte) error { +func (d *dockerImageDestination) putSignaturesToAPIExtension(ctx context.Context, signatures [][]byte, instanceDigest *digest.Digest) error { // Skip dealing with the manifest digest, or reading the old state, if not necessary. if len(signatures) == 0 { return nil } - if d.manifestDigest.String() == "" { - // This shouldn’t happen, ImageDestination users are required to call PutManifest before PutSignatures - return errors.Errorf("Unknown manifest digest, can't add signatures") - } - // Because image signatures are a shared resource in Atomic Registry, the default upload // always adds signatures. Eventually we should also allow removing signatures, // but the X-Registry-Supports-Signatures API extension does not support that yet. - existingSignatures, err := d.c.getExtensionsSignatures(ctx, d.ref, d.manifestDigest) + existingSignatures, err := d.c.getExtensionsSignatures(ctx, d.ref, *instanceDigest) if err != nil { return err } @@ -567,7 +593,7 @@ sigExists: if err != nil || n != 16 { return errors.Wrapf(err, "Error generating random signature len %d", n) } - signatureName = fmt.Sprintf("%s@%032x", d.manifestDigest.String(), randBytes) + signatureName = fmt.Sprintf("%s@%032x", instanceDigest.String(), randBytes) if _, ok := existingSigNames[signatureName]; !ok { break } @@ -606,6 +632,6 @@ sigExists: // WARNING: This does not have any transactional semantics: // - Uploaded data MAY be visible to others before Commit() is called // - Uploaded data MAY be removed or MAY remain around if Close() is called without Commit() (i.e. rollback is allowed but not guaranteed) -func (d *dockerImageDestination) Commit(ctx context.Context) error { +func (d *dockerImageDestination) Commit(context.Context, types.UnparsedImage) error { return nil } diff --git a/docker/docker_image_src.go b/docker/docker_image_src.go index 353b1a6c59..23df287609 100644 --- a/docker/docker_image_src.go +++ b/docker/docker_image_src.go @@ -103,8 +103,15 @@ func (s *dockerImageSource) Close() error { return nil } -// LayerInfosForCopy() returns updated layer info that should be used when reading, in preference to values in the manifest, if specified. -func (s *dockerImageSource) LayerInfosForCopy(ctx context.Context) ([]types.BlobInfo, error) { +// LayerInfosForCopy returns either nil (meaning the values in the manifest are fine), or updated values for the layer +// blobsums that are listed in the image's manifest. If values are returned, they should be used when using GetBlob() +// to read the image's layers. +// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve BlobInfos for +// (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list +// (e.g. if the source never returns manifest lists). +// The Digest field is guaranteed to be provided; Size may be -1. +// WARNING: The list may contain duplicates, and they are semantically relevant. +func (s *dockerImageSource) LayerInfosForCopy(context.Context, *digest.Digest) ([]types.BlobInfo, error) { return nil, nil } diff --git a/docker/tarfile/dest.go b/docker/tarfile/dest.go index aec8404b67..eb10ed788c 100644 --- a/docker/tarfile/dest.go +++ b/docker/tarfile/dest.go @@ -195,10 +195,15 @@ func (d *Destination) createRepositoriesFile(rootLayerID string) error { } // PutManifest writes manifest to the destination. +// The instanceDigest value is expected to always be nil, because this transport does not support manifest lists, so +// there can be no secondary manifests. // FIXME? This should also receive a MIME type if known, to differentiate between schema versions. // If the destination is in principle available, refuses this manifest type (e.g. it does not recognize the schema), // but may accept a different manifest type, the returned error must be an ManifestTypeRejectedError. -func (d *Destination) PutManifest(ctx context.Context, m []byte) error { +func (d *Destination) PutManifest(ctx context.Context, m []byte, instanceDigest *digest.Digest) error { + if instanceDigest != nil { + return errors.New(`Manifest lists are not supported for docker tar files`) + } // We do not bother with types.ManifestTypeRejectedError; our .SupportedManifestMIMETypes() above is already providing only one alternative, // so the caller trying a different manifest kind would be pointless. var man manifest.Schema2 @@ -390,10 +395,13 @@ func (d *Destination) sendFile(path string, expectedSize int64, stream io.Reader return nil } -// PutSignatures adds the given signatures to the docker tarfile (currently not -// supported). MUST be called after PutManifest (signatures reference manifest -// contents) -func (d *Destination) PutSignatures(ctx context.Context, signatures [][]byte) error { +// PutSignatures would add the given signatures to the docker tarfile (currently not supported). +// The instanceDigest value is expected to always be nil, because this transport does not support manifest lists, so +// there can be no secondary manifests. MUST be called after PutManifest (signatures reference manifest contents). +func (d *Destination) PutSignatures(ctx context.Context, signatures [][]byte, instanceDigest *digest.Digest) error { + if instanceDigest != nil { + return errors.Errorf(`Manifest lists are not supported for docker tar files`) + } if len(signatures) != 0 { return errors.Errorf("Storing signatures for docker tar files is not supported") } diff --git a/docker/tarfile/src.go b/docker/tarfile/src.go index 78e4d6f656..b84afd6aa8 100644 --- a/docker/tarfile/src.go +++ b/docker/tarfile/src.go @@ -349,10 +349,12 @@ func (s *Source) prepareLayerData(tarManifest *ManifestItem, parsedConfig *manif // It may use a remote (= slow) service. // If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve (when the primary manifest is a manifest list); // this never happens if the primary manifest is not a manifest list (e.g. if the source never returns manifest lists). +// This source implementation does not support manifest lists, so the passed-in instanceDigest should always be nil, +// as the primary manifest can not be a list, so there can be no secondary instances. func (s *Source) GetManifest(ctx context.Context, instanceDigest *digest.Digest) ([]byte, string, error) { if instanceDigest != nil { // How did we even get here? GetManifest(ctx, nil) has returned a manifest.DockerV2Schema2MediaType. - return nil, "", errors.Errorf(`Manifest lists are not supported by "docker-daemon:"`) + return nil, "", errors.New(`Manifest lists are not supported by "docker-daemon:"`) } if s.generatedManifest == nil { if err := s.ensureCachedDataIsPresent(); err != nil { @@ -466,9 +468,8 @@ func (s *Source) GetBlob(ctx context.Context, info types.BlobInfo, cache types.B } // GetSignatures returns the image's signatures. It may use a remote (= slow) service. -// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve signatures for -// (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list -// (e.g. if the source never returns manifest lists). +// This source implementation does not support manifest lists, so the passed-in instanceDigest should always be nil, +// as there can be no secondary manifests. func (s *Source) GetSignatures(ctx context.Context, instanceDigest *digest.Digest) ([][]byte, error) { if instanceDigest != nil { // How did we even get here? GetManifest(ctx, nil) has returned a manifest.DockerV2Schema2MediaType. @@ -476,3 +477,14 @@ func (s *Source) GetSignatures(ctx context.Context, instanceDigest *digest.Diges } return [][]byte{}, nil } + +// LayerInfosForCopy returns either nil (meaning the values in the manifest are fine), or updated values for the layer +// blobsums that are listed in the image's manifest. If values are returned, they should be used when using GetBlob() +// to read the image's layers. +// This source implementation does not support manifest lists, so the passed-in instanceDigest should always be nil, +// as the primary manifest can not be a list, so there can be no secondary manifests. +// The Digest field is guaranteed to be provided; Size may be -1. +// WARNING: The list may contain duplicates, and they are semantically relevant. +func (s *Source) LayerInfosForCopy(context.Context, *digest.Digest) ([]types.BlobInfo, error) { + return nil, nil +} diff --git a/image/docker_list.go b/image/docker_list.go index a11cd06b99..7e22489840 100644 --- a/image/docker_list.go +++ b/image/docker_list.go @@ -2,69 +2,24 @@ package image import ( "context" - "encoding/json" - "fmt" - "runtime" "github.com/containers/image/v4/manifest" "github.com/containers/image/v4/types" - "github.com/opencontainers/go-digest" "github.com/pkg/errors" ) -type platformSpec struct { - Architecture string `json:"architecture"` - OS string `json:"os"` - OSVersion string `json:"os.version,omitempty"` - OSFeatures []string `json:"os.features,omitempty"` - Variant string `json:"variant,omitempty"` - Features []string `json:"features,omitempty"` // removed in OCI -} - -// A manifestDescriptor references a platform-specific manifest. -type manifestDescriptor struct { - manifest.Schema2Descriptor - Platform platformSpec `json:"platform"` -} - -type manifestList struct { - SchemaVersion int `json:"schemaVersion"` - MediaType string `json:"mediaType"` - Manifests []manifestDescriptor `json:"manifests"` -} - -// chooseDigestFromManifestList parses blob as a schema2 manifest list, -// and returns the digest of the image appropriate for the current environment. -func chooseDigestFromManifestList(sys *types.SystemContext, blob []byte) (digest.Digest, error) { - wantedArch := runtime.GOARCH - if sys != nil && sys.ArchitectureChoice != "" { - wantedArch = sys.ArchitectureChoice - } - wantedOS := runtime.GOOS - if sys != nil && sys.OSChoice != "" { - wantedOS = sys.OSChoice - } - - list := manifestList{} - if err := json.Unmarshal(blob, &list); err != nil { - return "", err - } - for _, d := range list.Manifests { - if d.Platform.Architecture == wantedArch && d.Platform.OS == wantedOS { - return d.Digest, nil - } - } - return "", fmt.Errorf("no image found in manifest list for architecture %s, OS %s", wantedArch, wantedOS) -} - func manifestSchema2FromManifestList(ctx context.Context, sys *types.SystemContext, src types.ImageSource, manblob []byte) (genericManifest, error) { - targetManifestDigest, err := chooseDigestFromManifestList(sys, manblob) + list, err := manifest.Schema2ListFromManifest(manblob) + if err != nil { + return nil, errors.Wrapf(err, "Error parsing schema2 manifest list") + } + targetManifestDigest, err := list.ChooseInstance(sys) if err != nil { - return nil, err + return nil, errors.Wrapf(err, "Error choosing image instance") } manblob, mt, err := src.GetManifest(ctx, &targetManifestDigest) if err != nil { - return nil, err + return nil, errors.Wrapf(err, "Error loading manifest for target platform") } matches, err := manifest.MatchesDigest(manblob, targetManifestDigest) @@ -72,23 +27,8 @@ func manifestSchema2FromManifestList(ctx context.Context, sys *types.SystemConte return nil, errors.Wrap(err, "Error computing manifest digest") } if !matches { - return nil, errors.Errorf("Manifest image does not match selected manifest digest %s", targetManifestDigest) + return nil, errors.Errorf("Image manifest does not match selected manifest digest %s", targetManifestDigest) } return manifestInstanceFromBlob(ctx, sys, src, manblob, mt) } - -// ChooseManifestInstanceFromManifestList returns a digest of a manifest appropriate -// for the current system from the manifest available from src. -func ChooseManifestInstanceFromManifestList(ctx context.Context, sys *types.SystemContext, src types.UnparsedImage) (digest.Digest, error) { - // For now this only handles manifest.DockerV2ListMediaType; we can generalize it later, - // probably along with manifest list editing. - blob, mt, err := src.Manifest(ctx) - if err != nil { - return "", err - } - if mt != manifest.DockerV2ListMediaType { - return "", fmt.Errorf("Internal error: Trying to select an image from a non-manifest-list manifest type %s", mt) - } - return chooseDigestFromManifestList(sys, blob) -} diff --git a/image/docker_list_test.go b/image/docker_list_test.go deleted file mode 100644 index 6bfbae7151..0000000000 --- a/image/docker_list_test.go +++ /dev/null @@ -1,44 +0,0 @@ -package image - -import ( - "bytes" - "io/ioutil" - "path/filepath" - "testing" - - "github.com/containers/image/v4/types" - "github.com/opencontainers/go-digest" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestChooseDigestFromManifestList(t *testing.T) { - manifest, err := ioutil.ReadFile(filepath.Join("fixtures", "schema2list.json")) - require.NoError(t, err) - - // Match found - for arch, expected := range map[string]digest.Digest{ - "amd64": "sha256:030fcb92e1487b18c974784dcc110a93147c9fc402188370fbfd17efabffc6af", - "s390x": "sha256:e5aa1b0a24620228b75382997a0977f609b3ca3a95533dafdef84c74cc8df642", - // There are several "arm" images with different variants; - // the current code returns the first match. NOTE: This is NOT an API promise. - "arm": "sha256:9142d97ef280a7953cf1a85716de49a24cc1dd62776352afad67e635331ff77a", - } { - digest, err := chooseDigestFromManifestList(&types.SystemContext{ - ArchitectureChoice: arch, - OSChoice: "linux", - }, manifest) - require.NoError(t, err, arch) - assert.Equal(t, expected, digest) - } - - // Invalid manifest list - _, err = chooseDigestFromManifestList(&types.SystemContext{ - ArchitectureChoice: "amd64", OSChoice: "linux", - }, bytes.Join([][]byte{manifest, []byte("!INVALID")}, nil)) - assert.Error(t, err) - - // Not found - _, err = chooseDigestFromManifestList(&types.SystemContext{OSChoice: "Unmatched"}, manifest) - assert.Error(t, err) -} diff --git a/image/docker_schema2_test.go b/image/docker_schema2_test.go index 8a901cbd5f..89e1526a89 100644 --- a/image/docker_schema2_test.go +++ b/image/docker_schema2_test.go @@ -41,7 +41,7 @@ func (f unusedImageSource) GetBlob(context.Context, types.BlobInfo, types.BlobIn func (f unusedImageSource) GetSignatures(context.Context, *digest.Digest) ([][]byte, error) { panic("Unexpected call to a mock function") } -func (f unusedImageSource) LayerInfosForCopy(ctx context.Context) ([]types.BlobInfo, error) { +func (f unusedImageSource) LayerInfosForCopy(context.Context, *digest.Digest) ([]types.BlobInfo, error) { panic("Unexpected call to a mock function") } @@ -430,13 +430,13 @@ func (d *memoryImageDest) PutBlob(ctx context.Context, stream io.Reader, inputIn func (d *memoryImageDest) TryReusingBlob(context.Context, types.BlobInfo, types.BlobInfoCache, bool) (bool, types.BlobInfo, error) { panic("Unexpected call to a mock function") } -func (d *memoryImageDest) PutManifest(ctx context.Context, m []byte) error { +func (d *memoryImageDest) PutManifest(context.Context, []byte, *digest.Digest) error { panic("Unexpected call to a mock function") } -func (d *memoryImageDest) PutSignatures(ctx context.Context, signatures [][]byte) error { +func (d *memoryImageDest) PutSignatures(ctx context.Context, signatures [][]byte, instanceDigest *digest.Digest) error { panic("Unexpected call to a mock function") } -func (d *memoryImageDest) Commit(ctx context.Context) error { +func (d *memoryImageDest) Commit(context.Context, types.UnparsedImage) error { panic("Unexpected call to a mock function") } diff --git a/image/fixtures/oci1index.json b/image/fixtures/oci1index.json new file mode 100644 index 0000000000..066f058db1 --- /dev/null +++ b/image/fixtures/oci1index.json @@ -0,0 +1,30 @@ +{ + "schemaVersion": 2, + "manifests": [ + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "size": 7143, + "digest": "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f", + "platform": { + "architecture": "ppc64le", + "os": "linux" + } + }, + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "size": 7682, + "digest": "sha256:5b0bcabd1ed22e9fb1310cf6c2dec7cdef19f0ad69efa1f392e94a4333501270", + "platform": { + "architecture": "amd64", + "os": "linux", + "os.features": [ + "sse4" + ] + } + } + ], + "annotations": { + "com.example.key1": "value1", + "com.example.key2": "value2" + } +} diff --git a/image/manifest.go b/image/manifest.go index f384d2fb8a..fa5fd6f4dd 100644 --- a/image/manifest.go +++ b/image/manifest.go @@ -58,6 +58,8 @@ func manifestInstanceFromBlob(ctx context.Context, sys *types.SystemContext, src return manifestSchema2FromManifest(src, manblob) case manifest.DockerV2ListMediaType: return manifestSchema2FromManifestList(ctx, sys, src, manblob) + case imgspecv1.MediaTypeImageIndex: + return manifestOCI1FromImageIndex(ctx, sys, src, manblob) default: // Note that this may not be reachable, manifest.NormalizedMIMEType has a default for unknown values. return nil, fmt.Errorf("Unimplemented manifest MIME type %s", mt) } diff --git a/image/oci_index.go b/image/oci_index.go new file mode 100644 index 0000000000..32a3b75051 --- /dev/null +++ b/image/oci_index.go @@ -0,0 +1,34 @@ +package image + +import ( + "context" + + "github.com/containers/image/v4/manifest" + "github.com/containers/image/v4/types" + "github.com/pkg/errors" +) + +func manifestOCI1FromImageIndex(ctx context.Context, sys *types.SystemContext, src types.ImageSource, manblob []byte) (genericManifest, error) { + index, err := manifest.OCI1IndexFromManifest(manblob) + if err != nil { + return nil, errors.Wrapf(err, "Error parsing OCI1 index") + } + targetManifestDigest, err := index.ChooseInstance(sys) + if err != nil { + return nil, errors.Wrapf(err, "Error choosing image instance") + } + manblob, mt, err := src.GetManifest(ctx, &targetManifestDigest) + if err != nil { + return nil, errors.Wrapf(err, "Error loading manifest for target platform") + } + + matches, err := manifest.MatchesDigest(manblob, targetManifestDigest) + if err != nil { + return nil, errors.Wrap(err, "Error computing manifest digest") + } + if !matches { + return nil, errors.Errorf("Image manifest does not match selected manifest digest %s", targetManifestDigest) + } + + return manifestInstanceFromBlob(ctx, sys, src, manblob, mt) +} diff --git a/image/sourced.go b/image/sourced.go index d2a3e2ee62..6d59b6df02 100644 --- a/image/sourced.go +++ b/image/sourced.go @@ -100,5 +100,5 @@ func (i *sourcedImage) Manifest(ctx context.Context) ([]byte, string, error) { } func (i *sourcedImage) LayerInfosForCopy(ctx context.Context) ([]types.BlobInfo, error) { - return i.UnparsedImage.src.LayerInfosForCopy(ctx) + return i.UnparsedImage.src.LayerInfosForCopy(ctx, i.UnparsedImage.instanceDigest) } diff --git a/manifest/docker_schema2_list.go b/manifest/docker_schema2_list.go new file mode 100644 index 0000000000..4b41bc453b --- /dev/null +++ b/manifest/docker_schema2_list.go @@ -0,0 +1,216 @@ +package manifest + +import ( + "encoding/json" + "fmt" + "runtime" + + "github.com/containers/image/v4/types" + "github.com/opencontainers/go-digest" + imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" +) + +// Schema2PlatformSpec describes the platform which a particular manifest is +// specialized for. +type Schema2PlatformSpec struct { + Architecture string `json:"architecture"` + OS string `json:"os"` + OSVersion string `json:"os.version,omitempty"` + OSFeatures []string `json:"os.features,omitempty"` + Variant string `json:"variant,omitempty"` + Features []string `json:"features,omitempty"` // removed in OCI +} + +// Schema2ManifestDescriptor references a platform-specific manifest. +type Schema2ManifestDescriptor struct { + Schema2Descriptor + Platform Schema2PlatformSpec `json:"platform"` +} + +// Schema2List is a list of platform-specific manifests. +type Schema2List struct { + SchemaVersion int `json:"schemaVersion"` + MediaType string `json:"mediaType"` + Manifests []Schema2ManifestDescriptor `json:"manifests"` +} + +// MIMEType returns the MIME type of this particular manifest list. +func (list *Schema2List) MIMEType() string { + return list.MediaType +} + +// Instances returns a slice of digests of the manifests that this list knows of. +func (list *Schema2List) Instances() []digest.Digest { + results := make([]digest.Digest, len(list.Manifests)) + for i, m := range list.Manifests { + results[i] = m.Digest + } + return results +} + +// Instance returns the ListUpdate of a particular instance in the list. +func (list *Schema2List) Instance(instanceDigest digest.Digest) (ListUpdate, error) { + for _, manifest := range list.Manifests { + if manifest.Digest == instanceDigest { + return ListUpdate{ + Digest: manifest.Digest, + Size: manifest.Size, + MediaType: manifest.MediaType, + }, nil + } + } + return ListUpdate{}, errors.Errorf("unable to find instance %s passed to Schema2List.Instances", instanceDigest) +} + +// UpdateInstances updates the sizes, digests, and media types of the manifests +// which the list catalogs. +func (list *Schema2List) UpdateInstances(updates []ListUpdate) error { + if len(updates) != len(list.Manifests) { + return errors.Errorf("incorrect number of update entries passed to Schema2List.UpdateInstances: expected %d, got %d", len(list.Manifests), len(updates)) + } + for i := range updates { + if err := updates[i].Digest.Validate(); err != nil { + return errors.Wrapf(err, "update %d of %d passed to Schema2List.UpdateInstances contained an invalid digest", i+1, len(updates)) + } + list.Manifests[i].Digest = updates[i].Digest + if updates[i].Size < 0 { + return errors.Errorf("update %d of %d passed to Schema2List.UpdateInstances had an invalid size (%d)", i+1, len(updates), updates[i].Size) + } + list.Manifests[i].Size = updates[i].Size + if updates[i].MediaType == "" { + return errors.Errorf("update %d of %d passed to Schema2List.UpdateInstances had no media type (was %q)", i+1, len(updates), list.Manifests[i].MediaType) + } + if err := SupportedSchema2MediaType(updates[i].MediaType); err != nil && SupportedOCI1MediaType(updates[i].MediaType) != nil { + return errors.Wrapf(err, "update %d of %d passed to Schema2List.UpdateInstances had an unsupported media type (was %q): %q", i+1, len(updates), list.Manifests[i].MediaType, updates[i].MediaType) + } + list.Manifests[i].MediaType = updates[i].MediaType + } + return nil +} + +// ChooseInstance parses blob as a schema2 manifest list, and returns the digest +// of the image which is appropriate for the current environment. +func (list *Schema2List) ChooseInstance(ctx *types.SystemContext) (digest.Digest, error) { + wantedArch := runtime.GOARCH + if ctx != nil && ctx.ArchitectureChoice != "" { + wantedArch = ctx.ArchitectureChoice + } + wantedOS := runtime.GOOS + if ctx != nil && ctx.OSChoice != "" { + wantedOS = ctx.OSChoice + } + + for _, d := range list.Manifests { + if d.Platform.Architecture == wantedArch && d.Platform.OS == wantedOS { + return d.Digest, nil + } + } + return "", fmt.Errorf("no image found in manifest list for architecture %s, OS %s", wantedArch, wantedOS) +} + +// Serialize returns the list in a blob format. +// NOTE: Serialize() does not in general reproduce the original blob if this object was loaded from one, even if no modifications were made! +func (list *Schema2List) Serialize() ([]byte, error) { + buf, err := json.Marshal(list) + if err != nil { + return nil, errors.Wrapf(err, "error marshaling Schema2List %#v", list) + } + return buf, nil +} + +// Schema2ListFromComponents creates a Schema2 manifest list instance from the +// supplied data. +func Schema2ListFromComponents(components []Schema2ManifestDescriptor) *Schema2List { + list := Schema2List{ + SchemaVersion: 2, + MediaType: DockerV2ListMediaType, + Manifests: make([]Schema2ManifestDescriptor, len(components)), + } + for i, component := range components { + m := Schema2ManifestDescriptor{ + Schema2Descriptor{ + MediaType: component.MediaType, + Size: component.Size, + Digest: component.Digest, + URLs: dupStringSlice(component.URLs), + }, + Schema2PlatformSpec{ + Architecture: component.Platform.Architecture, + OS: component.Platform.OS, + OSVersion: component.Platform.OSVersion, + OSFeatures: dupStringSlice(component.Platform.OSFeatures), + Variant: component.Platform.Variant, + Features: dupStringSlice(component.Platform.Features), + }, + } + list.Manifests[i] = m + } + return &list +} + +// Schema2ListClone creates a deep copy of the passed-in list. +func Schema2ListClone(list *Schema2List) *Schema2List { + return Schema2ListFromComponents(list.Manifests) +} + +// ToOCI1Index returns the list encoded as an OCI1 index. +func (list *Schema2List) ToOCI1Index() (*OCI1Index, error) { + components := make([]imgspecv1.Descriptor, 0, len(list.Manifests)) + for _, manifest := range list.Manifests { + converted := imgspecv1.Descriptor{ + MediaType: manifest.MediaType, + Size: manifest.Size, + Digest: manifest.Digest, + URLs: dupStringSlice(manifest.URLs), + Platform: &imgspecv1.Platform{ + OS: manifest.Platform.OS, + Architecture: manifest.Platform.Architecture, + OSFeatures: dupStringSlice(manifest.Platform.OSFeatures), + OSVersion: manifest.Platform.OSVersion, + Variant: manifest.Platform.Variant, + }, + } + components = append(components, converted) + } + oci := OCI1IndexFromComponents(components, nil) + return oci, nil +} + +// ToSchema2List returns the list encoded as a Schema2 list. +func (list *Schema2List) ToSchema2List() (*Schema2List, error) { + return Schema2ListClone(list), nil +} + +// Schema2ListFromManifest creates a Schema2 manifest list instance from marshalled +// JSON, presumably generated by encoding a Schema2 manifest list. +func Schema2ListFromManifest(manifest []byte) (*Schema2List, error) { + list := Schema2List{ + Manifests: []Schema2ManifestDescriptor{}, + } + if err := json.Unmarshal(manifest, &list); err != nil { + return nil, errors.Wrapf(err, "error unmarshaling Schema2List %q", string(manifest)) + } + return &list, nil +} + +// Clone returns a deep copy of this list and its contents. +func (list *Schema2List) Clone() List { + return Schema2ListClone(list) +} + +// ConvertToMIMEType converts the passed-in manifest list to a manifest +// list of the specified type. +func (list *Schema2List) ConvertToMIMEType(manifestMIMEType string) (List, error) { + switch normalized := NormalizedMIMEType(manifestMIMEType); normalized { + case DockerV2ListMediaType: + return list.Clone(), nil + case imgspecv1.MediaTypeImageIndex: + return list.ToOCI1Index() + case DockerV2Schema1MediaType, DockerV2Schema1SignedMediaType, imgspecv1.MediaTypeImageManifest, DockerV2Schema2MediaType: + return nil, fmt.Errorf("Can not convert manifest list to MIME type %q, which is not a list type", manifestMIMEType) + default: + // Note that this may not be reachable, NormalizedMIMEType has a default for unknown values. + return nil, fmt.Errorf("Unimplemented manifest list MIME type %s", manifestMIMEType) + } +} diff --git a/manifest/list.go b/manifest/list.go new file mode 100644 index 0000000000..2db5584645 --- /dev/null +++ b/manifest/list.go @@ -0,0 +1,106 @@ +package manifest + +import ( + "fmt" + + "github.com/containers/image/v4/types" + digest "github.com/opencontainers/go-digest" + imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" +) + +var ( + // SupportedListMIMETypes is a list of the manifest list types that we know how to + // read/manipulate/write. + SupportedListMIMETypes = []string{ + DockerV2ListMediaType, + imgspecv1.MediaTypeImageIndex, + } +) + +// List is an interface for parsing, modifying lists of image manifests. +// Callers can either use this abstract interface without understanding the details of the formats, +// or instantiate a specific implementation (e.g. manifest.OCI1Index) and access the public members +// directly. +type List interface { + // MIMEType returns the MIME type of this particular manifest list. + MIMEType() string + + // Instances returns a list of the manifests that this list knows of, other than its own. + Instances() []digest.Digest + + // Update information about the list's instances. The length of the passed-in slice must + // match the length of the list of instances which the list already contains, and every field + // must be specified. + UpdateInstances([]ListUpdate) error + + // Instance returns the size and MIME type of a particular instance in the list. + Instance(digest.Digest) (ListUpdate, error) + + // ChooseInstance selects which manifest is most appropriate for the platform described by the + // SystemContext, or for the current platform if the SystemContext doesn't specify any details. + ChooseInstance(ctx *types.SystemContext) (digest.Digest, error) + + // Serialize returns the list in a blob format. + // NOTE: Serialize() does not in general reproduce the original blob if this object was loaded + // from, even if no modifications were made! + Serialize() ([]byte, error) + + // ConvertToMIMEType returns the list rebuilt to the specified MIME type, or an error. + ConvertToMIMEType(mimeType string) (List, error) + + // Clone returns a deep copy of this list and its contents. + Clone() List +} + +// ListUpdate includes the fields which a List's UpdateInstances() method will modify. +type ListUpdate struct { + Digest digest.Digest + Size int64 + MediaType string +} + +// dupStringSlice returns a deep copy of a slice of strings, or nil if the +// source slice is empty. +func dupStringSlice(list []string) []string { + if len(list) == 0 { + return nil + } + dup := make([]string, len(list)) + for i := range list { + dup[i] = list[i] + } + return dup +} + +// dupStringStringMap returns a deep copy of a map[string]string, or nil if the +// passed-in map is nil or has no keys. +func dupStringStringMap(m map[string]string) map[string]string { + if len(m) == 0 { + return nil + } + result := make(map[string]string) + for k, v := range m { + result[k] = v + } + return result +} + +// ListFromBlob parses a list of manifests. +func ListFromBlob(manifest []byte, manifestMIMEType string) (List, error) { + normalized := NormalizedMIMEType(manifestMIMEType) + switch normalized { + case DockerV2ListMediaType: + return Schema2ListFromManifest(manifest) + case imgspecv1.MediaTypeImageIndex: + return OCI1IndexFromManifest(manifest) + case DockerV2Schema1MediaType, DockerV2Schema1SignedMediaType, imgspecv1.MediaTypeImageManifest, DockerV2Schema2MediaType: + return nil, fmt.Errorf("Treating single images as manifest lists is not implemented") + } + return nil, fmt.Errorf("Unimplemented manifest list MIME type %s (normalized as %s)", manifestMIMEType, normalized) +} + +// ConvertListToMIMEType converts the passed-in manifest list to a manifest +// list of the specified type. +func ConvertListToMIMEType(list List, manifestMIMEType string) (List, error) { + return list.ConvertToMIMEType(manifestMIMEType) +} diff --git a/manifest/list_test.go b/manifest/list_test.go new file mode 100644 index 0000000000..8659fd0752 --- /dev/null +++ b/manifest/list_test.go @@ -0,0 +1,119 @@ +package manifest + +import ( + "io/ioutil" + "path/filepath" + "testing" + + "github.com/containers/image/v4/types" + digest "github.com/opencontainers/go-digest" + imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func pare(m List) { + if impl, ok := m.(*OCI1Index); ok { + impl.Annotations = nil + } + if impl, ok := m.(*Schema2List); ok { + for i := range impl.Manifests { + impl.Manifests[i].Platform.Features = nil + } + } + return +} + +func TestParseLists(t *testing.T) { + cases := []struct { + path string + mimeType string + }{ + {"ociv1.image.index.json", imgspecv1.MediaTypeImageIndex}, + {"v2list.manifest.json", DockerV2ListMediaType}, + } + for _, c := range cases { + manifest, err := ioutil.ReadFile(filepath.Join("fixtures", c.path)) + require.NoError(t, err, "error reading file %q", filepath.Join("fixtures", c.path)) + assert.Equal(t, GuessMIMEType(manifest), c.mimeType) + + _, err = FromBlob(manifest, c.mimeType) + require.Error(t, err, "manifest list %q should not parse as single images", c.path) + + m, err := ListFromBlob(manifest, c.mimeType) + require.NoError(t, err, "manifest list %q should parse as list types", c.path) + assert.Equal(t, m.MIMEType(), c.mimeType, "manifest %q is not of the expected MIME type", c.path) + + clone := m.Clone() + assert.Equal(t, clone, m, "manifest %q is missing some fields after being cloned", c.path) + + pare(m) + + index, err := m.ConvertToMIMEType(imgspecv1.MediaTypeImageIndex) + require.NoError(t, err, "error converting %q to an OCI1Index", c.path) + + list, err := m.ConvertToMIMEType(DockerV2ListMediaType) + require.NoError(t, err, "error converting %q to an Schema2List", c.path) + + index2, err := list.ConvertToMIMEType(imgspecv1.MediaTypeImageIndex) + assert.Equal(t, index, index2, "index %q lost data in conversion", c.path) + + list2, err := index.ConvertToMIMEType(DockerV2ListMediaType) + assert.Equal(t, list, list2, "list %q lost data in conversion", c.path) + } +} + +func TestChooseInstance(t *testing.T) { + for _, manifestList := range []struct { + listFile string + matchedInstances map[string]digest.Digest + unmatchedInstances []string + }{ + { + listFile: "schema2list.json", + matchedInstances: map[string]digest.Digest{ + "amd64": "sha256:030fcb92e1487b18c974784dcc110a93147c9fc402188370fbfd17efabffc6af", + "s390x": "sha256:e5aa1b0a24620228b75382997a0977f609b3ca3a95533dafdef84c74cc8df642", + // There are several "arm" images with different variants; + // the current code returns the first match. NOTE: This is NOT an API promise. + "arm": "sha256:9142d97ef280a7953cf1a85716de49a24cc1dd62776352afad67e635331ff77a", + }, + unmatchedInstances: []string{ + "unmatched", + }, + }, + { + listFile: "oci1index.json", + matchedInstances: map[string]digest.Digest{ + "amd64": "sha256:5b0bcabd1ed22e9fb1310cf6c2dec7cdef19f0ad69efa1f392e94a4333501270", + "ppc64le": "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f", + }, + unmatchedInstances: []string{ + "unmatched", + }, + }, + } { + man, err := ioutil.ReadFile(filepath.Join("..", "image", "fixtures", manifestList.listFile)) + require.NoError(t, err) + rawManifest := man + list, err := ListFromBlob(rawManifest, GuessMIMEType(rawManifest)) + require.NoError(t, err) + // Match found + for arch, expected := range manifestList.matchedInstances { + digest, err := list.ChooseInstance(&types.SystemContext{ + ArchitectureChoice: arch, + OSChoice: "linux", + }) + require.NoError(t, err, arch) + assert.Equal(t, expected, digest) + } + // Not found + for _, arch := range manifestList.unmatchedInstances { + _, err := list.ChooseInstance(&types.SystemContext{ + ArchitectureChoice: arch, + OSChoice: "linux", + }) + assert.Error(t, err) + } + } +} diff --git a/manifest/manifest.go b/manifest/manifest.go index 32af97ea81..2daf1b80eb 100644 --- a/manifest/manifest.go +++ b/manifest/manifest.go @@ -52,6 +52,7 @@ var DefaultRequestedManifestMIMETypes = []string{ DockerV2Schema1SignedMediaType, DockerV2Schema1MediaType, DockerV2ListMediaType, + imgspecv1.MediaTypeImageIndex, } // Manifest is an interface for parsing, modifying image manifests in isolation. @@ -140,8 +141,11 @@ func GuessMIMEType(manifest []byte) string { if err := json.Unmarshal(manifest, &ociIndex); err != nil { return "" } - if len(ociIndex.Manifests) != 0 && ociIndex.Manifests[0].MediaType == imgspecv1.MediaTypeImageManifest { - return imgspecv1.MediaTypeImageIndex + if len(ociIndex.Manifests) != 0 { + if ociMan.Config.MediaType == "" { + return imgspecv1.MediaTypeImageIndex + } + return ociMan.Config.MediaType } return DockerV2Schema2MediaType } @@ -199,7 +203,7 @@ func AddDummyV2S1Signature(manifest []byte) ([]byte, error) { // MIMETypeIsMultiImage returns true if mimeType is a list of images func MIMETypeIsMultiImage(mimeType string) bool { - return mimeType == DockerV2ListMediaType + return mimeType == DockerV2ListMediaType || mimeType == imgspecv1.MediaTypeImageIndex } // NormalizedMIMEType returns the effective MIME type of a manifest MIME type returned by a server, @@ -213,6 +217,7 @@ func NormalizedMIMEType(input string) string { return DockerV2Schema1SignedMediaType case DockerV2Schema1MediaType, DockerV2Schema1SignedMediaType, imgspecv1.MediaTypeImageManifest, + imgspecv1.MediaTypeImageIndex, DockerV2Schema2MediaType, DockerV2ListMediaType: return input @@ -232,18 +237,19 @@ func NormalizedMIMEType(input string) string { // FromBlob returns a Manifest instance for the specified manifest blob and the corresponding MIME type func FromBlob(manblob []byte, mt string) (Manifest, error) { - switch NormalizedMIMEType(mt) { + nmt := NormalizedMIMEType(mt) + switch nmt { case DockerV2Schema1MediaType, DockerV2Schema1SignedMediaType: return Schema1FromManifest(manblob) case imgspecv1.MediaTypeImageManifest: return OCI1FromManifest(manblob) case DockerV2Schema2MediaType: return Schema2FromManifest(manblob) - case DockerV2ListMediaType: + case DockerV2ListMediaType, imgspecv1.MediaTypeImageIndex: return nil, fmt.Errorf("Treating manifest lists as individual manifests is not implemented") - default: // Note that this may not be reachable, NormalizedMIMEType has a default for unknown values. - return nil, fmt.Errorf("Unimplemented manifest MIME type %s", mt) } + // Note that this may not be reachable, NormalizedMIMEType has a default for unknown values. + return nil, fmt.Errorf("Unimplemented manifest MIME type %s (normalized as %s)", mt, nmt) } // layerInfosToStrings converts a list of layer infos, presumably obtained from a Manifest.LayerInfos() diff --git a/manifest/manifest_test.go b/manifest/manifest_test.go index 7ed9e042c6..739a6f5af1 100644 --- a/manifest/manifest_test.go +++ b/manifest/manifest_test.go @@ -134,6 +134,8 @@ func TestMIMETypeIsMultiImage(t *testing.T) { {DockerV2Schema1MediaType, false}, {DockerV2Schema1SignedMediaType, false}, {DockerV2Schema2MediaType, false}, + {imgspecv1.MediaTypeImageIndex, true}, + {imgspecv1.MediaTypeImageManifest, false}, } { res := MIMETypeIsMultiImage(c.mt) assert.Equal(t, c.expected, res, c.mt) diff --git a/manifest/oci_index.go b/manifest/oci_index.go new file mode 100644 index 0000000000..4776ee88cb --- /dev/null +++ b/manifest/oci_index.go @@ -0,0 +1,221 @@ +package manifest + +import ( + "encoding/json" + "fmt" + "runtime" + + "github.com/containers/image/v4/types" + "github.com/opencontainers/go-digest" + imgspec "github.com/opencontainers/image-spec/specs-go" + imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" +) + +// OCI1Index is just an alias for the OCI index type, but one which we can +// provide methods for. +type OCI1Index struct { + imgspecv1.Index +} + +// MIMEType returns the MIME type of this particular manifest index. +func (index *OCI1Index) MIMEType() string { + return imgspecv1.MediaTypeImageIndex +} + +// Instances returns a slice of digests of the manifests that this index knows of. +func (index *OCI1Index) Instances() []digest.Digest { + results := make([]digest.Digest, len(index.Manifests)) + for i, m := range index.Manifests { + results[i] = m.Digest + } + return results +} + +// Instance returns the ListUpdate of a particular instance in the index. +func (index *OCI1Index) Instance(instanceDigest digest.Digest) (ListUpdate, error) { + for _, manifest := range index.Manifests { + if manifest.Digest == instanceDigest { + return ListUpdate{ + Digest: manifest.Digest, + Size: manifest.Size, + MediaType: manifest.MediaType, + }, nil + } + } + return ListUpdate{}, errors.Errorf("unable to find instance %s in OCI1Index", instanceDigest) +} + +// UpdateInstances updates the sizes, digests, and media types of the manifests +// which the list catalogs. +func (index *OCI1Index) UpdateInstances(updates []ListUpdate) error { + if len(updates) != len(index.Manifests) { + return errors.Errorf("incorrect number of update entries passed to OCI1Index.UpdateInstances: expected %d, got %d", len(index.Manifests), len(updates)) + } + for i := range updates { + if err := updates[i].Digest.Validate(); err != nil { + return errors.Wrapf(err, "update %d of %d passed to OCI1Index.UpdateInstances contained an invalid digest", i+1, len(updates)) + } + index.Manifests[i].Digest = updates[i].Digest + if updates[i].Size < 0 { + return errors.Errorf("update %d of %d passed to OCI1Index.UpdateInstances had an invalid size (%d)", i+1, len(updates), updates[i].Size) + } + index.Manifests[i].Size = updates[i].Size + if updates[i].MediaType == "" { + return errors.Errorf("update %d of %d passed to OCI1Index.UpdateInstances had no media type (was %q)", i+1, len(updates), index.Manifests[i].MediaType) + } + if err := SupportedOCI1MediaType(updates[i].MediaType); err != nil && SupportedSchema2MediaType(updates[i].MediaType) != nil && updates[i].MediaType != imgspecv1.MediaTypeImageIndex { + return errors.Wrapf(err, "update %d of %d passed to OCI1Index.UpdateInstances had an unsupported media type (was %q): %q", i+1, len(updates), index.Manifests[i].MediaType, updates[i].MediaType) + } + index.Manifests[i].MediaType = updates[i].MediaType + } + return nil +} + +// ChooseInstance parses blob as an oci v1 manifest index, and returns the digest +// of the image which is appropriate for the current environment. +func (index *OCI1Index) ChooseInstance(ctx *types.SystemContext) (digest.Digest, error) { + wantedArch := runtime.GOARCH + if ctx != nil && ctx.ArchitectureChoice != "" { + wantedArch = ctx.ArchitectureChoice + } + wantedOS := runtime.GOOS + if ctx != nil && ctx.OSChoice != "" { + wantedOS = ctx.OSChoice + } + + for _, d := range index.Manifests { + if d.Platform != nil && d.Platform.Architecture == wantedArch && d.Platform.OS == wantedOS { + return d.Digest, nil + } + } + for _, d := range index.Manifests { + if d.Platform == nil { + return d.Digest, nil + } + } + return "", fmt.Errorf("no image found in image index for architecture %s, OS %s", wantedArch, wantedOS) +} + +// Serialize returns the index in a blob format. +// NOTE: Serialize() does not in general reproduce the original blob if this object was loaded from one, even if no modifications were made! +func (index *OCI1Index) Serialize() ([]byte, error) { + buf, err := json.Marshal(index) + if err != nil { + return nil, errors.Wrapf(err, "error marshaling OCI1Index %#v", index) + } + return buf, nil +} + +// OCI1IndexFromComponents creates an OCI1 image index instance from the +// supplied data. +func OCI1IndexFromComponents(components []imgspecv1.Descriptor, annotations map[string]string) *OCI1Index { + index := OCI1Index{ + imgspecv1.Index{ + Versioned: imgspec.Versioned{SchemaVersion: 2}, + Manifests: make([]imgspecv1.Descriptor, len(components)), + Annotations: dupStringStringMap(annotations), + }, + } + for i, component := range components { + var platform *imgspecv1.Platform + if component.Platform != nil { + platform = &imgspecv1.Platform{ + Architecture: component.Platform.Architecture, + OS: component.Platform.OS, + OSVersion: component.Platform.OSVersion, + OSFeatures: dupStringSlice(component.Platform.OSFeatures), + Variant: component.Platform.Variant, + } + } + m := imgspecv1.Descriptor{ + MediaType: component.MediaType, + Size: component.Size, + Digest: component.Digest, + URLs: dupStringSlice(component.URLs), + Annotations: dupStringStringMap(component.Annotations), + Platform: platform, + } + index.Manifests[i] = m + } + return &index +} + +// OCI1IndexClone creates a deep copy of the passed-in index. +func OCI1IndexClone(index *OCI1Index) *OCI1Index { + return OCI1IndexFromComponents(index.Manifests, index.Annotations) +} + +// ToOCI1Index returns the index encoded as an OCI1 index. +func (index *OCI1Index) ToOCI1Index() (*OCI1Index, error) { + return OCI1IndexClone(index), nil +} + +// ToSchema2List returns the index encoded as a Schema2 list. +func (index *OCI1Index) ToSchema2List() (*Schema2List, error) { + components := make([]Schema2ManifestDescriptor, 0, len(index.Manifests)) + for _, manifest := range index.Manifests { + platform := manifest.Platform + if platform == nil { + platform = &imgspecv1.Platform{ + OS: runtime.GOOS, + Architecture: runtime.GOARCH, + } + } + converted := Schema2ManifestDescriptor{ + Schema2Descriptor{ + MediaType: manifest.MediaType, + Size: manifest.Size, + Digest: manifest.Digest, + URLs: dupStringSlice(manifest.URLs), + }, + Schema2PlatformSpec{ + OS: platform.OS, + Architecture: platform.Architecture, + OSFeatures: dupStringSlice(platform.OSFeatures), + OSVersion: platform.OSVersion, + Variant: platform.Variant, + }, + } + components = append(components, converted) + } + s2 := Schema2ListFromComponents(components) + return s2, nil +} + +// OCI1IndexFromManifest creates an OCI1 manifest index instance from marshalled +// JSON, presumably generated by encoding a OCI1 manifest index. +func OCI1IndexFromManifest(manifest []byte) (*OCI1Index, error) { + index := OCI1Index{ + Index: imgspecv1.Index{ + Versioned: imgspec.Versioned{SchemaVersion: 2}, + Manifests: []imgspecv1.Descriptor{}, + Annotations: make(map[string]string), + }, + } + if err := json.Unmarshal(manifest, &index); err != nil { + return nil, errors.Wrapf(err, "error unmarshaling OCI1Index %q", string(manifest)) + } + return &index, nil +} + +// Clone returns a deep copy of this list and its contents. +func (index *OCI1Index) Clone() List { + return OCI1IndexClone(index) +} + +// ConvertToMIMEType converts the passed-in image index to a manifest list of +// the specified type. +func (index *OCI1Index) ConvertToMIMEType(manifestMIMEType string) (List, error) { + switch normalized := NormalizedMIMEType(manifestMIMEType); normalized { + case DockerV2ListMediaType: + return index.ToSchema2List() + case imgspecv1.MediaTypeImageIndex: + return index.Clone(), nil + case DockerV2Schema1MediaType, DockerV2Schema1SignedMediaType, imgspecv1.MediaTypeImageManifest, DockerV2Schema2MediaType: + return nil, fmt.Errorf("Can not convert image index to MIME type %q, which is not a list type", manifestMIMEType) + default: + // Note that this may not be reachable, NormalizedMIMEType has a default for unknown values. + return nil, fmt.Errorf("Unimplemented manifest MIME type %s", manifestMIMEType) + } +} diff --git a/oci/archive/oci_dest.go b/oci/archive/oci_dest.go index 2455ed575a..26921b8bda 100644 --- a/oci/archive/oci_dest.go +++ b/oci/archive/oci_dest.go @@ -7,6 +7,7 @@ import ( "github.com/containers/image/v4/types" "github.com/containers/storage/pkg/archive" + digest "github.com/opencontainers/go-digest" "github.com/pkg/errors" ) @@ -105,19 +106,26 @@ func (d *ociArchiveImageDestination) TryReusingBlob(ctx context.Context, info ty return d.unpackedDest.TryReusingBlob(ctx, info, cache, canSubstitute) } -// PutManifest writes manifest to the destination -func (d *ociArchiveImageDestination) PutManifest(ctx context.Context, m []byte) error { - return d.unpackedDest.PutManifest(ctx, m) +// PutManifest writes the manifest to the destination. +// If instanceDigest is not nil, it contains a digest of the specific manifest instance to overwrite the manifest for (when +// the primary manifest is a manifest list); this should always be nil if the primary manifest is not a manifest list. +// It is expected but not enforced that the instanceDigest, when specified, matches the digest of `manifest` as generated +// by `manifest.Digest()`. +func (d *ociArchiveImageDestination) PutManifest(ctx context.Context, m []byte, instanceDigest *digest.Digest) error { + return d.unpackedDest.PutManifest(ctx, m, instanceDigest) } -func (d *ociArchiveImageDestination) PutSignatures(ctx context.Context, signatures [][]byte) error { - return d.unpackedDest.PutSignatures(ctx, signatures) +// PutSignatures writes a set of signatures to the destination. +// If instanceDigest is not nil, it contains a digest of the specific manifest instance to write or overwrite the signatures for +// (when the primary manifest is a manifest list); this should always be nil if the primary manifest is not a manifest list. +func (d *ociArchiveImageDestination) PutSignatures(ctx context.Context, signatures [][]byte, instanceDigest *digest.Digest) error { + return d.unpackedDest.PutSignatures(ctx, signatures, instanceDigest) } // Commit marks the process of storing the image as successful and asks for the image to be persisted // after the directory is made, it is tarred up into a file and the directory is deleted -func (d *ociArchiveImageDestination) Commit(ctx context.Context) error { - if err := d.unpackedDest.Commit(ctx); err != nil { +func (d *ociArchiveImageDestination) Commit(ctx context.Context, unparsedToplevel types.UnparsedImage) error { + if err := d.unpackedDest.Commit(ctx, unparsedToplevel); err != nil { return errors.Wrapf(err, "error storing image %q", d.ref.image) } diff --git a/oci/archive/oci_src.go b/oci/archive/oci_src.go index 8a479883fa..e54b423e41 100644 --- a/oci/archive/oci_src.go +++ b/oci/archive/oci_src.go @@ -96,7 +96,14 @@ func (s *ociArchiveImageSource) GetSignatures(ctx context.Context, instanceDiges return s.unpackedSrc.GetSignatures(ctx, instanceDigest) } -// LayerInfosForCopy() returns updated layer info that should be used when reading, in preference to values in the manifest, if specified. -func (s *ociArchiveImageSource) LayerInfosForCopy(ctx context.Context) ([]types.BlobInfo, error) { - return nil, nil +// LayerInfosForCopy returns either nil (meaning the values in the manifest are fine), or updated values for the layer +// blobsums that are listed in the image's manifest. If values are returned, they should be used when using GetBlob() +// to read the image's layers. +// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve BlobInfos for +// (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list +// (e.g. if the source never returns manifest lists). +// The Digest field is guaranteed to be provided; Size may be -1. +// WARNING: The list may contain duplicates, and they are semantically relevant. +func (s *ociArchiveImageSource) LayerInfosForCopy(ctx context.Context, instanceDigest *digest.Digest) ([]types.BlobInfo, error) { + return s.unpackedSrc.LayerInfosForCopy(ctx, instanceDigest) } diff --git a/oci/layout/oci_dest.go b/oci/layout/oci_dest.go index 20925d3dca..62f5e09141 100644 --- a/oci/layout/oci_dest.go +++ b/oci/layout/oci_dest.go @@ -38,6 +38,7 @@ func newImageDestination(sys *types.SystemContext, ref ociReference) (types.Imag Versioned: imgspec.Versioned{ SchemaVersion: 2, }, + Annotations: make(map[string]string), } } @@ -73,6 +74,7 @@ func (d *ociImageDestination) Close() error { func (d *ociImageDestination) SupportedManifestMIMETypes() []string { return []string{ imgspecv1.MediaTypeImageManifest, + imgspecv1.MediaTypeImageIndex, } } @@ -205,20 +207,27 @@ func (d *ociImageDestination) TryReusingBlob(ctx context.Context, info types.Blo return true, types.BlobInfo{Digest: info.Digest, Size: finfo.Size()}, nil } -// PutManifest writes manifest to the destination. +// PutManifest writes a manifest to the destination. Per our list of supported manifest MIME types, +// this should be either an OCI manifest (possibly converted to this format by the caller) or index, +// neither of which we'll need to modify further. +// If instanceDigest is not nil, it contains a digest of the specific manifest instance to overwrite the manifest for (when +// the primary manifest is a manifest list); this should always be nil if the primary manifest is not a manifest list. +// It is expected but not enforced that the instanceDigest, when specified, matches the digest of `manifest` as generated +// by `manifest.Digest()`. // FIXME? This should also receive a MIME type if known, to differentiate between schema versions. // If the destination is in principle available, refuses this manifest type (e.g. it does not recognize the schema), // but may accept a different manifest type, the returned error must be an ManifestTypeRejectedError. -func (d *ociImageDestination) PutManifest(ctx context.Context, m []byte) error { - digest, err := manifest.Digest(m) - if err != nil { - return err +func (d *ociImageDestination) PutManifest(ctx context.Context, m []byte, instanceDigest *digest.Digest) error { + var digest digest.Digest + var err error + if instanceDigest != nil { + digest = *instanceDigest + } else { + digest, err = manifest.Digest(m) + if err != nil { + return err + } } - desc := imgspecv1.Descriptor{} - desc.Digest = digest - // TODO(runcom): beaware and add support for OCI manifest list - desc.MediaType = imgspecv1.MediaTypeImageManifest - desc.Size = int64(len(m)) blobPath, err := d.ref.blobPath(digest, d.sharedBlobDir) if err != nil { @@ -231,32 +240,59 @@ func (d *ociImageDestination) PutManifest(ctx context.Context, m []byte) error { return err } - if d.ref.image != "" { - annotations := make(map[string]string) - annotations["org.opencontainers.image.ref.name"] = d.ref.image - desc.Annotations = annotations + if instanceDigest != nil { + return nil } - desc.Platform = &imgspecv1.Platform{ - Architecture: runtime.GOARCH, - OS: runtime.GOOS, + + // If we had platform information, we'd build an imgspecv1.Platform structure here. + + // Start filling out the descriptor for this entry + desc := imgspecv1.Descriptor{} + desc.Digest = digest + desc.Size = int64(len(m)) + if d.ref.image != "" { + desc.Annotations = make(map[string]string) + desc.Annotations[imgspecv1.AnnotationRefName] = d.ref.image } + + // If we knew the MIME type, we wouldn't have to guess here. + desc.MediaType = manifest.GuessMIMEType(m) + d.addManifest(&desc) return nil } func (d *ociImageDestination) addManifest(desc *imgspecv1.Descriptor) { + // If the new entry has a name, remove any conflicting names which we already have. + if desc.Annotations != nil && desc.Annotations[imgspecv1.AnnotationRefName] != "" { + // The name is being set on a new entry, so remove any older ones that had the same name. + // We might be storing an index and all of its component images, and we'll want to attach + // the name to the last one, which is the index. + for i, manifest := range d.index.Manifests { + if manifest.Annotations[imgspecv1.AnnotationRefName] == desc.Annotations[imgspecv1.AnnotationRefName] { + delete(d.index.Manifests[i].Annotations, imgspecv1.AnnotationRefName) + break + } + } + } + // If it has the same digest as another entry in the index, we already overwrote the file, + // so just pick up the other information. for i, manifest := range d.index.Manifests { - if manifest.Annotations["org.opencontainers.image.ref.name"] == desc.Annotations["org.opencontainers.image.ref.name"] { - // TODO Should there first be a cleanup based on the descriptor we are going to replace? + if manifest.Digest == desc.Digest { + // Replace it completely. d.index.Manifests[i] = *desc return } } + // It's a new entry to be added to the index. d.index.Manifests = append(d.index.Manifests, *desc) } -func (d *ociImageDestination) PutSignatures(ctx context.Context, signatures [][]byte) error { +// PutSignatures would add the given signatures to the oci layout (currently not supported). +// If instanceDigest is not nil, it contains a digest of the specific manifest instance to write or overwrite the signatures for +// (when the primary manifest is a manifest list); this should always be nil if the primary manifest is not a manifest list. +func (d *ociImageDestination) PutSignatures(ctx context.Context, signatures [][]byte, instanceDigest *digest.Digest) error { if len(signatures) != 0 { return errors.Errorf("Pushing signatures for OCI images is not supported") } @@ -267,7 +303,7 @@ func (d *ociImageDestination) PutSignatures(ctx context.Context, signatures [][] // WARNING: This does not have any transactional semantics: // - Uploaded data MAY be visible to others before Commit() is called // - Uploaded data MAY be removed or MAY remain around if Close() is called without Commit() (i.e. rollback is allowed but not guaranteed) -func (d *ociImageDestination) Commit(ctx context.Context) error { +func (d *ociImageDestination) Commit(context.Context, types.UnparsedImage) error { if err := ioutil.WriteFile(d.ref.ociLayoutPath(), []byte(`{"imageLayoutVersion": "1.0.0"}`), 0644); err != nil { return err } diff --git a/oci/layout/oci_dest_test.go b/oci/layout/oci_dest_test.go index edc510b3ba..5e1adfa6ba 100644 --- a/oci/layout/oci_dest_test.go +++ b/oci/layout/oci_dest_test.go @@ -1,13 +1,16 @@ package layout import ( + "bytes" "context" + "io/ioutil" "os" "path/filepath" "testing" "github.com/containers/image/v4/pkg/blobinfocache/memory" "github.com/containers/image/v4/types" + digest "github.com/opencontainers/go-digest" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -56,7 +59,7 @@ func TestPutBlobDigestFailure(t *testing.T) { _, err = dest.PutBlob(context.Background(), reader, types.BlobInfo{Digest: blobDigest, Size: -1}, cache, false) assert.Error(t, err) assert.Contains(t, digestErrorString, err.Error()) - err = dest.Commit(context.Background()) + err = dest.Commit(context.Background(), nil) assert.NoError(t, err) _, err = os.Lstat(blobPath) @@ -96,23 +99,49 @@ func TestPutManifestTwice(t *testing.T) { ociRef, ok := ref.(ociReference) require.True(t, ok) + putTestConfig(t, ociRef, tmpDir) putTestManifest(t, ociRef, tmpDir) putTestManifest(t, ociRef, tmpDir) index, err := ociRef.getIndex() assert.NoError(t, err) - assert.Equal(t, 1, len(index.Manifests), "Unexpected number of manifests") + assert.Equal(t, 2, len(index.Manifests), "Unexpected number of manifests") +} + +func putTestConfig(t *testing.T, ociRef ociReference, tmpDir string) { + data, err := ioutil.ReadFile("../../image/fixtures/oci1-config.json") + assert.NoError(t, err) + imageDest, err := newImageDestination(nil, ociRef) + assert.NoError(t, err) + + cache := memory.New() + + _, err = imageDest.PutBlob(context.Background(), bytes.NewReader(data), types.BlobInfo{Size: int64(len(data)), Digest: digest.FromBytes(data)}, cache, true) + assert.NoError(t, err) + + err = imageDest.Commit(context.Background(), nil) + assert.NoError(t, err) + + paths := []string{} + filepath.Walk(tmpDir, func(path string, info os.FileInfo, err error) error { + paths = append(paths, path) + return nil + }) + + digest := digest.FromBytes(data).Encoded() + assert.Contains(t, paths, filepath.Join(tmpDir, "blobs", "sha256", digest), "The OCI directory does not contain the new config data") } func putTestManifest(t *testing.T, ociRef ociReference, tmpDir string) { + data, err := ioutil.ReadFile("../../image/fixtures/oci1.json") + assert.NoError(t, err) imageDest, err := newImageDestination(nil, ociRef) assert.NoError(t, err) - data := []byte("abc") - err = imageDest.PutManifest(context.Background(), data) + err = imageDest.PutManifest(context.Background(), data, nil) assert.NoError(t, err) - err = imageDest.Commit(context.Background()) + err = imageDest.Commit(context.Background(), nil) assert.NoError(t, err) paths := []string{} @@ -121,6 +150,6 @@ func putTestManifest(t *testing.T, ociRef ociReference, tmpDir string) { return nil }) - digest := "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + digest := digest.FromBytes(data).Encoded() assert.Contains(t, paths, filepath.Join(tmpDir, "blobs", "sha256", digest), "The OCI directory does not contain the new manifest data") } diff --git a/oci/layout/oci_src.go b/oci/layout/oci_src.go index dd6c6c4a66..f9977f8136 100644 --- a/oci/layout/oci_src.go +++ b/oci/layout/oci_src.go @@ -8,6 +8,7 @@ import ( "os" "strconv" + "github.com/containers/image/v4/manifest" "github.com/containers/image/v4/pkg/tlsclientconfig" "github.com/containers/image/v4/types" "github.com/docker/go-connections/tlsconfig" @@ -18,6 +19,7 @@ import ( type ociImageSource struct { ref ociReference + index *imgspecv1.Index descriptor imgspecv1.Descriptor client *http.Client sharedBlobDir string @@ -41,7 +43,11 @@ func newImageSource(sys *types.SystemContext, ref ociReference) (types.ImageSour if err != nil { return nil, err } - d := &ociImageSource{ref: ref, descriptor: descriptor, client: client} + index, err := ref.getIndex() + if err != nil { + return nil, err + } + d := &ociImageSource{ref: ref, index: index, descriptor: descriptor, client: client} if sys != nil { // TODO(jonboulle): check dir existence? d.sharedBlobDir = sys.OCISharedBlobDirPath @@ -66,28 +72,33 @@ func (s *ociImageSource) Close() error { func (s *ociImageSource) GetManifest(ctx context.Context, instanceDigest *digest.Digest) ([]byte, string, error) { var dig digest.Digest var mimeType string + var err error + if instanceDigest == nil { dig = digest.Digest(s.descriptor.Digest) mimeType = s.descriptor.MediaType } else { dig = *instanceDigest - // XXX: instanceDigest means that we don't immediately have the context of what - // mediaType the manifest has. In OCI this means that we don't know - // what reference it came from, so we just *assume* that its - // MediaTypeImageManifest. - // FIXME: We should actually be able to look up the manifest in the index, - // and see the MIME type there. - mimeType = imgspecv1.MediaTypeImageManifest + for _, md := range s.index.Manifests { + if md.Digest == dig { + mimeType = md.MediaType + break + } + } } manifestPath, err := s.ref.blobPath(dig, s.sharedBlobDir) if err != nil { return nil, "", err } + m, err := ioutil.ReadFile(manifestPath) if err != nil { return nil, "", err } + if mimeType == "" { + mimeType = manifest.GuessMIMEType(m) + } return m, mimeType, nil } @@ -157,8 +168,15 @@ func (s *ociImageSource) getExternalBlob(ctx context.Context, urls []string) (io return nil, 0, errWrap } -// LayerInfosForCopy() returns updated layer info that should be used when reading, in preference to values in the manifest, if specified. -func (s *ociImageSource) LayerInfosForCopy(ctx context.Context) ([]types.BlobInfo, error) { +// LayerInfosForCopy returns either nil (meaning the values in the manifest are fine), or updated values for the layer +// blobsums that are listed in the image's manifest. If values are returned, they should be used when using GetBlob() +// to read the image's layers. +// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve BlobInfos for +// (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list +// (e.g. if the source never returns manifest lists). +// The Digest field is guaranteed to be provided; Size may be -1. +// WARNING: The list may contain duplicates, and they are semantically relevant. +func (s *ociImageSource) LayerInfosForCopy(context.Context, *digest.Digest) ([]types.BlobInfo, error) { return nil, nil } diff --git a/oci/layout/oci_transport.go b/oci/layout/oci_transport.go index 259852b4dc..8b6be8ead8 100644 --- a/oci/layout/oci_transport.go +++ b/oci/layout/oci_transport.go @@ -195,10 +195,10 @@ func (ref ociReference) getManifestDescriptor() (imgspecv1.Descriptor, error) { } else { // if image specified, look through all manifests for a match for _, md := range index.Manifests { - if md.MediaType != imgspecv1.MediaTypeImageManifest { + if md.MediaType != imgspecv1.MediaTypeImageManifest && md.MediaType != imgspecv1.MediaTypeImageIndex { continue } - refName, ok := md.Annotations["org.opencontainers.image.ref.name"] + refName, ok := md.Annotations[imgspecv1.AnnotationRefName] if !ok { continue } diff --git a/openshift/openshift.go b/openshift/openshift.go index 51fff6269c..8294f70ca2 100644 --- a/openshift/openshift.go +++ b/openshift/openshift.go @@ -231,16 +231,16 @@ func (s *openshiftImageSource) GetBlob(ctx context.Context, info types.BlobInfo, // (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list // (e.g. if the source never returns manifest lists). func (s *openshiftImageSource) GetSignatures(ctx context.Context, instanceDigest *digest.Digest) ([][]byte, error) { - var imageName string + var imageStreamImageName string if instanceDigest == nil { if err := s.ensureImageIsResolved(ctx); err != nil { return nil, err } - imageName = s.imageStreamImageName + imageStreamImageName = s.imageStreamImageName } else { - imageName = instanceDigest.String() + imageStreamImageName = instanceDigest.String() } - image, err := s.client.getImage(ctx, imageName) + image, err := s.client.getImage(ctx, imageStreamImageName) if err != nil { return nil, err } @@ -253,8 +253,15 @@ func (s *openshiftImageSource) GetSignatures(ctx context.Context, instanceDigest return sigs, nil } -// LayerInfosForCopy() returns updated layer info that should be used when reading, in preference to values in the manifest, if specified. -func (s *openshiftImageSource) LayerInfosForCopy(ctx context.Context) ([]types.BlobInfo, error) { +// LayerInfosForCopy returns either nil (meaning the values in the manifest are fine), or updated values for the layer +// blobsums that are listed in the image's manifest. If values are returned, they should be used when using GetBlob() +// to read the image's layers. +// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve BlobInfos for +// (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list +// (e.g. if the source never returns manifest lists). +// The Digest field is guaranteed to be provided; Size may be -1. +// WARNING: The list may contain duplicates, and they are semantically relevant. +func (s *openshiftImageSource) LayerInfosForCopy(ctx context.Context, instanceDigest *digest.Digest) ([]types.BlobInfo, error) { return nil, nil } @@ -414,20 +421,28 @@ func (d *openshiftImageDestination) TryReusingBlob(ctx context.Context, info typ // FIXME? This should also receive a MIME type if known, to differentiate between schema versions. // If the destination is in principle available, refuses this manifest type (e.g. it does not recognize the schema), // but may accept a different manifest type, the returned error must be an ManifestTypeRejectedError. -func (d *openshiftImageDestination) PutManifest(ctx context.Context, m []byte) error { - manifestDigest, err := manifest.Digest(m) - if err != nil { - return err +func (d *openshiftImageDestination) PutManifest(ctx context.Context, m []byte, instanceDigest *digest.Digest) error { + if instanceDigest == nil { + manifestDigest, err := manifest.Digest(m) + if err != nil { + return err + } + d.imageStreamImageName = manifestDigest.String() } - d.imageStreamImageName = manifestDigest.String() - - return d.docker.PutManifest(ctx, m) + return d.docker.PutManifest(ctx, m, instanceDigest) } -func (d *openshiftImageDestination) PutSignatures(ctx context.Context, signatures [][]byte) error { - if d.imageStreamImageName == "" { - return errors.Errorf("Internal error: Unknown manifest digest, can't add signatures") +func (d *openshiftImageDestination) PutSignatures(ctx context.Context, signatures [][]byte, instanceDigest *digest.Digest) error { + var imageStreamName string + if instanceDigest == nil { + if d.imageStreamImageName == "" { + return errors.Errorf("Internal error: Unknown manifest digest, can't add signatures") + } + imageStreamName = d.imageStreamImageName + } else { + imageStreamName = instanceDigest.String() } + // Because image signatures are a shared resource in Atomic Registry, the default upload // always adds signatures. Eventually we should also allow removing signatures. @@ -435,7 +450,7 @@ func (d *openshiftImageDestination) PutSignatures(ctx context.Context, signature return nil // No need to even read the old state. } - image, err := d.client.getImage(ctx, d.imageStreamImageName) + image, err := d.client.getImage(ctx, imageStreamName) if err != nil { return err } @@ -460,7 +475,7 @@ sigExists: if err != nil || n != 16 { return errors.Wrapf(err, "Error generating random signature len %d", n) } - signatureName = fmt.Sprintf("%s@%032x", d.imageStreamImageName, randBytes) + signatureName = fmt.Sprintf("%s@%032x", imageStreamName, randBytes) if _, ok := existingSigNames[signatureName]; !ok { break } @@ -489,8 +504,8 @@ sigExists: // WARNING: This does not have any transactional semantics: // - Uploaded data MAY be visible to others before Commit() is called // - Uploaded data MAY be removed or MAY remain around if Close() is called without Commit() (i.e. rollback is allowed but not guaranteed) -func (d *openshiftImageDestination) Commit(ctx context.Context) error { - return d.docker.Commit(ctx) +func (d *openshiftImageDestination) Commit(ctx context.Context, unparsedToplevel types.UnparsedImage) error { + return d.docker.Commit(ctx, unparsedToplevel) } // These structs are subsets of github.com/openshift/origin/pkg/image/api/v1 and its dependencies. diff --git a/ostree/ostree_dest.go b/ostree/ostree_dest.go index 9e1436e29e..09de76f99e 100644 --- a/ostree/ostree_dest.go +++ b/ostree/ostree_dest.go @@ -376,10 +376,16 @@ func (d *ostreeImageDestination) TryReusingBlob(ctx context.Context, info types. } // PutManifest writes manifest to the destination. +// The instanceDigest value is expected to always be nil, because this transport does not support manifest lists, so +// there can be no secondary manifests. // FIXME? This should also receive a MIME type if known, to differentiate between schema versions. // If the destination is in principle available, refuses this manifest type (e.g. it does not recognize the schema), // but may accept a different manifest type, the returned error must be an ManifestTypeRejectedError. -func (d *ostreeImageDestination) PutManifest(ctx context.Context, manifestBlob []byte) error { +func (d *ostreeImageDestination) PutManifest(ctx context.Context, manifestBlob []byte, instanceDigest *digest.Digest) error { + if instanceDigest != nil { + return errors.New(`Manifest lists are not supported by "ostree:"`) + } + d.manifest = string(manifestBlob) if err := json.Unmarshal(manifestBlob, &d.schema); err != nil { @@ -400,7 +406,14 @@ func (d *ostreeImageDestination) PutManifest(ctx context.Context, manifestBlob [ return ioutil.WriteFile(manifestPath, manifestBlob, 0644) } -func (d *ostreeImageDestination) PutSignatures(ctx context.Context, signatures [][]byte) error { +// PutSignatures writes signatures to the destination. +// The instanceDigest value is expected to always be nil, because this transport does not support manifest lists, so +// there can be no secondary manifests. +func (d *ostreeImageDestination) PutSignatures(ctx context.Context, signatures [][]byte, instanceDigest *digest.Digest) error { + if instanceDigest != nil { + return errors.New(`Manifest lists are not supported by "ostree:"`) + } + path := filepath.Join(d.tmpDirPath, d.ref.signaturePath(0)) if err := ensureParentDirectoryExists(path); err != nil { return err @@ -416,7 +429,7 @@ func (d *ostreeImageDestination) PutSignatures(ctx context.Context, signatures [ return nil } -func (d *ostreeImageDestination) Commit(ctx context.Context) error { +func (d *ostreeImageDestination) Commit(context.Context, types.UnparsedImage) error { runtime.LockOSThread() defer runtime.UnlockOSThread() diff --git a/ostree/ostree_src.go b/ostree/ostree_src.go index ecb6e3f84a..62182b1bc0 100644 --- a/ostree/ostree_src.go +++ b/ostree/ostree_src.go @@ -98,9 +98,11 @@ func (s *ostreeImageSource) getTarSplitData(blob string) ([]byte, error) { // GetManifest returns the image's manifest along with its MIME type (which may be empty when it can't be determined but the manifest is available). // It may use a remote (= slow) service. +// This source implementation does not support manifest lists, so the passed-in instanceDigest should always be nil, +// as the primary manifest can not be a list, so there can be non-default instances. func (s *ostreeImageSource) GetManifest(ctx context.Context, instanceDigest *digest.Digest) ([]byte, string, error) { if instanceDigest != nil { - return nil, "", errors.Errorf(`Manifest lists are not supported by "ostree:"`) + return nil, "", errors.New(`Manifest lists are not supported by "ostree:"`) } if s.repo == nil { repo, err := openRepo(s.ref.repo) @@ -275,7 +277,7 @@ func (s *ostreeImageSource) GetBlob(ctx context.Context, info types.BlobInfo, ca // Ensure s.compressed is initialized. It is build by LayerInfosForCopy. if s.compressed == nil { - _, err := s.LayerInfosForCopy(ctx) + _, err := s.LayerInfosForCopy(ctx, nil) if err != nil { return nil, -1, err } @@ -337,9 +339,12 @@ func (s *ostreeImageSource) GetBlob(ctx context.Context, info types.BlobInfo, ca return rc, layerSize, nil } +// GetSignatures returns the image's signatures. It may use a remote (= slow) service. +// This source implementation does not support manifest lists, so the passed-in instanceDigest should always be nil, +// as there can be no secondary manifests. func (s *ostreeImageSource) GetSignatures(ctx context.Context, instanceDigest *digest.Digest) ([][]byte, error) { if instanceDigest != nil { - return nil, errors.New("manifest lists are not supported by this transport") + return nil, errors.New(`Manifest lists are not supported by "ostree:"`) } lenSignatures, err := s.getLenSignatures() if err != nil { @@ -372,9 +377,18 @@ func (s *ostreeImageSource) GetSignatures(ctx context.Context, instanceDigest *d return signatures, nil } -// LayerInfosForCopy() returns the list of layer blobs that make up the root filesystem of -// the image, after they've been decompressed. -func (s *ostreeImageSource) LayerInfosForCopy(ctx context.Context) ([]types.BlobInfo, error) { +// LayerInfosForCopy returns either nil (meaning the values in the manifest are fine), or updated values for the layer +// blobsums that are listed in the image's manifest. If values are returned, they should be used when using GetBlob() +// to read the image's layers. +// This source implementation does not support manifest lists, so the passed-in instanceDigest should always be nil, +// as the primary manifest can not be a list, so there can be secondary manifests. +// The Digest field is guaranteed to be provided; Size may be -1. +// WARNING: The list may contain duplicates, and they are semantically relevant. +func (s *ostreeImageSource) LayerInfosForCopy(ctx context.Context, instanceDigest *digest.Digest) ([]types.BlobInfo, error) { + if instanceDigest != nil { + return nil, errors.New(`Manifest lists are not supported by "ostree:"`) + } + updatedBlobInfos := []types.BlobInfo{} manifestBlob, manifestType, err := s.GetManifest(ctx, nil) if err != nil { diff --git a/storage/storage_image.go b/storage/storage_image.go index 4e913b84c0..366a9edfa8 100644 --- a/storage/storage_image.go +++ b/storage/storage_image.go @@ -6,6 +6,7 @@ import ( "bytes" "context" "encoding/json" + stderrors "errors" "fmt" "io" "io/ioutil" @@ -32,38 +33,38 @@ import ( var ( // ErrBlobDigestMismatch is returned when PutBlob() is given a blob // with a digest-based name that doesn't match its contents. - ErrBlobDigestMismatch = errors.New("blob digest mismatch") + ErrBlobDigestMismatch = stderrors.New("blob digest mismatch") // ErrBlobSizeMismatch is returned when PutBlob() is given a blob // with an expected size that doesn't match the reader. - ErrBlobSizeMismatch = errors.New("blob size mismatch") - // ErrNoManifestLists is returned when GetManifest() is called. - // with a non-nil instanceDigest. - ErrNoManifestLists = errors.New("manifest lists are not supported by this transport") + ErrBlobSizeMismatch = stderrors.New("blob size mismatch") // ErrNoSuchImage is returned when we attempt to access an image which // doesn't exist in the storage area. ErrNoSuchImage = storage.ErrNotAnImage ) type storageImageSource struct { - imageRef storageReference - image *storage.Image - layerPosition map[digest.Digest]int // Where we are in reading a blob's layers - cachedManifest []byte // A cached copy of the manifest, if already known, or nil - getBlobMutex sync.Mutex // Mutex to sync state for parallel GetBlob executions - SignatureSizes []int `json:"signature-sizes,omitempty"` // List of sizes of each signature slice + imageRef storageReference + image *storage.Image + layerPosition map[digest.Digest]int // Where we are in reading a blob's layers + cachedManifest []byte // A cached copy of the manifest, if already known, or nil + getBlobMutex sync.Mutex // Mutex to sync state for parallel GetBlob executions + SignatureSizes []int `json:"signature-sizes,omitempty"` // List of sizes of each signature slice + SignaturesSizes map[digest.Digest][]int `json:"signatures-sizes,omitempty"` // List of sizes of each signature slice } type storageImageDestination struct { - imageRef storageReference - directory string // Temporary directory where we store blobs until Commit() time - nextTempFileID int32 // A counter that we use for computing filenames to assign to blobs - manifest []byte // Manifest contents, temporary - signatures []byte // Signature contents, temporary - putBlobMutex sync.Mutex // Mutex to sync state for parallel PutBlob executions - blobDiffIDs map[digest.Digest]digest.Digest // Mapping from layer blobsums to their corresponding DiffIDs - fileSizes map[digest.Digest]int64 // Mapping from layer blobsums to their sizes - filenames map[digest.Digest]string // Mapping from layer blobsums to names of files we used to hold them - SignatureSizes []int `json:"signature-sizes,omitempty"` // List of sizes of each signature slice + imageRef storageReference + directory string // Temporary directory where we store blobs until Commit() time + nextTempFileID int32 // A counter that we use for computing filenames to assign to blobs + manifest []byte // Manifest contents, temporary + signatures []byte // Signature contents, temporary + signatureses map[digest.Digest][]byte // Instance signature contents, temporary + putBlobMutex sync.Mutex // Mutex to sync state for parallel PutBlob executions + blobDiffIDs map[digest.Digest]digest.Digest // Mapping from layer blobsums to their corresponding DiffIDs + fileSizes map[digest.Digest]int64 // Mapping from layer blobsums to their sizes + filenames map[digest.Digest]string // Mapping from layer blobsums to names of files we used to hold them + SignatureSizes []int `json:"signature-sizes,omitempty"` // List of sizes of each signature slice + SignaturesSizes map[digest.Digest][]int `json:"signatures-sizes,omitempty"` // Sizes of each manifest's signature slice } type storageImageCloser struct { @@ -72,26 +73,33 @@ type storageImageCloser struct { } // manifestBigDataKey returns a key suitable for recording a manifest with the specified digest using storage.Store.ImageBigData and related functions. -// If a specific manifest digest is explicitly requested by the user, the key retruned function should be used preferably; +// If a specific manifest digest is explicitly requested by the user, the key returned by this function should be used preferably; // for compatibility, if a manifest is not available under this key, check also storage.ImageDigestBigDataKey func manifestBigDataKey(digest digest.Digest) string { return storage.ImageDigestManifestBigDataNamePrefix + "-" + digest.String() } +// signatureBigDataKey returns a key suitable for recording the signatures associated with the manifest with the specified digest using storage.Store.ImageBigData and related functions. +// If a specific manifest digest is explicitly requested by the user, the key returned by this function should be used preferably; +func signatureBigDataKey(digest digest.Digest) string { + return "signature-" + digest.Encoded() +} + // newImageSource sets up an image for reading. -func newImageSource(imageRef storageReference) (*storageImageSource, error) { +func newImageSource(ctx context.Context, sys *types.SystemContext, imageRef storageReference) (*storageImageSource, error) { // First, locate the image. - img, err := imageRef.resolveImage() + img, err := imageRef.resolveImage(sys) if err != nil { return nil, err } // Build the reader object. image := &storageImageSource{ - imageRef: imageRef, - image: img, - layerPosition: make(map[digest.Digest]int), - SignatureSizes: []int{}, + imageRef: imageRef, + image: img, + layerPosition: make(map[digest.Digest]int), + SignatureSizes: []int{}, + SignaturesSizes: make(map[digest.Digest][]int), } if img.Metadata != "" { if err := json.Unmarshal([]byte(img.Metadata), image); err != nil { @@ -182,7 +190,12 @@ func (s *storageImageSource) getBlobAndLayerID(info types.BlobInfo) (rc io.ReadC // GetManifest() reads the image's manifest. func (s *storageImageSource) GetManifest(ctx context.Context, instanceDigest *digest.Digest) (manifestBlob []byte, MIMEType string, err error) { if instanceDigest != nil { - return nil, "", ErrNoManifestLists + key := manifestBigDataKey(*instanceDigest) + blob, err := s.imageRef.transport.store.ImageBigData(s.image.ID, key) + if err != nil { + return nil, "", errors.Wrapf(err, "error reading manifest for image instance %q", *instanceDigest) + } + return blob, manifest.GuessMIMEType(blob), err } if len(s.cachedManifest) == 0 { // The manifest is stored as a big data item. @@ -214,11 +227,14 @@ func (s *storageImageSource) GetManifest(ctx context.Context, instanceDigest *di // LayerInfosForCopy() returns the list of layer blobs that make up the root filesystem of // the image, after they've been decompressed. -func (s *storageImageSource) LayerInfosForCopy(ctx context.Context) ([]types.BlobInfo, error) { - manifestBlob, manifestType, err := s.GetManifest(ctx, nil) +func (s *storageImageSource) LayerInfosForCopy(ctx context.Context, instanceDigest *digest.Digest) ([]types.BlobInfo, error) { + manifestBlob, manifestType, err := s.GetManifest(ctx, instanceDigest) if err != nil { return nil, errors.Wrapf(err, "error reading image manifest for %q", s.image.ID) } + if manifest.MIMETypeIsMultiImage(manifestType) { + return nil, errors.Errorf("can't copy layers for a manifest list (shouldn't be attempted)") + } man, err := manifest.FromBlob(manifestBlob, manifestType) if err != nil { return nil, errors.Wrapf(err, "error parsing image manifest for %q", s.image.ID) @@ -292,25 +308,33 @@ func buildLayerInfosForCopy(manifestInfos []manifest.LayerInfo, physicalInfos [] // GetSignatures() parses the image's signatures blob into a slice of byte slices. func (s *storageImageSource) GetSignatures(ctx context.Context, instanceDigest *digest.Digest) (signatures [][]byte, err error) { - if instanceDigest != nil { - return nil, ErrNoManifestLists - } var offset int sigslice := [][]byte{} signature := []byte{} - if len(s.SignatureSizes) > 0 { - signatureBlob, err := s.imageRef.transport.store.ImageBigData(s.image.ID, "signatures") + signatureSizes := s.SignatureSizes + key := "signatures" + instance := "default instance" + if instanceDigest != nil { + signatureSizes = s.SignaturesSizes[*instanceDigest] + key = signatureBigDataKey(*instanceDigest) + instance = instanceDigest.Encoded() + } + if len(signatureSizes) > 0 { + signatureBlob, err := s.imageRef.transport.store.ImageBigData(s.image.ID, key) if err != nil { - return nil, errors.Wrapf(err, "error looking up signatures data for image %q", s.image.ID) + return nil, errors.Wrapf(err, "error looking up signatures data for image %q (%s)", s.image.ID, instance) } signature = signatureBlob } - for _, length := range s.SignatureSizes { + for _, length := range signatureSizes { + if offset+length > len(signature) { + return nil, errors.Wrapf(err, "error looking up signatures data for image %q (%s): expected at least %d bytes, only found %d", s.image.ID, instance, len(signature), offset+length) + } sigslice = append(sigslice, signature[offset:offset+length]) offset += length } if offset != len(signature) { - return nil, errors.Errorf("signatures data contained %d extra bytes", len(signatures)-offset) + return nil, errors.Errorf("signatures data (%s) contained %d extra bytes", instance, len(signatures)-offset) } return sigslice, nil } @@ -323,12 +347,14 @@ func newImageDestination(imageRef storageReference) (*storageImageDestination, e return nil, errors.Wrapf(err, "error creating a temporary directory") } image := &storageImageDestination{ - imageRef: imageRef, - directory: directory, - blobDiffIDs: make(map[digest.Digest]digest.Digest), - fileSizes: make(map[digest.Digest]int64), - filenames: make(map[digest.Digest]string), - SignatureSizes: []int{}, + imageRef: imageRef, + directory: directory, + signatureses: make(map[digest.Digest][]byte), + blobDiffIDs: make(map[digest.Digest]digest.Digest), + fileSizes: make(map[digest.Digest]int64), + filenames: make(map[digest.Digest]string), + SignatureSizes: []int{}, + SignaturesSizes: make(map[digest.Digest][]int), } return image, nil } @@ -404,10 +430,10 @@ func (s *storageImageDestination) PutBlob(ctx context.Context, stream io.Reader, } // Ensure that any information that we were given about the blob is correct. if blobinfo.Digest.Validate() == nil && blobinfo.Digest != hasher.Digest() { - return errorBlobInfo, ErrBlobDigestMismatch + return errorBlobInfo, errors.WithStack(ErrBlobDigestMismatch) } if blobinfo.Size >= 0 && blobinfo.Size != counter.Count { - return errorBlobInfo, ErrBlobSizeMismatch + return errorBlobInfo, errors.WithStack(ErrBlobSizeMismatch) } // Record information about the blob. s.putBlobMutex.Lock() @@ -579,7 +605,34 @@ func (s *storageImageDestination) getConfigBlob(info types.BlobInfo) ([]byte, er return nil, errors.New("blob not found") } -func (s *storageImageDestination) Commit(ctx context.Context) error { +func (s *storageImageDestination) Commit(ctx context.Context, unparsedToplevel types.UnparsedImage) error { + if len(s.manifest) == 0 { + return errors.New("Internal error: storageImageDestination.Commit() called without PutManifest()") + } + toplevelManifest, _, err := unparsedToplevel.Manifest(ctx) + if err != nil { + return errors.Wrapf(err, "error retrieving top-level manifest") + } + // If the name we're saving to includes a digest, then check that the + // manifests that we're about to save all either match the one from the + // unparsedToplevel, or match the digest in the name that we're using. + if s.imageRef.named != nil { + if digested, ok := s.imageRef.named.(reference.Digested); ok { + matches, err := manifest.MatchesDigest(s.manifest, digested.Digest()) + if err != nil { + return err + } + if !matches { + matches, err = manifest.MatchesDigest(toplevelManifest, digested.Digest()) + if err != nil { + return err + } + } + if !matches { + return fmt.Errorf("Manifest to be saved does not match expected digest %s", digested.Digest()) + } + } + } // Find the list of layer blobs. if len(s.manifest) == 0 { return errors.New("Internal error: storageImageDestination.Commit() called without PutManifest()") @@ -747,7 +800,8 @@ func (s *storageImageDestination) Commit(ctx context.Context) error { return errors.Wrapf(err, "error saving big data %q for image %q", blob.String(), img.ID) } } - // Set the reference's name on the image. + // Set the reference's name on the image. We don't need to worry about avoiding duplicate + // values because SetNames() will deduplicate the list that we pass to it. if name := s.imageRef.DockerReference(); len(oldNames) > 0 || name != nil { names := []string{} if name != nil { @@ -765,26 +819,43 @@ func (s *storageImageDestination) Commit(ctx context.Context) error { } logrus.Debugf("set names of image %q to %v", img.ID, names) } - // Save the manifest. Allow looking it up by digest by using the key convention defined by the Store. + // Save the unparsedToplevel's manifest. + if len(toplevelManifest) != 0 { + manifestDigest, err := manifest.Digest(toplevelManifest) + if err != nil { + return errors.Wrapf(err, "error digesting top-level manifest") + } + key := manifestBigDataKey(manifestDigest) + if err := s.imageRef.transport.store.SetImageBigData(img.ID, key, toplevelManifest, manifest.Digest); err != nil { + if _, err2 := s.imageRef.transport.store.DeleteImage(img.ID, true); err2 != nil { + logrus.Debugf("error deleting incomplete image %q: %v", img.ID, err2) + } + logrus.Debugf("error saving top-level manifest for image %q: %v", img.ID, err) + return errors.Wrapf(err, "error saving top-level manifest for image %q", img.ID) + } + } + // Save the image's manifest. Allow looking it up by digest by using the key convention defined by the Store. // Record the manifest twice: using a digest-specific key to allow references to that specific digest instance, // and using storage.ImageDigestBigDataKey for future users that don’t specify any digest and for compatibility with older readers. manifestDigest, err := manifest.Digest(s.manifest) if err != nil { return errors.Wrapf(err, "error computing manifest digest") } - if err := s.imageRef.transport.store.SetImageBigData(img.ID, manifestBigDataKey(manifestDigest), s.manifest, manifest.Digest); err != nil { + key := manifestBigDataKey(manifestDigest) + if err := s.imageRef.transport.store.SetImageBigData(img.ID, key, s.manifest, manifest.Digest); err != nil { if _, err2 := s.imageRef.transport.store.DeleteImage(img.ID, true); err2 != nil { logrus.Debugf("error deleting incomplete image %q: %v", img.ID, err2) } logrus.Debugf("error saving manifest for image %q: %v", img.ID, err) - return err + return errors.Wrapf(err, "error saving manifest for image %q", img.ID) } - if err := s.imageRef.transport.store.SetImageBigData(img.ID, storage.ImageDigestBigDataKey, s.manifest, manifest.Digest); err != nil { + key = storage.ImageDigestBigDataKey + if err := s.imageRef.transport.store.SetImageBigData(img.ID, key, s.manifest, manifest.Digest); err != nil { if _, err2 := s.imageRef.transport.store.DeleteImage(img.ID, true); err2 != nil { logrus.Debugf("error deleting incomplete image %q: %v", img.ID, err2) } logrus.Debugf("error saving manifest for image %q: %v", img.ID, err) - return err + return errors.Wrapf(err, "error saving manifest for image %q", img.ID) } // Save the signatures, if we have any. if len(s.signatures) > 0 { @@ -793,7 +864,17 @@ func (s *storageImageDestination) Commit(ctx context.Context) error { logrus.Debugf("error deleting incomplete image %q: %v", img.ID, err2) } logrus.Debugf("error saving signatures for image %q: %v", img.ID, err) - return err + return errors.Wrapf(err, "error saving signatures for image %q", img.ID) + } + } + for instanceDigest, signatures := range s.signatureses { + key := signatureBigDataKey(instanceDigest) + if err := s.imageRef.transport.store.SetImageBigData(img.ID, key, signatures, manifest.Digest); err != nil { + if _, err2 := s.imageRef.transport.store.DeleteImage(img.ID, true); err2 != nil { + logrus.Debugf("error deleting incomplete image %q: %v", img.ID, err2) + } + logrus.Debugf("error saving signatures for image %q: %v", img.ID, err) + return errors.Wrapf(err, "error saving signatures for image %q", img.ID) } } // Save our metadata. @@ -803,7 +884,7 @@ func (s *storageImageDestination) Commit(ctx context.Context) error { logrus.Debugf("error deleting incomplete image %q: %v", img.ID, err2) } logrus.Debugf("error encoding metadata for image %q: %v", img.ID, err) - return err + return errors.Wrapf(err, "error encoding metadata for image %q", img.ID) } if len(metadata) != 0 { if err = s.imageRef.transport.store.SetMetadata(img.ID, string(metadata)); err != nil { @@ -811,7 +892,7 @@ func (s *storageImageDestination) Commit(ctx context.Context) error { logrus.Debugf("error deleting incomplete image %q: %v", img.ID, err2) } logrus.Debugf("error saving metadata for image %q: %v", img.ID, err) - return err + return errors.Wrapf(err, "error saving metadata for image %q", img.ID) } logrus.Debugf("saved image metadata %q", string(metadata)) } @@ -830,21 +911,10 @@ func (s *storageImageDestination) SupportedManifestMIMETypes() []string { } // PutManifest writes the manifest to the destination. -func (s *storageImageDestination) PutManifest(ctx context.Context, manifestBlob []byte) error { - if s.imageRef.named != nil { - if digested, ok := s.imageRef.named.(reference.Digested); ok { - matches, err := manifest.MatchesDigest(manifestBlob, digested.Digest()) - if err != nil { - return err - } - if !matches { - return fmt.Errorf("Manifest does not match expected digest %s", digested.Digest()) - } - } - } - - s.manifest = make([]byte, len(manifestBlob)) - copy(s.manifest, manifestBlob) +func (s *storageImageDestination) PutManifest(ctx context.Context, manifestBlob []byte, instanceDigest *digest.Digest) error { + newBlob := make([]byte, len(manifestBlob)) + copy(newBlob, manifestBlob) + s.manifest = newBlob return nil } @@ -873,7 +943,7 @@ func (s *storageImageDestination) IgnoresEmbeddedDockerReference() bool { } // PutSignatures records the image's signatures for committing as a single data blob. -func (s *storageImageDestination) PutSignatures(ctx context.Context, signatures [][]byte) error { +func (s *storageImageDestination) PutSignatures(ctx context.Context, signatures [][]byte, instanceDigest *digest.Digest) error { sizes := []int{} sigblob := []byte{} for _, sig := range signatures { @@ -883,8 +953,21 @@ func (s *storageImageDestination) PutSignatures(ctx context.Context, signatures copy(newblob[len(sigblob):], sig) sigblob = newblob } - s.signatures = sigblob - s.SignatureSizes = sizes + if instanceDigest == nil { + s.signatures = sigblob + s.SignatureSizes = sizes + } + if instanceDigest == nil && len(s.manifest) > 0 { + manifestDigest, err := manifest.Digest(s.manifest) + if err != nil { + return err + } + instanceDigest = &manifestDigest + } + if instanceDigest != nil { + s.signatureses[*instanceDigest] = sigblob + s.SignaturesSizes[*instanceDigest] = sizes + } return nil } @@ -940,7 +1023,7 @@ func (s *storageImageCloser) Size() (int64, error) { // newImage creates an image that also knows its size func newImage(ctx context.Context, sys *types.SystemContext, s storageReference) (types.ImageCloser, error) { - src, err := newImageSource(s) + src, err := newImageSource(ctx, sys, s) if err != nil { return nil, err } diff --git a/storage/storage_reference.go b/storage/storage_reference.go index 7ad20817b8..335fb5d950 100644 --- a/storage/storage_reference.go +++ b/storage/storage_reference.go @@ -7,8 +7,11 @@ import ( "strings" "github.com/containers/image/v4/docker/reference" + "github.com/containers/image/v4/manifest" "github.com/containers/image/v4/types" "github.com/containers/storage" + digest "github.com/opencontainers/go-digest" + imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -51,9 +54,79 @@ func imageMatchesRepo(image *storage.Image, ref reference.Named) bool { return false } +// imageMatchesSystemContext checks if the passed-in image both contains a +// manifest that matches the passed-in digest, and identifies itself as being +// appropriate for running on the system that matches sys. +// If we somehow ended up sharing the same storage among multiple types of +// systems, and managed to download multiple images from the same manifest +// list, their image records will all contain copies of the manifest list, and +// this check will help us decide which of them we want to return when we've +// been asked to resolve an image reference that uses the list's digest to a +// specific image ID. +func imageMatchesSystemContext(store storage.Store, img *storage.Image, manifestDigest digest.Digest, sys *types.SystemContext) bool { + // First, check if the image record has a manifest that matches the + // specified digest. + key := manifestBigDataKey(manifestDigest) + manifestBytes, err := store.ImageBigData(img.ID, key) + if err != nil { + return false + } + // The manifest is either a list, or not a list. If it's a list, find + // the digest of the instance that matches the current system, and try + // to load that manifest from the image record, and use it. + manifestType := manifest.GuessMIMEType(manifestBytes) + if manifest.MIMETypeIsMultiImage(manifestType) { + list, err := manifest.ListFromBlob(manifestBytes, manifestType) + if err != nil { + return false + } + manifestDigest, err = list.ChooseInstance(sys) + if err != nil { + return false + } + key = manifestBigDataKey(manifestDigest) + manifestBytes, err = store.ImageBigData(img.ID, key) + if err != nil { + return false + } + manifestType = manifest.GuessMIMEType(manifestBytes) + } + // Load the image's configuration blob. + m, err := manifest.FromBlob(manifestBytes, manifestType) + getConfig := func(blobInfo types.BlobInfo) ([]byte, error) { + return store.ImageBigData(img.ID, blobInfo.Digest.String()) + } + ii, err := m.Inspect(getConfig) + if err != nil { + return false + } + // Build a dummy index containing one instance and information about + // the image's target system from the image's configuration. + index := manifest.OCI1IndexFromComponents([]imgspecv1.Descriptor{{ + MediaType: imgspecv1.MediaTypeImageManifest, + Digest: manifestDigest, + Size: int64(len(manifestBytes)), + Platform: &imgspecv1.Platform{ + OS: ii.Os, + Architecture: ii.Architecture, + }, + }}, nil) + // Check that ChooseInstance() would select this image for this system, + // from a list of images. + instanceDigest, err := index.ChooseInstance(sys) + if err != nil { + return false + } + // Double-check that we can read the runnable image's manifest from the + // image record. + key = manifestBigDataKey(instanceDigest) + _, err = store.ImageBigData(img.ID, key) + return err == nil +} + // Resolve the reference's name to an image ID in the store, if there's already // one present with the same name or ID, and return the image. -func (s *storageReference) resolveImage() (*storage.Image, error) { +func (s *storageReference) resolveImage(sys *types.SystemContext) (*storage.Image, error) { var loadedImage *storage.Image if s.id == "" && s.named != nil { // Look for an image that has the expanded reference name as an explicit Name value. @@ -72,9 +145,10 @@ func (s *storageReference) resolveImage() (*storage.Image, error) { if err == nil && len(images) > 0 { for _, image := range images { if imageMatchesRepo(image, s.named) { - loadedImage = image - s.id = image.ID - break + if loadedImage == nil || imageMatchesSystemContext(s.transport.store, image, digested.Digest(), sys) { + loadedImage = image + s.id = image.ID + } } } } @@ -202,7 +276,7 @@ func (s storageReference) NewImage(ctx context.Context, sys *types.SystemContext } func (s storageReference) DeleteImage(ctx context.Context, sys *types.SystemContext) error { - img, err := s.resolveImage() + img, err := s.resolveImage(sys) if err != nil { return err } @@ -217,7 +291,7 @@ func (s storageReference) DeleteImage(ctx context.Context, sys *types.SystemCont } func (s storageReference) NewImageSource(ctx context.Context, sys *types.SystemContext) (types.ImageSource, error) { - return newImageSource(s) + return newImageSource(ctx, sys, s) } func (s storageReference) NewImageDestination(ctx context.Context, sys *types.SystemContext) (types.ImageDestination, error) { diff --git a/storage/storage_test.go b/storage/storage_test.go index 0cf4b51680..e9aaba7437 100644 --- a/storage/storage_test.go +++ b/storage/storage_test.go @@ -14,10 +14,12 @@ import ( "io/ioutil" "os" "path/filepath" + "reflect" "strings" "testing" "time" + imanifest "github.com/containers/image/v4/manifest" "github.com/containers/image/v4/pkg/blobinfocache/memory" "github.com/containers/image/v4/types" "github.com/containers/storage" @@ -271,6 +273,10 @@ func makeLayer(t *testing.T, compression archive.Compression) (ddigest.Digest, i if n != len(buf) { t.Fatalf("Short write writing tar header: %d < %d", n, len(buf)) } + err = twriter.Flush() + if err != nil { + t.Fatalf("Error flushing output to tar archive: %v", err) + } }() _, err = io.Copy(&tbuffer, preader) if err != nil { @@ -394,13 +400,19 @@ func TestWriteRead(t *testing.T) { manifest = strings.Replace(manifest, "%li", li, -1) manifest = strings.Replace(manifest, "%ci", sum.Hex(), -1) t.Logf("this manifest is %q", manifest) - if err := dest.PutManifest(context.Background(), []byte(manifest)); err != nil { + if err := dest.PutManifest(context.Background(), []byte(manifest), nil); err != nil { t.Fatalf("Error saving manifest to destination: %v", err) } - if err := dest.PutSignatures(context.Background(), signatures); err != nil { + if err := dest.PutSignatures(context.Background(), signatures, nil); err != nil { t.Fatalf("Error saving signatures to destination: %v", err) } - if err := dest.Commit(context.Background()); err != nil { + unparsedToplevel := unparsedImage{ + imageReference: nil, + manifestBytes: []byte(manifest), + manifestType: imanifest.GuessMIMEType([]byte(manifest)), + signatures: signatures, + } + if err := dest.Commit(context.Background(), &unparsedToplevel); err != nil { t.Fatalf("Error committing changes to destination: %v", err) } dest.Close() @@ -454,10 +466,16 @@ func TestWriteRead(t *testing.T) { t.Fatalf("GetManifest(%q) returned error %v", ref.StringWithinTransport(), err) } t.Logf("this manifest's type appears to be %q", manifestType) - sum = ddigest.SHA256.FromBytes([]byte(manifest)) - _, _, err = src.GetManifest(context.Background(), &sum) - if err == nil { - t.Fatalf("GetManifest(%q) with an instanceDigest is supposed to fail", ref.StringWithinTransport()) + sum, err = imanifest.Digest([]byte(manifest)) + if err != nil { + t.Fatalf("manifest.Digest() returned error %v", err) + } + retrieved, _, err := src.GetManifest(context.Background(), &sum) + if err != nil { + t.Fatalf("GetManifest(%q) with an instanceDigest is supposed to succeed", ref.StringWithinTransport()) + } + if string(retrieved) != string(manifest) { + t.Fatalf("GetManifest(%q) with an instanceDigest retrieved a different manifest", ref.StringWithinTransport()) } sigs, err := src.GetSignatures(context.Background(), nil) if err != nil { @@ -474,9 +492,12 @@ func TestWriteRead(t *testing.T) { t.Fatalf("Signature %d was corrupted", i) } } - _, err = src.GetSignatures(context.Background(), &sum) - if err == nil { - t.Fatalf("GetSignatures(%q) with instanceDigest is supposed to fail", ref.StringWithinTransport()) + sigs2, err := src.GetSignatures(context.Background(), &sum) + if err != nil { + t.Fatalf("GetSignatures(%q) with instance %s returned error %v", ref.StringWithinTransport(), sum.String(), err) + } + if !reflect.DeepEqual(sigs, sigs2) { + t.Fatalf("GetSignatures(%q) with instance %s returned a different result", ref.StringWithinTransport(), sum.String()) } for _, layerInfo := range layerInfos { buf := bytes.Buffer{} @@ -493,6 +514,7 @@ func TestWriteRead(t *testing.T) { t.Fatalf("Error decompressing layer %q from %q", layerInfo.Digest, ref.StringWithinTransport()) } n, err := io.Copy(&buf, decompressed) + layer.Close() if layerInfo.Size >= 0 && compressed.Count != layerInfo.Size { t.Fatalf("Blob size is different than expected: %d != %d, read %d", compressed.Count, layerInfo.Size, n) } @@ -556,10 +578,16 @@ func TestDuplicateName(t *testing.T) { ] } `, digest, size) - if err := dest.PutManifest(context.Background(), []byte(manifest)); err != nil { + if err := dest.PutManifest(context.Background(), []byte(manifest), nil); err != nil { t.Fatalf("Error storing manifest to destination: %v", err) } - if err := dest.Commit(context.Background()); err != nil { + unparsedToplevel := unparsedImage{ + imageReference: nil, + manifestBytes: []byte(manifest), + manifestType: imanifest.GuessMIMEType([]byte(manifest)), + signatures: nil, + } + if err := dest.Commit(context.Background(), &unparsedToplevel); err != nil { t.Fatalf("Error committing changes to destination, first pass: %v", err) } dest.Close() @@ -591,10 +619,16 @@ func TestDuplicateName(t *testing.T) { ] } `, digest, size) - if err := dest.PutManifest(context.Background(), []byte(manifest)); err != nil { + if err := dest.PutManifest(context.Background(), []byte(manifest), nil); err != nil { t.Fatalf("Error storing manifest to destination: %v", err) } - if err := dest.Commit(context.Background()); err != nil { + unparsedToplevel = unparsedImage{ + imageReference: nil, + manifestBytes: []byte(manifest), + manifestType: imanifest.GuessMIMEType([]byte(manifest)), + signatures: nil, + } + if err := dest.Commit(context.Background(), &unparsedToplevel); err != nil { t.Fatalf("Error committing changes to destination, second pass: %v", err) } dest.Close() @@ -643,10 +677,16 @@ func TestDuplicateID(t *testing.T) { ] } `, digest, size) - if err := dest.PutManifest(context.Background(), []byte(manifest)); err != nil { + if err := dest.PutManifest(context.Background(), []byte(manifest), nil); err != nil { t.Fatalf("Error storing manifest to destination: %v", err) } - if err := dest.Commit(context.Background()); err != nil { + unparsedToplevel := unparsedImage{ + imageReference: nil, + manifestBytes: []byte(manifest), + manifestType: imanifest.GuessMIMEType([]byte(manifest)), + signatures: nil, + } + if err := dest.Commit(context.Background(), &unparsedToplevel); err != nil { t.Fatalf("Error committing changes to destination, first pass: %v", err) } dest.Close() @@ -678,10 +718,16 @@ func TestDuplicateID(t *testing.T) { ] } `, digest, size) - if err := dest.PutManifest(context.Background(), []byte(manifest)); err != nil { + if err := dest.PutManifest(context.Background(), []byte(manifest), nil); err != nil { t.Fatalf("Error storing manifest to destination: %v", err) } - if err := dest.Commit(context.Background()); errors.Cause(err) != storage.ErrDuplicateID { + unparsedToplevel = unparsedImage{ + imageReference: nil, + manifestBytes: []byte(manifest), + manifestType: imanifest.GuessMIMEType([]byte(manifest)), + signatures: nil, + } + if err := dest.Commit(context.Background(), &unparsedToplevel); errors.Cause(err) != storage.ErrDuplicateID { if err != nil { t.Fatalf("Wrong error committing changes to destination, second pass: %v", err) } @@ -733,10 +779,16 @@ func TestDuplicateNameID(t *testing.T) { ] } `, digest, size) - if err := dest.PutManifest(context.Background(), []byte(manifest)); err != nil { + if err := dest.PutManifest(context.Background(), []byte(manifest), nil); err != nil { t.Fatalf("Error storing manifest to destination: %v", err) } - if err := dest.Commit(context.Background()); err != nil { + unparsedToplevel := unparsedImage{ + imageReference: nil, + manifestBytes: []byte(manifest), + manifestType: imanifest.GuessMIMEType([]byte(manifest)), + signatures: nil, + } + if err := dest.Commit(context.Background(), &unparsedToplevel); err != nil { t.Fatalf("Error committing changes to destination, first pass: %v", err) } dest.Close() @@ -768,10 +820,16 @@ func TestDuplicateNameID(t *testing.T) { ] } `, digest, size) - if err := dest.PutManifest(context.Background(), []byte(manifest)); err != nil { + if err := dest.PutManifest(context.Background(), []byte(manifest), nil); err != nil { t.Fatalf("Error storing manifest to destination: %v", err) } - if err := dest.Commit(context.Background()); errors.Cause(err) != storage.ErrDuplicateID { + unparsedToplevel = unparsedImage{ + imageReference: nil, + manifestBytes: []byte(manifest), + manifestType: imanifest.GuessMIMEType([]byte(manifest)), + signatures: nil, + } + if err := dest.Commit(context.Background(), &unparsedToplevel); errors.Cause(err) != storage.ErrDuplicateID { if err != nil { t.Fatalf("Wrong error committing changes to destination, second pass: %v", err) } @@ -889,10 +947,16 @@ func TestSize(t *testing.T) { ] } `, configInfo.Size, configInfo.Digest, digest1, size1, digest2, size2) - if err := dest.PutManifest(context.Background(), []byte(manifest)); err != nil { + if err := dest.PutManifest(context.Background(), []byte(manifest), nil); err != nil { t.Fatalf("Error storing manifest to destination: %v", err) } - if err := dest.Commit(context.Background()); err != nil { + unparsedToplevel := unparsedImage{ + imageReference: nil, + manifestBytes: []byte(manifest), + manifestType: imanifest.GuessMIMEType([]byte(manifest)), + signatures: nil, + } + if err := dest.Commit(context.Background(), &unparsedToplevel); err != nil { t.Fatalf("Error committing changes to destination: %v", err) } dest.Close() @@ -905,8 +969,8 @@ func TestSize(t *testing.T) { if usize == -1 || err != nil { t.Fatalf("Error calculating image size: %v", err) } - if int(usize) != len(config)+int(usize1)+int(usize2)+len(manifest) { - t.Fatalf("Unexpected image size: %d != %d + %d + %d + %d", usize, len(config), usize1, usize2, len(manifest)) + if int(usize) != len(config)+int(usize1)+int(usize2)+2*len(manifest) { + t.Fatalf("Unexpected image size: %d != %d + %d + %d + %d (%d)", usize, len(config), usize1, usize2, len(manifest), len(config)+int(usize1)+int(usize2)+2*len(manifest)) } img.Close() } @@ -1000,10 +1064,16 @@ func TestDuplicateBlob(t *testing.T) { ] } `, configInfo.Size, configInfo.Digest, digest1, size1, digest2, size2, digest1, size1, digest2, size2) - if err := dest.PutManifest(context.Background(), []byte(manifest)); err != nil { + if err := dest.PutManifest(context.Background(), []byte(manifest), nil); err != nil { t.Fatalf("Error storing manifest to destination: %v", err) } - if err := dest.Commit(context.Background()); err != nil { + unparsedToplevel := unparsedImage{ + imageReference: nil, + manifestBytes: []byte(manifest), + manifestType: imanifest.GuessMIMEType([]byte(manifest)), + signatures: nil, + } + if err := dest.Commit(context.Background(), &unparsedToplevel); err != nil { t.Fatalf("Error committing changes to destination: %v", err) } dest.Close() @@ -1047,3 +1117,20 @@ func TestDuplicateBlob(t *testing.T) { src.Close() img.Close() } + +type unparsedImage struct { + imageReference types.ImageReference + manifestBytes []byte + manifestType string + signatures [][]byte +} + +func (u *unparsedImage) Reference() types.ImageReference { + return u.imageReference +} +func (u *unparsedImage) Manifest(context.Context) ([]byte, string, error) { + return u.manifestBytes, u.manifestType, nil +} +func (u *unparsedImage) Signatures(context.Context) ([][]byte, error) { + return u.signatures, nil +} diff --git a/storage/storage_transport.go b/storage/storage_transport.go index 48b909c035..3f87d0c275 100644 --- a/storage/storage_transport.go +++ b/storage/storage_transport.go @@ -288,7 +288,7 @@ func (s storageTransport) GetStoreImage(store storage.Store, ref types.ImageRefe } if sref, ok := ref.(*storageReference); ok { tmpRef := *sref - if img, err := tmpRef.resolveImage(); err == nil { + if img, err := tmpRef.resolveImage(&types.SystemContext{}); err == nil { return img, nil } } diff --git a/tarball/tarball_src.go b/tarball/tarball_src.go index ead1a50bd3..f5c3e958ab 100644 --- a/tarball/tarball_src.go +++ b/tarball/tarball_src.go @@ -248,9 +248,8 @@ func (is *tarballImageSource) GetManifest(ctx context.Context, instanceDigest *d } // GetSignatures returns the image's signatures. It may use a remote (= slow) service. -// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve signatures for -// (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list -// (e.g. if the source never returns manifest lists). +// This source implementation does not support manifest lists, so the passed-in instanceDigest should always be nil, +// as there can be no secondary manifests. func (*tarballImageSource) GetSignatures(ctx context.Context, instanceDigest *digest.Digest) ([][]byte, error) { if instanceDigest != nil { return nil, fmt.Errorf("manifest lists are not supported by the %q transport", transportName) @@ -262,7 +261,14 @@ func (is *tarballImageSource) Reference() types.ImageReference { return &is.reference } -// LayerInfosForCopy() returns updated layer info that should be used when reading, in preference to values in the manifest, if specified. -func (*tarballImageSource) LayerInfosForCopy(ctx context.Context) ([]types.BlobInfo, error) { +// LayerInfosForCopy returns either nil (meaning the values in the manifest are fine), or updated values for the layer +// blobsums that are listed in the image's manifest. If values are returned, they should be used when using GetBlob() +// to read the image's layers. +// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve BlobInfos for +// (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list +// (e.g. if the source never returns manifest lists). +// The Digest field is guaranteed to be provided; Size may be -1. +// WARNING: The list may contain duplicates, and they are semantically relevant. +func (*tarballImageSource) LayerInfosForCopy(context.Context, *digest.Digest) ([]types.BlobInfo, error) { return nil, nil } diff --git a/types/types.go b/types/types.go index edb1cf66df..0be3a3a70f 100644 --- a/types/types.go +++ b/types/types.go @@ -227,10 +227,15 @@ type ImageSource interface { // (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list // (e.g. if the source never returns manifest lists). GetSignatures(ctx context.Context, instanceDigest *digest.Digest) ([][]byte, error) - // LayerInfosForCopy returns either nil (meaning the values in the manifest are fine), or updated values for the layer blobsums that are listed in the image's manifest. + // LayerInfosForCopy returns either nil (meaning the values in the manifest are fine), or updated values for the layer + // blobsums that are listed in the image's manifest. If values are returned, they should be used when using GetBlob() + // to read the image's layers. + // If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve BlobInfos for + // (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list + // (e.g. if the source never returns manifest lists). // The Digest field is guaranteed to be provided; Size may be -1. // WARNING: The list may contain duplicates, and they are semantically relevant. - LayerInfosForCopy(ctx context.Context) ([]BlobInfo, error) + LayerInfosForCopy(ctx context.Context, instanceDigest *digest.Digest) ([]BlobInfo, error) } // ImageDestination is a service, possibly remote (= slow), to store components of a single image. @@ -286,16 +291,24 @@ type ImageDestination interface { // May use and/or update cache. TryReusingBlob(ctx context.Context, info BlobInfo, cache BlobInfoCache, canSubstitute bool) (bool, BlobInfo, error) // PutManifest writes manifest to the destination. + // If instanceDigest is not nil, it contains a digest of the specific manifest instance to write the manifest for + // (when the primary manifest is a manifest list); this should always be nil if the primary manifest is not a manifest list. + // It is expected but not enforced that the instanceDigest, when specified, matches the digest of `manifest` as generated + // by `manifest.Digest()`. // FIXME? This should also receive a MIME type if known, to differentiate between schema versions. // If the destination is in principle available, refuses this manifest type (e.g. it does not recognize the schema), // but may accept a different manifest type, the returned error must be an ManifestTypeRejectedError. - PutManifest(ctx context.Context, manifest []byte) error - PutSignatures(ctx context.Context, signatures [][]byte) error + PutManifest(ctx context.Context, manifest []byte, instanceDigest *digest.Digest) error + // PutSignatures writes a set of signatures to the destination. + // If instanceDigest is not nil, it contains a digest of the specific manifest instance to write or overwrite the signatures for + // (when the primary manifest is a manifest list); this should always be nil if the primary manifest is not a manifest list. + // MUST be called after PutManifest (signatures may reference manifest contents). + PutSignatures(ctx context.Context, signatures [][]byte, instanceDigest *digest.Digest) error // Commit marks the process of storing the image as successful and asks for the image to be persisted. // WARNING: This does not have any transactional semantics: // - Uploaded data MAY be visible to others before Commit() is called // - Uploaded data MAY be removed or MAY remain around if Close() is called without Commit() (i.e. rollback is allowed but not guaranteed) - Commit(ctx context.Context) error + Commit(ctx context.Context, unparsedToplevel UnparsedImage) error } // ManifestTypeRejectedError is returned by ImageDestination.PutManifest if the destination is in principle available,