Skip to content

Commit

Permalink
Fix handling of references with digests
Browse files Browse the repository at this point in the history 
References can contain digests or tags, but not both, so fix our parser
to handle either type of name for use as image names.

Update PolicyConfigurationNamespaces() to discard tags and digests when
computing the list of alternate namespaces which can match an image.

Drop digests from a storageImageSource's copies of references, in case
we're reading and writing images using references which contain digests,
since schema1 images tend to fail verification of those.

Signed-off-by: Nalin Dahyabhai <[email protected]>
  • Loading branch information
@nalind
nalind committed Jul 31, 2017
1 parent 032b94b commit 3fbfcb4
Show file tree
Hide file tree
Showing 6 changed files with 245 additions and 110 deletions.
138 changes: 83 additions & 55 deletions storage/storage_image.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"sync/atomic"

"github.com/Sirupsen/logrus"
"github.com/containers/image/docker/reference"
"github.com/containers/image/image"
"github.com/containers/image/manifest"
"github.com/containers/image/types"
Expand Down Expand Up @@ -55,29 +56,36 @@ type storageImageDestination struct {
SignatureSizes []int `json:"signature-sizes"` // List of sizes of each signature slice
}

type storageImageUnnamedDestination struct {
named *storageImageDestination
}

type storageImage struct {
types.Image
reader *storageImageReader
size int64
imageRef storageReference
reader *storageUnnamedImageReader
size int64
}

type storageImageReader struct {
type storageUnnamedImageReader struct {
ID string
imageRef storageReference
transport storageTransport
layerPosition map[digest.Digest]int // Where we are in reading a blob's layers
SignatureSizes []int `json:"signature-sizes"` // List of sizes of each signature slice
}

// newImageReader sets us up to read out an image without making any changes to what we read before
// handing it back to the caller.
func newImageReader(imageRef storageReference) (*storageImageReader, error) {
// newUnnamedImageReader sets us up to read out an image without making any changes to what we read before
// handing it back to the caller, without caring about the image's name.
func newUnnamedImageReader(imageRef storageReference) (*storageUnnamedImageReader, error) {
// First, locate the image using the original reference.
img, err := imageRef.resolveImage()
if err != nil {
return nil, err
}
image := &storageImageReader{
// Build the reader object, reading the image referred to by a possibly-updated reference.
image := &storageUnnamedImageReader{
ID: img.ID,
imageRef: imageRef,
transport: imageRef.transport,
layerPosition: make(map[digest.Digest]int),
SignatureSizes: []int{},
}
Expand All @@ -87,25 +95,26 @@ func newImageReader(imageRef storageReference) (*storageImageReader, error) {
return image, nil
}

// Reference returns the image reference that we used to find this image.
func (s storageImageReader) Reference() types.ImageReference {
return s.imageRef
// Reference returns the image reference that we used to find this image. Since we're officially
// unnamed, return a reference with no naming information.
func (s storageUnnamedImageReader) Reference() types.ImageReference {
return newReference(s.transport, "", s.ID, nil, "", "")
}

// Close cleans up any resources we tied up while reading the image.
func (s storageImageReader) Close() error {
func (s storageUnnamedImageReader) Close() error {
return nil
}

// GetBlob reads the data blob or filesystem layer which matches the digest and size, if given.
func (s *storageImageReader) GetBlob(info types.BlobInfo) (rc io.ReadCloser, n int64, err error) {
func (s *storageUnnamedImageReader) GetBlob(info types.BlobInfo) (rc io.ReadCloser, n int64, err error) {
rc, n, _, err = s.getBlobAndLayerID(info, false)
return rc, n, err
}

// GetBlob reads the data blob or filesystem layer which matches the digest and size, if given, and returns
// the layer ID for the layer, if it was a layer.
func (s *storageImageReader) getBlobAndLayerID(info types.BlobInfo, decompress bool) (rc io.ReadCloser, n int64, layerID string, err error) {
func (s *storageUnnamedImageReader) getBlobAndLayerID(info types.BlobInfo, decompress bool) (rc io.ReadCloser, n int64, layerID string, err error) {
var layer storage.Layer
var diffOptions *storage.DiffOptions
// We need a valid digest value.
Expand All @@ -115,11 +124,11 @@ func (s *storageImageReader) getBlobAndLayerID(info types.BlobInfo, decompress b
}
// Check if the blob corresponds to a diff that was used to initialize any layers, either
// before or after decompression, since we don't care.
uncompressedLayers, err := s.imageRef.transport.store.LayersByUncompressedDigest(info.Digest)
compressedLayers, err := s.imageRef.transport.store.LayersByCompressedDigest(info.Digest)
uncompressedLayers, err := s.transport.store.LayersByUncompressedDigest(info.Digest)
compressedLayers, err := s.transport.store.LayersByCompressedDigest(info.Digest)
// If it's not a layer, then it must be a data item.
if len(uncompressedLayers) == 0 && len(compressedLayers) == 0 {
b, err := s.imageRef.transport.store.ImageBigData(s.ID, info.Digest.String())
b, err := s.transport.store.ImageBigData(s.ID, info.Digest.String())
if err != nil {
return nil, -1, "", err
}
Expand Down Expand Up @@ -158,29 +167,29 @@ func (s *storageImageReader) getBlobAndLayerID(info types.BlobInfo, decompress b
}
logrus.Debugf("exporting filesystem layer %q with default compression (%v) for blob %q", layer.CompressionType, layerID, info.Digest)
}
rc, err = s.imageRef.transport.store.Diff("", layerID, diffOptions)
rc, err = s.transport.store.Diff("", layerID, diffOptions)
if err != nil {
return nil, -1, "", err
}
return rc, n, layerID, err
}

// GetManifest() reads the image's manifest.
func (s *storageImageReader) GetManifest() (manifestBlob []byte, MIMEType string, err error) {
manifestBlob, err = s.imageRef.transport.store.ImageBigData(s.ID, "manifest")
func (s *storageUnnamedImageReader) GetManifest() (manifestBlob []byte, MIMEType string, err error) {
manifestBlob, err = s.transport.store.ImageBigData(s.ID, "manifest")
return manifestBlob, manifest.GuessMIMEType(manifestBlob), err
}

// GetTargetManifest() is not supported.
func (s *storageImageReader) GetTargetManifest(d digest.Digest) (manifestBlob []byte, MIMEType string, err error) {
func (s *storageUnnamedImageReader) GetTargetManifest(d digest.Digest) (manifestBlob []byte, MIMEType string, err error) {
return nil, "", ErrNoManifestLists
}

// GetSignatures() parses the image's signatures blob into a slice of byte slices.
func (s *storageImageReader) GetSignatures() (signatures [][]byte, err error) {
func (s *storageUnnamedImageReader) GetSignatures() (signatures [][]byte, err error) {
var offset int
sigslice := [][]byte{}
signature, err := s.imageRef.transport.store.ImageBigData(s.ID, "signatures")
signature, err := s.transport.store.ImageBigData(s.ID, "signatures")
if err != nil {
if !os.IsNotExist(errors.Cause(err)) {
logrus.Debugf("got error %v looking up signatures for image %q", err, s.ID)
Expand All @@ -199,15 +208,15 @@ func (s *storageImageReader) GetSignatures() (signatures [][]byte, err error) {

// getSize() adds up the sizes of the image's data blobs (which includes the configuration blob), the
// signatures, and the uncompressed sizes of all of the image's layers.
func (s *storageImageReader) getSize() (int64, error) {
func (s *storageUnnamedImageReader) getSize() (int64, error) {
var sum int64
// Size up the data blobs.
dataNames, err := s.imageRef.transport.store.ListImageBigData(s.ID)
dataNames, err := s.transport.store.ListImageBigData(s.ID)
if err != nil {
return -1, errors.Wrapf(err, "error reading image %q", s.ID)
}
for _, dataName := range dataNames {
bigSize, err := s.imageRef.transport.store.ImageBigDataSize(s.ID, dataName)
bigSize, err := s.transport.store.ImageBigDataSize(s.ID, dataName)
if err != nil {
return -1, errors.Wrapf(err, "error reading data blob size %q for %q", dataName, s.ID)
}
Expand All @@ -218,14 +227,14 @@ func (s *storageImageReader) getSize() (int64, error) {
sum += int64(sigSize)
}
// Prepare to walk the layer list.
img, err := s.imageRef.transport.store.Image(s.ID)
img, err := s.transport.store.Image(s.ID)
if err != nil {
return -1, errors.Wrapf(err, "error reading image info %q", s.ID)
}
// Walk the layer list.
layerID := img.TopLayer
for layerID != "" {
layer, err := s.imageRef.transport.store.Layer(layerID)
layer, err := s.transport.store.Layer(layerID)
if err != nil {
return -1, err
}
Expand All @@ -244,7 +253,7 @@ func (s *storageImageReader) getSize() (int64, error) {
// newImage creates an image that knows its size and always refers to its layer blobs using
// uncompressed digests and sizes
func newImage(s storageReference) (*storageImage, error) {
reader, err := newImageReader(s)
reader, err := newUnnamedImageReader(s)
if err != nil {
return nil, err
}
Expand All @@ -254,15 +263,15 @@ func newImage(s storageReference) (*storageImage, error) {
return nil, err
}
// Build the updated information that we want for the manifest.
simg, err := reader.imageRef.transport.store.Image(reader.ID)
simg, err := reader.transport.store.Image(reader.ID)
if err != nil {
return nil, err
}
updatedBlobInfos := []types.BlobInfo{}
diffIDs := []digest.Digest{}
layerID := simg.TopLayer
for layerID != "" {
layer, err := reader.imageRef.transport.store.Layer(layerID)
layer, err := reader.transport.store.Layer(layerID)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -298,7 +307,14 @@ func newImage(s storageReference) (*storageImage, error) {
if err != nil {
return nil, err
}
return &storageImage{Image: updated, reader: reader, size: size}, nil
// Discard a digest in the image reference, if there is one, since the updated image manifest
// won't match it.
newName := s.name
if s.digest != "" {
newName = reference.TrimNamed(newName)
}
newRef := newReference(reader.transport, verboseName(newName), reader.ID, newName, "", "")
return &storageImage{Image: updated, imageRef: *newRef, reader: reader, size: size}, nil
}

// Size returns the image's previously-computed size.
Expand Down Expand Up @@ -338,9 +354,9 @@ func (s *storageImageSource) GetSignatures() ([][]byte, error) {
return s.image.reader.GetSignatures()
}

// Reference returns the image reference that we used to find this image.
// Reference returns a trimmed copy of the image reference that we used to find this image.
func (s storageImageSource) Reference() types.ImageReference {
return s.image.reader.imageRef
return s.image.imageRef
}

// Close cleans up any resources we tied up while reading the image.
Expand Down Expand Up @@ -505,15 +521,15 @@ func (s *storageImageDestination) ReapplyBlob(blobinfo types.BlobInfo) (types.Bl

// GetBlob() shouldn't really be called, but include an implementation in case other parts of the library
// start needing it.
func (s *storageImageDestination) GetBlob(blobinfo types.BlobInfo) (rc io.ReadCloser, n int64, err error) {
func (s *storageImageUnnamedDestination) GetBlob(blobinfo types.BlobInfo) (rc io.ReadCloser, n int64, err error) {
if blobinfo.Digest == "" {
return nil, -1, errors.Errorf(`can't read a blob with unknown digest`)
}
if err := blobinfo.Digest.Validate(); err != nil {
return nil, -1, errors.Wrapf(err, `can't check for a blob with invalid digest`)
}
// Check if we've already cached the blob as a file.
if filename, ok := s.filenames[blobinfo.Digest]; ok {
if filename, ok := s.named.filenames[blobinfo.Digest]; ok {
f, err := os.Open(filename)
if err != nil {
return nil, -1, errors.Wrapf(err, `can't read file %q`, filename)
Expand All @@ -522,22 +538,22 @@ func (s *storageImageDestination) GetBlob(blobinfo types.BlobInfo) (rc io.ReadCl
}
// Check if we have a wasn't-compressed layer in storage that's based on that blob. If we have one,
// start reading it.
layers, err := s.imageRef.transport.store.LayersByUncompressedDigest(blobinfo.Digest)
layers, err := s.named.imageRef.transport.store.LayersByUncompressedDigest(blobinfo.Digest)
if err != nil {
return nil, -1, errors.Wrapf(err, `error looking for layers with digest %q`, blobinfo.Digest)
}
if len(layers) > 0 {
rc, err := s.imageRef.transport.store.Diff("", layers[0].ID, nil)
rc, err := s.named.imageRef.transport.store.Diff("", layers[0].ID, nil)
return rc, -1, err
}
// Check if we have a was-compressed layer in storage that's based on that blob. If we have one,
// start reading it.
layers, err = s.imageRef.transport.store.LayersByCompressedDigest(blobinfo.Digest)
layers, err = s.named.imageRef.transport.store.LayersByCompressedDigest(blobinfo.Digest)
if err != nil {
return nil, -1, errors.Wrapf(err, `error looking for compressed layers with digest %q`, blobinfo.Digest)
}
if len(layers) > 0 {
rc, err := s.imageRef.transport.store.Diff("", layers[0].ID, nil)
rc, err := s.named.imageRef.transport.store.Diff("", layers[0].ID, nil)
return rc, -1, err
}
// Nope, we don't have it.
Expand All @@ -549,7 +565,7 @@ func (s *storageImageDestination) Commit() error {
// parse the manifest to get a list of which blobs are filesystem layers, leaving any cached
// files that aren't filesystem layers to be saved as data items.
if s.image == nil {
img, err := image.FromSource(s)
img, err := image.FromSource(&storageImageUnnamedDestination{named: s})
if err != nil {
return errors.Wrapf(err, "error locating manifest for layer blob list")
}
Expand Down Expand Up @@ -707,7 +723,7 @@ func (s *storageImageDestination) Commit() error {
logrus.Debugf("set names of image %q to %v", img.ID, names)
}
// Save the manifest.
manifest, _, err := s.GetManifest()
manifest, _, err := s.image.Manifest()
if err != nil {
manifest = s.manifest
}
Expand Down Expand Up @@ -763,15 +779,15 @@ func (s *storageImageDestination) SupportedManifestMIMETypes() []string {

// GetManifest reads the manifest that we intend to store. If we haven't been given one (yet?),
// generate one.
func (s *storageImageDestination) GetManifest() ([]byte, string, error) {
if len(s.manifest) == 0 {
func (s *storageImageUnnamedDestination) GetManifest() ([]byte, string, error) {
if len(s.named.manifest) == 0 {
m := imgspecv1.Manifest{
Versioned: imgspec.Versioned{
SchemaVersion: 2,
},
Annotations: make(map[string]string),
}
for _, blob := range s.blobOrder {
for _, blob := range s.named.blobOrder {
desc := imgspecv1.Descriptor{
MediaType: imgspecv1.MediaTypeImageLayer,
Digest: blob,
Expand All @@ -783,20 +799,20 @@ func (s *storageImageDestination) GetManifest() ([]byte, string, error) {
if err != nil {
return nil, "", errors.Wrapf(err, "no manifest written yet, and got an error encoding a temporary one")
}
s.manifest = encoded
s.named.manifest = encoded
}
return s.manifest, manifest.GuessMIMEType(s.manifest), nil
return s.named.manifest, manifest.GuessMIMEType(s.named.manifest), nil
}

// GetTargetManifest reads a manifest among several that we might intend to store.
func (s *storageImageDestination) GetTargetManifest(targetDigest digest.Digest) ([]byte, string, error) {
if len(s.manifest) == 0 {
func (s *storageImageUnnamedDestination) GetTargetManifest(targetDigest digest.Digest) ([]byte, string, error) {
if len(s.named.manifest) == 0 {
return nil, "", errors.Errorf("no manifest written yet")
}
if digest.Canonical.FromBytes(s.manifest) != targetDigest {
if digest.Canonical.FromBytes(s.named.manifest) != targetDigest {
return nil, "", errors.Errorf("no matching target manifest")
}
return s.manifest, manifest.GuessMIMEType(s.manifest), nil
return s.named.manifest, manifest.GuessMIMEType(s.named.manifest), nil
}

// PutManifest writes the manifest to the destination.
Expand Down Expand Up @@ -839,12 +855,24 @@ func (s *storageImageDestination) PutSignatures(signatures [][]byte) error {
return nil
}

// Close cleans up any resources we tied up while writing the image.
func (s *storageImageUnnamedDestination) Close() error {
return s.named.Close()
}

// Reference returns the image reference that we used to initialize this image. Since we're officially
// unnamed, return a reference with no naming information.
func (s *storageImageUnnamedDestination) Reference() types.ImageReference {
ref := newReference(s.named.imageRef.transport, "", "", nil, "", "")
return ref
}

// GetSignatures splits up the signature blob and returns a slice of byte slices.
func (s *storageImageDestination) GetSignatures() ([][]byte, error) {
func (s *storageImageUnnamedDestination) GetSignatures() ([][]byte, error) {
sigs := [][]byte{}
first := 0
for _, length := range s.SignatureSizes {
sigs = append(sigs, s.signatures[first:first+length])
for _, length := range s.named.SignatureSizes {
sigs = append(sigs, s.named.signatures[first:first+length])
first += length
}
if first == 0 {
Expand Down
Loading

0 comments on commit 3fbfcb4

Please sign in to comment.