docker-archive: read+write

docker/archive/dest.go

@@ -3,47 +3,48 @@ package archive
 import (
 	"context"
 	"io"
-	"os"
 
-	"github.com/containers/image/v5/docker/tarfile"
+	"github.com/containers/image/v5/docker/internal/tarfile"
 	"github.com/containers/image/v5/types"
 	"github.com/pkg/errors"
 )
 
 type archiveImageDestination struct {
 	*tarfile.Destination // Implements most of types.ImageDestination
 	ref                  archiveReference
-	writer               io.Closer
+	archive              *tarfile.Writer // Should only be closed if writer != nil
+	writer               io.Closer       // May be nil if the archive is shared
 }
 
 func newImageDestination(sys *types.SystemContext, ref archiveReference) (types.ImageDestination, error) {
-	// ref.path can be either a pipe or a regular file
-	// in the case of a pipe, we require that we can open it for write
-	// in the case of a regular file, we don't want to overwrite any pre-existing file
-	// so we check for Size() == 0 below (This is racy, but using O_EXCL would also be racy,
-	// only in a different way. Either way, it’s up to the user to not have two writers to the same path.)
-	fh, err := os.OpenFile(ref.path, os.O_WRONLY|os.O_CREATE, 0644)
-	if err != nil {
-		return nil, errors.Wrapf(err, "error opening file %q", ref.path)
+	if ref.sourceIndex != -1 {
+		return nil, errors.Errorf("Destination reference must not contain a manifest index @%d", ref.sourceIndex)
 	}
 
-	fhStat, err := fh.Stat()
-	if err != nil {
-		return nil, errors.Wrapf(err, "error statting file %q", ref.path)
-	}
+	var archive *tarfile.Writer
+	var writer io.Closer
+	if ref.archiveWriter != nil {
+		archive = ref.archiveWriter
+		writer = nil
+	} else {
+		fh, err := openArchiveForWriting(ref.path)
+		if err != nil {
+			return nil, err
+		}
 
-	if fhStat.Mode().IsRegular() && fhStat.Size() != 0 {
-		return nil, errors.New("docker-archive doesn't support modifying existing images")
+		archive = tarfile.NewWriter(fh)
+		writer = fh
 	}
 
-	tarDest := tarfile.NewDestinationWithContext(sys, fh, ref.destinationRef)
+	tarDest := tarfile.NewDestination(sys, archive, ref.ref)
 	if sys != nil && sys.DockerArchiveAdditionalTags != nil {
 		tarDest.AddRepoTags(sys.DockerArchiveAdditionalTags)
 	}
 	return &archiveImageDestination{
 		Destination: tarDest,
 		ref:         ref,
-		writer:      fh,
+		archive:     archive,
+		writer:      writer,
 	}, nil
 }
 
@@ -60,13 +61,19 @@ func (d *archiveImageDestination) Reference() types.ImageReference {
 
 // Close removes resources associated with an initialized ImageDestination, if any.
 func (d *archiveImageDestination) Close() error {
-	return d.writer.Close()
+	if d.writer != nil {
+		return d.writer.Close()
+	}
+	return nil
 }
 
 // Commit marks the process of storing the image as successful and asks for the image to be persisted.
 // WARNING: This does not have any transactional semantics:
 // - Uploaded data MAY be visible to others before Commit() is called
 // - Uploaded data MAY be removed or MAY remain around if Close() is called without Commit() (i.e. rollback is allowed but not guaranteed)
 func (d *archiveImageDestination) Commit(ctx context.Context, unparsedToplevel types.UnparsedImage) error {
-	return d.Destination.Commit(ctx)
+	if d.writer != nil {
+		return d.archive.Close()
+	}
+	return nil
 }

docker/archive/reader.go

@@ -0,0 +1,68 @@
+package archive
+
+import (
+	"github.com/containers/image/v5/docker/internal/tarfile"
+	"github.com/containers/image/v5/docker/reference"
+	"github.com/containers/image/v5/types"
+	"github.com/pkg/errors"
+)
+
+// Reader manages a single Docker archive, allows listing its contents and accessing
+// individual images with less overhead than creating image references individually
+// (because the archive is, if necessary, copied or decompressed only once).
+type Reader struct {
+	path    string // The original, user-specified path; not the maintained temporary file, if any
+	archive *tarfile.Reader
+}
+
+// NewReader returns a Reader for path.
+// The caller should call .Close() on the returned object.
+func NewReader(sys *types.SystemContext, path string) (*Reader, error) {
+	archive, err := tarfile.NewReaderFromFile(sys, path)
+	if err != nil {
+		return nil, err
+	}
+	return &Reader{
+		path:    path,
+		archive: archive,
+	}, nil
+}
+
+// Close deletes temporary files associated with the Reader, if any.
+func (r *Reader) Close() error {
+	return r.archive.Close()
+}
+
+// List returns the a set of references for images in the Reader,
+// grouped by the image the references point to.
+// The references are valid only until the Reader is closed.
+func (r *Reader) List() ([][]types.ImageReference, error) {
+	res := [][]types.ImageReference{}
+	for imageIndex, image := range r.archive.Manifest {
+		refs := []types.ImageReference{}
+		for _, tag := range image.RepoTags {
+			parsedTag, err := reference.ParseNormalizedNamed(tag)
+			if err != nil {
+				return nil, errors.Wrapf(err, "Invalid tag %#v in manifest item @%d", tag, imageIndex)
+			}
+			nt, ok := parsedTag.(reference.NamedTagged)
+			if !ok {
+				return nil, errors.Errorf("Invalid tag %s (%s): does not contain a tag", tag, parsedTag.String())
+			}
+			ref, err := newReference(r.path, nt, -1, r.archive, nil)
+			if err != nil {
+				return nil, errors.Wrapf(err, "Error creating a reference for tag %#v in manifest item @%d", tag, imageIndex)
+			}
+			refs = append(refs, ref)
+		}
+		if len(refs) == 0 {
+			ref, err := newReference(r.path, nil, imageIndex, r.archive, nil)
+			if err != nil {
+				return nil, errors.Wrapf(err, "Error creating a reference for manifest item @%d", imageIndex)
+			}
+			refs = append(refs, ref)
+		}
+		res = append(res, refs)
+	}
+	return res, nil
+}

docker/archive/src.go

@@ -3,9 +3,8 @@ package archive
 import (
 	"context"
 
-	"github.com/containers/image/v5/docker/tarfile"
+	"github.com/containers/image/v5/docker/internal/tarfile"
 	"github.com/containers/image/v5/types"
-	"github.com/sirupsen/logrus"
 )
 
 type archiveImageSource struct {
@@ -16,13 +15,20 @@ type archiveImageSource struct {
 // newImageSource returns a types.ImageSource for the specified image reference.
 // The caller must call .Close() on the returned ImageSource.
 func newImageSource(ctx context.Context, sys *types.SystemContext, ref archiveReference) (types.ImageSource, error) {
-	if ref.destinationRef != nil {
-		logrus.Warnf("docker-archive: references are not supported for sources (ignoring)")
-	}
-	src, err := tarfile.NewSourceFromFileWithContext(sys, ref.path)
-	if err != nil {
-		return nil, err
+	var archive *tarfile.Reader
+	var closeArchive bool
+	if ref.archiveReader != nil {
+		archive = ref.archiveReader
+		closeArchive = false
+	} else {
+		a, err := tarfile.NewReaderFromFile(sys, ref.path)
+		if err != nil {
+			return nil, err
+		}
+		archive = a
+		closeArchive = true
 	}
+	src := tarfile.NewSource(archive, closeArchive, ref.ref, ref.sourceIndex)
 	return &archiveImageSource{
 		Source: src,
 		ref:    ref,

docker/archive/transport.go

@@ -3,8 +3,10 @@ package archive
 import (
 	"context"
 	"fmt"
+	"strconv"
 	"strings"
 
+	"github.com/containers/image/v5/docker/internal/tarfile"
 	"github.com/containers/image/v5/docker/reference"
 	ctrImage "github.com/containers/image/v5/image"
 	"github.com/containers/image/v5/transports"
@@ -42,9 +44,16 @@ func (t archiveTransport) ValidatePolicyConfigurationScope(scope string) error {
 // archiveReference is an ImageReference for Docker images.
 type archiveReference struct {
 	path string
-	// only used for destinations,
-	// archiveReference.destinationRef is optional and can be nil for destinations as well.
-	destinationRef reference.NamedTagged
+	// May be nil to read the only image in an archive, or to create an untagged image.
+	ref reference.NamedTagged
+	// If not -1, a zero-based index of the image in the manifest. Valid only for sources.
+	// Must not be set if ref is set.
+	sourceIndex int
+	// If not nil, must have been created from path (but archiveReader.path may point at a temporary
+	// file, not necesarily path precisely).
+	archiveReader *tarfile.Reader
+	// If not nil, must have been created for path
+	archiveWriter *tarfile.Writer
 }
 
 // ParseReference converts a string, which should not start with the ImageTransport.Name prefix, into an Docker ImageReference.
@@ -55,37 +64,69 @@ func ParseReference(refString string) (types.ImageReference, error) {
 
 	parts := strings.SplitN(refString, ":", 2)
 	path := parts[0]
-	var destinationRef reference.NamedTagged
+	var nt reference.NamedTagged
+	sourceIndex := -1
 
-	// A :tag was specified, which is only necessary for destinations.
 	if len(parts) == 2 {
-		ref, err := reference.ParseNormalizedNamed(parts[1])
-		if err != nil {
-			return nil, errors.Wrapf(err, "docker-archive parsing reference")
+		// A :tag or :@index was specified.
+		if len(parts[1]) > 0 && parts[1][0] == '@' {
+			i, err := strconv.Atoi(parts[1][1:])
+			if err != nil {
+				return nil, errors.Wrapf(err, "Invalid source index %s", parts[1])
+			}
+			if i < 0 {
+				return nil, errors.Errorf("Invalid source index @%d: must not be negative", i)
+			}
+			sourceIndex = i
+		} else {
+			ref, err := reference.ParseNormalizedNamed(parts[1])
+			if err != nil {
+				return nil, errors.Wrapf(err, "docker-archive parsing reference")
+			}
+			ref = reference.TagNameOnly(ref)
+			refTagged, isTagged := ref.(reference.NamedTagged)
+			if !isTagged { // If ref contains a digest, TagNameOnly does not change it
+				return nil, errors.Errorf("reference does not include a tag: %s", ref.String())
+			}
+			nt = refTagged
 		}
-		ref = reference.TagNameOnly(ref)
-		refTagged, isTagged := ref.(reference.NamedTagged)
-		if !isTagged {
-			// Really shouldn't be hit...
-			return nil, errors.Errorf("internal error: reference is not tagged even after reference.TagNameOnly: %s", refString)
-		}
-		destinationRef = refTagged
 	}
 
-	return NewReference(path, destinationRef)
+	return newReference(path, nt, sourceIndex, nil, nil)
+}
+
+// NewReference returns a Docker archive reference for a path and an optional reference.
+func NewReference(path string, ref reference.NamedTagged) (types.ImageReference, error) {
+	return newReference(path, ref, -1, nil, nil)
 }
 
-// NewReference rethrns a Docker archive reference for a path and an optional destination reference.
-func NewReference(path string, destinationRef reference.NamedTagged) (types.ImageReference, error) {
+// NewIndexReference returns a Docker archive reference for a path and a zero-based source manifest index.
+func NewIndexReference(path string, sourceIndex int) (types.ImageReference, error) {
+	return newReference(path, nil, sourceIndex, nil, nil)
+}
+
+// newReference returns a docker archive reference for a path, an optional reference or sourceIndex,
+// and optionally a tarfile.Reader and/or a tarfile.Writer matching path.
+func newReference(path string, ref reference.NamedTagged, sourceIndex int,
+	archiveReader *tarfile.Reader, archiveWriter *tarfile.Writer) (types.ImageReference, error) {
 	if strings.Contains(path, ":") {
 		return nil, errors.Errorf("Invalid docker-archive: reference: colon in path %q is not supported", path)
 	}
-	if _, isDigest := destinationRef.(reference.Canonical); isDigest {
-		return nil, errors.Errorf("docker-archive doesn't support digest references: %s", destinationRef.String())
+	if ref != nil && sourceIndex != -1 {
+		return nil, errors.Errorf("Invalid docker-archive: reference: cannot use both a tag and a source index")
+	}
+	if _, isDigest := ref.(reference.Canonical); isDigest {
+		return nil, errors.Errorf("docker-archive doesn't support digest references: %s", ref.String())
+	}
+	if sourceIndex != -1 && sourceIndex < 0 {
+		return nil, errors.Errorf("Invalid docker-archive: reference: index @%d must not be negative", sourceIndex)
 	}
 	return archiveReference{
-		path:           path,
-		destinationRef: destinationRef,
+		path:          path,
+		ref:           ref,
+		sourceIndex:   sourceIndex,
+		archiveReader: archiveReader,
+		archiveWriter: archiveWriter,
 	}, nil
 }
 
@@ -99,17 +140,21 @@ func (ref archiveReference) Transport() types.ImageTransport {
 // e.g. default attribute values omitted by the user may be filled in in the return value, or vice versa.
 // WARNING: Do not use the return value in the UI to describe an image, it does not contain the Transport().Name() prefix.
 func (ref archiveReference) StringWithinTransport() string {
-	if ref.destinationRef == nil {
+	switch {
+	case ref.ref != nil:
+		return fmt.Sprintf("%s:%s", ref.path, ref.ref.String())
+	case ref.sourceIndex != -1:
+		return fmt.Sprintf("%s:@%d", ref.path, ref.sourceIndex)
+	default:
 		return ref.path
 	}
-	return fmt.Sprintf("%s:%s", ref.path, ref.destinationRef.String())
 }
 
 // DockerReference returns a Docker reference associated with this reference
 // (fully explicit, i.e. !reference.IsNameOnly, but reflecting user intent,
 // not e.g. after redirect or alias processing), or nil if unknown/not applicable.
 func (ref archiveReference) DockerReference() reference.Named {
-	return ref.destinationRef
+	return ref.ref
 }
 
 // PolicyConfigurationIdentity returns a string representation of the reference, suitable for policy lookup.