Skip pulling layers reusable from additional layer store

This commit adds support for "Additional Layer Store".

Pull is one of the time-consuming steps in the container lifecycle. Additional
Layer Store enables runtimes (e.g. Podman, CRI-O, etc) to startup containers
using layers stored in a specified directory, instead of pulling them from the
registry.

One of the expected use cases of this feature is "lazy pulling". This enables
the runtimes to startup containers without waiting for the entire image contents
to be locally available, but necessary chunks of the contents are fetched
on-demand (lazily).

There are several image formats and filesystems to enable lazy pulling in the
community. They includes stargz/eStargz, zstd:chunked, CVMFS, etc. Additional
Layer Store makes it easy to integrate with these filesystems for performing
lazy pulling.

Signed-off-by: Kohei Tokunaga <[email protected]>

copy/copy.go

@@ -910,7 +910,7 @@ func (ic *imageCopier) copyLayers(ctx context.Context) error {
 	}
 
 	data := make([]copyLayerData, numLayers)
-	copyLayerHelper := func(index int, srcLayer types.BlobInfo, toEncrypt bool, pool *mpb.Progress) {
+	copyLayerHelper := func(index int, srcLayer types.BlobInfo, toEncrypt bool, pool *mpb.Progress, srcRef reference.Named) {
 		defer copySemaphore.Release(1)
 		defer copyGroup.Done()
 		cld := copyLayerData{}
@@ -925,7 +925,7 @@ func (ic *imageCopier) copyLayers(ctx context.Context) error {
 				logrus.Debugf("Skipping foreign layer %q copy to %s", cld.destInfo.Digest, ic.c.dest.Reference().Transport().Name())
 			}
 		} else {
-			cld.destInfo, cld.diffID, cld.err = ic.copyLayer(ctx, srcLayer, toEncrypt, pool, index)
+			cld.destInfo, cld.diffID, cld.err = ic.copyLayer(ctx, srcLayer, toEncrypt, pool, index, srcRef)
 		}
 		data[index] = cld
 	}
@@ -962,7 +962,7 @@ func (ic *imageCopier) copyLayers(ctx context.Context) error {
 				return errors.Wrapf(err, "Can't acquire semaphore")
 			}
 			copyGroup.Add(1)
-			go copyLayerHelper(i, srcLayer, encLayerBitmap[i], progressPool)
+			go copyLayerHelper(i, srcLayer, encLayerBitmap[i], progressPool, ic.c.rawSource.Reference().DockerReference())
 		}
 
 		// A call to copyGroup.Wait() is done at this point by the defer above.
@@ -1147,7 +1147,8 @@ type diffIDResult struct {
 
 // copyLayer copies a layer with srcInfo (with known Digest and Annotations and possibly known Size) in src to dest, perhaps (de/re/)compressing it,
 // and returns a complete blobInfo of the copied layer, and a value for LayerDiffIDs if diffIDIsNeeded
-func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, toEncrypt bool, pool *mpb.Progress, layerIndex int) (types.BlobInfo, digest.Digest, error) {
+// srcRef can be used as an additional hint to the destination during checking whehter a layer can be reused but srcRef can be nil.
+func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, toEncrypt bool, pool *mpb.Progress, layerIndex int, srcRef reference.Named) (types.BlobInfo, digest.Digest, error) {
 	// If the srcInfo doesn't contain compression information, try to compute it from the
 	// MediaType, which was either read from a manifest by way of LayerInfos() or constructed
 	// by LayerInfosForCopy(), if it was supplied at all.  If we succeed in copying the blob,
@@ -1189,11 +1190,14 @@ func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, to
 		// layers which requires passing the index of the layer.
 		// Hence, we need to special case and cast.
 		dest, ok := ic.c.dest.(internalTypes.ImageDestinationWithOptions)
-		if ok && enableEarlyCommit {
+		if ok {
 			options := internalTypes.TryReusingBlobOptions{
 				Cache:         ic.c.blobInfoCache,
 				CanSubstitute: ic.canSubstituteBlobs,
-				LayerIndex:    &layerIndex,
+				SrcRef:        srcRef,
+			}
+			if enableEarlyCommit {
+				options.LayerIndex = &layerIndex
 			}
 			reused, blobInfo, err = dest.TryReusingBlobWithOptions(ctx, srcInfo, options)
 		} else {
@@ -1550,12 +1554,12 @@ func (c *copier) copyBlobFromStream(ctx context.Context, srcStream io.Reader, sr
 	// which requires passing the index of the layer.  Hence, we need to
 	// special case and cast.
 	dest, ok := c.dest.(internalTypes.ImageDestinationWithOptions)
-	if ok && enableEarlyCommit {
+	if ok {
 		options := internalTypes.PutBlobOptions{
 			Cache:    c.blobInfoCache,
 			IsConfig: isConfig,
 		}
-		if !isConfig {
+		if !isConfig && enableEarlyCommit {
 			options.LayerIndex = &layerIndex
 		}
 		uploadedInfo, err = dest.PutBlobWithOptions(ctx, &errorAnnotationReader{destStream}, inputInfo, options)

go.mod

@@ -4,7 +4,6 @@ go 1.11
 
 require (
 	github.com/14rcole/gopopulate v0.0.0-20180821133914-b175b219e774 // indirect
-	github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect
 	github.com/BurntSushi/toml v0.3.1
 	github.com/containers/libtrust v0.0.0-20190913040956-14b96171aa3b
 	github.com/containers/ocicrypt v1.1.0
@@ -13,16 +12,13 @@ require (
 	github.com/docker/docker v1.4.2-0.20191219165747-a9416c67da9f
 	github.com/docker/docker-credential-helpers v0.6.3
 	github.com/docker/go-connections v0.4.0
-	github.com/docker/go-metrics v0.0.1 // indirect
 	github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect
 	github.com/ghodss/yaml v1.0.0
 	github.com/gorilla/mux v1.7.4 // indirect
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/imdario/mergo v0.3.11
 	github.com/klauspost/compress v1.12.1
 	github.com/klauspost/pgzip v1.2.5
-	github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect
-	github.com/magefile/mage v1.10.0 // indirect
 	github.com/manifoldco/promptui v0.8.0
 	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/mtrmac/gpgme v0.1.2
@@ -44,5 +40,4 @@ require (
 	golang.org/x/net v0.0.0-20201224014010-6772e930b67b
 	golang.org/x/sync v0.0.0-20201207232520-09787c993a3a
 	golang.org/x/sys v0.0.0-20210324051608-47abb6519492
-	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
 )