overlay, composefs: use data-only lower layers

cmd/containers-storage/diff.go

@@ -1,12 +1,17 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"io"
 	"os"
 
 	"github.com/containers/storage"
+	graphdriver "github.com/containers/storage/drivers"
 	"github.com/containers/storage/pkg/archive"
+	"github.com/containers/storage/pkg/chunked"
+	"github.com/containers/storage/pkg/chunked/compressor"
+	"github.com/containers/storage/pkg/ioutils"
 	"github.com/containers/storage/pkg/mflag"
 )
 
@@ -96,6 +101,93 @@ func diff(flags *mflag.FlagSet, action string, m storage.Store, args []string) (
 	return 0, nil
 }
 
+type fileFetcher struct {
+	file *os.File
+}
+
+func sendFileParts(f *fileFetcher, chunks []chunked.ImageSourceChunk, streams chan io.ReadCloser, errors chan error) {
+	defer close(streams)
+	defer close(errors)
+
+	for _, chunk := range chunks {
+		l := io.NewSectionReader(f.file, int64(chunk.Offset), int64(chunk.Length))
+		streams <- ioutils.NewReadCloserWrapper(l, func() error {
+			return nil
+		})
+	}
+}
+
+func (f fileFetcher) GetBlobAt(chunks []chunked.ImageSourceChunk) (chan io.ReadCloser, chan error, error) {
+	streams := make(chan io.ReadCloser)
+	errs := make(chan error)
+	go sendFileParts(&f, chunks, streams, errs)
+	return streams, errs, nil
+}
+
+func applyDiffUsingStagingDirectory(flags *mflag.FlagSet, action string, m storage.Store, args []string) (int, error) {
+	if len(args) < 2 {
+		return 2, nil
+	}
+
+	layer := args[0]
+	sourceDirectory := args[1]
+
+	tOptions := archive.TarOptions{}
+	tr, err := archive.TarWithOptions(sourceDirectory, &tOptions)
+	if err != nil {
+		return 1, err
+	}
+	defer tr.Close()
+
+	tar, err := os.CreateTemp("", "layer-diff-tar-")
+	if err != nil {
+		return 1, err
+	}
+	defer os.Remove(tar.Name())
+	defer tar.Close()
+
+	// we go through the zstd:chunked compressor first so that it generates the metadata required to mount
+	// a composefs image.
+
+	metadata := make(map[string]string)
+	compressor, err := compressor.ZstdCompressor(tar, metadata, nil)
+	if err != nil {
+		return 1, err
+	}
+
+	if _, err := io.Copy(compressor, tr); err != nil {
+		return 1, err
+	}
+	if err := compressor.Close(); err != nil {
+		return 1, err
+	}
+
+	size, err := tar.Seek(0, io.SeekCurrent)
+	if err != nil {
+		return 1, err
+	}
+
+	fetcher := fileFetcher{
+		file: tar,
+	}
+
+	differ, err := chunked.GetDiffer(context.Background(), m, size, metadata, &fetcher)
+	if err != nil {
+		return 1, err
+	}
+
+	var options graphdriver.ApplyDiffOpts
+	out, err := m.ApplyDiffWithDiffer("", &options, differ)
+	if err != nil {
+		return 1, err
+	}
+	if err := m.ApplyDiffFromStagingDirectory(layer, out.Target, out, &options); err != nil {
+		m.CleanupStagingDirectory(out.Target)
+		return 1, err
+	}
+	return 0, nil
+}
+
 func applyDiff(flags *mflag.FlagSet, action string, m storage.Store, args []string) (int, error) {
 	if len(args) < 1 {
 		return 1, nil
@@ -179,4 +271,12 @@ func init() {
 			flags.StringVar(&applyDiffFile, []string{"-file", "f"}, "", "Read from file instead of stdin")
 		},
 	})
+	commands = append(commands, command{
+		names:       []string{"applydiff-using-staging-dir"},
+		optionsHelp: "layerNameOrID directory",
+		usage:       "Apply a diff to a layer using a staging directory",
+		minArgs:     2,
+		maxArgs:     2,
+		action:      applyDiffUsingStagingDirectory,
+	})
 }

docs/containers-storage-applydiff-using-staging-dir.md

@@ -0,0 +1,27 @@
+## containers-storage-applydiff-using-staging-dir 1 "September 2023"
+
+## NAME
+containers-storage applydiff-using-staging-dir - Apply a layer diff to a layer using a staging directory
+
+## SYNOPSIS
+**containers-storage** **applydiff-using-staging-dir** *layerNameOrID* *source*
+
+## DESCRIPTION
+When a layer is first created, it contains no changes relative to its parent
+layer.  The layer can either be mounted read-write and its contents modified
+directly, or contents can be added (or removed) by applying a layer diff.  A
+layer diff takes the form of a (possibly compressed) tar archive with
+additional information present in its headers, and can be produced by running
+*containers-storage diff* or an equivalent.
+
+Differently than **apply-diff**, the command **applydiff-using-staging-dir**
+first creates a staging directory and then moves the final result to the destination.
+
+## EXAMPLE
+**containers-storage applydiff-using-staging-dir 5891b5b522 /path/to/diff**
+
+## SEE ALSO
+containers-storage-apply-diff(1)
+containers-storage-changes(1)
+containers-storage-diff(1)
+containers-storage-diffsize(1)

docs/containers-storage.md

@@ -52,79 +52,81 @@ configuration will be stored as data items.
 
 ## SUB-COMMANDS
 The *containers-storage* command's features are broken down into several subcommands:
- **containers-storage add-names(1)**           Add layer, image, or container name or names
+ **containers-storage add-names(1)**                   Add layer, image, or container name or names
 
- **containers-storage applydiff(1)**           Apply a diff to a layer
+ **containers-storage applydiff(1)**                   Apply a diff to a layer
 
- **containers-storage changes(1)**             Compare two layers
+ **containers-storage applydiff-using-staging-dir(1)** Apply a diff to a layer staging the new content first.
 
- **containers-storage check(1)**               Check for and possibly remove damaged layers/images/containers
+ **containers-storage changes(1)**                     Compare two layers
 
- **containers-storage container(1)**           Examine a container
+ **containers-storage check(1)**                       Check for and possibly remove damaged layers/images/containers
 
- **containers-storage containers(1)**          List containers
+ **containers-storage container(1)**                   Examine a container
 
- **containers-storage create-container(1)**    Create a new container from an image
+ **containers-storage containers(1)**                  List containers
 
- **containers-storage create-image(1)**        Create a new image using layers
+ **containers-storage create-container(1)**            Create a new container from an image
 
- **containers-storage create-layer(1)**        Create a new layer
+ **containers-storage create-image(1)**                Create a new image using layers
 
- **containers-storage create-storage-layer(1)** Create a new layer in the lower-level storage driver
+ **containers-storage create-layer(1)**                Create a new layer
 
- **containers-storage delete(1)**              Delete a layer or image or container, with no safety checks
+ **containers-storage create-storage-layer(1)**        Create a new layer in the lower-level storage driver
 
- **containers-storage delete-container(1)**    Delete a container, with safety checks
+ **containers-storage delete(1)**                      Delete a layer or image or container, with no safety checks
 
- **containers-storage delete-image(1)**        Delete an image, with safety checks
+ **containers-storage delete-container(1)**            Delete a container, with safety checks
 
- **containers-storage delete-layer(1)**        Delete a layer, with safety checks
+ **containers-storage delete-image(1)**                Delete an image, with safety checks
 
- **containers-storage diff(1)**                Compare two layers
+ **containers-storage delete-layer(1)**                Delete a layer, with safety checks
 
- **containers-storage diffsize(1)**            Compare two layers
+ **containers-storage diff(1)**                        Compare two layers
 
- **containers-storage exists(1)**              Check if a layer or image or container exists
+ **containers-storage diffsize(1)**                    Compare two layers
 
- **containers-storage get-container-data(1)**  Get data that is attached to a container
+ **containers-storage exists(1)**                      Check if a layer or image or container exists
 
- **containers-storage get-image-data(1)**      Get data that is attached to an image
+ **containers-storage get-container-data(1)**          Get data that is attached to a container
 
- **containers-storage image(1)**               Examine an image
+ **containers-storage get-image-data(1)**              Get data that is attached to an image
 
- **containers-storage images(1)**              List images
+ **containers-storage image(1)**                       Examine an image
 
- **containers-storage layers(1)**              List layers
+ **containers-storage images(1)**                      List images
 
- **containers-storage list-container-data(1)** List data items that are attached to a container
+ **containers-storage layers(1)**                      List layers
 
- **containers-storage list-image-data(1)**     List data items that are attached to an image
+ **containers-storage list-container-data(1)**         List data items that are attached to a container
 
- **containers-storage metadata(1)**            Retrieve layer, image, or container metadata
+ **containers-storage list-image-data(1)**             List data items that are attached to an image
 
- **containers-storage mount(1)**               Mount a layer or container
+ **containers-storage metadata(1)**                    Retrieve layer, image, or container metadata
 
- **containers-storage mounted(1)**             Check if a file system is mounted
+ **containers-storage mount(1)**                       Mount a layer or container
 
- **containers-storage set-container-data(1)**  Set data that is attached to a container
+ **containers-storage mounted(1)**                     Check if a file system is mounted
 
- **containers-storage set-image-data(1)**      Set data that is attached to an image
+ **containers-storage set-container-data(1)**          Set data that is attached to a container
 
- **containers-storage set-metadata(1)**        Set layer, image, or container metadata
+ **containers-storage set-image-data(1)**              Set data that is attached to an image
 
- **containers-storage set-names(1)**           Set layer, image, or container name or names
+ **containers-storage set-metadata(1)**                Set layer, image, or container metadata
 
- **containers-storage shutdown(1)**            Shut down graph driver
+ **containers-storage set-names(1)**                   Set layer, image, or container name or names
 
- **containers-storage status(1)**              Check on graph driver status
+ **containers-storage shutdown(1)**                    Shut down graph driver
 
- **containers-storage unmount(1)**             Unmount a layer or container
+ **containers-storage status(1)**                      Check on graph driver status
 
- **containers-storage unshare(1)**             Run a command in a user namespace
+ **containers-storage unmount(1)**                     Unmount a layer or container
 
- **containers-storage version(1)**             Return containers-storage version information
+ **containers-storage unshare(1)**                     Run a command in a user namespace
 
- **containers-storage wipe(1)**                Wipe all layers, images, and containers
+ **containers-storage version(1)**                     Return containers-storage version information
+
+ **containers-storage wipe(1)**                        Wipe all layers, images, and containers
 
 ## OPTIONS
 **--help**

drivers/overlay/check.go

@@ -275,3 +275,36 @@ func supportsIdmappedLowerLayers(home string) (bool, error) {
 	}()
 	return true, nil
 }
+
+// supportsDataOnlyLayers checks if the kernel supports mounting a overlay file system
+// that uses data-only layers.
+func supportsDataOnlyLayers(home string) (bool, error) {
+	layerDir, err := os.MkdirTemp(home, "compat")
+	if err != nil {
+		return false, err
+	}
+	defer func() {
+		_ = os.RemoveAll(layerDir)
+	}()
+
+	mergedDir := filepath.Join(layerDir, "merged")
+	lowerDir := filepath.Join(layerDir, "lower")
+	lowerDirDataOnly := filepath.Join(layerDir, "lower-data")
+	upperDir := filepath.Join(layerDir, "upper")
+	workDir := filepath.Join(layerDir, "work")
+
+	_ = idtools.MkdirAs(mergedDir, 0o700, 0, 0)
+	_ = idtools.MkdirAs(lowerDir, 0o700, 0, 0)
+	_ = idtools.MkdirAs(lowerDirDataOnly, 0o700, 0, 0)
+	_ = idtools.MkdirAs(upperDir, 0o700, 0, 0)
+	_ = idtools.MkdirAs(workDir, 0o700, 0, 0)
+
+	opts := fmt.Sprintf("lowerdir=%s::%s,upperdir=%s,workdir=%s,metacopy=on", lowerDir, lowerDirDataOnly, upperDir, workDir)
+	flags := uintptr(0)
+	if err := unix.Mount("overlay", mergedDir, "overlay", flags, opts); err != nil {
+		return false, err
+	}
+	_ = unix.Unmount(mergedDir, unix.MNT_DETACH)
+
+	return true, nil
+}

drivers/overlay/composefs_supported.go

@@ -93,7 +93,7 @@ func generateComposeFsBlob(toc []byte, composefsDir string) error {
 
 	fd, err := unix.Openat(unix.AT_FDCWD, destFile, unix.O_WRONLY|unix.O_CREAT|unix.O_TRUNC|unix.O_EXCL|unix.O_CLOEXEC, 0o644)
 	if err != nil {
-		return fmt.Errorf("failed to open output file: %w", err)
+		return fmt.Errorf("failed to open output file %q: %w", destFile, err)
 	}
 	outFd := os.NewFile(uintptr(fd), "outFd")
 

drivers/overlay/mount.go

@@ -141,14 +141,27 @@ func mountOverlayFromMain() {
 	// the new value for the list of lowers, because it's shorter.
 	if lowerv != "" {
 		lowers := strings.Split(lowerv, ":")
-		for i := range lowers {
-			lowerFd, err := unix.Open(lowers[i], unix.O_RDONLY, 0)
+		var newLowers []string
+		dataOnly := false
+		for _, lowerPath := range lowers {
+			if lowerPath == "" {
+				dataOnly = true
+				continue
+			}
+			lowerFd, err := unix.Open(lowerPath, unix.O_RDONLY, 0)
 			if err != nil {
 				fatal(err)
 			}
-			lowers[i] = fmt.Sprintf("%d", lowerFd)
+			var lower string
+			if dataOnly {
+				lower = fmt.Sprintf(":%d", lowerFd)
+				dataOnly = false
+			} else {
+				lower = fmt.Sprintf("%d", lowerFd)
+			}
+			newLowers = append(newLowers, lower)
 		}
-		lowerv = strings.Join(lowers, ":")
+		lowerv = strings.Join(newLowers, ":")
 	}
 
 	// Reconstruct the Label field.