overlay, composefs: use data-only lower layers

use the new overlay data-only feature to mount the composefs data
directory so there is no need for upper layers to create whiteouts to
hide payload files.

The feature was added to Linux 6.5.

Signed-off-by: Giuseppe Scrivano <[email protected]>

drivers/overlay/check.go

@@ -275,3 +275,36 @@ func supportsIdmappedLowerLayers(home string) (bool, error) {
 	}()
 	return true, nil
 }
+
+// supportsDataOnlyLayers checks if the kernel supports mounting a overlay file system
+// that uses data-only layers.
+func supportsDataOnlyLayers(home string) (bool, error) {
+	layerDir, err := os.MkdirTemp(home, "compat")
+	if err != nil {
+		return false, err
+	}
+	defer func() {
+		_ = os.RemoveAll(layerDir)
+	}()
+
+	mergedDir := filepath.Join(layerDir, "merged")
+	lowerDir := filepath.Join(layerDir, "lower")
+	lowerDirDataOnly := filepath.Join(layerDir, "lower-data")
+	upperDir := filepath.Join(layerDir, "upper")
+	workDir := filepath.Join(layerDir, "work")
+
+	_ = idtools.MkdirAs(mergedDir, 0o700, 0, 0)
+	_ = idtools.MkdirAs(lowerDir, 0o700, 0, 0)
+	_ = idtools.MkdirAs(lowerDirDataOnly, 0o700, 0, 0)
+	_ = idtools.MkdirAs(upperDir, 0o700, 0, 0)
+	_ = idtools.MkdirAs(workDir, 0o700, 0, 0)
+
+	opts := fmt.Sprintf("lowerdir=%s::%s,upperdir=%s,workdir=%s,metacopy=on", lowerDir, lowerDirDataOnly, upperDir, workDir)
+	flags := uintptr(0)
+	if err := unix.Mount("overlay", mergedDir, "overlay", flags, opts); err != nil {
+		return false, err
+	}
+	_ = unix.Unmount(mergedDir, unix.MNT_DETACH)
+
+	return true, nil
+}

drivers/overlay/mount.go

@@ -141,14 +141,27 @@ func mountOverlayFromMain() {
 	// the new value for the list of lowers, because it's shorter.
 	if lowerv != "" {
 		lowers := strings.Split(lowerv, ":")
+		var newLowers []string
+		dataOnly := false
 		for i := range lowers {
+			if lowers[i] == "" {
+				dataOnly = true
+				continue
+			}
 			lowerFd, err := unix.Open(lowers[i], unix.O_RDONLY, 0)
 			if err != nil {
 				fatal(err)
 			}
-			lowers[i] = fmt.Sprintf("%d", lowerFd)
+			var lower string
+			if dataOnly {
+				lower = fmt.Sprintf(":%d", lowerFd)
+				dataOnly = false
+			} else {
+				lower = fmt.Sprintf("%d", lowerFd)
+			}
+			newLowers = append(newLowers, lower)
 		}
-		lowerv = strings.Join(lowers, ":")
+		lowerv = strings.Join(newLowers, ":")
 	}
 
 	// Reconstruct the Label field.

drivers/overlay/overlay.go

@@ -1657,8 +1657,6 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO
 		optsList = append(optsList, "metacopy=on", "redirect_dir=on")
 	}
 
-	absLowers = append(absLowers, composeFsLayers...)
-
 	if len(absLowers) == 0 {
 		absLowers = append(absLowers, path.Join(dir, "empty"))
 	}
@@ -1752,11 +1750,20 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO
 		absLowers = newAbsDir
 	}
 
+	lowerDirs := strings.Join(absLowers, ":")
+	if len(composeFsLayers) > 0 {
+		composeFsLayersLowerDirs := strings.Join(composeFsLayers, "::")
+		lowerDirs = lowerDirs + "::" + composeFsLayersLowerDirs
+	}
+	// absLowers is not valid anymore now as we have added composeFsLayers to it, so prevent
+	// its usage.
+	absLowers = nil //nolint:ineffassign
+
 	var opts string
 	if readWrite {
-		opts = fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", strings.Join(absLowers, ":"), diffDir, workdir)
+		opts = fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", lowerDirs, diffDir, workdir)
 	} else {
-		opts = fmt.Sprintf("lowerdir=%s:%s", diffDir, strings.Join(absLowers, ":"))
+		opts = fmt.Sprintf("lowerdir=%s:%s", diffDir, lowerDirs)
 	}
 	if len(optsList) > 0 {
 		opts = fmt.Sprintf("%s,%s", opts, strings.Join(optsList, ","))
@@ -1800,9 +1807,9 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO
 		if readWrite {
 			diffDir := path.Join(id, "diff")
 			workDir := path.Join(id, "work")
-			opts = fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", strings.Join(absLowers, ":"), diffDir, workDir)
+			opts = fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", lowerDirs, diffDir, workDir)
 		} else {
-			opts = fmt.Sprintf("lowerdir=%s:%s", diffDir, strings.Join(absLowers, ":"))
+			opts = fmt.Sprintf("lowerdir=%s:%s", diffDir, lowerDirs)
 		}
 		if len(optsList) > 0 {
 			opts = strings.Join(append([]string{opts}, optsList...), ",")
@@ -2009,11 +2016,34 @@ func (d *Driver) CleanupStagingDirectory(stagingDirectory string) error {
 	return os.RemoveAll(stagingDirectory)
 }
 
+func (d *Driver) supportsDataOnlyLayers() (bool, error) {
+	feature := "dataonly-layers"
+	overlayCacheResult, overlayCacheText, err := cachedFeatureCheck(d.runhome, feature)
+	if err == nil {
+		if overlayCacheResult {
+			logrus.Debugf("Cached value indicated that data-only layers for overlay are supported")
+			return true, nil
+		}
+		logrus.Debugf("Cached value indicated that data-only layers for overlay are not supported")
+		return false, errors.New(overlayCacheText)
+	}
+	supportsDataOnly, err := supportsDataOnlyLayers(d.home)
+	if err2 := cachedFeatureRecord(d.runhome, feature, supportsDataOnly, ""); err2 != nil {
+		return false, fmt.Errorf("recording overlay data-only layers support status: %w", err2)
+	}
+	return supportsDataOnly, err
+}
+
 func (d *Driver) useComposeFs() bool {
 	if !composeFsSupported() || unshare.IsRootless() {
 		return false
 	}
-	return true
+	supportsDataOnlyLayers, err := d.supportsDataOnlyLayers()
+	if err != nil {
+		logrus.Debugf("Check for data-only layers failed with: %v", err)
+		return false
+	}
+	return supportsDataOnlyLayers
 }
 
 // ApplyDiff applies the changes in the new layer using the specified function