From 9f95ca3629667bc2b9e7dd7acd01e34be5517ccc Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Fri, 23 Jun 2023 13:27:21 +0200 Subject: [PATCH] overlay: integrate ComposeFS This commit introduces support for ComposeFS using the EROFS filesystem to mount the file system metadata. The current implementation allows each layer to be mounted individually. Only images that are using the zstd:chunked and eStargz format can be used in this way since the metadata is stored in the image itself. In future support for arbitrary images can be added. Signed-off-by: Giuseppe Scrivano --- drivers/overlay/composefs_notsupported.go | 20 ++++ drivers/overlay/composefs_supported.go | 66 +++++++++++ drivers/overlay/overlay.go | 134 +++++++++++++++++++++- 3 files changed, 214 insertions(+), 6 deletions(-) create mode 100644 drivers/overlay/composefs_notsupported.go create mode 100644 drivers/overlay/composefs_supported.go diff --git a/drivers/overlay/composefs_notsupported.go b/drivers/overlay/composefs_notsupported.go new file mode 100644 index 0000000000..273ef4699d --- /dev/null +++ b/drivers/overlay/composefs_notsupported.go @@ -0,0 +1,20 @@ +//go:build !linux || !composefs || !cgo +// +build !linux !composefs !cgo + +package overlay + +import ( + "fmt" +) + +func composeFsSupported() bool { + return false +} + +func generateComposeFsBlob(toc []byte, destFile string) error { + return fmt.Errorf("composefs is not supported") +} + +func mountErofsBlob(blobFile, mountPoint string) error { + return fmt.Errorf("composefs is not supported") +} diff --git a/drivers/overlay/composefs_supported.go b/drivers/overlay/composefs_supported.go new file mode 100644 index 0000000000..7ecde856d1 --- /dev/null +++ b/drivers/overlay/composefs_supported.go @@ -0,0 +1,66 @@ +//go:build linux && composefs && cgo +// +build linux,composefs,cgo + +package overlay + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "sync" + + "github.com/containers/storage/pkg/loopback" + "golang.org/x/sys/unix" +) + +var ( + composeFsHelperOnce sync.Once + composeFsHelperPath string + composeFsHelperErr error +) + +func getComposeFsHelper() (string, error) { + composeFsHelperOnce.Do(func() { + composeFsHelperPath, composeFsHelperErr = exec.LookPath("composefs-from-json") + }) + return composeFsHelperPath, composeFsHelperErr +} + +func composeFsSupported() bool { + _, err := getComposeFsHelper() + return err == nil +} + +func generateComposeFsBlob(toc []byte, destFile string) error { + writerJson, err := getComposeFsHelper() + if err != nil { + return fmt.Errorf("failed to find composefs-from-json: %w", err) + } + + fd, err := unix.Openat(unix.AT_FDCWD, destFile, unix.O_WRONLY|unix.O_CREAT|unix.O_TRUNC|unix.O_EXCL|unix.O_CLOEXEC, 0o644) + if err != nil { + return fmt.Errorf("failed to open output file: %w", err) + } + outFd := os.NewFile(uintptr(fd), "outFd") + + defer outFd.Close() + cmd := exec.Command(writerJson, "--format=erofs", "--out=/proc/self/fd/3", "/proc/self/fd/0") + cmd.ExtraFiles = []*os.File{outFd} + cmd.Stdin = bytes.NewReader(toc) + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to convert json to erofs: %w", err) + } + return nil +} + +func mountErofsBlob(blobFile, mountPoint string) error { + loop, err := loopback.AttachLoopDevice(blobFile) + if err != nil { + return err + } + defer loop.Close() + + return unix.Mount(loop.Name(), mountPoint, "erofs", unix.MS_RDONLY, "ro,noacl") +} diff --git a/drivers/overlay/overlay.go b/drivers/overlay/overlay.go index e924e2c34e..a4e6b7d3ca 100644 --- a/drivers/overlay/overlay.go +++ b/drivers/overlay/overlay.go @@ -82,6 +82,8 @@ const ( lowerFile = "lower" maxDepth = 500 + zstdChunkedManifest = "zstd-chunked-manifest" + // idLength represents the number of random characters // which can be used to create the unique link identifier // for every layer. If this value is too long then the @@ -780,6 +782,10 @@ func supportsOverlay(home string, homeMagic graphdriver.FsMagic, rootUID, rootGI } func (d *Driver) useNaiveDiff() bool { + if d.useComposeFs() { + return true + } + useNaiveDiffLock.Do(func() { if d.options.mountProgram != "" { useNaiveDiffOnly = true @@ -1512,12 +1518,66 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO defer cleanupFunc() } + erofsLayers := filepath.Join(workDirBase, "erofs-layers") + if err := os.MkdirAll(erofsLayers, 0o700); err != nil { + return "", err + } + + skipIDMappingLayers := make(map[string]string) + + composeFsLayers := []string{} + + erofsMounts := []string{} + defer func() { + for _, m := range erofsMounts { + defer unix.Unmount(m, unix.MNT_DETACH) + } + }() + + maybeAddErofsMount := func(lowerID string, i int) (string, error) { + erofsBlob := d.getErofsBlob(lowerID) + _, err = os.Stat(erofsBlob) + if err != nil { + if os.IsNotExist(err) { + return "", nil + } + return "", err + } + logrus.Debugf("overlay: using erofs blob %s for lower %s", erofsBlob, lowerID) + + dest := filepath.Join(erofsLayers, fmt.Sprintf("%d", i)) + if err := os.MkdirAll(dest, 0o700); err != nil { + return "", err + } + + if err := mountErofsBlob(erofsBlob, dest); err != nil { + return "", err + } + erofsMounts = append(erofsMounts, dest) + composeFsPath, err := d.getDiffPath(lowerID) + if err != nil { + return "", err + } + composeFsLayers = append(composeFsLayers, composeFsPath) + skipIDMappingLayers[composeFsPath] = composeFsPath + return dest, nil + } + + diffDir := path.Join(workDirBase, "diff") + + if dest, err := maybeAddErofsMount(id, 0); err != nil { + return "", err + } else if dest != "" { + diffDir = dest + } + // For each lower, resolve its path, and append it and any additional diffN // directories to the lowers list. - for _, l := range splitLowers { + for i, l := range splitLowers { if l == "" { continue } + lower := "" newpath := path.Join(d.home, l) if st, err := os.Stat(newpath); err != nil { @@ -1551,6 +1611,30 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO } lower = newpath } + + linkContent, err := os.Readlink(lower) + if err != nil { + return "", err + } + lowerID := filepath.Base(filepath.Dir(linkContent)) + erofsMount, err := maybeAddErofsMount(lowerID, i+1) + if err != nil { + return "", err + } + if erofsMount != "" { + if needsIDMapping { + if err := idmap.CreateIDMappedMount(erofsMount, erofsMount, idmappedMountProcessPid); err != nil { + return "", fmt.Errorf("create mapped mount for %q: %w", erofsMount, err) + } + skipIDMappingLayers[erofsMount] = erofsMount + // overlay takes a reference on the mount, so it is safe to unmount + // the mapped idmounts as soon as the final overlay file system is mounted. + defer unix.Unmount(erofsMount, unix.MNT_DETACH) + } + absLowers = append(absLowers, erofsMount) + continue + } + absLowers = append(absLowers, lower) diffN = 1 _, err = os.Stat(dumbJoin(lower, "..", nameWithSuffix("diff", diffN))) @@ -1561,15 +1645,22 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO } } + if len(composeFsLayers) > 0 { + optsList = append(optsList, "metacopy=on", "redirect_dir=on") + } + + absLowers = append(absLowers, composeFsLayers...) + if len(absLowers) == 0 { absLowers = append(absLowers, path.Join(dir, "empty")) } + // user namespace requires this to move a directory from lower to upper. rootUID, rootGID, err := idtools.GetRootUIDGID(options.UidMaps, options.GidMaps) if err != nil { return "", err } - diffDir := path.Join(workDirBase, "diff") + if err := idtools.MkdirAllAs(diffDir, perms, rootUID, rootGID); err != nil { return "", err } @@ -1623,6 +1714,11 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO for _, absLower := range absLowers { mappedMountSrc := getMappedMountRoot(absLower) + if _, ok := skipIDMappingLayers[absLower]; ok { + newAbsDir = append(newAbsDir, absLower) + continue + } + root, found := idMappedMounts[mappedMountSrc] if !found { root = filepath.Join(mappedRoot, fmt.Sprintf("%d", c)) @@ -1903,6 +1999,13 @@ func (d *Driver) CleanupStagingDirectory(stagingDirectory string) error { return os.RemoveAll(stagingDirectory) } +func (d *Driver) useComposeFs() bool { + if !composeFsSupported() || unshare.IsRootless() { + return false + } + return true +} + // ApplyDiff applies the changes in the new layer using the specified function func (d *Driver) ApplyDiffWithDiffer(id, parent string, options *graphdriver.ApplyDiffOpts, differ graphdriver.Differ) (output graphdriver.DriverWithDifferOutput, err error) { var idMappings *idtools.IDMappings @@ -1935,14 +2038,22 @@ func (d *Driver) ApplyDiffWithDiffer(id, parent string, options *graphdriver.App logrus.Debugf("Applying differ in %s", applyDir) + differOptions := graphdriver.DifferOptions{ + Format: graphdriver.DifferOutputFormatDir, + } + if d.useComposeFs() { + differOptions.Format = graphdriver.DifferOutputFormatFlat + } out, err := differ.ApplyDiff(applyDir, &archive.TarOptions{ UIDMaps: idMappings.UIDs(), GIDMaps: idMappings.GIDs(), IgnoreChownErrors: d.options.ignoreChownErrors, WhiteoutFormat: d.getWhiteoutFormat(), InUserNS: unshare.IsRootless(), - }, nil) + }, &differOptions) + out.Target = applyDir + return out, err } @@ -1952,17 +2063,23 @@ func (d *Driver) ApplyDiffFromStagingDirectory(id, parent, stagingDirectory stri return fmt.Errorf("%q is not a staging directory", stagingDirectory) } - diff, err := d.getDiffPath(id) + if d.useComposeFs() { + toc := diffOutput.BigData[zstdChunkedManifest] + if err := generateComposeFsBlob(toc, d.getErofsBlob(id)); err != nil { + return err + } + } + diffPath, err := d.getDiffPath(id) if err != nil { return err } - if err := os.RemoveAll(diff); err != nil && !os.IsNotExist(err) { + if err := os.RemoveAll(diffPath); err != nil && !os.IsNotExist(err) { return err } diffOutput.UncompressedDigest = diffOutput.TOCDigest - return os.Rename(stagingDirectory, diff) + return os.Rename(stagingDirectory, diffPath) } // DifferTarget gets the location where files are stored for the layer. @@ -2008,6 +2125,11 @@ func (d *Driver) ApplyDiff(id, parent string, options graphdriver.ApplyDiffOpts) return directory.Size(applyDir) } +func (d *Driver) getErofsBlob(id string) string { + dir := d.dir(id) + return path.Join(dir, "erofs-blob") +} + func (d *Driver) getDiffPath(id string) (string, error) { dir, imagestore, _ := d.dir2(id) base := dir