overlay: new option to support ostree deduplication

usage example:

skopeo copy docker-daemon:busybox:latest containers-storage:\[overlay2@/var/lib/containers/storage+/var/run/containers/storage:overlay2.ostree_repo=/var/lib/containers/storage/overlay2/ostree/.repo,overlay2.override_kernel_check=1\]\busybox

Signed-off-by: Giuseppe Scrivano <[email protected]>

Makefile

@@ -5,7 +5,7 @@ GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD 2>/dev/null)
 GIT_BRANCH_CLEAN := $(shell echo $(GIT_BRANCH) | sed -e "s/[^[:alnum:]]/-/g")
 EPOCH_TEST_COMMIT := 0418ebf59f9e1f564831c0ba9378b7f8e40a1c73
 NATIVETAGS := exclude_graphdriver_devicemapper exclude_graphdriver_btrfs exclude_graphdriver_overlay
-AUTOTAGS := $(shell ./hack/btrfs_tag.sh) $(shell ./hack/libdm_tag.sh)
+AUTOTAGS := $(shell ./hack/btrfs_tag.sh) $(shell ./hack/libdm_tag.sh) $(shell ./hack/ostree_tag.sh)
 BUILDFLAGS := -tags "$(AUTOTAGS) $(TAGS)" $(FLAGS)
 GO := go
 

docs/containers-storage.conf.5.md

@@ -129,6 +129,9 @@ Marks thinpool device for deferred deletion. If the thinpool is in use when the
 
 Specifies the maximum number of retries XFS should attempt to complete IO when ENOSPC (no space) error is returned by underlying storage device. (default: 0, which means to try continuously.)
 
+**ostree_repo=""**
+  Tell storage drivers to use the specified OSTree repository.  Some storage drivers, such as overlay, might use
+
 # HISTORY
 May 2017, Originally compiled by Dan Walsh <[email protected]>
 Format copied from crio.conf man page created by Aleksa Sarai <[email protected]>

drivers/overlay/overlay.go

@@ -24,6 +24,7 @@ import (
 	"github.com/containers/storage/pkg/idtools"
 	"github.com/containers/storage/pkg/locker"
 	"github.com/containers/storage/pkg/mount"
+	"github.com/containers/storage/pkg/ostree"
 	"github.com/containers/storage/pkg/parsers"
 	"github.com/containers/storage/pkg/system"
 	units "github.com/docker/go-units"
@@ -85,6 +86,7 @@ type overlayOptions struct {
 	imageStores         []string
 	quota               quota.Quota
 	fuseProgram         string
+	ostreeRepo          string
 }
 
 // Driver contains information about the home directory and the list of active mounts that are created using this driver.
@@ -99,6 +101,7 @@ type Driver struct {
 	naiveDiff     graphdriver.DiffDriver
 	supportsDType bool
 	locker        *locker.Locker
+	convert       map[string]bool
 }
 
 var (
@@ -164,6 +167,12 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap
 		return nil, err
 	}
 
+	if opts.ostreeRepo != "" {
+		if err := ostree.CreateOSTreeRepository(opts.ostreeRepo, rootUID, rootGID); err != nil {
+			return nil, err
+		}
+	}
+
 	d := &Driver{
 		name:          "overlay",
 		home:          home,
@@ -173,6 +182,7 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap
 		supportsDType: supportsDType,
 		locker:        locker.New(),
 		options:       *opts,
+		convert:       make(map[string]bool),
 	}
 
 	d.naiveDiff = graphdriver.NewNaiveDiffDriver(d, d)
@@ -240,6 +250,12 @@ func parseOptions(options []string) (*overlayOptions, error) {
 				return nil, fmt.Errorf("overlay: can't stat FUSE program %s: %v", val, err)
 			}
 			o.fuseProgram = val
+		case "overlay2.ostree_repo", "overlay.ostree_repo", ".ostree_repo":
+			logrus.Debugf("overlay: ostree_repo=%s", val)
+			if !ostree.OstreeSupport() {
+				return nil, fmt.Errorf("overlay: ostree_repo specified but support for ostree is missing")
+			}
+			o.ostreeRepo = val
 		default:
 			return nil, fmt.Errorf("overlay: Unknown option %s", key)
 		}
@@ -394,6 +410,11 @@ func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) (retErr
 			return fmt.Errorf("--storage-opt size is only supported for ReadWrite Layers")
 		}
 	}
+
+	if d.options.ostreeRepo != "" {
+		d.convert[id] = true
+	}
+
 	return d.create(id, parent, opts)
 }
 
@@ -561,6 +582,12 @@ func (d *Driver) getLowerDirs(id string) ([]string, error) {
 func (d *Driver) Remove(id string) error {
 	d.locker.Lock(id)
 	defer d.locker.Unlock(id)
+
+	// Ignore errors, we don't want to fail if the ostree branch doesn't exist,
+	if d.options.ostreeRepo != "" {
+		ostree.DeleteOSTree(d.options.ostreeRepo, id)
+	}
+
 	dir := d.dir(id)
 	lid, err := ioutil.ReadFile(path.Join(dir, "link"))
 	if err == nil {
@@ -786,6 +813,13 @@ func (d *Driver) ApplyDiff(id string, idMappings *idtools.IDMappings, parent str
 		return 0, err
 	}
 
+	_, convert := d.convert[id]
+	if convert {
+		if err := ostree.ConvertToOSTree(d.options.ostreeRepo, applyDir, id); err != nil {
+			return 0, err
+		}
+	}
+
 	return directory.Size(applyDir)
 }
 

hack/ostree_tag.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+cc -E - $(pkg-config --cflags ostree-1) > /dev/null 2> /dev/null << EOF
+#include <ostree-1/ostree.h>
+EOF
+if test $? -eq 0 ; then
+	echo ostree
+fi

storage.conf

@@ -120,3 +120,6 @@ override_kernel_check = "false"
 # attempt to complete IO when ENOSPC (no space) error is returned by
 # underlying storage device.
 # xfs_nospace_max_retries = "0"
+
+# If specified, use OSTree to deduplicate files with the overlay backend
+ostree_repo = ""

store.go

@@ -2987,6 +2987,8 @@ type OptionsConfig struct {
 	RemapGroup string `toml:"remap-group"`
 	// Thinpool container options to be handed to thinpool drivers
 	Thinpool struct{ ThinpoolOptionsConfig } `toml:"thinpool"`
+	// OSTree repository
+	OstreeRepo string `toml:"ostree_repo"`
 }
 
 // TOML-friendly explicit tables used for conversions.
@@ -3077,6 +3079,9 @@ func init() {
 	if config.Storage.Options.Size != "" {
 		DefaultStoreOptions.GraphDriverOptions = append(DefaultStoreOptions.GraphDriverOptions, fmt.Sprintf("%s.size=%s", config.Storage.Driver, config.Storage.Options.Size))
 	}
+	if config.Storage.Options.OstreeRepo != "" {
+		DefaultStoreOptions.GraphDriverOptions = append(DefaultStoreOptions.GraphDriverOptions, fmt.Sprintf("%s.ostree_repo=%s", config.Storage.Driver, config.Storage.Options.OstreeRepo))
+	}
 	if config.Storage.Options.OverrideKernelCheck != "" {
 		DefaultStoreOptions.GraphDriverOptions = append(DefaultStoreOptions.GraphDriverOptions, fmt.Sprintf("%s.override_kernel_check=%s", config.Storage.Driver, config.Storage.Options.OverrideKernelCheck))
 	}