overlay: new option to support ostree deduplication

usage example:

skopeo copy docker-daemon:busybox:latest containers-storage:\[overlay2@/var/lib/containers/storage+/var/run/containers/storage:overlay2.ostree_repo=/var/lib/containers/storage/overlay2/ostree/.repo,overlay2.override_kernel_check=1\]\busybox

Skopeo needs a bugfix from:

containers/image#305

which is not merged yet.  Just apply this patch:

diff --git a/vendor/github.com/containers/image/storage/storage_image.go b/vendor/github.com/containers/image/storage/storage_image.go
index 08fa71b..5edccce 100644
--- a/vendor/github.com/containers/image/storage/storage_image.go
+++ b/vendor/github.com/containers/image/storage/storage_image.go
@@ -174,7 +174,7 @@ func (s *storageImageDestination) putBlob(stream io.Reader, blobinfo types.BlobI
                        id = ddigest.Canonical.FromBytes([]byte(parentLayer + "+" + digest.String())).Hex()
                }
                // Attempt to create the identified layer and import its contents.
-               layer, uncompressedSize, err := s.imageRef.transport.store.PutLayer(id, parentLayer, nil, "", true, multi)
+               layer, uncompressedSize, err := s.imageRef.transport.store.PutLayer(id, parentLayer, nil, "", false, multi)
                if err != nil && errors.Cause(err) != storage.ErrDuplicateID {
                        logrus.Debugf("error importing layer blob %q as %q: %v", blobinfo.Digest, id, err)
                        return errorBlobInfo, err

Signed-off-by: Giuseppe Scrivano <[email protected]>

.travis.yml

@@ -5,10 +5,12 @@ go:
     - 1.7
 dist: trusty
 sudo: required
+env:
+  - TAGS='exclude_ostree'
 before_install:
     - sudo apt-get -qq update
     - sudo apt-get -qq install btrfs-tools libdevmapper-dev
 script:
     - make install.tools
     - make local-binary docs local-cross local-validate
-    - sudo PATH="$PATH" make local-test-unit local-test-integration
+    - sudo -E PATH="$PATH" make local-test-unit local-test-integration

drivers/driver_linux.go

@@ -59,6 +59,7 @@ var (
 		"btrfs",
 		"zfs",
 		"vfs",
+		"ostree",
 	}
 
 	// FsNames maps filesystem id to name of the filesystem.

drivers/overlay/no_ostree.go

@@ -0,0 +1,19 @@
+// +build exclude_ostree
+
+package overlay
+
+func ostreeSupport() bool {
+	return false
+}
+
+func deleteOSTree(repoLocation, id string) error {
+	return nil
+}
+
+func createOSTreeRepository(repoLocation string, rootUID int, rootGID int) error {
+	return nil
+}
+
+func convertToOSTree(repoLocation, root, id string) error {
+	return nil
+}

drivers/overlay/ostree.go

@@ -0,0 +1,78 @@
+// +build !exclude_ostree
+
+package overlay
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"time"
+
+	"github.com/containers/storage/pkg/idtools"
+	"github.com/containers/storage/pkg/system"
+	"github.com/ostreedev/ostree-go/pkg/otbuiltin"
+	"github.com/pkg/errors"
+)
+
+func ostreeSupport() bool {
+	return true
+}
+
+// Create prepares the filesystem for the OSTREE driver and copies the directory for the given id under the parent.
+func convertToOSTree(repoLocation, root, id string) error {
+	repo, err := otbuiltin.OpenRepo(repoLocation)
+	if err != nil {
+		return errors.Wrap(err, "could not open the OSTree repository")
+	}
+
+	if _, err := repo.PrepareTransaction(); err != nil {
+		return errors.Wrap(err, "could not prepare the OSTree transaction")
+	}
+	commitOpts := otbuiltin.NewCommitOptions()
+	commitOpts.Timestamp = time.Now()
+	commitOpts.Parent = "0000000000000000000000000000000000000000000000000000000000000000"
+	branch := fmt.Sprintf("ocilayer/%s", id)
+
+	if _, err := repo.Commit(root, branch, commitOpts); err != nil {
+		return errors.Wrap(err, "could not commit the layer")
+	}
+
+	if _, err := repo.CommitTransaction(); err != nil {
+		return errors.Wrap(err, "could not complete the OSTree transaction")
+	}
+
+	if err := system.EnsureRemoveAll(root); err != nil {
+		return err
+	}
+
+	checkoutOpts := otbuiltin.NewCheckoutOptions()
+	checkoutOpts.RequireHardlinks = true
+	if err := otbuiltin.Checkout(repoLocation, root, branch, checkoutOpts); err != nil {
+		return errors.Wrap(err, "could not checkout from OSTree")
+	}
+	return nil
+}
+
+func createOSTreeRepository(repoLocation string, rootUID int, rootGID int) error {
+	_, err := os.Stat(repoLocation)
+	if err != nil && !os.IsNotExist(err) {
+		return err
+	} else if err != nil {
+		if err := idtools.MkdirAllAs(repoLocation, 0700, rootUID, rootGID); err != nil {
+			return errors.Wrap(err, "could not create OSTree repository directory: %v")
+		}
+
+		if err := exec.Command("ostree", fmt.Sprintf("--repo=%s", repoLocation), "init").Run(); err != nil {
+			return errors.Wrap(err, "could not create OSTree repository")
+		}
+	}
+	return nil
+}
+
+func deleteOSTree(repoLocation, id string) error {
+	branch := fmt.Sprintf("ocilayer/%s", id)
+	if err := exec.Command("ostree", fmt.Sprintf("--repo=%s", repoLocation), "refs", "--delete", branch).Run(); err != nil {
+		return errors.Wrap(err, "could not delete OSTree branch")
+	}
+	return nil
+}

drivers/overlay/overlay.go

@@ -84,6 +84,7 @@ type overlayOptions struct {
 	overrideKernelCheck bool
 	imageStores         []string
 	quota               quota.Quota
+	ostreeRepo          string
 }
 
 // Driver contains information about the home directory and the list of active mounts that are created using this driver.
@@ -98,6 +99,7 @@ type Driver struct {
 	naiveDiff     graphdriver.DiffDriver
 	supportsDType bool
 	locker        *locker.Locker
+	convert       map[string]bool
 }
 
 var (
@@ -156,6 +158,12 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap
 		return nil, err
 	}
 
+	if opts.ostreeRepo != "" {
+		if err := createOSTreeRepository(opts.ostreeRepo, rootUID, rootGID); err != nil {
+			return nil, err
+		}
+	}
+
 	d := &Driver{
 		name:          "overlay",
 		home:          home,
@@ -165,6 +173,7 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap
 		supportsDType: supportsDType,
 		locker:        locker.New(),
 		options:       *opts,
+		convert:       make(map[string]bool),
 	}
 
 	d.naiveDiff = graphdriver.NewNaiveDiffDriver(d, uidMaps, gidMaps)
@@ -225,6 +234,12 @@ func parseOptions(options []string) (*overlayOptions, error) {
 				}
 				o.imageStores = append(o.imageStores, store)
 			}
+		case "overlay2.ostree_repo", ".ostree_repo", "ostree_repo":
+			logrus.Debugf("overlay: ostree_repo=%s", val)
+			if !ostreeSupport() {
+				return nil, fmt.Errorf("overlay: ostree_repo specified but support for ostree is missing")
+			}
+			o.ostreeRepo = val
 		default:
 			return nil, fmt.Errorf("overlay: Unknown option %s", key)
 		}
@@ -374,6 +389,7 @@ func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) (retErr
 			return fmt.Errorf("--storage-opt size is only supported for ReadWrite Layers")
 		}
 	}
+	d.convert[id] = true
 	return d.create(id, parent, opts)
 }
 
@@ -533,6 +549,14 @@ func (d *Driver) getLowerDirs(id string) ([]string, error) {
 func (d *Driver) Remove(id string) error {
 	d.locker.Lock(id)
 	defer d.locker.Unlock(id)
+
+	if d.options.ostreeRepo != "" {
+		err := deleteOSTree(d.options.ostreeRepo, id)
+		if err != nil {
+			return err
+		}
+	}
+
 	dir := d.dir(id)
 	lid, err := ioutil.ReadFile(path.Join(dir, "link"))
 	if err == nil {
@@ -718,6 +742,13 @@ func (d *Driver) ApplyDiff(id string, parent string, diff io.Reader) (size int64
 		return 0, err
 	}
 
+	_, convert := d.convert[id]
+	if convert {
+		if err := convertToOSTree(d.options.ostreeRepo, applyDir, id); err != nil {
+			return 0, err
+		}
+	}
+
 	return directory.Size(applyDir)
 }