Merge pull request #137 from giuseppe/ostree-storage

ostree: new option for the overlay driver

Makefile

@@ -5,7 +5,7 @@ GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD 2>/dev/null)
 GIT_BRANCH_CLEAN := $(shell echo $(GIT_BRANCH) | sed -e "s/[^[:alnum:]]/-/g")
 EPOCH_TEST_COMMIT := 0418ebf59f9e1f564831c0ba9378b7f8e40a1c73
 NATIVETAGS := exclude_graphdriver_devicemapper exclude_graphdriver_btrfs exclude_graphdriver_overlay
-AUTOTAGS := $(shell ./hack/btrfs_tag.sh) $(shell ./hack/libdm_tag.sh)
+AUTOTAGS := $(shell ./hack/btrfs_tag.sh) $(shell ./hack/libdm_tag.sh) $(shell ./hack/ostree_tag.sh)
 BUILDFLAGS := -tags "$(AUTOTAGS) $(TAGS)" $(FLAGS)
 GO := go
 

Vagrantfile

@@ -1,13 +1,13 @@
 # -*- mode: ruby -*-
 # vi: set ft=ruby :
 #
-#  The fedora/25-cloud-base and debian/jessie64 boxes are also available for
+#  The fedora/28-cloud-base and debian/jessie64 boxes are also available for
 #  the "virtualbox" provider.  Set the VAGRANT_PROVIDER environment variable to
 #  "virtualbox" to use them instead.
 #
 Vagrant.configure("2") do |config|
   config.vm.define "fedora" do |c|
-    c.vm.box = "fedora/25-cloud-base"
+    c.vm.box = "fedora/28-cloud-base"
     c.vm.synced_folder ".", "/vagrant", type: "rsync",
       rsync__exclude: "bundles", rsync__args: ["-vadz", "--delete"]
     c.vm.provision "shell", inline: <<-SHELL

docs/containers-storage.conf.5.md

@@ -129,6 +129,12 @@ Marks thinpool device for deferred deletion. If the thinpool is in use when the
 
 Specifies the maximum number of retries XFS should attempt to complete IO when ENOSPC (no space) error is returned by underlying storage device. (default: 0, which means to try continuously.)
 
+**ostree_repo=""**
+  Tell storage drivers to use the specified OSTree repository.  Some storage drivers, such as overlay, might use
+
+**skip_mount_home=""**
+  Tell storage drivers to not create a PRIVATE bind mount on their home directory.
+
 # HISTORY
 May 2017, Originally compiled by Dan Walsh <[email protected]>
 Format copied from crio.conf man page created by Aleksa Sarai <[email protected]>

drivers/overlay/overlay.go

@@ -24,6 +24,7 @@ import (
 	"github.com/containers/storage/pkg/idtools"
 	"github.com/containers/storage/pkg/locker"
 	"github.com/containers/storage/pkg/mount"
+	"github.com/containers/storage/pkg/ostree"
 	"github.com/containers/storage/pkg/parsers"
 	"github.com/containers/storage/pkg/system"
 	units "github.com/docker/go-units"
@@ -85,6 +86,8 @@ type overlayOptions struct {
 	imageStores         []string
 	quota               quota.Quota
 	fuseProgram         string
+	ostreeRepo          string
+	skipMountHome       bool
 }
 
 // Driver contains information about the home directory and the list of active mounts that are created using this driver.
@@ -99,6 +102,7 @@ type Driver struct {
 	naiveDiff     graphdriver.DiffDriver
 	supportsDType bool
 	locker        *locker.Locker
+	convert       map[string]bool
 }
 
 var (
@@ -160,8 +164,16 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap
 		}
 	}
 
-	if err := mount.MakePrivate(home); err != nil {
-		return nil, err
+	if !opts.skipMountHome {
+		if err := mount.MakePrivate(home); err != nil {
+			return nil, err
+		}
+	}
+
+	if opts.ostreeRepo != "" {
+		if err := ostree.CreateOSTreeRepository(opts.ostreeRepo, rootUID, rootGID); err != nil {
+			return nil, err
+		}
 	}
 
 	d := &Driver{
@@ -173,6 +185,7 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap
 		supportsDType: supportsDType,
 		locker:        locker.New(),
 		options:       *opts,
+		convert:       make(map[string]bool),
 	}
 
 	d.naiveDiff = graphdriver.NewNaiveDiffDriver(d, d)
@@ -240,6 +253,18 @@ func parseOptions(options []string) (*overlayOptions, error) {
 				return nil, fmt.Errorf("overlay: can't stat FUSE program %s: %v", val, err)
 			}
 			o.fuseProgram = val
+		case "overlay2.ostree_repo", "overlay.ostree_repo", ".ostree_repo":
+			logrus.Debugf("overlay: ostree_repo=%s", val)
+			if !ostree.OstreeSupport() {
+				return nil, fmt.Errorf("overlay: ostree_repo specified but support for ostree is missing")
+			}
+			o.ostreeRepo = val
+		case "overlay2.skip_mount_home", "overlay.skip_mount_home", ".skip_mount_home":
+			logrus.Debugf("overlay: skip_mount_home=%s", val)
+			o.skipMountHome, err = strconv.ParseBool(val)
+			if err != nil {
+				return nil, err
+			}
 		default:
 			return nil, fmt.Errorf("overlay: Unknown option %s", key)
 		}
@@ -394,6 +419,11 @@ func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) (retErr
 			return fmt.Errorf("--storage-opt size is only supported for ReadWrite Layers")
 		}
 	}
+
+	if d.options.ostreeRepo != "" {
+		d.convert[id] = true
+	}
+
 	return d.create(id, parent, opts)
 }
 
@@ -561,6 +591,12 @@ func (d *Driver) getLowerDirs(id string) ([]string, error) {
 func (d *Driver) Remove(id string) error {
 	d.locker.Lock(id)
 	defer d.locker.Unlock(id)
+
+	// Ignore errors, we don't want to fail if the ostree branch doesn't exist,
+	if d.options.ostreeRepo != "" {
+		ostree.DeleteOSTree(d.options.ostreeRepo, id)
+	}
+
 	dir := d.dir(id)
 	lid, err := ioutil.ReadFile(path.Join(dir, "link"))
 	if err == nil {
@@ -786,6 +822,13 @@ func (d *Driver) ApplyDiff(id string, idMappings *idtools.IDMappings, parent str
 		return 0, err
 	}
 
+	_, convert := d.convert[id]
+	if convert {
+		if err := ostree.ConvertToOSTree(d.options.ostreeRepo, applyDir, id); err != nil {
+			return 0, err
+		}
+	}
+
 	return directory.Size(applyDir)
 }
 

drivers/vfs/driver.go

@@ -9,6 +9,7 @@ import (
 	"github.com/containers/storage/drivers"
 	"github.com/containers/storage/pkg/chrootarchive"
 	"github.com/containers/storage/pkg/idtools"
+	"github.com/containers/storage/pkg/ostree"
 	"github.com/containers/storage/pkg/system"
 	"github.com/opencontainers/selinux/go-selinux/label"
 )
@@ -42,6 +43,27 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap
 			d.homes = append(d.homes, strings.Split(option[12:], ",")...)
 			continue
 		}
+		if strings.HasPrefix(option, "vfs.ostree_repo=") {
+			if !ostree.OstreeSupport() {
+				return nil, fmt.Errorf("vfs: ostree_repo specified but support for ostree is missing")
+			}
+			d.ostreeRepo = option[16:]
+		}
+		if strings.HasPrefix(option, ".ostree_repo=") {
+			if !ostree.OstreeSupport() {
+				return nil, fmt.Errorf("vfs: ostree_repo specified but support for ostree is missing")
+			}
+			d.ostreeRepo = option[13:]
+		}
+	}
+	if d.ostreeRepo != "" {
+		rootUID, rootGID, err := idtools.GetRootUIDGID(uidMaps, gidMaps)
+		if err != nil {
+			return nil, err
+		}
+		if err := ostree.CreateOSTreeRepository(d.ostreeRepo, rootUID, rootGID); err != nil {
+			return nil, err
+		}
 	}
 	return graphdriver.NewNaiveDiffDriver(d, graphdriver.NewNaiveLayerIDMapUpdater(d)), nil
 }
@@ -53,6 +75,7 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap
 type Driver struct {
 	homes      []string
 	idMappings *idtools.IDMappings
+	ostreeRepo string
 }
 
 func (d *Driver) String() string {
@@ -77,11 +100,15 @@ func (d *Driver) Cleanup() error {
 // CreateReadWrite creates a layer that is writable for use as a container
 // file system.
 func (d *Driver) CreateReadWrite(id, parent string, opts *graphdriver.CreateOpts) error {
-	return d.Create(id, parent, opts)
+	return d.create(id, parent, opts, false)
 }
 
 // Create prepares the filesystem for the VFS driver and copies the directory for the given id under the parent.
 func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) error {
+	return d.create(id, parent, opts, true)
+}
+
+func (d *Driver) create(id, parent string, opts *graphdriver.CreateOpts, ro bool) error {
 	if opts != nil && len(opts.StorageOpt) != 0 {
 		return fmt.Errorf("--storage-opt is not supported for vfs")
 	}
@@ -106,14 +133,23 @@ func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) error {
 	if _, mountLabel, err := label.InitLabels(labelOpts); err == nil {
 		label.SetFileLabel(dir, mountLabel)
 	}
-	if parent == "" {
-		return nil
+	if parent != "" {
+		parentDir, err := d.Get(parent, "")
+		if err != nil {
+			return fmt.Errorf("%s: %s", parent, err)
+		}
+		if err := CopyWithTar(parentDir, dir); err != nil {
+			return err
+		}
 	}
-	parentDir, err := d.Get(parent, "")
-	if err != nil {
-		return fmt.Errorf("%s: %s", parent, err)
+
+	if ro && d.ostreeRepo != "" {
+		if err := ostree.ConvertToOSTree(d.ostreeRepo, dir, id); err != nil {
+			return err
+		}
 	}
-	return CopyWithTar(parentDir, dir)
+	return nil
+
 }
 
 func (d *Driver) dir(id string) string {
@@ -132,6 +168,10 @@ func (d *Driver) dir(id string) string {
 
 // Remove deletes the content from the directory for a given id.
 func (d *Driver) Remove(id string) error {
+	if d.ostreeRepo != "" {
+		// Ignore errors, we don't want to fail if the ostree branch doesn't exist,
+		ostree.DeleteOSTree(d.ostreeRepo, id)
+	}
 	return system.EnsureRemoveAll(d.dir(id))
 }
 

hack/ostree_tag.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+cc -E - $(pkg-config --cflags ostree-1) > /dev/null 2> /dev/null << EOF
+#include <ostree-1/ostree.h>
+EOF
+if test $? -eq 0 ; then
+	echo ostree
+fi

hack/spc_ci_test.sh

@@ -13,15 +13,15 @@ fi
 # Additional packages needed ontop of the base (generic) image
 case "$DISTRO" in
     *ubuntu*)
-        export INSTALL_CMD="apt-get -qq install bats btrfs-tools libdevmapper-dev"
+        export INSTALL_CMD="apt-get -qq install bats btrfs-tools libdevmapper-dev ostree libostree-dev"
         ;;
     *fedora*)
         export INSTALL_CMD="dnf -y install bats btrfs-progs btrfs-progs-devel
-                            e2fsprogs xfsprogs device-mapper-devel"
+                            e2fsprogs xfsprogs device-mapper-devel ostree ostree-devel"
         ;;
     *centos*)
         export INSTALL_CMD="yum install -y bats btrfs-progs btrfs-progs-devel
-                            e2fsprogs xfsprogs device-mapper-devel"
+                            e2fsprogs xfsprogs device-mapper-devel ostree ostree-devel"
         ;;
     *)
         echo "Unknown/unsupported \$DISTRO=$DISTRO"

pkg/ostree/no_ostree.go

@@ -0,0 +1,19 @@
+// +build !ostree
+
+package ostree
+
+func OstreeSupport() bool {
+	return false
+}
+
+func DeleteOSTree(repoLocation, id string) error {
+	return nil
+}
+
+func CreateOSTreeRepository(repoLocation string, rootUID int, rootGID int) error {
+	return nil
+}
+
+func ConvertToOSTree(repoLocation, root, id string) error {
+	return nil
+}