buildkit: add from field to bind and cache mounts so images can be us…

…ed as source Following commit adds buildkit like support for `from` field to `--mount=type=bind` and `--mount=type=cache` so images and stage can be used as mount source. Usage looks like ```dockerfile RUN --mount=type=bind,source=.,from=<your-image>,target=/path ls /path ``` and ```dockerfile RUN --mount=type=cache,from=<your-image>,target=/path ls /path ``` Signed-off-by: Aditya Rajan <[email protected]>
containers · Oct 20, 2021 · a2f101b · a2f101b
1 parent 762cf6f
commit a2f101b
Show file tree

Hide file tree

Showing 12 changed files with 190 additions and 61 deletions.
diff --git a/cmd/buildah/run.go b/cmd/buildah/run.go
@@ -149,7 +149,7 @@ func runCmd(c *cobra.Command, args []string, iopts runInputOptions) error {
 		}
 	}
 
-	mounts, err := parse.GetVolumes(iopts.volumes, iopts.mounts, iopts.contextDir)
+	mounts, err := parse.GetVolumes(store, iopts.volumes, iopts.mounts, iopts.contextDir)
 	if err != nil {
 		return err
 	}

diff --git a/docs/Containerfile.5.md b/docs/Containerfile.5.md
@@ -114,7 +114,7 @@ Current supported mount TYPES are bind, cache, secret and tmpfs.
 
        Common Options:
 
-              · src, source: mount source spec for bind and volume. Mandatory for bind.
+              · src, source: mount source spec for bind and volume. Mandatory for bind. If `from` is specified, `src` is the subpath in the `from` field.
 
               · dst, destination, target: mount destination spec.
 
@@ -126,6 +126,8 @@ Current supported mount TYPES are bind, cache, secret and tmpfs.
 
               . bind-nonrecursive: do not setup a recursive bind mount.  By default it is recursive.
 
+              . from: Build stage or image name for the root of the source. Defaults to the build context.
+
        Options specific to tmpfs:
 
               · tmpfs-size: Size of the tmpfs mount in bytes. Unlimited by default in Linux.
@@ -146,6 +148,8 @@ Current supported mount TYPES are bind, cache, secret and tmpfs.
 
               · gid: gid for cache directory.
 
+              . from: Build stage or image name for the root of the source. Defaults to host cache directory.
+
 
 **RUN Secrets**
 

diff --git a/docs/buildah-run.1.md b/docs/buildah-run.1.md
@@ -115,7 +115,7 @@ Current supported mount TYPES are bind, cache, secret and tmpfs. <sup>[[1]](#Foo
 
        Common Options:
 
-              · src, source: mount source spec for bind and volume. Mandatory for bind.
+              · src, source: mount source spec for bind and volume. Mandatory for bind. If `from` is specified, `src` is the subpath in the `from` field.
 
               · dst, destination, target: mount destination spec.
 
@@ -127,6 +127,8 @@ Current supported mount TYPES are bind, cache, secret and tmpfs. <sup>[[1]](#Foo
 
               . bind-nonrecursive: do not setup a recursive bind mount.  By default it is recursive.
 
+              . from: Build stage or image name for the root of the source. Defaults to the build context.
+
        Options specific to tmpfs:
 
               · tmpfs-size: Size of the tmpfs mount in bytes. Unlimited by default in Linux.
@@ -151,6 +153,8 @@ Current supported mount TYPES are bind, cache, secret and tmpfs. <sup>[[1]](#Foo
 
               · gid: gid for cache directory.
 
+              . from: Build stage or image name for the root of the source. Defaults to host cache directory.
+
 **--network**, **--net**=*mode*
 
 Sets the configuration for the network namespace for the container.

diff --git a/pkg/parse/parse.go b/pkg/parse/parse.go
@@ -5,6 +5,7 @@ package parse
 // would be useful to projects vendoring buildah
 
 import (
+	"context"
 	"fmt"
 	"net"
 	"os"
@@ -16,8 +17,10 @@ import (
 	"github.com/containerd/containerd/platforms"
 	"github.com/containers/buildah/define"
 	"github.com/containers/buildah/pkg/sshagent"
+	"github.com/containers/buildah/util"
 	"github.com/containers/common/pkg/parse"
 	"github.com/containers/image/v5/types"
+	"github.com/containers/storage"
 	"github.com/containers/storage/pkg/idtools"
 	"github.com/containers/storage/pkg/unshare"
 	units "github.com/docker/go-units"
@@ -263,8 +266,8 @@ func getVolumeMounts(volumes []string) (map[string]specs.Mount, error) {
 }
 
 // GetVolumes gets the volumes from --volume and --mount
-func GetVolumes(volumes []string, mounts []string, contextDir string) ([]specs.Mount, error) {
-	unifiedMounts, err := getMounts(mounts, contextDir)
+func GetVolumes(store storage.Store, volumes []string, mounts []string, contextDir string) ([]specs.Mount, error) {
+	unifiedMounts, err := getMounts(store, mounts, contextDir)
 	if err != nil {
 		return nil, err
 	}
@@ -290,7 +293,7 @@ func GetVolumes(volumes []string, mounts []string, contextDir string) ([]specs.M
 // spec mounts.
 // buildah run --mount type=bind,src=/etc/resolv.conf,target=/etc/resolv.conf ...
 // buildah run --mount type=tmpfs,target=/dev/shm ...
-func getMounts(mounts []string, contextDir string) (map[string]specs.Mount, error) {
+func getMounts(store storage.Store, mounts []string, contextDir string) (map[string]specs.Mount, error) {
 	finalMounts := make(map[string]specs.Mount)
 
 	errInvalidSyntax := errors.Errorf("incorrect mount format: should be --mount type=<bind|tmpfs>,[src=<host-dir>,]target=<ctr-dir>[,options]")
@@ -313,7 +316,7 @@ func getMounts(mounts []string, contextDir string) (map[string]specs.Mount, erro
 		tokens := strings.Split(arr[1], ",")
 		switch kv[1] {
 		case TypeBind:
-			mount, err := GetBindMount(tokens, contextDir)
+			mount, _, err := GetBindMount(tokens, contextDir, store, "")
 			if err != nil {
 				return nil, err
 			}
@@ -322,7 +325,7 @@ func getMounts(mounts []string, contextDir string) (map[string]specs.Mount, erro
 			}
 			finalMounts[mount.Destination] = mount
 		case TypeCache:
-			mount, err := GetCacheMount(tokens)
+			mount, _, err := GetCacheMount(tokens, store, "")
 			if err != nil {
 				return nil, err
 			}
@@ -348,13 +351,14 @@ func getMounts(mounts []string, contextDir string) (map[string]specs.Mount, erro
 }
 
 // GetBindMount parses a single bind mount entry from the --mount flag.
-func GetBindMount(args []string, contextDir string) (specs.Mount, error) {
+func GetBindMount(args []string, contextDir string, store storage.Store, imageMountLabel string) (specs.Mount, string, error) {
 	newMount := specs.Mount{
 		Type: TypeBind,
 	}
 
 	setDest := false
 	bindNonRecursive := false
+	fromImage := ""
 
 	for _, val := range args {
 		kv := strings.SplitN(val, "=", 2)
@@ -373,25 +377,27 @@ func GetBindMount(args []string, contextDir string) (specs.Mount, error) {
 			newMount.Options = append(newMount.Options, "ro")
 		case "shared", "rshared", "private", "rprivate", "slave", "rslave", "Z", "z", "U":
 			newMount.Options = append(newMount.Options, kv[0])
+		case "from":
+			fromImage = kv[1]
 		case "bind-propagation":
 			if len(kv) == 1 {
-				return newMount, errors.Wrapf(optionArgError, kv[0])
+				return newMount, "", errors.Wrapf(optionArgError, kv[0])
 			}
 			newMount.Options = append(newMount.Options, kv[1])
 		case "src", "source":
 			if len(kv) == 1 {
-				return newMount, errors.Wrapf(optionArgError, kv[0])
+				return newMount, "", errors.Wrapf(optionArgError, kv[0])
 			}
 			if err := parse.ValidateVolumeHostDir(kv[1]); err != nil {
-				return newMount, err
+				return newMount, "", err
 			}
 			newMount.Source = kv[1]
 		case "target", "dst", "destination":
 			if len(kv) == 1 {
-				return newMount, errors.Wrapf(optionArgError, kv[0])
+				return newMount, "", errors.Wrapf(optionArgError, kv[0])
 			}
 			if err := parse.ValidateVolumeCtrDir(kv[1]); err != nil {
-				return newMount, err
+				return newMount, "", err
 			}
 			newMount.Destination = kv[1]
 			setDest = true
@@ -400,8 +406,22 @@ func GetBindMount(args []string, contextDir string) (specs.Mount, error) {
 			// and can thus be safely ignored.
 			// See also the handling of the equivalent "delegated" and "cached" in ValidateVolumeOpts
 		default:
-			return newMount, errors.Wrapf(errBadMntOption, kv[0])
+			return newMount, "", errors.Wrapf(errBadMntOption, kv[0])
+		}
+	}
+
+	if fromImage != "" {
+		image, err := util.LookupImage(store, fromImage)
+		if err != nil {
+			return newMount, "", err
+		}
+
+		mountPoint, err := image.Mount(context.Background(), nil, imageMountLabel)
+		if err != nil {
+			return newMount, "", err
 		}
+		contextDir = mountPoint
+		newMount.Options = append(newMount.Options, "ro")
 	}
 
 	// buildkit parity: default bind option must be `rbind`
@@ -411,7 +431,7 @@ func GetBindMount(args []string, contextDir string) (specs.Mount, error) {
 	}
 
 	if !setDest {
-		return newMount, noDestError
+		return newMount, fromImage, noDestError
 	}
 
 	// buildkit parity: support absolute path for sources from current build context
@@ -422,21 +442,22 @@ func GetBindMount(args []string, contextDir string) (specs.Mount, error) {
 
 	opts, err := parse.ValidateVolumeOpts(newMount.Options)
 	if err != nil {
-		return newMount, err
+		return newMount, fromImage, err
 	}
 	newMount.Options = opts
 
-	return newMount, nil
+	return newMount, fromImage, nil
 }
 
 // GetCacheMount parses a single cache mount entry from the --mount flag.
-func GetCacheMount(args []string) (specs.Mount, error) {
+func GetCacheMount(args []string, store storage.Store, imageMountLabel string) (specs.Mount, string, error) {
 	var err error
 	var (
 		setDest     bool
 		setShared   bool
 		setReadOnly bool
 	)
+	fromImage := ""
 	newMount := specs.Mount{
 		Type: TypeBind,
 	}
@@ -467,61 +488,93 @@ func GetCacheMount(args []string) (specs.Mount, error) {
 			setShared = true
 		case "bind-propagation":
 			if len(kv) == 1 {
-				return newMount, errors.Wrapf(optionArgError, kv[0])
+				return newMount, "", errors.Wrapf(optionArgError, kv[0])
 			}
 			newMount.Options = append(newMount.Options, kv[1])
 		case "id":
 			id = kv[1]
+		case "from":
+			fromImage = kv[1]
 		case "target", "dst", "destination":
 			if len(kv) == 1 {
-				return newMount, errors.Wrapf(optionArgError, kv[0])
+				return newMount, "", errors.Wrapf(optionArgError, kv[0])
 			}
 			if err := parse.ValidateVolumeCtrDir(kv[1]); err != nil {
-				return newMount, err
+				return newMount, "", err
 			}
 			newMount.Destination = kv[1]
 			setDest = true
+		case "src", "source":
+			if len(kv) == 1 {
+				return newMount, "", errors.Wrapf(optionArgError, kv[0])
+			}
+			if err := parse.ValidateVolumeHostDir(kv[1]); err != nil {
+				return newMount, "", err
+			}
+			newMount.Source = kv[1]
 		case "mode":
 			mode, err = strconv.Atoi(kv[1])
 			if err != nil {
-				return newMount, errors.Wrapf(err, "Unable to parse cache mode")
+				return newMount, "", errors.Wrapf(err, "Unable to parse cache mode")
 			}
 		case "uid":
 			uid, err = strconv.Atoi(kv[1])
 			if err != nil {
-				return newMount, errors.Wrapf(err, "Unable to parse cache uid")
+				return newMount, "", errors.Wrapf(err, "Unable to parse cache uid")
 			}
 		case "gid":
 			gid, err = strconv.Atoi(kv[1])
 			if err != nil {
-				return newMount, errors.Wrapf(err, "Unable to parse cache gid")
+				return newMount, "", errors.Wrapf(err, "Unable to parse cache gid")
 			}
 		default:
-			return newMount, errors.Wrapf(errBadMntOption, kv[0])
+			return newMount, "", errors.Wrapf(errBadMntOption, kv[0])
 		}
 	}
 
 	if !setDest {
-		return newMount, noDestError
+		return newMount, "", noDestError
 	}
 
-	// since type is cache and cache can be resused by consecutive builds
-	// create a common cache directory, which should persists on hosts within temp lifecycle
-	// add subdirectory if specified
-	if id != "" {
-		newMount.Source = filepath.Join(GetTempDir(), BuildahCacheDir, id)
+	if fromImage != "" {
+		// do not create cache on host
+		// instead use read-only mounted image as cache
+		image, err := util.LookupImage(store, fromImage)
+		if err != nil {
+			return newMount, "", err
+		}
+
+		mountPoint, err := image.Mount(context.Background(), nil, imageMountLabel)
+		if err != nil {
+			return newMount, "", err
+		}
+		contextDir := mountPoint
+		newMount.Options = append(newMount.Options, "ro")
+		if strings.HasPrefix(newMount.Source, ".") || newMount.Source == "" || !filepath.IsAbs(newMount.Source) {
+			// path should be /contextDir/specified path
+			newMount.Source = filepath.Join(contextDir, filepath.Clean(string(filepath.Separator)+newMount.Source))
+		}
 	} else {
-		newMount.Source = filepath.Join(GetTempDir(), BuildahCacheDir)
-	}
-	// create cache on host if not present
-	err = os.MkdirAll(newMount.Source, os.FileMode(mode))
-	if err != nil {
-		return newMount, errors.Wrapf(err, "Unable to create build cache directory")
-	}
-	//buidkit parity: change uid and gid if specificed otheriwise keep `0`
-	err = os.Chown(newMount.Source, uid, gid)
-	if err != nil {
-		return newMount, errors.Wrapf(err, "Unable to change uid,gid of cache directory")
+		// we need to create cache on host if no image is being used
+
+		// since type is cache and cache can be resused by consecutive builds
+		// create a common cache directory, which should persists on hosts within temp lifecycle
+		// add subdirectory if specified
+		if id != "" {
+			newMount.Source = filepath.Join(GetTempDir(), BuildahCacheDir, id)
+		} else {
+			newMount.Source = filepath.Join(GetTempDir(), BuildahCacheDir)
+		}
+		// create cache on host if not present
+		err = os.MkdirAll(newMount.Source, os.FileMode(mode))
+		if err != nil {
+			return newMount, "", errors.Wrapf(err, "Unable to create build cache directory")
+		}
+		//buidkit parity: change uid and gid if specificed otheriwise keep `0`
+		err = os.Chown(newMount.Source, uid, gid)
+		if err != nil {
+			return newMount, "", errors.Wrapf(err, "Unable to change uid,gid of cache directory")
+		}
 	}
 
 	// buildkit parity: default sharing should be shared
@@ -531,7 +584,7 @@ func GetCacheMount(args []string) (specs.Mount, error) {
 	}
 
 	// buildkit parity: cache must writable unless `ro` or `readonly` is configured explicitly
-	if !setReadOnly {
+	if !setReadOnly && fromImage == "" {
 		newMount.Options = append(newMount.Options, "rw")
 	}
 
@@ -541,11 +594,11 @@ func GetCacheMount(args []string) (specs.Mount, error) {
 
 	opts, err := parse.ValidateVolumeOpts(newMount.Options)
 	if err != nil {
-		return newMount, err
+		return newMount, fromImage, err
 	}
 	newMount.Options = opts
 
-	return newMount, nil
+	return newMount, fromImage, nil
 }
 
 // GetTmpfsMount parses a single tmpfs mount entry from the --mount flag

diff --git a/run.go b/run.go
@@ -153,6 +153,8 @@ type RunOptions struct {
 type runMountArtifacts struct {
 	// RunMountTargets are the run mount targets inside the container
 	RunMountTargets []string
+	// Any external images which were mounted inside container
+	MountedImages []string
 	// Agents are the ssh agents started
 	Agents []*sshagent.AgentServer
 	// SSHAuthSock is the path to the ssh auth sock inside the container