[WIP] Add support for mounting BlockCIM layers

Signed-off-by: Amit Barve <[email protected]>

internal/layers/helpers.go

@@ -75,8 +75,11 @@ const (
 	// parent layer CIMs
 	parentLayerCimPathsFlag = "parentCimPaths="
 
-	LegacyMountType string = "windows-layer"
-	CimFSMountType  string = "CimFS"
+	LegacyMountType    string = "windows-layer"
+	ForkedCIMMountType string = "CimFS"
+	BlockCIMMountType  string = "BlockCIM"
+	BlockCIMTypeFlag   string = "blockCIMType="
+	mergedCIMPathFlag  string = "mergedCIMPath="
 )
 
 // getOptionAsArray finds if there is an option which has the given prefix and if such an

internal/layers/wcow_mount.go

@@ -37,6 +37,11 @@ func MountWCOWLayers(ctx context.Context, containerID string, vm *uvm.UtilityVM,
 			return mountProcessIsolatedForkedCimLayers(ctx, containerID, l)
 		}
 		return nil, nil, fmt.Errorf("hyperv isolated containers aren't supported with forked cim layers")
+	case *wcowBlockCIMLayers:
+		if vm == nil {
+			return mountProcessIsolatedBlockCIMLayers(ctx, containerID, l)
+		}
+		return nil, nil, fmt.Errorf("hyperv isolated containers aren't supported with block cim layers")
 	default:
 		return nil, nil, fmt.Errorf("invalid layer type %T", wl)
 	}
@@ -188,37 +193,31 @@ func (l *wcowHostForkedCIMLayerCloser) Release(ctx context.Context) error {
 		return err
 	}
 
-	if err = cimlayer.CleanupContainerMounts(l.containerID); err != nil {
+	if err = wclayer.DeactivateLayer(ctx, l.scratchLayerPath); err != nil {
 		return err
 	}
-	return wclayer.DeactivateLayer(ctx, l.scratchLayerPath)
+
+	return cimlayer.CleanupContainerMounts(l.containerID)
 }
 
-func mountProcessIsolatedForkedCimLayers(ctx context.Context, containerID string, l *wcowForkedCIMLayers) (_ *MountedWCOWLayers, _ resources.ResourceCloser, err error) {
-	if err = wclayer.ActivateLayer(ctx, l.scratchLayerPath); err != nil {
+// Handles the common processing for mounting all 3 types of cimfs layers. This involves
+// mounting the scratch, attaching the filter and preparing the return values.
+// `volume` is the path to the volume at which read only layer CIMs are mounted.
+func mountProcessIsolatedCimLayersCommon(ctx context.Context, containerID string, volume string, s *scratchLayerData) (_ *MountedWCOWLayers, _ resources.ResourceCloser, err error) {
+	if err = wclayer.ActivateLayer(ctx, s.scratchLayerPath); err != nil {
 		return nil, nil, err
 	}
 	defer func() {
 		if err != nil {
-			_ = wclayer.DeactivateLayer(ctx, l.scratchLayerPath)
+			_ = wclayer.DeactivateLayer(ctx, s.scratchLayerPath)
 		}
 	}()
 
-	mountPath, err := wclayer.GetLayerMountPath(ctx, l.scratchLayerPath)
+	mountPath, err := wclayer.GetLayerMountPath(ctx, s.scratchLayerPath)
 	if err != nil {
 		return nil, nil, err
 	}
 
-	volume, err := cimlayer.MountForkedCimLayer(ctx, l.layers[0].cimPath, containerID)
-	if err != nil {
-		return nil, nil, fmt.Errorf("mount layer cim: %w", err)
-	}
-	defer func() {
-		if err != nil {
-			_ = cimlayer.UnmountCimLayer(ctx, volume)
-		}
-	}()
-
 	layerID, err := cimlayer.LayerID(volume)
 	if err != nil {
 		return nil, nil, err
@@ -253,10 +252,41 @@ func mountProcessIsolatedForkedCimLayers(ctx context.Context, containerID string
 			}},
 		}, &wcowHostForkedCIMLayerCloser{
 			containerID:      containerID,
-			scratchLayerData: l.scratchLayerData,
+			scratchLayerData: *s,
 		}, nil
 }
 
+func mountProcessIsolatedForkedCimLayers(ctx context.Context, containerID string, l *wcowForkedCIMLayers) (_ *MountedWCOWLayers, _ resources.ResourceCloser, err error) {
+	volume, err := cimlayer.MountForkedCimLayer(ctx, l.layers[0].cimPath, containerID)
+	if err != nil {
+		return nil, nil, fmt.Errorf("mount forked layer cim: %w", err)
+	}
+	defer func() {
+		if err != nil {
+			_ = cimlayer.UnmountCimLayer(ctx, volume)
+		}
+	}()
+	return mountProcessIsolatedCimLayersCommon(ctx, containerID, volume, &l.scratchLayerData)
+}
+
+func mountProcessIsolatedBlockCIMLayers(ctx context.Context, containerID string, l *wcowBlockCIMLayers) (_ *MountedWCOWLayers, _ resources.ResourceCloser, err error) {
+	var volume string
+	if len(l.parentLayers) > 1 {
+		volume, err = cimlayer.MergeMountBlockCIMLayer(ctx, l.mergedLayer, l.parentLayers, containerID)
+	} else {
+		volume, err = cimlayer.MountBlockCIMLayer(ctx, l.parentLayers[0], containerID)
+	}
+	if err != nil {
+		return nil, nil, fmt.Errorf("mount block CIM layers: %w", err)
+	}
+	defer func() {
+		if err != nil {
+			_ = cimlayer.UnmountCimLayer(ctx, volume)
+		}
+	}()
+	return mountProcessIsolatedCimLayersCommon(ctx, containerID, volume, &l.scratchLayerData)
+}
+
 type wcowIsolatedWCIFSLayerCloser struct {
 	uvm                     *uvm.UtilityVM
 	guestCombinedLayersPath string

internal/layers/wcow_parse.go

@@ -5,15 +5,18 @@ package layers
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/containerd/containerd/api/types"
 
 	"github.com/Microsoft/hcsshim/internal/copyfile"
 	"github.com/Microsoft/hcsshim/internal/uvm"
 	"github.com/Microsoft/hcsshim/internal/uvmfolder"
+	"github.com/Microsoft/hcsshim/pkg/cimfs"
 )
 
 // WCOW image layers is a tagging interface that all WCOW layers MUST implement. This is
@@ -67,6 +70,17 @@ type wcowForkedCIMLayers struct {
 	layers []forkedCIMLayer
 }
 
+// Represents CIM layers where each layer is stored in a block device or in a single file
+// and multiple such layer CIMs are merged before mounting them. Currently can only be
+// used for process isolated containers.
+type wcowBlockCIMLayers struct {
+	scratchLayerData
+	// parent layers in order [layerN (top-most), layerN-1,..layer0 (base)]
+	parentLayers []*cimfs.BlockCIM
+	// a merged layer is prepared by combining all parent layers
+	mergedLayer *cimfs.BlockCIM
+}
+
 func parseForkedCimMount(m *types.Mount) (*wcowForkedCIMLayers, error) {
 	parentLayerPaths, err := getOptionAsArray(m, parentLayerPathsFlag)
 	if err != nil {
@@ -94,6 +108,69 @@ func parseForkedCimMount(m *types.Mount) (*wcowForkedCIMLayers, error) {
 	}, nil
 }
 
+// TODO(ambarve): The code to parse mount should be in a common repo somewhere and then
+// should be consumed from there
+func parseBlockCIMMount(m *types.Mount) (*wcowBlockCIMLayers, error) {
+	var (
+		parentPaths   []string
+		layerType     cimfs.BlockCIMType
+		mergedCIMPath string
+	)
+
+	for _, option := range m.Options {
+		if val, ok := strings.CutPrefix(option, parentLayerCimPathsFlag); ok {
+			err := json.Unmarshal([]byte(val), &parentPaths)
+			if err != nil {
+				return nil, err
+			}
+		} else if val, ok = strings.CutPrefix(option, BlockCIMTypeFlag); ok {
+			if val == "device" {
+				layerType = cimfs.BlockCIMTypeDevice
+			} else if val == "file" {
+				layerType = cimfs.BlockCIMTypeSingleFile
+			} else {
+				return nil, fmt.Errorf("invalid block CIM type `%s`", val)
+			}
+		} else if val, ok = strings.CutPrefix(option, mergedCIMPathFlag); ok {
+			mergedCIMPath = val
+		}
+	}
+
+	if len(parentPaths) == 0 {
+		return nil, fmt.Errorf("need at least 1 parent layer")
+	}
+
+	var (
+		parentLayers []*cimfs.BlockCIM
+		mergedLayer  *cimfs.BlockCIM
+	)
+
+	if len(parentPaths) > 1 {
+		// for single parent layers merge won't be done
+		mergedLayer = &cimfs.BlockCIM{
+			Type:      layerType,
+			BlockPath: filepath.Dir(mergedCIMPath),
+			CimName:   filepath.Base(mergedCIMPath),
+		}
+	}
+
+	for _, p := range parentPaths {
+		parentLayers = append(parentLayers, &cimfs.BlockCIM{
+			Type:      layerType,
+			BlockPath: filepath.Dir(p),
+			CimName:   filepath.Base(p),
+		})
+	}
+
+	return &wcowBlockCIMLayers{
+		scratchLayerData: scratchLayerData{
+			scratchLayerPath: m.Source,
+		},
+		parentLayers: parentLayers,
+		mergedLayer:  mergedLayer,
+	}, nil
+}
+
 // ParseWCOWLayers parses the layers provided by containerd into the format understood by hcsshim and prepares
 // them for mounting.
 func ParseWCOWLayers(rootfs []*types.Mount, layerFolders []string) (WCOWLayers, error) {
@@ -123,8 +200,10 @@ func ParseWCOWLayers(rootfs []*types.Mount, layerFolders []string) (WCOWLayers,
 			},
 			layerPaths: parentLayers,
 		}, nil
-	case CimFSMountType:
+	case ForkedCIMMountType:
 		return parseForkedCimMount(m)
+	case BlockCIMMountType:
+		return parseBlockCIMMount(m)
 	default:
 		return nil, fmt.Errorf("invalid windows mount type: '%s'", m.Type)
 	}

internal/wclayer/cim/mount.go

@@ -6,6 +6,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"path/filepath"
 	"strings"
 
 	"github.com/Microsoft/go-winio/pkg/guid"
@@ -32,6 +33,51 @@ func MountForkedCimLayer(ctx context.Context, cimPath, containerID string) (stri
 	return vol, nil
 }
 
+// MountBlockCIMLayer mounts the given block cim and returns the mount
+// location of that cim. The containerID is used to generate the volumeID for the volume
+// at which this CIM is mounted.  containerID is used so that if the shim process crashes
+// for any reason, the mounted cim can be correctly cleaned up during `shim delete` call.
+func MountBlockCIMLayer(ctx context.Context, layer *cimfs.BlockCIM, containerID string) (string, error) {
+	volumeGUID, err := guid.NewV5(cimMountNamespace, []byte(containerID))
+	if err != nil {
+		return "", fmt.Errorf("generated cim mount GUID: %w", err)
+	}
+
+	cimPath := filepath.Join(layer.BlockPath, layer.CimName)
+	mountFlags := hcsschema.CimMountFlagCacheFiles
+	if layer.Type == cimfs.BlockCIMTypeDevice {
+		mountFlags |= hcsschema.CimMountBlockDeviceCim
+	} else {
+		mountFlags |= hcsschema.CimMountSingleFileCim
+	}
+	vol, err := cimfs.Mount(cimPath, volumeGUID, mountFlags)
+	if err != nil {
+		return "", err
+	}
+	return vol, nil
+}
+
+// MergeMountBlockCIMLayer mounts the given merged block cim and returns the mount
+// location of that cim. The containerID is used to generate the volumeID for the volume
+// at which this CIM is mounted.  containerID is used so that if the shim process crashes
+// for any reason, the mounted cim can be correctly cleaned up during `shim delete` call.
+// parentLayers MUST be in the base to topmost order. I.e base layer should be at index 0
+// and immediate parent MUST be at the last index.
+func MergeMountBlockCIMLayer(ctx context.Context, mergedLayer *cimfs.BlockCIM, parentLayers []*cimfs.BlockCIM, containerID string) (string, error) {
+	volumeGUID, err := guid.NewV5(cimMountNamespace, []byte(containerID))
+	if err != nil {
+		return "", fmt.Errorf("generated cim mount GUID: %w", err)
+	}
+	mountFlags := hcsschema.CimMountFlagCacheFiles
+	if mergedLayer.Type == cimfs.BlockCIMTypeDevice {
+		mountFlags |= hcsschema.CimMountBlockDeviceCim
+	} else {
+		mountFlags |= hcsschema.CimMountSingleFileCim
+	}
+	cimfs.MountMergedBlockCIMs(mergedLayer, parentLayers, mountFlags, volumeGUID)
+	return "", fmt.Errorf("not implemented")
+}
+
 // Unmounts the cim mounted at the given volume
 func UnmountCimLayer(ctx context.Context, volume string) error {
 	return cimfs.Unmount(volume)