Merge pull request #886 from dcantah/lcow-scratch-work

Set default scratch size for containers/UVMs and reuse LCOW scratch

cmd/containerd-shim-runhcs-v1/options/runhcs.proto

@@ -64,10 +64,25 @@ message Options {
 	// when a container requests a gpu
 	string GPUVHDPath = 10;
 
-        // scale_cpu_limits_to_sandbox indicates that container CPU limits should
-        // be adjusted to account for the difference in number of cores between the
-        // host and UVM.
-        bool scale_cpu_limits_to_sandbox = 11;
+	// scale_cpu_limits_to_sandbox indicates that container CPU limits should
+	// be adjusted to account for the difference in number of cores between the
+	// host and UVM.
+	bool scale_cpu_limits_to_sandbox = 11;
+
+	// default_container_scratch_size_in_gb is the default scratch size (sandbox.vhdx) 
+	// to be used for containers. Every container will get a sandbox of `size_in_gb` assigned
+	// instead of the default of 20GB. 
+	int32 default_container_scratch_size_in_gb = 12;
+
+	// default_vm_scratch_size_in_gb is the default scratch size (sandbox.vhdx) 
+	// to be used for the UVM. This only applies to WCOW as LCOW doesn't mount a scratch 
+	// specifically for the UVM.
+	int32 default_vm_scratch_size_in_gb = 13;
+
+	// share_scratch specifies if we'd like to reuse scratch space between multiple containers. 
+	// This currently only affects LCOW. The sandbox containers scratch space is re-used for all
+	// subsequent containers launched in the pod.  
+	bool share_scratch = 14;
 }
 
 // ProcessDetails contains additional information about a process. This is the additional

internal/hcsoci/create.go

@@ -118,10 +118,10 @@ func CreateContainer(ctx context.Context, createOptions *CreateOptions) (_ cow.C
 	}()
 
 	if coi.HostingSystem != nil {
-		n := coi.HostingSystem.ContainerCounter()
 		if coi.Spec.Linux != nil {
 			r.SetContainerRootInUVM(fmt.Sprintf(lcowRootInUVM, createOptions.ID))
 		} else {
+			n := coi.HostingSystem.ContainerCounter()
 			r.SetContainerRootInUVM(fmt.Sprintf(wcowRootInUVM, strconv.FormatUint(n, 16)))
 		}
 	}

internal/layers/layers.go

@@ -143,9 +143,13 @@ func MountContainerLayers(ctx context.Context, layerFolders []string, guestRoot
 		}
 	}
 
-	hostPath := filepath.Join(layerFolders[len(layerFolders)-1], "sandbox.vhdx")
 	containerScratchPathInUVM := ospath.Join(uvm.OS(), guestRoot)
+	hostPath, err := getScratchVHDPath(layerFolders)
+	if err != nil {
+		return "", fmt.Errorf("failed to get scratch VHD path in layer folders: %s", err)
+	}
 	log.G(ctx).WithField("hostPath", hostPath).Debug("mounting scratch VHD")
+
 	scsiMount, err := uvm.AddSCSI(ctx, hostPath, containerScratchPathInUVM, false, uvmpkg.VMAccessTypeIndividual)
 	if err != nil {
 		return "", fmt.Errorf("failed to add SCSI scratch VHD: %s", err)
@@ -284,7 +288,10 @@ func UnmountContainerLayers(ctx context.Context, layerFolders []string, containe
 
 	// Unload the SCSI scratch path
 	if (op & UnmountOperationSCSI) == UnmountOperationSCSI {
-		hostScratchFile := filepath.Join(layerFolders[len(layerFolders)-1], "sandbox.vhdx")
+		hostScratchFile, err := getScratchVHDPath(layerFolders)
+		if err != nil {
+			return fmt.Errorf("failed to get scratch VHD path in layer folders: %s", err)
+		}
 		if err := uvm.RemoveSCSI(ctx, hostScratchFile); err != nil {
 			log.G(ctx).WithError(err).Warn("failed to remove scratch")
 			if retError == nil {
@@ -353,3 +360,18 @@ func containerRootfsPath(uvm *uvm.UtilityVM, rootPath string) string {
 	}
 	return ospath.Join(uvm.OS(), rootPath, uvmpkg.RootfsPath)
 }
+
+func getScratchVHDPath(layerFolders []string) (string, error) {
+	hostPath := filepath.Join(layerFolders[len(layerFolders)-1], "sandbox.vhdx")
+	// For LCOW, we can reuse another container's scratch space (usually the sandbox container's).
+	//
+	// When sharing a scratch space, the `hostPath` will be a symlink to the sandbox.vhdx location to use.
+	// When not sharing a scratch space, `hostPath` will be the path to the sandbox.vhdx to use.
+	//
+	// Evaluate the symlink here (if there is one).
+	hostPath, err := filepath.EvalSymlinks(hostPath)
+	if err != nil {
+		return "", fmt.Errorf("failed to eval symlinks: %s", err)
+	}
+	return hostPath, nil
+}

internal/uvm/create_wcow.go

@@ -16,6 +16,7 @@ import (
 	hcsschema "github.com/Microsoft/hcsshim/internal/schema2"
 	"github.com/Microsoft/hcsshim/internal/schemaversion"
 	"github.com/Microsoft/hcsshim/internal/uvmfolder"
+	"github.com/Microsoft/hcsshim/internal/wclayer"
 	"github.com/Microsoft/hcsshim/internal/wcow"
 	"github.com/pkg/errors"
 	"go.opencensus.io/trace"
@@ -110,6 +111,11 @@ func CreateWCOW(ctx context.Context, opts *OptionsWCOW) (_ *UtilityVM, err error
 		if err := wcow.CreateUVMScratch(ctx, uvmFolder, scratchFolder, uvm.id); err != nil {
 			return nil, fmt.Errorf("failed to create scratch: %s", err)
 		}
+	} else {
+		// Sandbox.vhdx exists, just need to grant vm access to it.
+		if err := wclayer.GrantVmAccess(ctx, uvm.id, scratchPath); err != nil {
+			return nil, errors.Wrap(err, "failed to grant vm access to scratch")
+		}
 	}
 
 	processorTopology, err := hostProcessorInfo(ctx)

test/go.mod

@@ -3,7 +3,7 @@ module github.com/Microsoft/hcsshim/test
 go 1.13
 
 require (
-	github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5
+	github.com/Microsoft/go-winio v0.4.15-0.20200908182639-5b44b70ab3ab
 	github.com/Microsoft/hcsshim v0.8.7
 	github.com/blang/semver v3.1.0+incompatible // indirect
 	github.com/containerd/containerd v1.3.2

test/go.sum

@@ -6,6 +6,7 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5 h1:ygIc8M6trr62pF5DucadTWGdEB4mEyvzi0e2nbcmcyA=
 github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw=
+github.com/Microsoft/go-winio v0.4.15-0.20200908182639-5b44b70ab3ab/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw=
 github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ=
 github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0=
 github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=