Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Set default scratch size for containers/UVMs and reuse LCOW scratch #886

Merged
merged 1 commit into from
Nov 19, 2020
Merged

Set default scratch size for containers/UVMs and reuse LCOW scratch #886

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
465 changes: 343 additions & 122 deletions cmd/containerd-shim-runhcs-v1/options/runhcs.pb.go
View file
Open in desktop

Large diffs are not rendered by default.

23 changes: 19 additions & 4 deletions cmd/containerd-shim-runhcs-v1/options/runhcs.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,10 +64,25 @@ message Options {
// when a container requests a gpu
string GPUVHDPath = 10;

// scale_cpu_limits_to_sandbox indicates that container CPU limits should
// be adjusted to account for the difference in number of cores between the
// host and UVM.
bool scale_cpu_limits_to_sandbox = 11;
// scale_cpu_limits_to_sandbox indicates that container CPU limits should
// be adjusted to account for the difference in number of cores between the
// host and UVM.
bool scale_cpu_limits_to_sandbox = 11;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@dcantah here is the dup code

Copy link
Contributor Author

@dcantah dcantah Oct 23, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's right above in the diff

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah looks like it was just reformatted.


// default_container_scratch_size_in_gb is the default scratch size (sandbox.vhdx)
// to be used for containers. Every container will get a sandbox of `size_in_gb` assigned
// instead of the default of 20GB.
int32 default_container_scratch_size_in_gb = 12;

// default_vm_scratch_size_in_gb is the default scratch size (sandbox.vhdx)
// to be used for the UVM. This only applies to WCOW as LCOW doesn't mount a scratch
// specifically for the UVM.
int32 default_vm_scratch_size_in_gb = 13;

// share_scratch specifies if we'd like to reuse scratch space between multiple containers.
// This currently only affects LCOW. The sandbox containers scratch space is re-used for all
// subsequent containers launched in the pod.
bool share_scratch = 14;
}

// ProcessDetails contains additional information about a process. This is the additional
Expand Down
2 changes: 1 addition & 1 deletion internal/hcsoci/create.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,10 +118,10 @@ func CreateContainer(ctx context.Context, createOptions *CreateOptions) (_ cow.C
}()

if coi.HostingSystem != nil {
n := coi.HostingSystem.ContainerCounter()
if coi.Spec.Linux != nil {
r.SetContainerRootInUVM(fmt.Sprintf(lcowRootInUVM, createOptions.ID))
} else {
n := coi.HostingSystem.ContainerCounter()
r.SetContainerRootInUVM(fmt.Sprintf(wcowRootInUVM, strconv.FormatUint(n, 16)))
}
}
Expand Down
26 changes: 24 additions & 2 deletions internal/layers/layers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,9 +143,13 @@ func MountContainerLayers(ctx context.Context, layerFolders []string, guestRoot
}
}

hostPath := filepath.Join(layerFolders[len(layerFolders)-1], "sandbox.vhdx")
containerScratchPathInUVM := ospath.Join(uvm.OS(), guestRoot)
hostPath, err := getScratchVHDPath(layerFolders)
if err != nil {
return "", fmt.Errorf("failed to get scratch VHD path in layer folders: %s", err)
}
log.G(ctx).WithField("hostPath", hostPath).Debug("mounting scratch VHD")

scsiMount, err := uvm.AddSCSI(ctx, hostPath, containerScratchPathInUVM, false, uvmpkg.VMAccessTypeIndividual)
if err != nil {
return "", fmt.Errorf("failed to add SCSI scratch VHD: %s", err)
Expand Down Expand Up @@ -284,7 +288,10 @@ func UnmountContainerLayers(ctx context.Context, layerFolders []string, containe

// Unload the SCSI scratch path
if (op & UnmountOperationSCSI) == UnmountOperationSCSI {
hostScratchFile := filepath.Join(layerFolders[len(layerFolders)-1], "sandbox.vhdx")
hostScratchFile, err := getScratchVHDPath(layerFolders)
if err != nil {
return fmt.Errorf("failed to get scratch VHD path in layer folders: %s", err)
}
if err := uvm.RemoveSCSI(ctx, hostScratchFile); err != nil {
log.G(ctx).WithError(err).Warn("failed to remove scratch")
if retError == nil {
Expand Down Expand Up @@ -353,3 +360,18 @@ func containerRootfsPath(uvm *uvm.UtilityVM, rootPath string) string {
}
return ospath.Join(uvm.OS(), rootPath, uvmpkg.RootfsPath)
}

func getScratchVHDPath(layerFolders []string) (string, error) {
hostPath := filepath.Join(layerFolders[len(layerFolders)-1], "sandbox.vhdx")
// For LCOW, we can reuse another container's scratch space (usually the sandbox container's).
//
// When sharing a scratch space, the `hostPath` will be a symlink to the sandbox.vhdx location to use.
// When not sharing a scratch space, `hostPath` will be the path to the sandbox.vhdx to use.
//
// Evaluate the symlink here (if there is one).
hostPath, err := filepath.EvalSymlinks(hostPath)
if err != nil {
return "", fmt.Errorf("failed to eval symlinks: %s", err)
}
return hostPath, nil
}
6 changes: 6 additions & 0 deletions internal/uvm/create_wcow.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
hcsschema "github.com/Microsoft/hcsshim/internal/schema2"
"github.com/Microsoft/hcsshim/internal/schemaversion"
"github.com/Microsoft/hcsshim/internal/uvmfolder"
"github.com/Microsoft/hcsshim/internal/wclayer"
"github.com/Microsoft/hcsshim/internal/wcow"
"github.com/pkg/errors"
"go.opencensus.io/trace"
Expand Down Expand Up @@ -110,6 +111,11 @@ func CreateWCOW(ctx context.Context, opts *OptionsWCOW) (_ *UtilityVM, err error
if err := wcow.CreateUVMScratch(ctx, uvmFolder, scratchFolder, uvm.id); err != nil {
return nil, fmt.Errorf("failed to create scratch: %s", err)
}
} else {
ambarve marked this conversation as resolved.
Show resolved Hide resolved
// Sandbox.vhdx exists, just need to grant vm access to it.
if err := wclayer.GrantVmAccess(ctx, uvm.id, scratchPath); err != nil {
return nil, errors.Wrap(err, "failed to grant vm access to scratch")
}
}

processorTopology, err := hostProcessorInfo(ctx)
Expand Down
2 changes: 1 addition & 1 deletion test/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ module github.com/Microsoft/hcsshim/test
go 1.13

require (
github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5
github.com/Microsoft/go-winio v0.4.15-0.20200908182639-5b44b70ab3ab
github.com/Microsoft/hcsshim v0.8.7
github.com/blang/semver v3.1.0+incompatible // indirect
github.com/containerd/containerd v1.3.2
Expand Down
1 change: 1 addition & 0 deletions test/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5 h1:ygIc8M6trr62pF5DucadTWGdEB4mEyvzi0e2nbcmcyA=
github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw=
github.com/Microsoft/go-winio v0.4.15-0.20200908182639-5b44b70ab3ab/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw=
github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ=
github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0=
github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
Expand Down