diff --git a/internal/hcs/schema2/cpu_group_property.go b/internal/hcs/schema2/cpu_group_property.go index bbad6a2c45..31fe07c3aa 100644 --- a/internal/hcs/schema2/cpu_group_property.go +++ b/internal/hcs/schema2/cpu_group_property.go @@ -9,6 +9,14 @@ package hcsschema +type CPUGroupPropertyCode uint32 + +const ( + CPUCapacityProperty = 0x00010000 + CPUSchedulingPriorityProperty = 0x00020000 + IdleLPReserveProperty = 0x00030000 +) + type CpuGroupProperty struct { PropertyCode uint32 `json:"PropertyCode,omitempty"` PropertyValue uint32 `json:"PropertyValue,omitempty"` diff --git a/internal/uvm/cpugroups.go b/internal/uvm/cpugroups.go index 17d442c88e..f4c17fae2c 100644 --- a/internal/uvm/cpugroups.go +++ b/internal/uvm/cpugroups.go @@ -8,12 +8,10 @@ import ( "github.com/Microsoft/hcsshim/internal/cpugroup" "github.com/Microsoft/hcsshim/internal/hcs/resourcepaths" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + "github.com/Microsoft/hcsshim/osversion" ) -// Build that assigning a cpu group on creation of a vm is supported -const cpuGroupCreateBuild = 20124 - -var errCPUGroupCreateNotSupported = fmt.Errorf("cpu group assignment on create requires a build of %d or higher", cpuGroupCreateBuild) +var errCPUGroupCreateNotSupported = fmt.Errorf("cpu group assignment on create requires a build of %d or higher", osversion.V21H1) // ReleaseCPUGroup unsets the cpugroup from the VM func (uvm *UtilityVM) ReleaseCPUGroup(ctx context.Context) error { diff --git a/internal/uvm/create_lcow.go b/internal/uvm/create_lcow.go index e30791e881..3113a0cce8 100644 --- a/internal/uvm/create_lcow.go +++ b/internal/uvm/create_lcow.go @@ -216,7 +216,7 @@ func CreateLCOW(ctx context.Context, opts *OptionsLCOW) (_ *UtilityVM, err error } // We can set a cpu group for the VM at creation time in recent builds. if opts.CPUGroupID != "" { - if osversion.Build() < cpuGroupCreateBuild { + if osversion.Build() < osversion.V21H1 { return nil, errCPUGroupCreateNotSupported } processor.CpuGroup = &hcsschema.CpuGroup{Id: opts.CPUGroupID} diff --git a/internal/uvm/create_wcow.go b/internal/uvm/create_wcow.go index 3e8797768b..da414bd2a8 100644 --- a/internal/uvm/create_wcow.go +++ b/internal/uvm/create_wcow.go @@ -160,7 +160,7 @@ func prepareConfigDoc(ctx context.Context, uvm *UtilityVM, opts *OptionsWCOW, uv } // We can set a cpu group for the VM at creation time in recent builds. if opts.CPUGroupID != "" { - if osversion.Build() < cpuGroupCreateBuild { + if osversion.Build() < osversion.V21H1 { return nil, errCPUGroupCreateNotSupported } processor.CpuGroup = &hcsschema.CpuGroup{Id: opts.CPUGroupID} diff --git a/internal/uvm/update_uvm.go b/internal/uvm/update_uvm.go index 635c81ae81..1e1e07f274 100644 --- a/internal/uvm/update_uvm.go +++ b/internal/uvm/update_uvm.go @@ -4,10 +4,11 @@ import ( "context" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + "github.com/Microsoft/hcsshim/pkg/annotations" specs "github.com/opencontainers/runtime-spec/specs-go" ) -func (uvm *UtilityVM) UpdateConstraints(ctx context.Context, data interface{}, annotations map[string]string) error { +func (uvm *UtilityVM) UpdateConstraints(ctx context.Context, data interface{}, annots map[string]string) error { var memoryLimitInBytes *uint64 var processorLimits *hcsschema.ProcessorLimits @@ -52,5 +53,12 @@ func (uvm *UtilityVM) UpdateConstraints(ctx context.Context, data interface{}, a } } + // Check if an annotation was sent to update cpugroup membership + if cpuGroupID, ok := annots[annotations.CPUGroupID]; ok { + if err := uvm.SetCPUGroup(ctx, cpuGroupID); err != nil { + return err + } + } + return nil } diff --git a/test/cri-containerd/pod_update_test.go b/test/cri-containerd/pod_update_test.go index a80c536c36..2106ae0851 100644 --- a/test/cri-containerd/pod_update_test.go +++ b/test/cri-containerd/pod_update_test.go @@ -7,7 +7,11 @@ import ( "fmt" "testing" + "github.com/Microsoft/hcsshim/internal/cpugroup" + "github.com/Microsoft/hcsshim/internal/processorinfo" + "github.com/Microsoft/hcsshim/osversion" "github.com/Microsoft/hcsshim/pkg/annotations" + testutilities "github.com/Microsoft/hcsshim/test/functional/utilities" runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" ) @@ -215,3 +219,102 @@ func Test_Pod_UpdateResources_CPUShares(t *testing.T) { }) } } + +func Test_Pod_UpdateResources_CPUGroup(t *testing.T) { + testutilities.RequiresBuild(t, osversion.V21H1) + ctx := context.Background() + + processorTopology, err := processorinfo.HostProcessorInfo(ctx) + if err != nil { + t.Fatalf("failed to get host processor information: %s", err) + } + lpIndices := make([]uint32, processorTopology.LogicalProcessorCount) + for i, p := range processorTopology.LogicalProcessors { + lpIndices[i] = p.LpIndex + } + + startCPUGroupID := "FA22A12C-36B3-486D-A3E9-BC526C2B450B" + if err := cpugroup.Create(ctx, startCPUGroupID, lpIndices); err != nil { + t.Fatalf("failed to create test cpugroup with: %v", err) + } + + defer func() { + err := cpugroup.Delete(ctx, startCPUGroupID) + if err != nil && err != cpugroup.ErrHVStatusInvalidCPUGroupState { + t.Fatalf("failed to clean up test cpugroup with: %v", err) + } + }() + + updateCPUGroupID := "FA22A12C-36B3-486D-A3E9-BC526C2B450C" + if err := cpugroup.Create(ctx, updateCPUGroupID, lpIndices); err != nil { + t.Fatalf("failed to create test cpugroup with: %v", err) + } + + defer func() { + err := cpugroup.Delete(ctx, updateCPUGroupID) + if err != nil && err != cpugroup.ErrHVStatusInvalidCPUGroupState { + t.Fatalf("failed to clean up test cpugroup with: %v", err) + } + }() + + type config struct { + name string + requiredFeatures []string + runtimeHandler string + sandboxImage string + } + + tests := []config{ + { + name: "WCOW_Hypervisor", + requiredFeatures: []string{featureWCOWHypervisor}, + runtimeHandler: wcowHypervisorRuntimeHandler, + sandboxImage: imageWindowsNanoserver, + }, + { + name: "LCOW", + requiredFeatures: []string{featureLCOW}, + runtimeHandler: lcowRuntimeHandler, + sandboxImage: imageLcowK8sPause, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + requireFeatures(t, test.requiredFeatures...) + if test.runtimeHandler == lcowRuntimeHandler { + pullRequiredLCOWImages(t, []string{test.sandboxImage}) + } else { + pullRequiredImages(t, []string{test.sandboxImage}) + } + + podRequest := getRunPodSandboxRequest(t, test.runtimeHandler, WithSandboxAnnotations(map[string]string{ + annotations.CPUGroupID: startCPUGroupID, + })) + client := newTestRuntimeClient(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + podID := runPodSandbox(t, client, ctx, podRequest) + defer removePodSandbox(t, client, ctx, podID) + defer stopPodSandbox(t, client, ctx, podID) + + updateReq := &runtime.UpdateContainerResourcesRequest{ + ContainerId: podID, + Annotations: map[string]string{ + annotations.CPUGroupID: updateCPUGroupID, + }, + } + + if test.runtimeHandler == lcowRuntimeHandler { + updateReq.Linux = &runtime.LinuxContainerResources{} + } else { + updateReq.Windows = &runtime.WindowsContainerResources{} + } + + if _, err := client.UpdateContainerResources(ctx, updateReq); err != nil { + t.Fatalf("updating container resources for %s with %v", podID, err) + } + }) + } +} diff --git a/test/cri-containerd/runpodsandbox_test.go b/test/cri-containerd/runpodsandbox_test.go index 28261f8425..d549f02592 100644 --- a/test/cri-containerd/runpodsandbox_test.go +++ b/test/cri-containerd/runpodsandbox_test.go @@ -978,7 +978,7 @@ func Test_RunPodSandbox_Mount_SandboxDir_NoShare_WCOW(t *testing.T) { } func Test_RunPodSandbox_CPUGroup(t *testing.T) { - testutilities.RequiresBuild(t, 20124) + testutilities.RequiresBuild(t, osversion.V21H1) ctx := context.Background() presentID := "FA22A12C-36B3-486D-A3E9-BC526C2B450B" diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/hcs/schema2/cpu_group_property.go b/test/vendor/github.com/Microsoft/hcsshim/internal/hcs/schema2/cpu_group_property.go index bbad6a2c45..31fe07c3aa 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/hcs/schema2/cpu_group_property.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/hcs/schema2/cpu_group_property.go @@ -9,6 +9,14 @@ package hcsschema +type CPUGroupPropertyCode uint32 + +const ( + CPUCapacityProperty = 0x00010000 + CPUSchedulingPriorityProperty = 0x00020000 + IdleLPReserveProperty = 0x00030000 +) + type CpuGroupProperty struct { PropertyCode uint32 `json:"PropertyCode,omitempty"` PropertyValue uint32 `json:"PropertyValue,omitempty"` diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/uvm/cpugroups.go b/test/vendor/github.com/Microsoft/hcsshim/internal/uvm/cpugroups.go index 17d442c88e..f4c17fae2c 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/uvm/cpugroups.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/uvm/cpugroups.go @@ -8,12 +8,10 @@ import ( "github.com/Microsoft/hcsshim/internal/cpugroup" "github.com/Microsoft/hcsshim/internal/hcs/resourcepaths" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + "github.com/Microsoft/hcsshim/osversion" ) -// Build that assigning a cpu group on creation of a vm is supported -const cpuGroupCreateBuild = 20124 - -var errCPUGroupCreateNotSupported = fmt.Errorf("cpu group assignment on create requires a build of %d or higher", cpuGroupCreateBuild) +var errCPUGroupCreateNotSupported = fmt.Errorf("cpu group assignment on create requires a build of %d or higher", osversion.V21H1) // ReleaseCPUGroup unsets the cpugroup from the VM func (uvm *UtilityVM) ReleaseCPUGroup(ctx context.Context) error { diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/uvm/create_lcow.go b/test/vendor/github.com/Microsoft/hcsshim/internal/uvm/create_lcow.go index e30791e881..3113a0cce8 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/uvm/create_lcow.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/uvm/create_lcow.go @@ -216,7 +216,7 @@ func CreateLCOW(ctx context.Context, opts *OptionsLCOW) (_ *UtilityVM, err error } // We can set a cpu group for the VM at creation time in recent builds. if opts.CPUGroupID != "" { - if osversion.Build() < cpuGroupCreateBuild { + if osversion.Build() < osversion.V21H1 { return nil, errCPUGroupCreateNotSupported } processor.CpuGroup = &hcsschema.CpuGroup{Id: opts.CPUGroupID} diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/uvm/create_wcow.go b/test/vendor/github.com/Microsoft/hcsshim/internal/uvm/create_wcow.go index 3e8797768b..da414bd2a8 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/uvm/create_wcow.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/uvm/create_wcow.go @@ -160,7 +160,7 @@ func prepareConfigDoc(ctx context.Context, uvm *UtilityVM, opts *OptionsWCOW, uv } // We can set a cpu group for the VM at creation time in recent builds. if opts.CPUGroupID != "" { - if osversion.Build() < cpuGroupCreateBuild { + if osversion.Build() < osversion.V21H1 { return nil, errCPUGroupCreateNotSupported } processor.CpuGroup = &hcsschema.CpuGroup{Id: opts.CPUGroupID} diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/uvm/update_uvm.go b/test/vendor/github.com/Microsoft/hcsshim/internal/uvm/update_uvm.go index 635c81ae81..1e1e07f274 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/uvm/update_uvm.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/uvm/update_uvm.go @@ -4,10 +4,11 @@ import ( "context" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + "github.com/Microsoft/hcsshim/pkg/annotations" specs "github.com/opencontainers/runtime-spec/specs-go" ) -func (uvm *UtilityVM) UpdateConstraints(ctx context.Context, data interface{}, annotations map[string]string) error { +func (uvm *UtilityVM) UpdateConstraints(ctx context.Context, data interface{}, annots map[string]string) error { var memoryLimitInBytes *uint64 var processorLimits *hcsschema.ProcessorLimits @@ -52,5 +53,12 @@ func (uvm *UtilityVM) UpdateConstraints(ctx context.Context, data interface{}, a } } + // Check if an annotation was sent to update cpugroup membership + if cpuGroupID, ok := annots[annotations.CPUGroupID]; ok { + if err := uvm.SetCPUGroup(ctx, cpuGroupID); err != nil { + return err + } + } + return nil }