From f30bade7f82da6b97267ac89ff1d501f37b18418 Mon Sep 17 00:00:00 2001 From: Yuanyuan Lei Date: Sat, 30 Dec 2023 01:07:25 -0800 Subject: [PATCH 1/5] removed lcow tests --- test/cri-containerd/container_bench_test.go | 17 - .../container_fileshare_test.go | 94 - test/cri-containerd/container_network_test.go | 82 +- test/cri-containerd/container_test.go | 947 -------- test/cri-containerd/container_update_test.go | 107 +- .../container_virtual_device_test.go | 285 +-- test/cri-containerd/createcontainer_test.go | 381 +--- test/cri-containerd/execcontainer_test.go | 137 -- test/cri-containerd/extended_task_test.go | 16 +- test/cri-containerd/helper_container_test.go | 34 - test/cri-containerd/helper_exec_test.go | 9 - .../helper_update_utilities_test.go | 23 - test/cri-containerd/layer_integrity_test.go | 94 - test/cri-containerd/logging_binary_test.go | 24 +- test/cri-containerd/main_test.go | 57 - test/cri-containerd/pod_update_test.go | 94 +- test/cri-containerd/policy_test.go | 2026 ----------------- test/cri-containerd/pullimage_test.go | 8 - test/cri-containerd/removepodsandbox_test.go | 33 +- test/cri-containerd/runpodsandbox_test.go | 1071 +-------- test/cri-containerd/stats_test.go | 143 +- test/cri-containerd/stopcontainer_test.go | 161 -- .../imagepull_timestamp/Dockerfile | 2 + .../latest/delayed-shutdown.go | 2 +- .../latest/delayed-shutdown.go | 2 +- test/go.mod | 7 +- test/go.sum | 2 - 27 files changed, 103 insertions(+), 5755 deletions(-) delete mode 100644 test/cri-containerd/container_test.go delete mode 100644 test/cri-containerd/execcontainer_test.go delete mode 100644 test/cri-containerd/layer_integrity_test.go delete mode 100644 test/cri-containerd/policy_test.go diff --git a/test/cri-containerd/container_bench_test.go b/test/cri-containerd/container_bench_test.go index cd3e1985e0..560e9a0e55 100644 --- a/test/cri-containerd/container_bench_test.go +++ b/test/cri-containerd/container_bench_test.go @@ -19,13 +19,6 @@ var _containerBenchmarkTests = []struct { Image string Command []string }{ - { - Name: "LCOW", - Feature: featureLCOW, - Runtime: lcowRuntimeHandler, - Image: imageLcowAlpine, - Command: []string{"ash", "-c", "tail -f /dev/null"}, - }, { Name: "WCOW_Hypervisor", Feature: featureWCOWHypervisor, @@ -55,8 +48,6 @@ func BenchmarkPodCreate(b *testing.B) { requireFeatures(b, tt.Feature) switch tt.Feature { - case featureLCOW: - pullRequiredLCOWImages(b, append([]string{imageLcowK8sPause}, tt.Image)) case featureWCOWHypervisor, featureWCOWProcess: pullRequiredImages(b, []string{tt.Image}) } @@ -92,8 +83,6 @@ func BenchmarkContainerCreate(b *testing.B) { requireFeatures(b, tt.Feature) switch tt.Feature { - case featureLCOW: - pullRequiredLCOWImages(b, append([]string{imageLcowK8sPause}, tt.Image)) case featureWCOWHypervisor, featureWCOWProcess: pullRequiredImages(b, []string{tt.Image}) } @@ -127,8 +116,6 @@ func BenchmarkContainerCreate(b *testing.B) { requireFeatures(b, tt.Feature) switch tt.Feature { - case featureLCOW: - pullRequiredLCOWImages(b, append([]string{imageLcowK8sPause}, tt.Image)) case featureWCOWHypervisor, featureWCOWProcess: pullRequiredImages(b, []string{tt.Image}) } @@ -169,8 +156,6 @@ func BenchmarkContainerStart(b *testing.B) { requireFeatures(b, tt.Feature) switch tt.Feature { - case featureLCOW: - pullRequiredLCOWImages(b, append([]string{imageLcowK8sPause}, tt.Image)) case featureWCOWHypervisor, featureWCOWProcess: pullRequiredImages(b, []string{tt.Image}) } @@ -206,8 +191,6 @@ func BenchmarkContainerStart(b *testing.B) { requireFeatures(b, tt.Feature) switch tt.Feature { - case featureLCOW: - pullRequiredLCOWImages(b, append([]string{imageLcowK8sPause}, tt.Image)) case featureWCOWHypervisor, featureWCOWProcess: pullRequiredImages(b, []string{tt.Image}) } diff --git a/test/cri-containerd/container_fileshare_test.go b/test/cri-containerd/container_fileshare_test.go index 6b70157f87..4523e4347e 100644 --- a/test/cri-containerd/container_fileshare_test.go +++ b/test/cri-containerd/container_fileshare_test.go @@ -108,97 +108,3 @@ func Test_Container_File_Share_Writable_WCOW(t *testing.T) { t.Fatalf("did not correctly read file; got %q, expected %q", output, testContent) } } - -func Test_Container_File_Share_Writable_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - - client := newTestRuntimeClient(t) - ctx := context.Background() - - sbRequest := getRunPodSandboxRequest(t, lcowRuntimeHandler, - WithSandboxAnnotations(map[string]string{ - annotations.DisableWritableFileShares: "true", - }), - ) - podID := runPodSandbox(t, client, ctx, sbRequest) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - - var ( - tempDir = t.TempDir() - containerPath = "/mnt/test" - testFile = "t.txt" - testContent = "hello world" - ) - - if err := os.WriteFile( - filepath.Join(tempDir, testFile), - []byte(testContent), - 0644, - ); err != nil { - t.Fatalf("could not create test file: %v", err) - } - - cRequest := &runtime.CreateContainerRequest{ - Config: &runtime.ContainerConfig{ - Metadata: &runtime.ContainerMetadata{ - Name: t.Name(), - }, - Image: &runtime.ImageSpec{ - Image: imageLcowAlpine, - }, - Command: []string{ - "ash", - "-c", - "tail -f /dev/null", - }, - Mounts: []*runtime.Mount{ - { - HostPath: tempDir, - ContainerPath: containerPath, - Readonly: false, - }, - }, - }, - PodSandboxId: podID, - SandboxConfig: sbRequest.Config, - } - cID := createContainer(t, client, ctx, cRequest) - defer removeContainer(t, client, ctx, cID) - - // container should fail because of writable mount - _, err := client.StartContainer(ctx, &runtime.StartContainerRequest{ContainerId: cID}) - if err == nil { - stopContainer(t, client, ctx, cID) - } - // error is serialized over gRPC then embedded into "rpc error: code = %s desc = %s" - // so error.Is() wont work - if err == nil || !strings.Contains(err.Error(), fmt.Errorf("adding writable shares is denied: %w", hcs.ErrOperationDenied).Error()) { - t.Fatalf("StartContainer did not fail with writable fileshare: error is %v", err) - } - - // set it to read only - cRequest.Config.Metadata.Name = t.Name() + "_2" - cRequest.Config.Mounts[0].Readonly = true - - cID = createContainer(t, client, ctx, cRequest) - defer removeContainer(t, client, ctx, cID) - startContainer(t, client, ctx, cID) - defer stopContainer(t, client, ctx, cID) - - testExec := []string{ - "ash", - "-c", - // filepath.Join replaces `/` with `\`, so Join path manually - "cat " + containerPath + "/" + testFile, - } - output, errMsg, exitCode := execContainer(t, client, ctx, cID, testExec) - if exitCode != 0 { - t.Fatalf("could not find mounted file: %s %s", errMsg, output) - } - if output != testContent { - t.Fatalf("did not correctly read file; got %q, expected %q", output, testContent) - } -} diff --git a/test/cri-containerd/container_network_test.go b/test/cri-containerd/container_network_test.go index 269f7beeab..35a7fa0aae 100644 --- a/test/cri-containerd/container_network_test.go +++ b/test/cri-containerd/container_network_test.go @@ -4,87 +4,15 @@ package cri_containerd import ( - "bufio" "context" - "os" - "path/filepath" "strings" "testing" - "time" runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" ) -func Test_Container_Network_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - - // create a directory and log file - dir := t.TempDir() - log := filepath.Join(dir, "ping.txt") - - sandboxRequest := getRunPodSandboxRequest(t, lcowRuntimeHandler) - - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - podID := runPodSandbox(t, client, ctx, sandboxRequest) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - - request := &runtime.CreateContainerRequest{ - PodSandboxId: podID, - Config: &runtime.ContainerConfig{ - Metadata: &runtime.ContainerMetadata{ - Name: t.Name() + "-Container", - }, - Image: &runtime.ImageSpec{ - Image: imageLcowAlpine, - }, - Command: []string{ - "ping", - "-q", // -q outputs ping stats only. - "-c", - "10", - "google.com", - }, - LogPath: log, - Linux: &runtime.LinuxContainerConfig{}, - }, - SandboxConfig: sandboxRequest.Config, - } - - containerID := createContainer(t, client, ctx, request) - defer removeContainer(t, client, ctx, containerID) - - startContainer(t, client, ctx, containerID) - defer stopContainer(t, client, ctx, containerID) - - // wait a while for container to write to stdout - time.Sleep(3 * time.Second) - - // open the log and test for any packet loss - logFile, err := os.Open(log) - if err != nil { - t.Fatal(err) - } - defer logFile.Close() - - s := bufio.NewScanner(logFile) - for s.Scan() { - v := strings.Fields(s.Text()) - t.Logf("ping output: %v", v) - - if v != nil && v[len(v)-1] == "loss" && v[len(v)-3] != "0%" { - t.Fatalf("expected 0%% packet loss, got %v packet loss", v[len(v)-3]) - } - } -} - func Test_Container_Network_Hostname(t *testing.T) { - requireAnyFeature(t, featureWCOWProcess, featureWCOWHypervisor, featureLCOW) + requireAnyFeature(t, featureWCOWProcess, featureWCOWHypervisor) type config struct { name string @@ -111,14 +39,6 @@ func Test_Container_Network_Hostname(t *testing.T) { containerImage: imageWindowsNanoserver, cmd: []string{"cmd", "/c", "ping", "-t", "127.0.0.1"}, }, - { - name: "LCOW", - requiredFeatures: []string{featureLCOW}, - runtimeHandler: lcowRuntimeHandler, - sandboxImage: imageLcowK8sPause, - containerImage: imageLcowAlpine, - cmd: []string{"top"}, - }, } for _, test := range tests { diff --git a/test/cri-containerd/container_test.go b/test/cri-containerd/container_test.go deleted file mode 100644 index 7ff257ca00..0000000000 --- a/test/cri-containerd/container_test.go +++ /dev/null @@ -1,947 +0,0 @@ -//go:build windows && functional -// +build windows,functional - -package cri_containerd - -import ( - "bufio" - "context" - "fmt" - "io" - "os" - "path/filepath" - "strconv" - "strings" - "testing" - "time" - - "github.com/sirupsen/logrus" - runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" - - "github.com/Microsoft/hcsshim/pkg/annotations" - "github.com/Microsoft/hcsshim/test/pkg/definitions/guestpath" -) - -func runLogRotationContainer(t *testing.T, sandboxRequest *runtime.RunPodSandboxRequest, request *runtime.CreateContainerRequest, log string, logArchive string) { - t.Helper() - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - podID := runPodSandbox(t, client, ctx, sandboxRequest) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - - request.PodSandboxId = podID - request.SandboxConfig = sandboxRequest.Config - - containerID := createContainer(t, client, ctx, request) - defer removeContainer(t, client, ctx, containerID) - - startContainer(t, client, ctx, containerID) - defer stopContainer(t, client, ctx, containerID) - - // Give some time for log output to accumulate. - time.Sleep(3 * time.Second) - - // Rotate the logs. This is done by first renaming the existing log file, - // then calling ReopenContainerLog to cause containerd to start writing to - // a new log file. - - if err := os.Rename(log, logArchive); err != nil { - t.Fatalf("failed to rename log: %v", err) - } - - if _, err := client.ReopenContainerLog(ctx, &runtime.ReopenContainerLogRequest{ContainerId: containerID}); err != nil { - t.Fatalf("failed to reopen log: %v", err) - } - - // Give some time for log output to accumulate. - time.Sleep(3 * time.Second) -} - -func runContainerLifetime(t *testing.T, client runtime.RuntimeServiceClient, ctx context.Context, containerID string) { - t.Helper() - defer removeContainer(t, client, ctx, containerID) - startContainer(t, client, ctx, containerID) - stopContainer(t, client, ctx, containerID) -} - -func Test_RotateLogs_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - image := imageLcowAlpine - dir := t.TempDir() - log := filepath.Join(dir, "log.txt") - logArchive := filepath.Join(dir, "log-archive.txt") - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, image}) - logrus.SetLevel(logrus.DebugLevel) - - sandboxRequest := getRunPodSandboxRequest(t, lcowRuntimeHandler) - - request := &runtime.CreateContainerRequest{ - Config: &runtime.ContainerConfig{ - Metadata: &runtime.ContainerMetadata{ - Name: t.Name() + "-Container", - }, - Image: &runtime.ImageSpec{ - Image: image, - }, - Command: []string{ - "ash", - "-c", - "i=0; while true; do echo $i; i=$(expr $i + 1); sleep .1; done", - }, - LogPath: log, - Linux: &runtime.LinuxContainerConfig{}, - }, - } - - runLogRotationContainer(t, sandboxRequest, request, log, logArchive) - - // Make sure we didn't lose any values while rotating. First set of output - // should be in logArchive, followed by the output in log. - - logArchiveFile, err := os.Open(logArchive) - if err != nil { - t.Fatal(err) - } - defer logArchiveFile.Close() - - logFile, err := os.Open(log) - if err != nil { - t.Fatal(err) - } - defer logFile.Close() - - s := bufio.NewScanner(io.MultiReader(logArchiveFile, logFile)) - expected := 0 - for s.Scan() { - v := strings.Fields(s.Text()) - n, err := strconv.Atoi(v[len(v)-1]) - if err != nil { - t.Fatalf("failed to parse log value as integer: %v", err) - } - if n != expected { - t.Fatalf("missing expected output value: %v (got %v)", expected, n) - } - expected++ - } -} - -func Test_RunContainer_Events_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - client := newTestRuntimeClient(t) - - podctx, podcancel := context.WithCancel(context.Background()) - defer podcancel() - targetNamespace := "k8s.io" - - sandboxRequest := getRunPodSandboxRequest(t, lcowRuntimeHandler) - - podID := runPodSandbox(t, client, podctx, sandboxRequest) - defer removePodSandbox(t, client, podctx, podID) - defer stopPodSandbox(t, client, podctx, podID) - - request := &runtime.CreateContainerRequest{ - Config: &runtime.ContainerConfig{ - Metadata: &runtime.ContainerMetadata{ - Name: t.Name() + "-Container", - }, - Image: &runtime.ImageSpec{ - Image: imageLcowAlpine, - }, - Command: []string{ - "top", - }, - Linux: &runtime.LinuxContainerConfig{}, - }, - PodSandboxId: podID, - SandboxConfig: sandboxRequest.Config, - } - - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - topicNames, filters := getTargetRunTopics() - eventService := newTestEventService(t) - stream, errs := eventService.Subscribe(ctx, filters...) - - containerID := createContainer(t, client, podctx, request) - runContainerLifetime(t, client, podctx, containerID) - - for _, topic := range topicNames { - select { - case env := <-stream: - if topic != env.Topic { - t.Fatalf("event topic %v does not match expected topic %v", env.Topic, topic) - } - if targetNamespace != env.Namespace { - t.Fatalf("event namespace %v does not match expected namespace %v", env.Namespace, targetNamespace) - } - t.Logf("event topic seen: %v", env.Topic) - - id, _, err := convertEvent(env.Event) - if err != nil { - t.Fatalf("topic %v event: %v", env.Topic, err) - } - if id != containerID { - t.Fatalf("event topic %v belongs to container %v, not targeted container %v", env.Topic, id, containerID) - } - case e := <-errs: - t.Fatalf("event subscription err %v", e) - case <-ctx.Done(): - if ctx.Err() == context.DeadlineExceeded { //nolint:errorlint - t.Fatalf("event %v deadline exceeded", topic) - } - } - } -} - -func Test_RunContainer_ForksThenExits_ShowsAsExited_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - client := newTestRuntimeClient(t) - ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) - defer cancel() - - podRequest := getRunPodSandboxRequest(t, lcowRuntimeHandler) - podID := runPodSandbox(t, client, ctx, podRequest) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - - containerRequest := &runtime.CreateContainerRequest{ - Config: &runtime.ContainerConfig{ - Metadata: &runtime.ContainerMetadata{ - Name: t.Name() + "-Container", - }, - Image: &runtime.ImageSpec{ - Image: imageLcowAlpine, - }, - Command: []string{ - // Fork a background process (that runs forever), then exit. - "ash", - "-c", - "ash -c 'while true; do echo foo; sleep 1; done' &", - }, - Linux: &runtime.LinuxContainerConfig{}, - }, - PodSandboxId: podID, - SandboxConfig: podRequest.Config, - } - containerID := createContainer(t, client, ctx, containerRequest) - defer removeContainer(t, client, ctx, containerID) - startContainer(t, client, ctx, containerID) - defer stopContainer(t, client, ctx, containerID) - - requireContainerState(ctx, t, client, containerID, runtime.ContainerState_CONTAINER_EXITED) - - // Validate that the container shows as exited. Once the container init - // dies, the forked background process should be killed off. - statusResponse, err := client.ContainerStatus(ctx, &runtime.ContainerStatusRequest{ContainerId: containerID}) - if err != nil { - t.Fatalf("failed to get container status: %v", err) - } - if statusResponse.Status.State != runtime.ContainerState_CONTAINER_EXITED { - t.Fatalf("container expected to be exited but is in state %s", statusResponse.Status.State) - } -} - -func Test_RunContainer_ZeroVPMEM_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - sandboxRequest := getRunPodSandboxRequest( - t, - lcowRuntimeHandler, - WithSandboxAnnotations(map[string]string{ - annotations.PreferredRootFSType: "initrd", - annotations.VPMemCount: "0", - }), - ) - - podID := runPodSandbox(t, client, ctx, sandboxRequest) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - - request := &runtime.CreateContainerRequest{ - PodSandboxId: podID, - Config: &runtime.ContainerConfig{ - Metadata: &runtime.ContainerMetadata{ - Name: t.Name() + "-Container", - }, - Image: &runtime.ImageSpec{ - Image: imageLcowAlpine, - }, - Command: []string{ - "top", - }, - }, - SandboxConfig: sandboxRequest.Config, - } - - containerID := createContainer(t, client, ctx, request) - runContainerLifetime(t, client, ctx, containerID) -} - -func Test_RunContainer_ZeroVPMEM_Multiple_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - sandboxRequest := getRunPodSandboxRequest( - t, - lcowRuntimeHandler, - WithSandboxAnnotations(map[string]string{ - annotations.PreferredRootFSType: "initrd", - annotations.VPMemCount: "0", - }), - ) - - podID := runPodSandbox(t, client, ctx, sandboxRequest) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - - request := &runtime.CreateContainerRequest{ - PodSandboxId: podID, - Config: &runtime.ContainerConfig{ - Metadata: &runtime.ContainerMetadata{ - Name: "", - }, - Image: &runtime.ImageSpec{ - Image: imageLcowAlpine, - }, - Command: []string{ - "top", - }, - }, - SandboxConfig: sandboxRequest.Config, - } - - request.Config.Metadata.Name = "Container-1" - containerIDOne := createContainer(t, client, ctx, request) - defer removeContainer(t, client, ctx, containerIDOne) - startContainer(t, client, ctx, containerIDOne) - defer stopContainer(t, client, ctx, containerIDOne) - - request.Config.Metadata.Name = "Container-2" - containerIDTwo := createContainer(t, client, ctx, request) - defer removeContainer(t, client, ctx, containerIDTwo) - startContainer(t, client, ctx, containerIDTwo) - defer stopContainer(t, client, ctx, containerIDTwo) -} - -func Test_RunContainer_SandboxDevice_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - sandboxRequest := getRunPodSandboxRequest(t, lcowRuntimeHandler) - sandboxRequest.Config.Linux = &runtime.LinuxPodSandboxConfig{ - SecurityContext: &runtime.LinuxSandboxSecurityContext{ - Privileged: true, - }, - } - - podID := runPodSandbox(t, client, ctx, sandboxRequest) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - - request := &runtime.CreateContainerRequest{ - PodSandboxId: podID, - Config: &runtime.ContainerConfig{ - Metadata: &runtime.ContainerMetadata{ - Name: t.Name() + "-Container", - }, - Image: &runtime.ImageSpec{ - Image: imageLcowAlpine, - }, - Command: []string{ - "top", - }, - - Devices: []*runtime.Device{ - { - HostPath: "/dev/fuse", - }, - }, - }, - SandboxConfig: sandboxRequest.Config, - } - - containerID := createContainer(t, client, ctx, request) - defer removeContainer(t, client, ctx, containerID) - startContainer(t, client, ctx, containerID) - defer stopContainer(t, client, ctx, containerID) - - cmd := []string{"ls", "/dev/fuse"} - containerExecReq := &runtime.ExecSyncRequest{ - ContainerId: containerID, - Cmd: cmd, - Timeout: 20, - } - r := execSync(t, client, ctx, containerExecReq) - if r.ExitCode != 0 { - t.Fatalf("failed with exit code %d: %s", r.ExitCode, string(r.Stderr)) - } - if string(r.Stdout) == "" { - t.Fatal("did not find expected device /dev/fuse in container") - } -} - -func Test_RunContainer_NonDefault_User(t *testing.T) { - requireFeatures(t, featureLCOW) - - type config struct { - containerSecCtx *runtime.LinuxContainerSecurityContext - name string - } - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - - podReq := getRunPodSandboxRequest(t, lcowRuntimeHandler) - podID := runPodSandbox(t, client, ctx, podReq) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - - tests := []config{ - { - containerSecCtx: &runtime.LinuxContainerSecurityContext{ - RunAsUsername: "guest", - }, - name: "RunAsUsername", - }, - { - containerSecCtx: &runtime.LinuxContainerSecurityContext{ - RunAsUser: &runtime.Int64Value{ - Value: 10001, - }, - }, - name: "RunAsUserUID", - }, - } - - for _, test := range tests { - t.Run(test.name, func(_ *testing.T) { - conReq := &runtime.CreateContainerRequest{ - Config: &runtime.ContainerConfig{ - Metadata: &runtime.ContainerMetadata{ - Name: t.Name() + "-Container", - }, - Image: &runtime.ImageSpec{ - Image: imageLcowAlpine, - }, - Command: []string{ - "top", - }, - Linux: &runtime.LinuxContainerConfig{ - SecurityContext: test.containerSecCtx, - }, - }, - PodSandboxId: podID, - SandboxConfig: podReq.Config, - } - - containerID := createContainer(t, client, ctx, conReq) - defer removeContainer(t, client, ctx, containerID) - startContainer(t, client, ctx, containerID) - defer stopContainer(t, client, ctx, containerID) - }) - } -} - -func Test_RunContainer_ShareScratch_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - sandboxRequest := getRunPodSandboxRequest(t, lcowRuntimeHandler) - - podID := runPodSandbox(t, client, ctx, sandboxRequest) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - - request := &runtime.CreateContainerRequest{ - PodSandboxId: podID, - Config: &runtime.ContainerConfig{ - Metadata: &runtime.ContainerMetadata{ - Name: t.Name() + "-Container", - }, - Image: &runtime.ImageSpec{ - Image: imageLcowAlpine, - }, - Command: []string{ - "top", - }, - Annotations: map[string]string{ - "containerd.io/snapshot/io.microsoft.container.storage.reuse-scratch": "true", - }, - }, - SandboxConfig: sandboxRequest.Config, - } - - containerID := createContainer(t, client, ctx, request) - defer removeContainer(t, client, ctx, containerID) - startContainer(t, client, ctx, containerID) - defer stopContainer(t, client, ctx, containerID) - - // Verify that we only have one disk mounted. Generally we'd have sda + sdb for two containers - // (Pause + 1 workload container). - cmd := []string{"ls", "/dev/sdb"} - containerExecReq := &runtime.ExecSyncRequest{ - ContainerId: containerID, - Cmd: cmd, - Timeout: 20, - } - r := execSync(t, client, ctx, containerExecReq) - if r.ExitCode == 0 { - t.Fatalf("expected /dev/sdb to not be present %d: %s", r.ExitCode, string(r.Stdout)) - } -} - -func findOverlaySize(t *testing.T, ctx context.Context, client runtime.RuntimeServiceClient, cid string) []string { - t.Helper() - cmd := []string{"df"} - containerExecReq := &runtime.ExecSyncRequest{ - ContainerId: cid, - Cmd: cmd, - Timeout: 20, - } - r := execSync(t, client, ctx, containerExecReq) - - if r.ExitCode != 0 { - t.Fatalf("failed with exit code %d: %s", r.ExitCode, string(r.Stderr)) - } - - // Format of output for df is below - // Filesystem 1K-blocks Used Available Use% Mounted on - // overlay 20642524 36 19577528 0% / - // tmpfs 65536 0 65536 0% /dev - var ( - scanner = bufio.NewScanner(strings.NewReader(string(r.Stdout))) - cols []string - found bool - ) - for scanner.Scan() { - outputLine := scanner.Text() - if cols = strings.Fields(outputLine); cols[0] == "overlay" && cols[5] == "/" { - found = true - t.Log(outputLine) - break - } - } - - if !found { - t.Fatalf("could not find the correct output line for overlay mount on / n: error: %v, exitcode: %d", string(r.Stdout), r.ExitCode) - } - return cols -} - -func Test_RunContainer_ShareScratch_CheckSize_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - sandboxRequest := getRunPodSandboxRequest(t, lcowRuntimeHandler) - - podID := runPodSandbox(t, client, ctx, sandboxRequest) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - - request := &runtime.CreateContainerRequest{ - PodSandboxId: podID, - Config: &runtime.ContainerConfig{ - Metadata: &runtime.ContainerMetadata{ - Name: "", - }, - Image: &runtime.ImageSpec{ - Image: imageLcowAlpine, - }, - Command: []string{ - "top", - }, - Annotations: map[string]string{ - "containerd.io/snapshot/io.microsoft.container.storage.reuse-scratch": "true", - }, - }, - SandboxConfig: sandboxRequest.Config, - } - - request.Config.Metadata.Name = "Container-1" - containerIDOne := createContainer(t, client, ctx, request) - defer removeContainer(t, client, ctx, containerIDOne) - startContainer(t, client, ctx, containerIDOne) - defer stopContainer(t, client, ctx, containerIDOne) - - request.Config.Metadata.Name = "Container-2" - containerIDTwo := createContainer(t, client, ctx, request) - defer removeContainer(t, client, ctx, containerIDTwo) - startContainer(t, client, ctx, containerIDTwo) - defer stopContainer(t, client, ctx, containerIDTwo) - - cols := findOverlaySize(t, ctx, client, containerIDOne) - availableSizeContainerOne := cols[3] - - cols = findOverlaySize(t, ctx, client, containerIDTwo) - availableSizeContainerTwo := cols[3] - - // Check the initial size for both containers. - if availableSizeContainerOne != availableSizeContainerTwo { - t.Fatalf("expected available rootfs size to be the same, got: %s and %s", availableSizeContainerOne, availableSizeContainerTwo) - } - - // Write a 10MB file and then check size available on the root fs in both containers. - // It should be the same amount as it's being backed by the same disk. - cmd := []string{"dd", "if=/dev/urandom", "of=testfile", "bs=10MB", "count=1"} - containerExecReq := &runtime.ExecSyncRequest{ - ContainerId: containerIDOne, - Cmd: cmd, - Timeout: 20, - } - r := execSync(t, client, ctx, containerExecReq) - - if r.ExitCode != 0 { - t.Fatalf("failed with exit code %d: %s", r.ExitCode, string(r.Stderr)) - } - - // Grab the size available after the first write - cols = findOverlaySize(t, ctx, client, containerIDOne) - availableSizeContainerOne = cols[3] - - // Now the same for container two - cols = findOverlaySize(t, ctx, client, containerIDTwo) - availableSizeContainerTwo = cols[3] - - if availableSizeContainerOne != availableSizeContainerTwo { - t.Fatalf("expected available rootfs size to be the same, got: %s and %s", availableSizeContainerOne, availableSizeContainerTwo) - } -} - -func Test_CreateContainer_DevShmSize(t *testing.T) { - requireFeatures(t, featureLCOW) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - podReq := getRunPodSandboxRequest(t, lcowRuntimeHandler) - podID := runPodSandbox(t, client, ctx, podReq) - defer removePodSandbox(t, client, ctx, podID) - - cmd := []string{"ash", "-c", "while true; do sleep 1; done"} - contReq1 := getCreateContainerRequest(podID, "test-container-devshm-256", imageLcowAlpine, cmd, podReq.Config) - // the /dev/shm size is expected to be in KB, set it to 256 MB - size := 256 * 1024 - contReq1.Config.Annotations = map[string]string{ - annotations.LCOWDevShmSizeInKb: strconv.Itoa(size), - } - containerID1 := createContainer(t, client, ctx, contReq1) - defer removeContainer(t, client, ctx, containerID1) - - startContainer(t, client, ctx, containerID1) - defer stopContainer(t, client, ctx, containerID1) - - contReq2 := getCreateContainerRequest(podID, "test-container-devshm-default", imageLcowAlpine, cmd, podReq.Config) - containerID2 := createContainer(t, client, ctx, contReq2) - defer removeContainer(t, client, ctx, containerID2) - startContainer(t, client, ctx, containerID2) - defer stopContainer(t, client, ctx, containerID2) - - // check /dev/shm size on container 1, should be set to 256 MB - execResponse1 := execSync(t, client, ctx, &runtime.ExecSyncRequest{ - ContainerId: containerID1, - Cmd: []string{"df", "-h", "/dev/shm"}, - }) - if !strings.Contains(string(execResponse1.Stdout), "256.0M") { - t.Fatalf("expected the size of /dev/shm to be 256MB. Got output instead: %s", string(execResponse1.Stdout)) - } - - // check /dev/shm size on container 2, should be set to default 64 MB - execResponse2 := execSync(t, client, ctx, &runtime.ExecSyncRequest{ - ContainerId: containerID2, - Cmd: []string{"df", "-h", "/dev/shm"}, - }) - if !strings.Contains(string(execResponse2.Stdout), "64.0M") { - t.Fatalf("expected the size of /dev/shm to be 64MB. Got output instead: %s", string(execResponse1.Stdout)) - } -} - -func Test_CreateContainer_HugePageMount_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - - annots := map[string]string{ - annotations.FullyPhysicallyBacked: "true", - annotations.MemorySizeInMB: "2048", - annotations.KernelBootOptions: "hugepagesz=2M hugepages=10", - } - sandboxRequest := getRunPodSandboxRequest(t, lcowRuntimeHandler, WithSandboxAnnotations(annots)) - - podID := runPodSandbox(t, client, ctx, sandboxRequest) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - - request := &runtime.CreateContainerRequest{ - Config: &runtime.ContainerConfig{ - Metadata: &runtime.ContainerMetadata{ - Name: t.Name() + "-Container", - }, - Image: &runtime.ImageSpec{ - Image: imageLcowAlpine, - }, - // Hold this command open until killed - Command: []string{ - "top", - }, - Mounts: []*runtime.Mount{ - { - HostPath: fmt.Sprintf("%s2M/hugepage2M", guestpath.HugePagesMountPrefix), - ContainerPath: "/mnt/hugepage2M", - Readonly: false, - Propagation: runtime.MountPropagation_PROPAGATION_BIDIRECTIONAL, - }, - }, - }, - } - - request.PodSandboxId = podID - request.SandboxConfig = sandboxRequest.Config - - containerID := createContainer(t, client, ctx, request) - defer removeContainer(t, client, ctx, containerID) - startContainer(t, client, ctx, containerID) - defer stopContainer(t, client, ctx, containerID) - - execCommand := []string{"grep", "-i", "/mnt/hugepage2M", "/proc/mounts"} - - output, errorMsg, exitCode := execContainer(t, client, ctx, containerID, execCommand) - if exitCode != 0 || len(errorMsg) > 0 { - t.Fatalf("failed to exec in hugepage container errorMsg: %s, exitcode: %v\n", errorMsg, exitCode) - } - - if !strings.Contains(output, "hugetlbfs") { - t.Fatalf("output is supposed to contain hugetlbfs, output: %s", output) - } - - if !strings.Contains(output, "pagesize=2M") { - t.Fatalf("output is supposed to contain pagesize=2M, output: %s", output) - } -} - -func Test_RunContainer_ExecUser_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowCustomUser}) - - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - sandboxRequest := getRunPodSandboxRequest(t, lcowRuntimeHandler) - - podID := runPodSandbox(t, client, ctx, sandboxRequest) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - - cmd := []string{"sh", "-c", "while true; do sleep 1; done"} - request := &runtime.CreateContainerRequest{ - PodSandboxId: podID, - Config: &runtime.ContainerConfig{ - Metadata: &runtime.ContainerMetadata{ - Name: t.Name() + "-Container", - }, - Image: &runtime.ImageSpec{ - Image: imageLcowCustomUser, - }, - Command: cmd, - }, - SandboxConfig: sandboxRequest.Config, - } - - containerID := createContainer(t, client, ctx, request) - defer removeContainer(t, client, ctx, containerID) - startContainer(t, client, ctx, containerID) - defer stopContainer(t, client, ctx, containerID) - - // The `imageLcowCustomUser` image has a user created in the image named test that is set to run the init process as. This tests that - // any execed processes will honor the user set for the container also. - cmd = []string{"whoami"} - containerExecReq := &runtime.ExecSyncRequest{ - ContainerId: containerID, - Cmd: cmd, - Timeout: 20, - } - r := execSync(t, client, ctx, containerExecReq) - if r.ExitCode != 0 { - t.Fatalf("failed with exit code %d: %s", r.ExitCode, string(r.Stderr)) - } - - if !strings.Contains(string(r.Stdout), "test") { - t.Fatalf("expected user for exec to be 'test', got %q", string(r.Stdout)) - } -} - -func Test_RunContainer_ExecUser_Root_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowCustomUser}) - - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - sandboxRequest := getRunPodSandboxRequest(t, lcowRuntimeHandler) - - podID := runPodSandbox(t, client, ctx, sandboxRequest) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - - // Overide what user to run the container as and see if the exec also runs as root now. - cmd := []string{"sh", "-c", "while true; do sleep 1; done"} - request := &runtime.CreateContainerRequest{ - PodSandboxId: podID, - Config: &runtime.ContainerConfig{ - Metadata: &runtime.ContainerMetadata{ - Name: t.Name() + "-Container", - }, - Image: &runtime.ImageSpec{ - Image: imageLcowCustomUser, - }, - Command: cmd, - Linux: &runtime.LinuxContainerConfig{ - SecurityContext: &runtime.LinuxContainerSecurityContext{ - RunAsUsername: "root", - }, - }, - }, - SandboxConfig: sandboxRequest.Config, - } - - containerID := createContainer(t, client, ctx, request) - defer removeContainer(t, client, ctx, containerID) - startContainer(t, client, ctx, containerID) - defer stopContainer(t, client, ctx, containerID) - - // The `imageLcowCustomUser` image has a user created in the image named test that is set to run the init process as. This tests that - // any execed processes will honor the user set for the container also. - cmd = []string{"whoami"} - containerExecReq := &runtime.ExecSyncRequest{ - ContainerId: containerID, - Cmd: cmd, - Timeout: 20, - } - r := execSync(t, client, ctx, containerExecReq) - if r.ExitCode != 0 { - t.Fatalf("failed with exit code %d: %s", r.ExitCode, string(r.Stderr)) - } - - if !strings.Contains(string(r.Stdout), "root") { - t.Fatalf("expected user for exec to be 'root', got %q", string(r.Stdout)) - } -} - -// creates a linux container and attempts to mount a (non-existent) nfs share. Tests if the kernel has the -// required modules for supporting NFS mount. -func Test_Container_NFSMount_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowUbuntu}) - - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - // start a privileged pod & container. container must be privileged in order to be able to mount a NFS - // share. - sandboxRequest := getRunPodSandboxRequest(t, lcowRuntimeHandler) - sandboxRequest.Config.Linux = &runtime.LinuxPodSandboxConfig{ - SecurityContext: &runtime.LinuxSandboxSecurityContext{ - Privileged: true, - }, - } - podID := runPodSandbox(t, client, ctx, sandboxRequest) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - - requestTemplate := getCreateContainerRequest( - podID, - t.Name()+"-container", - imageLcowUbuntu, - []string{"bash", "-c", "while true; do echo 'hello'; sleep 1; done"}, - sandboxRequest.Config, - ) - requestTemplate.Config.Linux = &runtime.LinuxContainerConfig{ - SecurityContext: &runtime.LinuxContainerSecurityContext{ - Privileged: true, - }, - } - containerID := createContainer(t, client, ctx, requestTemplate) - defer removeContainer(t, client, ctx, containerID) - startContainer(t, client, ctx, containerID) - defer stopContainer(t, client, ctx, containerID) - - execHelper := func(ctrID string, cmd []string) { - t.Helper() - stdout, stderr, errcode := execContainer(t, client, ctx, ctrID, cmd) - if errcode != 0 { - t.Logf("stdout: %s \n\n stderr: %s\n\n", stdout, stderr) - t.Fatalf("failed to run '%v'\n: errcode: %d", cmd, errcode) - } - } - - // setup nfs client - nfsdir := "/mnt/nfstest" - execHelper(containerID, []string{"apt", "update"}) - execHelper(containerID, []string{"apt", "install", "-y", "nfs-common"}) - execHelper(containerID, []string{"mkdir", "-p", nfsdir}) - - // There is no NFS daemon running in the container, so it is expected that the mount call fails with - // the connection refused error. However getting upto the connection refused error verifies that the - // container has all the required NFS client modules to successfully mount a NFS. (This also means - // that the kernel was correctly built with the NFS client options). `retry=0` ensures it fails - // immediately instead of retrying. If the kernel isn't correctly configured the call would fail with - // "No Device" error. - stdout, stderr, errcode := execContainer(t, client, ctx, containerID, []string{"mount", "-v", "-t", "nfs", "localhost:/fake/nfs/mount", nfsdir, "-o", "vers=4,minorversion=1,sec=sys,retry=0"}) - if errcode != 32 { // 32 is mount failure - t.Logf("stdout: %s \n\n stderr: %s\n", stdout, stderr) - t.Fatalf("mount call is expected to fail with mount failure error code: 32, errcode was %d instead", errcode) - } else if !strings.Contains(stderr, "Connection refused") { - t.Logf("stdout: %s \n\n stderr: %s\n", stdout, stderr) - t.Fatalf("mount call is expected to fail with Connection refused error") - } -} diff --git a/test/cri-containerd/container_update_test.go b/test/cri-containerd/container_update_test.go index af3831ebc6..7942f692b2 100644 --- a/test/cri-containerd/container_update_test.go +++ b/test/cri-containerd/container_update_test.go @@ -34,7 +34,7 @@ func calculateJobCPURate(hostProcs uint32, processorCount uint32) uint32 { } func Test_Container_UpdateResources_CPUShare(t *testing.T) { - requireAnyFeature(t, featureWCOWProcess, featureWCOWHypervisor, featureLCOW) + requireAnyFeature(t, featureWCOWProcess, featureWCOWHypervisor) require.Build(t, osversion.V20H2) type config struct { @@ -62,23 +62,13 @@ func Test_Container_UpdateResources_CPUShare(t *testing.T) { containerImage: imageWindowsNanoserver, cmd: []string{"cmd", "/c", "ping", "-t", "127.0.0.1"}, }, - { - name: "LCOW", - requiredFeatures: []string{featureLCOW, featureCRIUpdateContainer}, - runtimeHandler: lcowRuntimeHandler, - sandboxImage: imageLcowK8sPause, - containerImage: imageLcowAlpine, - cmd: []string{"top"}, - }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { requireFeatures(t, test.requiredFeatures...) - if test.runtimeHandler == lcowRuntimeHandler { - pullRequiredLCOWImages(t, []string{test.sandboxImage}) - } else if test.runtimeHandler == wcowHypervisorRuntimeHandler { + if test.runtimeHandler == wcowHypervisorRuntimeHandler { pullRequiredImages(t, []string{test.sandboxImage}) } @@ -118,34 +108,24 @@ func Test_Container_UpdateResources_CPUShare(t *testing.T) { } var expected uint32 = 5000 - if test.runtimeHandler == lcowRuntimeHandler { - updateReq.Linux = &runtime.LinuxContainerResources{ - CpuShares: int64(expected), - } - } else { - updateReq.Windows = &runtime.WindowsContainerResources{ - CpuShares: int64(expected), - } + updateReq.Windows = &runtime.WindowsContainerResources{ + CpuShares: int64(expected), } if _, err := client.UpdateContainerResources(ctx, updateReq); err != nil { t.Fatalf("updating container resources for %s with %v", containerID, err) } - if test.runtimeHandler == lcowRuntimeHandler { - checkLCOWResourceLimit(t, ctx, client, containerID, "/sys/fs/cgroup/cpu/cpu.shares", uint64(expected)) - } else { - targetShimName := "k8s.io-" + podID - jobExpectedValue := calculateJobCPUWeight(expected) - checkWCOWResourceLimit(t, ctx, test.runtimeHandler, targetShimName, containerID, "cpu-weight", uint64(jobExpectedValue)) - } + targetShimName := "k8s.io-" + podID + jobExpectedValue := calculateJobCPUWeight(expected) + checkWCOWResourceLimit(t, ctx, test.runtimeHandler, targetShimName, containerID, "cpu-weight", uint64(jobExpectedValue)) }) } } func Test_Container_UpdateResources_CPUShare_NotRunning(t *testing.T) { requireFeatures(t, featureCRIUpdateContainer) - requireAnyFeature(t, featureWCOWProcess, featureWCOWHypervisor, featureLCOW) + requireAnyFeature(t, featureWCOWProcess, featureWCOWHypervisor) type config struct { name string @@ -172,25 +152,12 @@ func Test_Container_UpdateResources_CPUShare_NotRunning(t *testing.T) { containerImage: imageWindowsNanoserver, cmd: []string{"cmd", "/c", "ping", "-t", "127.0.0.1"}, }, - { - name: "LCOW", - requiredFeatures: []string{featureLCOW}, - runtimeHandler: lcowRuntimeHandler, - sandboxImage: imageLcowK8sPause, - containerImage: imageLcowAlpine, - cmd: []string{"top"}, - }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { requireFeatures(t, test.requiredFeatures...) - - if test.runtimeHandler == lcowRuntimeHandler { - pullRequiredLCOWImages(t, []string{test.sandboxImage}) - } else if test.runtimeHandler == wcowHypervisorRuntimeHandler { - pullRequiredImages(t, []string{test.sandboxImage}) - } + pullRequiredImages(t, []string{test.sandboxImage}) podRequest := getRunPodSandboxRequest(t, test.runtimeHandler) @@ -225,14 +192,8 @@ func Test_Container_UpdateResources_CPUShare_NotRunning(t *testing.T) { } var expected uint32 = 5000 - if test.runtimeHandler == lcowRuntimeHandler { - updateReq.Linux = &runtime.LinuxContainerResources{ - CpuShares: int64(expected), - } - } else { - updateReq.Windows = &runtime.WindowsContainerResources{ - CpuShares: int64(expected), - } + updateReq.Windows = &runtime.WindowsContainerResources{ + CpuShares: int64(expected), } if _, err := client.UpdateContainerResources(ctx, updateReq); err != nil { @@ -242,20 +203,16 @@ func Test_Container_UpdateResources_CPUShare_NotRunning(t *testing.T) { startContainer(t, client, ctx, containerID) defer stopContainer(t, client, ctx, containerID) - if test.runtimeHandler == lcowRuntimeHandler { - checkLCOWResourceLimit(t, ctx, client, containerID, "/sys/fs/cgroup/cpu/cpu.shares", uint64(expected)) - } else { - targetShimName := "k8s.io-" + podID - jobExpectedValue := calculateJobCPUWeight(expected) - checkWCOWResourceLimit(t, ctx, test.runtimeHandler, targetShimName, containerID, "cpu-weight", uint64(jobExpectedValue)) - } + targetShimName := "k8s.io-" + podID + jobExpectedValue := calculateJobCPUWeight(expected) + checkWCOWResourceLimit(t, ctx, test.runtimeHandler, targetShimName, containerID, "cpu-weight", uint64(jobExpectedValue)) }) } } func Test_Container_UpdateResources_Memory(t *testing.T) { requireFeatures(t, featureCRIUpdateContainer) - requireAnyFeature(t, featureWCOWProcess, featureWCOWHypervisor, featureLCOW) + requireAnyFeature(t, featureWCOWProcess, featureWCOWHypervisor) type config struct { name string @@ -282,25 +239,12 @@ func Test_Container_UpdateResources_Memory(t *testing.T) { containerImage: imageWindowsNanoserver, cmd: []string{"cmd", "/c", "ping", "-t", "127.0.0.1"}, }, - { - name: "LCOW", - requiredFeatures: []string{featureLCOW}, - runtimeHandler: lcowRuntimeHandler, - sandboxImage: imageLcowK8sPause, - containerImage: imageLcowAlpine, - cmd: []string{"top"}, - }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { requireFeatures(t, test.requiredFeatures...) - - if test.runtimeHandler == lcowRuntimeHandler { - pullRequiredLCOWImages(t, []string{test.sandboxImage}) - } else if test.runtimeHandler == wcowHypervisorRuntimeHandler { - pullRequiredImages(t, []string{test.sandboxImage}) - } + pullRequiredImages(t, []string{test.sandboxImage}) podRequest := getRunPodSandboxRequest(t, test.runtimeHandler) @@ -342,26 +286,15 @@ func Test_Container_UpdateResources_Memory(t *testing.T) { } newMemorySize := startingMemorySize / 2 - if test.runtimeHandler == lcowRuntimeHandler { - updateReq.Linux = &runtime.LinuxContainerResources{ - MemoryLimitInBytes: newMemorySize, - } - } else { - updateReq.Windows = &runtime.WindowsContainerResources{ - MemoryLimitInBytes: newMemorySize, - } + updateReq.Windows = &runtime.WindowsContainerResources{ + MemoryLimitInBytes: newMemorySize, } if _, err := client.UpdateContainerResources(ctx, updateReq); err != nil { t.Fatalf("updating container resources for %s with %v", containerID, err) } - - if test.runtimeHandler == lcowRuntimeHandler { - checkLCOWResourceLimit(t, ctx, client, containerID, "/sys/fs/cgroup/memory/memory.limit_in_bytes", uint64(newMemorySize)) - } else { - targetShimName := "k8s.io-" + podID - checkWCOWResourceLimit(t, ctx, test.runtimeHandler, targetShimName, containerID, "memory-limit", uint64(newMemorySize)) - } + targetShimName := "k8s.io-" + podID + checkWCOWResourceLimit(t, ctx, test.runtimeHandler, targetShimName, containerID, "memory-limit", uint64(newMemorySize)) }) } } diff --git a/test/cri-containerd/container_virtual_device_test.go b/test/cri-containerd/container_virtual_device_test.go index aeb24d55db..16fa909a69 100644 --- a/test/cri-containerd/container_virtual_device_test.go +++ b/test/cri-containerd/container_virtual_device_test.go @@ -37,18 +37,6 @@ func makeGPUExecCommand(os string, containerID string) *runtime.ExecSyncRequest // verifyGPUIsPresent is a helper function that runs a command in the container // to verify the existence of a GPU and fails the running test is none are found -func verifyGPUIsPresentLCOW(t *testing.T, client runtime.RuntimeServiceClient, ctx context.Context, containerID string) { - t.Helper() - execReq := makeGPUExecCommand("linux", containerID) - response := execSync(t, client, ctx, execReq) - if len(response.Stderr) != 0 { - t.Fatalf("expected to see no error, instead saw %s", string(response.Stderr)) - } - if len(response.Stdout) == 0 { - t.Fatal("expected to see GPU device on container, not present") - } -} - func isGPUPresentWCOW(t *testing.T, client runtime.RuntimeServiceClient, ctx context.Context, containerID string) bool { t.Helper() execReq := makeGPUExecCommand("windows", containerID) @@ -69,20 +57,6 @@ func isGPUPresentWCOW(t *testing.T, client runtime.RuntimeServiceClient, ctx con return false } -// verifyGPUIsNotPresent is a helper function that runs a command in the container -// to verify that there are no GPUs present in the container and fails the running test -// if any are found -func verifyGPUIsNotPresentLCOW(t *testing.T, client runtime.RuntimeServiceClient, ctx context.Context, containerID string) { - t.Helper() - execReq := makeGPUExecCommand("linux", containerID) - response := execSync(t, client, ctx, execReq) - if len(response.Stderr) == 0 { - t.Fatal("expected to see an error as file /dev/nvidia0 should not exist, instead saw none") - } else if len(response.Stdout) != 0 { - t.Fatal("expected to not see GPU device on container, but some are present") - } -} - // findTestNvidiaGPUDevice returns the first nvidia pcip device on the host func findTestNvidiaGPUDevice() (string, error) { out, err := exec.Command( @@ -121,43 +95,6 @@ func findTestVirtualDeviceID() (string, error) { return strings.TrimSpace(string(out)), nil } -var lcowPodGPUAnnotations = map[string]string{ - annotations.PreferredRootFSType: "initrd", - annotations.VPMemCount: "0", - annotations.VPCIEnabled: "true", - // we believe this is a sufficiently large high MMIO space amount for this test. - // if a given gpu device needs more, this test will fail to create the container - // and may hang. - annotations.MemoryHighMMIOGapInMB: "64000", - annotations.FullyPhysicallyBacked: "true", -} - -func getGPUContainerRequestLCOW(t *testing.T, podID string, podConfig *runtime.PodSandboxConfig, device *runtime.Device) *runtime.CreateContainerRequest { - t.Helper() - return &runtime.CreateContainerRequest{ - Config: &runtime.ContainerConfig{ - Metadata: &runtime.ContainerMetadata{ - Name: t.Name() + "-Container", - }, - Image: &runtime.ImageSpec{ - Image: imageLcowAlpine, - }, - Command: []string{ - "top", - }, - Devices: []*runtime.Device{ - device, - }, - Linux: &runtime.LinuxContainerConfig{}, - Annotations: map[string]string{ - annotations.ContainerGPUCapabilities: "utility", - }, - }, - PodSandboxId: podID, - SandboxConfig: podConfig, - } -} - func getGPUContainerRequestWCOW(t *testing.T, podID string, podConfig *runtime.PodSandboxConfig, device *runtime.Device) *runtime.CreateContainerRequest { t.Helper() return &runtime.CreateContainerRequest{ @@ -193,227 +130,11 @@ func getGPUContainerRequestWCOW(t *testing.T, podID string, podConfig *runtime.P } } -func Test_RunContainer_VirtualDevice_GPU_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW, featureGPU) - - if osversion.Build() < osversion.V20H1 { - t.Skip("Requires build +20H1") - } - - testDeviceInstanceID, err := findTestNvidiaGPUDevice() - if err != nil { - t.Fatalf("skipping test, failed to find assignable nvidia gpu on host with: %v", err) - } - if testDeviceInstanceID == "" { - t.Fatal("skipping test, host has no assignable nvidia gpu devices") - } - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - client := newTestRuntimeClient(t) - - podctx := context.Background() - sandboxRequest := getRunPodSandboxRequest( - t, - lcowRuntimeHandler, - WithSandboxAnnotations(lcowPodGPUAnnotations), - ) - - podID := runPodSandbox(t, client, podctx, sandboxRequest) - defer removePodSandbox(t, client, podctx, podID) - defer stopPodSandbox(t, client, podctx, podID) - - device := &runtime.Device{ - HostPath: "gpu://" + testDeviceInstanceID, - } - - containerRequest := getGPUContainerRequestLCOW(t, podID, sandboxRequest.Config, device) - ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) - defer cancel() - containerID := createContainer(t, client, ctx, containerRequest) +func runContainerLifetime(t *testing.T, client runtime.RuntimeServiceClient, ctx context.Context, containerID string) { + t.Helper() defer removeContainer(t, client, ctx, containerID) startContainer(t, client, ctx, containerID) - defer stopContainer(t, client, ctx, containerID) - - verifyGPUIsPresentLCOW(t, client, ctx, containerID) -} - -func Test_RunContainer_VirtualDevice_GPU_Multiple_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW, featureGPU) - - if osversion.Build() < osversion.V20H1 { - t.Skip("Requires build +20H1") - } - - numContainers := 2 - testDeviceInstanceID, err := findTestNvidiaGPUDevice() - if err != nil { - t.Fatalf("skipping test, failed to find assignable nvidia gpu on host with: %v", err) - } - if testDeviceInstanceID == "" { - t.Fatal("skipping test, host has no assignable nvidia gpu devices") - } - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - client := newTestRuntimeClient(t) - - podctx := context.Background() - sandboxRequest := getRunPodSandboxRequest( - t, - lcowRuntimeHandler, - WithSandboxAnnotations(lcowPodGPUAnnotations), - ) - - podID := runPodSandbox(t, client, podctx, sandboxRequest) - defer removePodSandbox(t, client, podctx, podID) - defer stopPodSandbox(t, client, podctx, podID) - - device := &runtime.Device{ - HostPath: "gpu://" + testDeviceInstanceID, - } - - containerRequest := getGPUContainerRequestLCOW(t, podID, sandboxRequest.Config, device) - - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - for i := 0; i < numContainers; i++ { - name := t.Name() + "-Container-" + fmt.Sprintf("%d", i) - containerRequest.Config.Metadata.Name = name - - containerID := createContainer(t, client, ctx, containerRequest) - defer removeContainer(t, client, ctx, containerID) - startContainer(t, client, ctx, containerID) - defer stopContainer(t, client, ctx, containerID) - - verifyGPUIsPresentLCOW(t, client, ctx, containerID) - } -} - -func Test_RunContainer_VirtualDevice_GPU_and_NoGPU_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW, featureGPU) - - if osversion.Build() < osversion.V20H1 { - t.Skip("Requires build +20H1") - } - - testDeviceInstanceID, err := findTestNvidiaGPUDevice() - if err != nil { - t.Fatalf("skipping test, failed to find assignable nvidia gpu on host with: %v", err) - } - if testDeviceInstanceID == "" { - t.Fatal("skipping test, host has no assignable nvidia gpu devices") - } - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - client := newTestRuntimeClient(t) - - podctx := context.Background() - sandboxRequest := getRunPodSandboxRequest( - t, - lcowRuntimeHandler, - WithSandboxAnnotations(lcowPodGPUAnnotations), - ) - - podID := runPodSandbox(t, client, podctx, sandboxRequest) - defer removePodSandbox(t, client, podctx, podID) - defer stopPodSandbox(t, client, podctx, podID) - - device := &runtime.Device{ - HostPath: "gpu://" + testDeviceInstanceID, - } - - containerGPURequest := getGPUContainerRequestLCOW(t, podID, sandboxRequest.Config, device) - - containerNoGPURequest := &runtime.CreateContainerRequest{ - Config: &runtime.ContainerConfig{ - Metadata: &runtime.ContainerMetadata{ - Name: "No-GPU-Container", - }, - Image: &runtime.ImageSpec{ - Image: imageLcowAlpine, - }, - Command: []string{ - "top", - }, - Linux: &runtime.LinuxContainerConfig{}, - }, - PodSandboxId: podID, - SandboxConfig: sandboxRequest.Config, - } - ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) - defer cancel() - - // create container with a GPU present - gpuContainerID := createContainer(t, client, ctx, containerGPURequest) - defer removeContainer(t, client, ctx, gpuContainerID) - startContainer(t, client, ctx, gpuContainerID) - defer stopContainer(t, client, ctx, gpuContainerID) - - // verify that we can access the GPU in the GPU-Container - verifyGPUIsPresentLCOW(t, client, ctx, gpuContainerID) - - // create container without a GPU - noGPUContainerID := createContainer(t, client, ctx, containerNoGPURequest) - defer removeContainer(t, client, ctx, noGPUContainerID) - startContainer(t, client, ctx, noGPUContainerID) - defer stopContainer(t, client, ctx, noGPUContainerID) - - // verify that we can't access the GPU in the No-GPU-Container - verifyGPUIsNotPresentLCOW(t, client, ctx, noGPUContainerID) -} - -func Test_RunContainer_VirtualDevice_GPU_Multiple_Removal_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW, featureGPU) - - if osversion.Build() < osversion.V20H1 { - t.Skip("Requires build +20H1") - } - - testDeviceInstanceID, err := findTestNvidiaGPUDevice() - if err != nil { - t.Fatalf("skipping test, failed to find assignable nvidia gpu on host with: %v", err) - } - if testDeviceInstanceID == "" { - t.Fatal("skipping test, host has no assignable nvidia gpu devices") - } - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - client := newTestRuntimeClient(t) - - podctx := context.Background() - sandboxRequest := getRunPodSandboxRequest( - t, - lcowRuntimeHandler, - WithSandboxAnnotations(lcowPodGPUAnnotations), - ) - - podID := runPodSandbox(t, client, podctx, sandboxRequest) - defer removePodSandbox(t, client, podctx, podID) - defer stopPodSandbox(t, client, podctx, podID) - - device := &runtime.Device{ - HostPath: "gpu://" + testDeviceInstanceID, - } - - containerRequest := getGPUContainerRequestLCOW(t, podID, sandboxRequest.Config, device) - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - // create and start up first container with GPU - containerRequest.Config.Metadata.Name = t.Name() + "-Container-1" - containerOneID := createContainer(t, client, ctx, containerRequest) - defer removeContainer(t, client, ctx, containerOneID) - startContainer(t, client, ctx, containerOneID) - defer stopContainer(t, client, ctx, containerOneID) - - // run full lifetime of second container with GPU - containerRequest.Config.Metadata.Name = t.Name() + "-Container-2" - containerTwoID := createContainer(t, client, ctx, containerRequest) - runContainerLifetime(t, client, ctx, containerTwoID) - - // verify after removing second container that we can still see - // the GPU on the first container - verifyGPUIsPresentLCOW(t, client, ctx, containerOneID) + stopContainer(t, client, ctx, containerID) } func Test_RunContainer_VirtualDevice_LocationPath_WCOW_Process(t *testing.T) { diff --git a/test/cri-containerd/createcontainer_test.go b/test/cri-containerd/createcontainer_test.go index 3815d92253..3cb1e847da 100644 --- a/test/cri-containerd/createcontainer_test.go +++ b/test/cri-containerd/createcontainer_test.go @@ -12,9 +12,8 @@ import ( "github.com/Microsoft/go-winio" "github.com/Microsoft/hcsshim/internal/memory" - "github.com/Microsoft/hcsshim/osversion" "github.com/Microsoft/hcsshim/pkg/annotations" - "github.com/Microsoft/hcsshim/test/pkg/require" + runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" ) @@ -96,28 +95,6 @@ func Test_CreateContainer_WCOW_Hypervisor(t *testing.T) { runCreateContainerTest(t, wcowHypervisorRuntimeHandler, request) } -func Test_CreateContainer_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - - request := &runtime.CreateContainerRequest{ - Config: &runtime.ContainerConfig{ - Metadata: &runtime.ContainerMetadata{ - Name: t.Name() + "-Container", - }, - Image: &runtime.ImageSpec{ - Image: imageLcowAlpine, - }, - // Hold this command open until killed - Command: []string{ - "top", - }, - }, - } - runCreateContainerTest(t, lcowRuntimeHandler, request) -} - func Test_CreateContainer_WCOW_Process_Tty(t *testing.T) { requireFeatures(t, featureWCOWProcess) @@ -164,97 +141,6 @@ func Test_CreateContainer_WCOW_Hypervisor_Tty(t *testing.T) { runCreateContainerTest(t, wcowHypervisorRuntimeHandler, request) } -func Test_CreateContainer_LCOW_Tty(t *testing.T) { - requireFeatures(t, featureLCOW) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - - request := &runtime.CreateContainerRequest{ - Config: &runtime.ContainerConfig{ - Metadata: &runtime.ContainerMetadata{ - Name: t.Name() + "-Container", - }, - Image: &runtime.ImageSpec{ - Image: imageLcowAlpine, - }, - // Tty will hold this open until killed. - Command: []string{ - "sh", - }, - Tty: true, - }, - } - runCreateContainerTest(t, lcowRuntimeHandler, request) -} - -func Test_CreateContainer_LCOW_Privileged(t *testing.T) { - requireFeatures(t, featureLCOW) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - - sandboxRequest := getRunPodSandboxRequest(t, lcowRuntimeHandler) - sandboxRequest.Config.Linux = &runtime.LinuxPodSandboxConfig{ - SecurityContext: &runtime.LinuxSandboxSecurityContext{ - Privileged: true, - }, - } - - request := &runtime.CreateContainerRequest{ - Config: &runtime.ContainerConfig{ - Metadata: &runtime.ContainerMetadata{ - Name: t.Name() + "-Container", - }, - Image: &runtime.ImageSpec{ - Image: imageLcowAlpine, - }, - // Hold this command open until killed - Command: []string{ - "top", - }, - Linux: &runtime.LinuxContainerConfig{ - SecurityContext: &runtime.LinuxContainerSecurityContext{ - Privileged: true, - }, - }, - }, - } - runCreateContainerTestWithSandbox(t, sandboxRequest, request) -} - -func Test_CreateContainer_SandboxDevice_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - - sandboxRequest := getRunPodSandboxRequest(t, lcowRuntimeHandler) - sandboxRequest.Config.Linux = &runtime.LinuxPodSandboxConfig{ - SecurityContext: &runtime.LinuxSandboxSecurityContext{ - Privileged: true, - }, - } - - request := &runtime.CreateContainerRequest{ - Config: &runtime.ContainerConfig{ - Metadata: &runtime.ContainerMetadata{ - Name: t.Name() + "-Container", - }, - Image: &runtime.ImageSpec{ - Image: imageLcowAlpine, - }, - // Hold this command open until killed - Command: []string{ - "top", - }, - Devices: []*runtime.Device{ - { - HostPath: "/dev/fuse", - }, - }, - }, - } - runCreateContainerTestWithSandbox(t, sandboxRequest, request) -} - func Test_CreateContainer_MemorySize_Config_WCOW_Process(t *testing.T) { requireFeatures(t, featureWCOWProcess) @@ -375,33 +261,6 @@ func Test_CreateContainer_MemorySize_Annotation_WCOW_Hypervisor(t *testing.T) { runCreateContainerTest(t, wcowHypervisorRuntimeHandler, request) } -func Test_CreateContainer_MemorySize_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - - request := &runtime.CreateContainerRequest{ - Config: &runtime.ContainerConfig{ - Metadata: &runtime.ContainerMetadata{ - Name: t.Name() + "-Container", - }, - Image: &runtime.ImageSpec{ - Image: imageLcowAlpine, - }, - // Hold this command open until killed - Command: []string{ - "top", - }, - Linux: &runtime.LinuxContainerConfig{ - Resources: &runtime.LinuxContainerResources{ - MemoryLimitInBytes: 768 * memory.MiB, // 768MB - }, - }, - }, - } - runCreateContainerTest(t, lcowRuntimeHandler, request) -} - func Test_CreateContainer_CPUCount_Config_WCOW_Process(t *testing.T) { requireFeatures(t, featureWCOWProcess) @@ -522,33 +381,6 @@ func Test_CreateContainer_CPUCount_Annotation_WCOW_Hypervisor(t *testing.T) { runCreateContainerTest(t, wcowHypervisorRuntimeHandler, request) } -func Test_CreateContainer_CPUCount_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - - request := &runtime.CreateContainerRequest{ - Config: &runtime.ContainerConfig{ - Metadata: &runtime.ContainerMetadata{ - Name: t.Name() + "-Container", - }, - Image: &runtime.ImageSpec{ - Image: imageLcowAlpine, - }, - // Hold this command open until killed - Command: []string{ - "top", - }, - Linux: &runtime.LinuxContainerConfig{ - Resources: &runtime.LinuxContainerResources{ - CpusetCpus: "0", - }, - }, - }, - } - runCreateContainerTest(t, lcowRuntimeHandler, request) -} - func Test_CreateContainer_CPULimit_Config_WCOW_Process(t *testing.T) { requireFeatures(t, featureWCOWProcess) @@ -669,34 +501,6 @@ func Test_CreateContainer_CPULimit_Annotation_WCOW_Hypervisor(t *testing.T) { runCreateContainerTest(t, wcowHypervisorRuntimeHandler, request) } -func Test_CreateContainer_CPUQuota_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - - request := &runtime.CreateContainerRequest{ - Config: &runtime.ContainerConfig{ - Metadata: &runtime.ContainerMetadata{ - Name: t.Name() + "-Container", - }, - Image: &runtime.ImageSpec{ - Image: imageLcowAlpine, - }, - // Hold this command open until killed - Command: []string{ - "top", - }, - Linux: &runtime.LinuxContainerConfig{ - Resources: &runtime.LinuxContainerResources{ - CpuQuota: 1000000, - CpuPeriod: 500000, - }, - }, - }, - } - runCreateContainerTest(t, lcowRuntimeHandler, request) -} - func Test_CreateContainer_CPUWeight_Config_WCOW_Process(t *testing.T) { requireFeatures(t, featureWCOWProcess) @@ -817,189 +621,6 @@ func Test_CreateContainer_CPUWeight_Annotation_WCOW_Hypervisor(t *testing.T) { runCreateContainerTest(t, wcowHypervisorRuntimeHandler, request) } -func Test_CreateContainer_CPUShares_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - - request := &runtime.CreateContainerRequest{ - Config: &runtime.ContainerConfig{ - Metadata: &runtime.ContainerMetadata{ - Name: t.Name() + "-Container", - }, - Image: &runtime.ImageSpec{ - Image: imageLcowAlpine, - }, - // Hold this command open until killed - Command: []string{ - "top", - }, - Linux: &runtime.LinuxContainerConfig{ - Resources: &runtime.LinuxContainerResources{ - CpuShares: 1024, - }, - }, - }, - } - runCreateContainerTest(t, lcowRuntimeHandler, request) -} - -func Test_CreateContainer_Mount_File_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - require.Build(t, osversion.V19H1) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - - tempFile, err := os.CreateTemp("", "test") - - if err != nil { - t.Fatalf("Failed to create temp file: %s", err) - } - - tempFile.Close() - - defer func() { - if err := os.Remove(tempFile.Name()); err != nil { - t.Fatalf("Failed to remove temp file: %s", err) - } - }() - - containerFilePath := "/foo/test.txt" - - request := &runtime.CreateContainerRequest{ - Config: &runtime.ContainerConfig{ - Metadata: &runtime.ContainerMetadata{ - Name: t.Name() + "-Container", - }, - Mounts: []*runtime.Mount{ - { - HostPath: tempFile.Name(), - ContainerPath: containerFilePath, - }, - }, - Image: &runtime.ImageSpec{ - Image: imageLcowAlpine, - }, - Command: []string{ - "top", - }, - Linux: &runtime.LinuxContainerConfig{}, - }, - } - runCreateContainerTest(t, lcowRuntimeHandler, request) -} - -func Test_CreateContainer_Mount_ReadOnlyFile_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - require.Build(t, osversion.V19H1) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - - tempFile, err := os.CreateTemp("", "test") - - if err != nil { - t.Fatalf("Failed to create temp file: %s", err) - } - - tempFile.Close() - - defer func() { - if err := os.Remove(tempFile.Name()); err != nil { - t.Fatalf("Failed to remove temp file: %s", err) - } - }() - - containerFilePath := "/foo/test.txt" - - request := &runtime.CreateContainerRequest{ - Config: &runtime.ContainerConfig{ - Metadata: &runtime.ContainerMetadata{ - Name: t.Name() + "-Container", - }, - Mounts: []*runtime.Mount{ - { - HostPath: tempFile.Name(), - ContainerPath: containerFilePath, - Readonly: true, - }, - }, - Image: &runtime.ImageSpec{ - Image: imageLcowAlpine, - }, - Command: []string{ - "top", - }, - Linux: &runtime.LinuxContainerConfig{}, - }, - } - runCreateContainerTest(t, lcowRuntimeHandler, request) -} - -func Test_CreateContainer_Mount_Dir_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - - tempDir := t.TempDir() - - containerFilePath := "/foo" - - request := &runtime.CreateContainerRequest{ - Config: &runtime.ContainerConfig{ - Metadata: &runtime.ContainerMetadata{ - Name: t.Name() + "-Container", - }, - Mounts: []*runtime.Mount{ - { - HostPath: tempDir, - ContainerPath: containerFilePath, - }, - }, - Image: &runtime.ImageSpec{ - Image: imageLcowAlpine, - }, - Command: []string{ - "top", - }, - Linux: &runtime.LinuxContainerConfig{}, - }, - } - runCreateContainerTest(t, lcowRuntimeHandler, request) -} - -func Test_CreateContainer_Mount_ReadOnlyDir_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - - tempDir := t.TempDir() - - containerFilePath := "/foo" - - request := &runtime.CreateContainerRequest{ - Config: &runtime.ContainerConfig{ - Metadata: &runtime.ContainerMetadata{ - Name: t.Name() + "-Container", - }, - Mounts: []*runtime.Mount{ - { - HostPath: tempDir, - ContainerPath: containerFilePath, - Readonly: true, - }, - }, - Image: &runtime.ImageSpec{ - Image: imageLcowAlpine, - }, - Command: []string{ - "top", - }, - Linux: &runtime.LinuxContainerConfig{}, - }, - } - runCreateContainerTest(t, lcowRuntimeHandler, request) -} - func Test_CreateContainer_Mount_File_WCOW(t *testing.T) { requireFeatures(t, featureWCOWHypervisor) pullRequiredImages(t, []string{imageWindowsNanoserver}) diff --git a/test/cri-containerd/execcontainer_test.go b/test/cri-containerd/execcontainer_test.go deleted file mode 100644 index bb940162cf..0000000000 --- a/test/cri-containerd/execcontainer_test.go +++ /dev/null @@ -1,137 +0,0 @@ -//go:build windows && functional -// +build windows,functional - -package cri_containerd - -import ( - "context" - "strconv" - "strings" - "testing" - - runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" -) - -func runexecContainerTestWithSandbox(t *testing.T, sandboxRequest *runtime.RunPodSandboxRequest, request *runtime.CreateContainerRequest, execReq *runtime.ExecSyncRequest) *runtime.ExecSyncResponse { - t.Helper() - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - podID := runPodSandbox(t, client, ctx, sandboxRequest) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - - request.PodSandboxId = podID - request.SandboxConfig = sandboxRequest.Config - - containerID := createContainer(t, client, ctx, request) - defer removeContainer(t, client, ctx, containerID) - - startContainer(t, client, ctx, containerID) - defer stopContainer(t, client, ctx, containerID) - - execReq.ContainerId = containerID - return execSync(t, client, ctx, execReq) -} - -func execContainerLCOW(t *testing.T, uid int64, cmd []string) *runtime.ExecSyncResponse { - t.Helper() - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowCosmos}) - - // run podsandbox request - sandboxRequest := getRunPodSandboxRequest(t, lcowRuntimeHandler) - sandboxRequest.Config.Linux = &runtime.LinuxPodSandboxConfig{ - SecurityContext: &runtime.LinuxSandboxSecurityContext{ - Privileged: true, - }, - } - - // create container request - request := &runtime.CreateContainerRequest{ - Config: &runtime.ContainerConfig{ - Metadata: &runtime.ContainerMetadata{ - Name: t.Name() + "-Container2", - }, - Image: &runtime.ImageSpec{ - Image: imageLcowCosmos, - }, - // Hold this command open until killed - Command: []string{ - "sleep", - "100000", - }, - Linux: &runtime.LinuxContainerConfig{ - SecurityContext: &runtime.LinuxContainerSecurityContext{ - // Our tests rely on the init process for the workload - // container being pid 1, but the CRI default is to use the - // pod's pid namespace. - NamespaceOptions: &runtime.NamespaceOption{ - Pid: runtime.NamespaceMode_CONTAINER, - }, - RunAsUser: &runtime.Int64Value{ - Value: uid, - }, - }, - }, - }, - } - - //exec request - execRequest := &runtime.ExecSyncRequest{ - ContainerId: "", - Cmd: cmd, - Timeout: 20, - } - - return runexecContainerTestWithSandbox(t, sandboxRequest, request, execRequest) -} - -func Test_ExecContainer_RunAs_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - // this is just saying 'give me the UID of the process with pid = 1; ignore headers' - r := execContainerLCOW(t, - 9000, // cosmosarno user - []string{ - "ps", - "-o", - "uid", - "-p", - "1", - "--no-headers", - }) - - output := strings.TrimSpace(string(r.Stdout)) - errorMsg := string(r.Stderr) - exitCode := int(r.ExitCode) - - t.Logf("exec request exited with code: %d", exitCode) - t.Logf("exec request output: %v", output) - - if exitCode != 0 { - t.Fatalf("Test %v exited with exit code: %d, Test_CreateContainer_RunAs_LCOW", errorMsg, exitCode) - } - - if output != "9000" { - t.Fatalf("failed to start container with runas option: error: %v, exitcode: %d", errorMsg, exitCode) - } -} - -func Test_ExecContainer_LCOW_HasEntropy(t *testing.T) { - requireFeatures(t, featureLCOW) - - r := execContainerLCOW(t, 9000, []string{"cat", "/proc/sys/kernel/random/entropy_avail"}) - if r.ExitCode != 0 { - t.Fatalf("failed with exit code %d to cat entropy_avail: %s", r.ExitCode, r.Stderr) - } - output := strings.TrimSpace(string(r.Stdout)) - bits, err := strconv.ParseInt(output, 10, 0) - if err != nil { - t.Fatalf("could not parse entropy output %s: %s", output, err) - } - if bits < 256 { - t.Fatalf("%d is fewer than 256 bits entropy", bits) - } - t.Logf("got %d bits entropy", bits) -} diff --git a/test/cri-containerd/extended_task_test.go b/test/cri-containerd/extended_task_test.go index b718153e5b..a880530431 100644 --- a/test/cri-containerd/extended_task_test.go +++ b/test/cri-containerd/extended_task_test.go @@ -24,7 +24,7 @@ func getPodProcessorInfo(ctx context.Context, podID string) (*extendedtask.Compu } func Test_ExtendedTask_ProcessorInfo(t *testing.T) { - requireAnyFeature(t, featureWCOWProcess, featureWCOWHypervisor, featureLCOW) + requireAnyFeature(t, featureWCOWProcess, featureWCOWHypervisor) type config struct { name string @@ -48,24 +48,12 @@ func Test_ExtendedTask_ProcessorInfo(t *testing.T) { sandboxImage: imageWindowsNanoserver, expectedError: false, }, - { - name: "LCOW returns correct cpu count", - requiredFeatures: []string{featureLCOW}, - runtimeHandler: lcowRuntimeHandler, - sandboxImage: imageLcowAlpine, - expectedError: false, - }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { requireFeatures(t, test.requiredFeatures...) - - if test.runtimeHandler == lcowRuntimeHandler { - pullRequiredLCOWImages(t, []string{test.sandboxImage}) - } else { - pullRequiredImages(t, []string{test.sandboxImage}) - } + pullRequiredImages(t, []string{test.sandboxImage}) request := getRunPodSandboxRequest( t, diff --git a/test/cri-containerd/helper_container_test.go b/test/cri-containerd/helper_container_test.go index a56326a9e6..300849252f 100644 --- a/test/cri-containerd/helper_container_test.go +++ b/test/cri-containerd/helper_container_test.go @@ -5,11 +5,8 @@ package cri_containerd import ( "context" - "fmt" "testing" - "time" - "github.com/stretchr/testify/require" runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" ) @@ -97,37 +94,6 @@ func getContainerStatusFull(tb testing.TB, client runtime.RuntimeServiceClient, return response.Status } -// requireContainerState periodically checks the state of a container, returns -// an error if the expected container state isn't reached within 30 seconds. -func requireContainerState( - ctx context.Context, - tb testing.TB, - client runtime.RuntimeServiceClient, - containerID string, - expectedState runtime.ContainerState, -) { - tb.Helper() - require.NoError(tb, func() error { - start := time.Now() - var lastState runtime.ContainerState - for { - lastState = getContainerStatus(tb, client, ctx, containerID) - if lastState == expectedState { - return nil - } - if time.Since(start) >= 30*time.Second { - break - } - time.Sleep(500 * time.Millisecond) - } - return fmt.Errorf( - "expected state %q, last reported state %q", - runtime.ContainerState_name[int32(expectedState)], - runtime.ContainerState_name[int32(lastState)], - ) - }()) -} - func cleanupContainer(t *testing.T, client runtime.RuntimeServiceClient, ctx context.Context, containerID *string) { t.Helper() if *containerID == "" { diff --git a/test/cri-containerd/helper_exec_test.go b/test/cri-containerd/helper_exec_test.go index 96410834e1..db0c97c56d 100644 --- a/test/cri-containerd/helper_exec_test.go +++ b/test/cri-containerd/helper_exec_test.go @@ -26,15 +26,6 @@ func execSync(tb testing.TB, client runtime.RuntimeServiceClient, ctx context.Co return response } -func execRequest(tb testing.TB, client runtime.RuntimeServiceClient, ctx context.Context, request *runtime.ExecRequest) string { - tb.Helper() - response, err := client.Exec(ctx, request) - if err != nil { - tb.Fatalf("failed Exec request with: %v", err) - } - return response.Url -} - // execInHost executes command in the container's host. // `stdinBuf` is an optional parameter to specify an io.Reader that can be used as stdin for the executed program. // `stdoutBuf` is an optional parameter to specify an io.Writer that can be used as stdout for the executed program. diff --git a/test/cri-containerd/helper_update_utilities_test.go b/test/cri-containerd/helper_update_utilities_test.go index 7c3cc5e5ad..56cef66dfb 100644 --- a/test/cri-containerd/helper_update_utilities_test.go +++ b/test/cri-containerd/helper_update_utilities_test.go @@ -12,7 +12,6 @@ import ( "testing" "github.com/Microsoft/hcsshim/test/pkg/definitions/shimdiag" - runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" ) const podJobObjectUtilPath = "C:\\jobobject-util.exe" @@ -28,28 +27,6 @@ func createJobObjectsGetUtilArgs(ctx context.Context, cid, toolPath string, opti return args } -func checkLCOWResourceLimit(tb testing.TB, ctx context.Context, client runtime.RuntimeServiceClient, cid, path string, expected uint64) { - tb.Helper() - cmd := []string{"cat", path} - containerExecReq := &runtime.ExecSyncRequest{ - ContainerId: cid, - Cmd: cmd, - Timeout: 20, - } - r := execSync(tb, client, ctx, containerExecReq) - if r.ExitCode != 0 { - tb.Fatalf("failed with exit code %d to cat path: %s", r.ExitCode, r.Stderr) - } - output := strings.TrimSpace(string(r.Stdout)) - bytesActual, err := strconv.ParseUint(output, 10, 0) - if err != nil { - tb.Fatalf("could not parse output %s: %s", output, err) - } - if bytesActual != expected { - tb.Fatalf("expected to have a memory limit of %v, instead got %v", expected, bytesActual) - } -} - func checkWCOWResourceLimit(tb testing.TB, ctx context.Context, runtimeHandler, shimName, cid, query string, expected uint64) { tb.Helper() shim, err := shimdiag.GetShim(shimName) diff --git a/test/cri-containerd/layer_integrity_test.go b/test/cri-containerd/layer_integrity_test.go deleted file mode 100644 index 509154fe60..0000000000 --- a/test/cri-containerd/layer_integrity_test.go +++ /dev/null @@ -1,94 +0,0 @@ -//go:build windows && functional -// +build windows,functional - -package cri_containerd - -import ( - "context" - "fmt" - "strconv" - "strings" - "testing" - - "github.com/Microsoft/hcsshim/pkg/annotations" -) - -func Test_LCOW_Layer_Integrity(t *testing.T) { - requireFeatures(t, featureLCOW, featureLCOWIntegrity) - - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause}) - - // Delete container image in case it already exists. - removeImages(t, []string{imageLcowAlpine}) - - // Pull image with dm-verity enabled. - pullRequiredLCOWImages( - t, - []string{imageLcowAlpine}, - WithSandboxLabels(map[string]string{ - "containerd.io/diff/io.microsoft.lcow.append-dm-verity": "true", - }), - ) - - type config struct { - layerType string - vPMemCount int - rootFSType string - } - - for _, scenario := range []config{ - { - layerType: "scsi", - vPMemCount: 0, - rootFSType: "initrd", - }, - { - layerType: "pmem", - vPMemCount: 16, - rootFSType: "initrd", - }, - { - layerType: "pmem", - vPMemCount: 16, - rootFSType: "vhd", - }, - } { - t.Run(fmt.Sprintf("Integrity-For-%s-%s", scenario.layerType, scenario.rootFSType), func(t *testing.T) { - podReq := getRunPodSandboxRequest( - t, - lcowRuntimeHandler, - WithSandboxAnnotations(map[string]string{ - annotations.VPMemCount: strconv.Itoa(scenario.vPMemCount), - annotations.PreferredRootFSType: scenario.rootFSType, - }), - ) - podID := runPodSandbox(t, client, ctx, podReq) - defer removePodSandbox(t, client, ctx, podID) - - // Launch container - cmd := []string{"ash", "-c", "while true; do sleep 1; done"} - contReq := getCreateContainerRequest( - podID, - fmt.Sprintf("alpine-%s-%s", scenario.layerType, scenario.rootFSType), - imageLcowAlpine, - cmd, - podReq.Config, - ) - contID := createContainer(t, client, ctx, contReq) - defer removeContainer(t, client, ctx, contID) - startContainer(t, client, ctx, contID) - defer stopContainer(t, client, ctx, contID) - - // Validate that verity target(s) present - output := shimDiagExecOutput(ctx, t, podID, []string{"ls", "-l", "/dev/mapper"}) - filtered := filterStrings(strings.Split(output, "\n"), fmt.Sprintf("dm-verity-%s", scenario.layerType)) - if len(filtered) == 0 { - t.Fatalf("expected verity targets for %s devices, none found.\n%s\n", scenario.layerType, output) - } - }) - } -} diff --git a/test/cri-containerd/logging_binary_test.go b/test/cri-containerd/logging_binary_test.go index 450147bb62..aa69e599d6 100644 --- a/test/cri-containerd/logging_binary_test.go +++ b/test/cri-containerd/logging_binary_test.go @@ -22,7 +22,7 @@ import ( // which this test will use to construct logPath for CreateContainerRequest and as // the location of stdout artifacts created by the binary func Test_Run_Container_With_Binary_Logger(t *testing.T) { - requireAnyFeature(t, featureWCOWProcess, featureWCOWHypervisor, featureLCOW) + requireAnyFeature(t, featureWCOWProcess, featureWCOWHypervisor) binaryPath := require.Binary(t, "sample-logging-driver.exe") client := newTestRuntimeClient(t) @@ -63,16 +63,6 @@ func Test_Run_Container_With_Binary_Logger(t *testing.T) { cmd: []string{"ping", "-t", "127.0.0.1"}, expectedContent: "Pinging 127.0.0.1 with 32 bytes of data", }, - { - name: "LCOW", - containerName: t.Name() + "-Container-LCOW", - requiredFeatures: []string{featureLCOW}, - runtimeHandler: lcowRuntimeHandler, - sandboxImage: imageLcowK8sPause, - containerImage: imageLcowAlpine, - cmd: []string{"ash", "-c", "while true; do echo 'Hello, World!'; sleep 1; done"}, - expectedContent: "Hello, World!", - }, } // Positive tests @@ -81,11 +71,7 @@ func Test_Run_Container_With_Binary_Logger(t *testing.T) { requireFeatures(t, test.requiredFeatures...) requiredImages := []string{test.sandboxImage, test.containerImage} - if test.runtimeHandler == lcowRuntimeHandler { - pullRequiredLCOWImages(t, requiredImages) - } else { - pullRequiredImages(t, requiredImages) - } + pullRequiredImages(t, requiredImages) podReq := getRunPodSandboxRequest(t, test.runtimeHandler) podID := runPodSandbox(t, client, ctx, podReq) @@ -119,11 +105,7 @@ func Test_Run_Container_With_Binary_Logger(t *testing.T) { requireFeatures(t, test.requiredFeatures...) requiredImages := []string{test.sandboxImage, test.containerImage} - if test.runtimeHandler == lcowRuntimeHandler { - pullRequiredLCOWImages(t, requiredImages) - } else { - pullRequiredImages(t, requiredImages) - } + pullRequiredImages(t, requiredImages) podReq := getRunPodSandboxRequest(t, test.runtimeHandler) podID := runPodSandbox(t, client, ctx, podReq) diff --git a/test/cri-containerd/main_test.go b/test/cri-containerd/main_test.go index e89dd37aa9..e869e678ce 100644 --- a/test/cri-containerd/main_test.go +++ b/test/cri-containerd/main_test.go @@ -5,19 +5,12 @@ package cri_containerd import ( "context" - "errors" "flag" - "fmt" "os" "testing" "time" - "github.com/containerd/containerd" - eventtypes "github.com/containerd/containerd/api/events" - eventsapi "github.com/containerd/containerd/api/services/events/v1" kubeutil "github.com/containerd/containerd/integration/remote/util" - eventruntime "github.com/containerd/containerd/runtime" - typeurl "github.com/containerd/typeurl/v2" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" @@ -98,7 +91,6 @@ var ( flagVirtstack = flag.String("virtstack", "", "Virtstack to use for hypervisor isolated containers") flagVMServiceBinary = flag.String("vmservice-binary", "", "Path to a binary implementing the vmservice ttrpc service") flagContainerdServiceName = flag.String("containerd-service-name", "containerd", "Name of the containerd Windows service") - flagSevSnp = flag.Bool("sev-snp", false, "Indicates that the tests are running on hardware with SEV-SNP support") ) // Features @@ -191,17 +183,6 @@ func newTestRuntimeClient(tb testing.TB) runtime.RuntimeServiceClient { return runtime.NewRuntimeServiceClient(conn) } -func newTestEventService(tb testing.TB) containerd.EventService { - tb.Helper() - ctx, cancel := context.WithTimeout(context.Background(), connectTimeout) - defer cancel() - conn, err := createGRPCConn(ctx) - if err != nil { - tb.Fatalf("Failed to create a client connection %v", err) - } - return containerd.NewEventServiceFromClient(eventsapi.NewEventsClient(conn)) -} - func newTestImageClient(tb testing.TB) runtime.ImageServiceClient { tb.Helper() ctx, cancel := context.WithTimeout(context.Background(), connectTimeout) @@ -213,44 +194,6 @@ func newTestImageClient(tb testing.TB) runtime.ImageServiceClient { return runtime.NewImageServiceClient(conn) } -func getTargetRunTopics() (topicNames []string, filters []string) { - topicNames = []string{ - eventruntime.TaskCreateEventTopic, - eventruntime.TaskStartEventTopic, - eventruntime.TaskExitEventTopic, - eventruntime.TaskDeleteEventTopic, - } - - filters = make([]string, len(topicNames)) - - for i, name := range topicNames { - filters[i] = fmt.Sprintf(`topic=="%v"`, name) - } - return topicNames, filters -} - -func convertEvent(e typeurl.Any) (string, interface{}, error) { - id := "" - evt, err := typeurl.UnmarshalAny(e) - if err != nil { - return "", nil, err - } - - switch event := evt.(type) { - case *eventtypes.TaskCreate: - id = event.ContainerID - case *eventtypes.TaskStart: - id = event.ContainerID - case *eventtypes.TaskDelete: - id = event.ContainerID - case *eventtypes.TaskExit: - id = event.ContainerID - default: - return "", nil, errors.New("test does not support this event") - } - return id, evt, nil -} - func pullRequiredImages(tb testing.TB, images []string, opts ...SandboxConfigOpt) { tb.Helper() opts = append(opts, WithSandboxLabels(map[string]string{ diff --git a/test/cri-containerd/pod_update_test.go b/test/cri-containerd/pod_update_test.go index 6deaa5e6b0..05595e2f21 100644 --- a/test/cri-containerd/pod_update_test.go +++ b/test/cri-containerd/pod_update_test.go @@ -17,7 +17,7 @@ import ( ) func Test_Pod_UpdateResources_Memory(t *testing.T) { - requireAnyFeature(t, featureLCOW, featureWCOWHypervisor) + requireAnyFeature(t, featureWCOWHypervisor) type config struct { name string @@ -32,23 +32,12 @@ func Test_Pod_UpdateResources_Memory(t *testing.T) { runtimeHandler: wcowHypervisorRuntimeHandler, sandboxImage: imageWindowsNanoserver, }, - { - name: "LCOW", - requiredFeatures: []string{featureLCOW}, - runtimeHandler: lcowRuntimeHandler, - sandboxImage: imageLcowK8sPause, - }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { requireFeatures(t, test.requiredFeatures...) - - if test.runtimeHandler == lcowRuntimeHandler { - pullRequiredLCOWImages(t, []string{test.sandboxImage}) - } else { - pullRequiredImages(t, []string{test.sandboxImage}) - } + pullRequiredImages(t, []string{test.sandboxImage}) var startingMemorySize int64 = 768 * memory.MiB podRequest := getRunPodSandboxRequest( t, @@ -72,14 +61,8 @@ func Test_Pod_UpdateResources_Memory(t *testing.T) { ContainerId: podID, } - if test.runtimeHandler == lcowRuntimeHandler { - updateReq.Linux = &runtime.LinuxContainerResources{ - MemoryLimitInBytes: newMemorySize, - } - } else { - updateReq.Windows = &runtime.WindowsContainerResources{ - MemoryLimitInBytes: newMemorySize, - } + updateReq.Windows = &runtime.WindowsContainerResources{ + MemoryLimitInBytes: newMemorySize, } if _, err := client.UpdateContainerResources(ctx, updateReq); err != nil { @@ -90,7 +73,7 @@ func Test_Pod_UpdateResources_Memory(t *testing.T) { } func Test_Pod_UpdateResources_Memory_PA(t *testing.T) { - requireAnyFeature(t, featureLCOW, featureWCOWHypervisor) + requireAnyFeature(t, featureWCOWHypervisor) type config struct { name string @@ -105,23 +88,13 @@ func Test_Pod_UpdateResources_Memory_PA(t *testing.T) { runtimeHandler: wcowHypervisorRuntimeHandler, sandboxImage: imageWindowsNanoserver, }, - { - name: "LCOW", - requiredFeatures: []string{featureLCOW}, - runtimeHandler: lcowRuntimeHandler, - sandboxImage: imageLcowK8sPause, - }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { requireFeatures(t, test.requiredFeatures...) + pullRequiredImages(t, []string{test.sandboxImage}) - if test.runtimeHandler == lcowRuntimeHandler { - pullRequiredLCOWImages(t, []string{test.sandboxImage}) - } else { - pullRequiredImages(t, []string{test.sandboxImage}) - } var startingMemorySize int64 = 200 * memory.MiB podRequest := getRunPodSandboxRequest( t, @@ -146,14 +119,8 @@ func Test_Pod_UpdateResources_Memory_PA(t *testing.T) { ContainerId: podID, } - if test.runtimeHandler == lcowRuntimeHandler { - updateReq.Linux = &runtime.LinuxContainerResources{ - MemoryLimitInBytes: newMemorySize, - } - } else { - updateReq.Windows = &runtime.WindowsContainerResources{ - MemoryLimitInBytes: newMemorySize, - } + updateReq.Windows = &runtime.WindowsContainerResources{ + MemoryLimitInBytes: newMemorySize, } if _, err := client.UpdateContainerResources(ctx, updateReq); err != nil { @@ -164,7 +131,7 @@ func Test_Pod_UpdateResources_Memory_PA(t *testing.T) { } func Test_Pod_UpdateResources_CPUShares(t *testing.T) { - requireAnyFeature(t, featureLCOW, featureWCOWHypervisor) + requireAnyFeature(t, featureWCOWHypervisor) type config struct { name string @@ -179,23 +146,13 @@ func Test_Pod_UpdateResources_CPUShares(t *testing.T) { runtimeHandler: wcowHypervisorRuntimeHandler, sandboxImage: imageWindowsNanoserver, }, - { - name: "LCOW", - requiredFeatures: []string{featureLCOW}, - runtimeHandler: lcowRuntimeHandler, - sandboxImage: imageLcowK8sPause, - }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { requireFeatures(t, test.requiredFeatures...) + pullRequiredImages(t, []string{test.sandboxImage}) - if test.runtimeHandler == lcowRuntimeHandler { - pullRequiredLCOWImages(t, []string{test.sandboxImage}) - } else { - pullRequiredImages(t, []string{test.sandboxImage}) - } podRequest := getRunPodSandboxRequest(t, test.runtimeHandler) client := newTestRuntimeClient(t) @@ -210,14 +167,8 @@ func Test_Pod_UpdateResources_CPUShares(t *testing.T) { ContainerId: podID, } - if test.runtimeHandler == lcowRuntimeHandler { - updateReq.Linux = &runtime.LinuxContainerResources{ - CpuShares: 2000, - } - } else { - updateReq.Windows = &runtime.WindowsContainerResources{ - CpuShares: 2000, - } + updateReq.Windows = &runtime.WindowsContainerResources{ + CpuShares: 2000, } if _, err := client.UpdateContainerResources(ctx, updateReq); err != nil { @@ -229,7 +180,7 @@ func Test_Pod_UpdateResources_CPUShares(t *testing.T) { func Test_Pod_UpdateResources_CPUGroup(t *testing.T) { t.Skip("Skipping for now") - requireAnyFeature(t, featureLCOW, featureWCOWHypervisor) + requireAnyFeature(t, featureWCOWHypervisor) ctx := context.Background() @@ -280,22 +231,12 @@ func Test_Pod_UpdateResources_CPUGroup(t *testing.T) { runtimeHandler: wcowHypervisorRuntimeHandler, sandboxImage: imageWindowsNanoserver, }, - { - name: "LCOW", - requiredFeatures: []string{featureLCOW}, - runtimeHandler: lcowRuntimeHandler, - sandboxImage: imageLcowK8sPause, - }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { requireFeatures(t, test.requiredFeatures...) - if test.runtimeHandler == lcowRuntimeHandler { - pullRequiredLCOWImages(t, []string{test.sandboxImage}) - } else { - pullRequiredImages(t, []string{test.sandboxImage}) - } + pullRequiredImages(t, []string{test.sandboxImage}) podRequest := getRunPodSandboxRequest(t, test.runtimeHandler, WithSandboxAnnotations(map[string]string{ annotations.CPUGroupID: startCPUGroupID, @@ -314,12 +255,7 @@ func Test_Pod_UpdateResources_CPUGroup(t *testing.T) { annotations.CPUGroupID: updateCPUGroupID, }, } - - if test.runtimeHandler == lcowRuntimeHandler { - updateReq.Linux = &runtime.LinuxContainerResources{} - } else { - updateReq.Windows = &runtime.WindowsContainerResources{} - } + updateReq.Windows = &runtime.WindowsContainerResources{} if _, err := client.UpdateContainerResources(ctx, updateReq); err != nil { t.Fatalf("updating container resources for %s with %v", podID, err) diff --git a/test/cri-containerd/policy_test.go b/test/cri-containerd/policy_test.go deleted file mode 100644 index b6e906dd79..0000000000 --- a/test/cri-containerd/policy_test.go +++ /dev/null @@ -1,2026 +0,0 @@ -//go:build windows && functional -// +build windows,functional - -package cri_containerd - -import ( - "context" - _ "embed" - "encoding/base64" - "fmt" - "os" - "path/filepath" - "strconv" - "strings" - "testing" - "time" - - "github.com/Microsoft/hcsshim/internal/shimdiag" - securityPolicyTest "github.com/Microsoft/hcsshim/test/pkg/securitypolicy" - runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" - - "github.com/Microsoft/hcsshim/pkg/annotations" - "github.com/Microsoft/hcsshim/pkg/securitypolicy" - "github.com/Microsoft/hcsshim/test/pkg/definitions/tools/securitypolicy/helpers" - "github.com/Microsoft/hcsshim/test/pkg/require" -) - -var validPolicyAlpineCommand = []string{"ash", "-c", "echo 'Hello'"} - -type configSideEffect func(*runtime.CreateContainerRequest) error - -var defaultExternalProcesses = []securitypolicy.ExternalProcessConfig{ - { - Command: []string{"ls", "-l", "/dev/mapper"}, - WorkingDir: "/", - AllowStdioAccess: true, - }, - { - Command: []string{"bash"}, - WorkingDir: "/", - AllowStdioAccess: true, - }, -} - -func alpineSecurityPolicy(t *testing.T, policyType string, allowEnvironmentVariableDropping bool, allowCapabilityDropping bool, opts ...securitypolicy.ContainerConfigOpt) string { - t.Helper() - containerConfigOpts := append( - []securitypolicy.ContainerConfigOpt{ - securitypolicy.WithCommand(validPolicyAlpineCommand), - securitypolicy.WithAllowPrivilegeEscalation(true), - }, - opts..., - ) - return securityPolicyTest.PolicyFromImageWithOpts( - t, - imageLcowAlpine, - policyType, - containerConfigOpts, - []securitypolicy.PolicyConfigOpt{ - securitypolicy.WithAllowEnvVarDropping(allowEnvironmentVariableDropping), - securitypolicy.WithAllowCapabilityDropping(allowCapabilityDropping), - securitypolicy.WithAllowUnencryptedScratch(!*flagSevSnp), - }, - ) -} - -func sandboxRequestWithPolicy(t *testing.T, policy string) *runtime.RunPodSandboxRequest { - t.Helper() - return getRunPodSandboxRequest( - t, - lcowRuntimeHandler, - WithSandboxAnnotations( - map[string]string{ - annotations.NoSecurityHardware: strconv.FormatBool(!*flagSevSnp), - annotations.SecurityPolicy: policy, - annotations.VPMemNoMultiMapping: "true", - annotations.VPMemCount: "0", - // This allows for better experience for policy only tests. - annotations.EncryptedScratchDisk: strconv.FormatBool(*flagSevSnp), - }, - ), - ) -} - -type policyConfig struct { - enforcer string - input string -} - -var policyTestMatrix = []policyConfig{ - { - enforcer: "rego", - input: "rego", - }, - { - enforcer: "rego", - input: "json", - }, - { - enforcer: "standard", - input: "json", - }, -} - -func Test_RunPodSandbox_WithPolicy_Allowed(t *testing.T) { - requireFeatures(t, featureLCOW, featureLCOWIntegrity) - pullRequiredLCOWImages(t, []string{imageLcowK8sPause}) - - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - for _, pc := range policyTestMatrix { - t.Run(t.Name()+fmt.Sprintf("_Enforcer_%s_Input_%s", pc.enforcer, pc.input), func(t *testing.T) { - sandboxPolicy := securityPolicyTest.PolicyWithOpts( - t, - pc.input, - securitypolicy.WithAllowUnencryptedScratch(true), - securitypolicy.WithAllowEnvVarDropping(true), - securitypolicy.WithAllowCapabilityDropping(true), - ) - sandboxRequest := sandboxRequestWithPolicy(t, sandboxPolicy) - sandboxRequest.Config.Annotations[annotations.SecurityPolicyEnforcer] = pc.enforcer - - podID := runPodSandbox(t, client, ctx, sandboxRequest) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - }) - } -} - -func Test_RunSimpleAlpineContainer_WithPolicy_Allowed(t *testing.T) { - requireFeatures(t, featureLCOW, featureLCOWIntegrity) - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - for _, pc := range policyTestMatrix { - t.Run(t.Name()+fmt.Sprintf("_Enforcer_%s_Input_%s", pc.enforcer, pc.input), func(t *testing.T) { - alpinePolicy := alpineSecurityPolicy(t, pc.input, false, false) - sandboxRequest := sandboxRequestWithPolicy(t, alpinePolicy) - sandboxRequest.Config.Annotations[annotations.SecurityPolicyEnforcer] = pc.enforcer - - podID := runPodSandbox(t, client, ctx, sandboxRequest) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - - containerRequest := getCreateContainerRequest( - podID, - "alpine-with-policy", - imageLcowAlpine, - validPolicyAlpineCommand, - sandboxRequest.Config, - ) - - containerID := createContainer(t, client, ctx, containerRequest) - defer removeContainer(t, client, ctx, containerID) - - startContainer(t, client, ctx, containerID) - stopContainer(t, client, ctx, containerID) - }) - } -} - -func Test_RunContainer_WithPolicy_And_ValidConfigs(t *testing.T) { - type sideEffect func(*runtime.CreateContainerRequest) - type config struct { - name string - sf sideEffect - opts []securitypolicy.ContainerConfigOpt - } - - requireFeatures(t, featureLCOW, featureLCOWIntegrity) - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - for _, testConfig := range []config{ - { - name: "WorkingDir", - sf: func(req *runtime.CreateContainerRequest) { - req.Config.WorkingDir = "/root" - }, - opts: []securitypolicy.ContainerConfigOpt{securitypolicy.WithWorkingDir("/root")}, - }, - { - name: "EnvironmentVariable", - sf: func(req *runtime.CreateContainerRequest) { - req.Config.Envs = append( - req.Config.Envs, &runtime.KeyValue{ - Key: "KEY", - Value: "VALUE", - }, - ) - }, - opts: []securitypolicy.ContainerConfigOpt{ - securitypolicy.WithEnvVarRules( - []securitypolicy.EnvRuleConfig{ - { - Strategy: securitypolicy.EnvVarRuleString, - Rule: "KEY=VALUE", - }, - }, - ), - }, - }, - } { - for _, pc := range policyTestMatrix { - t.Run(testConfig.name+fmt.Sprintf("_Enforcer_%s_Input_%s", pc.enforcer, pc.input), func(t *testing.T) { - alpinePolicy := alpineSecurityPolicy(t, pc.input, false, false, testConfig.opts...) - sandboxRequest := sandboxRequestWithPolicy(t, alpinePolicy) - sandboxRequest.Config.Annotations[annotations.SecurityPolicyEnforcer] = pc.enforcer - - podID := runPodSandbox(t, client, ctx, sandboxRequest) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - - containerRequest := getCreateContainerRequest( - podID, - "alpine-with-policy", - imageLcowAlpine, - validPolicyAlpineCommand, - sandboxRequest.Config, - ) - testConfig.sf(containerRequest) - - containerID := createContainer(t, client, ctx, containerRequest) - startContainer(t, client, ctx, containerID) - defer removeContainer(t, client, ctx, containerID) - defer stopContainer(t, client, ctx, containerID) - }) - } - } -} - -func Test_RunContainer_WithPolicy_And_InvalidConfigs(t *testing.T) { - type config struct { - name string - sf configSideEffect - expectedError string - } - - requireFeatures(t, featureLCOW, featureLCOWIntegrity) - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - for _, testConfig := range []config{ - { - name: "InvalidWorkingDir", - sf: func(req *runtime.CreateContainerRequest) error { - req.Config.WorkingDir = "/non/existent" - return nil - }, - expectedError: "invalid working directory", - }, - { - name: "InvalidCommand", - sf: func(req *runtime.CreateContainerRequest) error { - req.Config.Command = []string{"ash", "-c", "echo 'invalid command'"} - return nil - }, - expectedError: "invalid command", - }, - { - name: "InvalidEnvironmentVariable", - sf: func(req *runtime.CreateContainerRequest) error { - req.Config.Envs = append( - req.Config.Envs, &runtime.KeyValue{ - Key: "KEY", - Value: "VALUE", - }, - ) - return nil - }, - expectedError: "invalid env list", - }, - } { - t.Run(testConfig.name, func(t *testing.T) { - alpinePolicy := alpineSecurityPolicy(t, "rego", false, false) - sandboxRequest := sandboxRequestWithPolicy(t, alpinePolicy) - - podID := runPodSandbox(t, client, ctx, sandboxRequest) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - - containerRequest := getCreateContainerRequest( - podID, - "alpine-with-policy", - imageLcowAlpine, - validPolicyAlpineCommand, - sandboxRequest.Config, - ) - - if err := testConfig.sf(containerRequest); err != nil { - t.Fatalf("failed to apply containerRequest side effect: %s", err) - } - - containerID := createContainer(t, client, ctx, containerRequest) - _, err := client.StartContainer( - ctx, &runtime.StartContainerRequest{ - ContainerId: containerID, - }, - ) - if err == nil { - t.Fatal("expected container start failure") - } - if !securityPolicyTest.AssertErrorContains(t, err, testConfig.expectedError) { - t.Fatalf("expected %q in error message, got: %q", testConfig.expectedError, err) - } - }) - } -} - -func Test_RunContainer_WithPolicy_And_MountConstraints_Allowed(t *testing.T) { - requireFeatures(t, featureLCOW, featureLCOWIntegrity) - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - type config struct { - name string - sideEffect configSideEffect - opts []securitypolicy.ContainerConfigOpt - } - - for _, testConfig := range []config{ - { - name: "DefaultMounts", - sideEffect: func(_ *runtime.CreateContainerRequest) error { - return nil - }, - opts: []securitypolicy.ContainerConfigOpt{}, - }, - { - name: "SandboxMountRW", - sideEffect: func(req *runtime.CreateContainerRequest) error { - req.Config.Mounts = append( - req.Config.Mounts, &runtime.Mount{ - HostPath: "sandbox:///sandbox/path", - ContainerPath: "/container/path", - Propagation: runtime.MountPropagation_PROPAGATION_BIDIRECTIONAL, - }, - ) - return nil - }, - opts: []securitypolicy.ContainerConfigOpt{ - securitypolicy.WithMountConstraints( - []securitypolicy.MountConfig{ - { - HostPath: "sandbox:///sandbox/path", - ContainerPath: "/container/path", - }, - }, - )}, - }, - { - name: "SandboxMountRO", - sideEffect: func(req *runtime.CreateContainerRequest) error { - req.Config.Mounts = append( - req.Config.Mounts, &runtime.Mount{ - HostPath: "sandbox:///sandbox/path", - ContainerPath: "/container/path", - Propagation: runtime.MountPropagation_PROPAGATION_BIDIRECTIONAL, - Readonly: true, - }, - ) - return nil - }, - opts: []securitypolicy.ContainerConfigOpt{ - securitypolicy.WithMountConstraints( - []securitypolicy.MountConfig{ - { - HostPath: "sandbox:///sandbox/path", - ContainerPath: "/container/path", - Readonly: true, - }, - }, - )}, - }, - { - name: "SandboxMountRegex", - sideEffect: func(req *runtime.CreateContainerRequest) error { - req.Config.Mounts = append( - req.Config.Mounts, &runtime.Mount{ - HostPath: "sandbox:///sandbox/path/regexp", - ContainerPath: "/container/path", - Propagation: runtime.MountPropagation_PROPAGATION_BIDIRECTIONAL, - }, - ) - return nil - }, - opts: []securitypolicy.ContainerConfigOpt{ - securitypolicy.WithMountConstraints( - []securitypolicy.MountConfig{ - { - HostPath: "sandbox:///sandbox/path/r.+", - ContainerPath: "/container/path", - }, - }, - )}, - }, - } { - for _, pc := range policyTestMatrix { - t.Run(testConfig.name+fmt.Sprintf("_Enforcer_%s_Input_%s", pc.enforcer, pc.input), func(t *testing.T) { - alpinePolicy := alpineSecurityPolicy(t, pc.input, false, false, testConfig.opts...) - sandboxRequest := sandboxRequestWithPolicy(t, alpinePolicy) - sandboxRequest.Config.Annotations[annotations.SecurityPolicyEnforcer] = pc.enforcer - - podID := runPodSandbox(t, client, ctx, sandboxRequest) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - - containerRequest := getCreateContainerRequest( - podID, - "alpine-with-policy", - imageLcowAlpine, - validPolicyAlpineCommand, - sandboxRequest.Config, - ) - - if err := testConfig.sideEffect(containerRequest); err != nil { - t.Fatalf("failed to apply containerRequest side effect: %s", err) - } - - containerID := createContainer(t, client, ctx, containerRequest) - startContainer(t, client, ctx, containerID) - defer removeContainer(t, client, ctx, containerID) - defer stopContainer(t, client, ctx, containerID) - }) - } - } -} - -func Test_RunContainer_WithPolicy_And_MountConstraints_NotAllowed(t *testing.T) { - requireFeatures(t, featureLCOW, featureLCOWIntegrity) - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - type config struct { - name string - sideEffect configSideEffect - opts []securitypolicy.ContainerConfigOpt - expectedError string - } - - testSandboxMountOpts := []securitypolicy.ContainerConfigOpt{ - securitypolicy.WithMountConstraints( - []securitypolicy.MountConfig{ - { - HostPath: "sandbox:///sandbox/path", - ContainerPath: "/container/path", - }, - }, - ), - } - for _, testConfig := range []config{ - { - name: "InvalidSandboxMountSource", - sideEffect: func(req *runtime.CreateContainerRequest) error { - req.Config.Mounts = append( - req.Config.Mounts, &runtime.Mount{ - HostPath: "sandbox:///sandbox/invalid/path", - ContainerPath: "/container/path", - Propagation: runtime.MountPropagation_PROPAGATION_BIDIRECTIONAL, - }, - ) - return nil - }, - opts: testSandboxMountOpts, - expectedError: "invalid mount list", - }, - { - name: "InvalidSandboxMountDestination", - sideEffect: func(req *runtime.CreateContainerRequest) error { - req.Config.Mounts = append( - req.Config.Mounts, &runtime.Mount{ - HostPath: "sandbox:///sandbox/path", - ContainerPath: "/container/path/invalid", - Propagation: runtime.MountPropagation_PROPAGATION_BIDIRECTIONAL, - }, - ) - return nil - }, - opts: testSandboxMountOpts, - expectedError: "invalid mount list", - }, - { - name: "InvalidSandboxMountFlagRO", - sideEffect: func(req *runtime.CreateContainerRequest) error { - req.Config.Mounts = append( - req.Config.Mounts, &runtime.Mount{ - HostPath: "sandbox:///sandbox/path", - ContainerPath: "/container/path", - Propagation: runtime.MountPropagation_PROPAGATION_BIDIRECTIONAL, - Readonly: true, - }, - ) - return nil - }, - opts: testSandboxMountOpts, - expectedError: "invalid mount list", - }, - { - name: "InvalidSandboxMountFlagRW", - sideEffect: func(req *runtime.CreateContainerRequest) error { - req.Config.Mounts = append( - req.Config.Mounts, &runtime.Mount{ - HostPath: "sandbox:///sandbox/path", - ContainerPath: "/container/path", - Propagation: runtime.MountPropagation_PROPAGATION_BIDIRECTIONAL, - }, - ) - return nil - }, - opts: []securitypolicy.ContainerConfigOpt{ - securitypolicy.WithMountConstraints( - []securitypolicy.MountConfig{ - { - HostPath: "sandbox:///sandbox/path", - ContainerPath: "/container/path", - Readonly: true, - }, - }, - )}, - expectedError: "invalid mount list", - }, - { - name: "InvalidHostPathForRegex", - sideEffect: func(req *runtime.CreateContainerRequest) error { - req.Config.Mounts = append( - req.Config.Mounts, &runtime.Mount{ - HostPath: "sandbox:///sandbox/path/regex/no/match", - ContainerPath: "/container/path", - Propagation: runtime.MountPropagation_PROPAGATION_BIDIRECTIONAL, - }, - ) - return nil - }, - opts: []securitypolicy.ContainerConfigOpt{ - securitypolicy.WithMountConstraints( - []securitypolicy.MountConfig{ - { - HostPath: "sandbox:///sandbox/path/R.+", - ContainerPath: "/container/path", - }, - }, - )}, - expectedError: "invalid mount list", - }, - } { - t.Run(testConfig.name, func(t *testing.T) { - alpinePolicy := alpineSecurityPolicy( - t, - "rego", - false, - false, - testConfig.opts...) - sandboxRequest := sandboxRequestWithPolicy(t, alpinePolicy) - - podID := runPodSandbox(t, client, ctx, sandboxRequest) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - - containerRequest := getCreateContainerRequest( - podID, - "alpine-with-policy", - imageLcowAlpine, - validPolicyAlpineCommand, - sandboxRequest.Config, - ) - - if err := testConfig.sideEffect(containerRequest); err != nil { - t.Fatalf("failed to apply containerRequest side effect: %s", err) - } - - containerID := createContainer(t, client, ctx, containerRequest) - _, err := client.StartContainer( - ctx, &runtime.StartContainerRequest{ - ContainerId: containerID, - }, - ) - if err == nil { - t.Fatal("expected container start failure") - } - if !securityPolicyTest.AssertErrorContains(t, err, testConfig.expectedError) { - t.Fatalf("expected %q in error message, got: %q", testConfig.expectedError, err) - } - }) - } -} - -func Test_RunPrivilegedContainer_WithPolicy_And_AllowElevated_Set(t *testing.T) { - requireFeatures(t, featureLCOWIntegrity) - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - for _, pc := range policyTestMatrix { - t.Run(t.Name()+fmt.Sprintf("_Enforcer_%s_Input_%s", pc.enforcer, pc.input), func(t *testing.T) { - alpinePolicy := alpineSecurityPolicy(t, pc.input, false, false, securitypolicy.WithAllowElevated(true)) - sandboxRequest := sandboxRequestWithPolicy(t, alpinePolicy) - sandboxRequest.Config.Linux = &runtime.LinuxPodSandboxConfig{ - SecurityContext: &runtime.LinuxSandboxSecurityContext{ - Privileged: true, - }, - } - sandboxRequest.Config.Annotations[annotations.SecurityPolicyEnforcer] = pc.enforcer - - podID := runPodSandbox(t, client, ctx, sandboxRequest) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - - contRequest := getCreateContainerRequest( - podID, - "alpine-privileged", - imageLcowAlpine, - validPolicyAlpineCommand, - sandboxRequest.Config, - ) - contRequest.Config.Linux = &runtime.LinuxContainerConfig{ - SecurityContext: &runtime.LinuxContainerSecurityContext{ - Privileged: true, - }, - } - containerID := createContainer(t, client, ctx, contRequest) - defer removeContainer(t, client, ctx, containerID) - startContainer(t, client, ctx, containerID) - defer stopContainer(t, client, ctx, containerID) - }) - } -} - -func Test_RunPrivilegedContainer_WithPolicy_And_AllowElevated_NotSet(t *testing.T) { - requireFeatures(t, featureLCOWIntegrity) - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - alpinePolicy := alpineSecurityPolicy(t, "rego", false, false) - sandboxRequest := sandboxRequestWithPolicy(t, alpinePolicy) - sandboxRequest.Config.Linux = &runtime.LinuxPodSandboxConfig{ - SecurityContext: &runtime.LinuxSandboxSecurityContext{ - Privileged: true, - }, - } - - podID := runPodSandbox(t, client, ctx, sandboxRequest) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - - contRequest := getCreateContainerRequest( - podID, - "alpine-privileged", - imageLcowAlpine, - validPolicyAlpineCommand, - sandboxRequest.Config, - ) - contRequest.Config.Linux = &runtime.LinuxContainerConfig{ - SecurityContext: &runtime.LinuxContainerSecurityContext{ - Privileged: true, - }, - } - containerID := createContainer(t, client, ctx, contRequest) - defer removeContainer(t, client, ctx, containerID) - if _, err := client.StartContainer( - ctx, - &runtime.StartContainerRequest{ContainerId: containerID}, - ); err == nil { - t.Fatalf("expected to fail") - } else { - expectedErrStr := "privileged escalation not allowed" - if !securityPolicyTest.AssertErrorContains(t, err, expectedErrStr) { - t.Fatalf("expected different error: %s", err) - } - } -} - -func Test_RunContainer_WithPolicy_CannotSet_AllowAll_And_Containers(t *testing.T) { - requireFeatures(t, featureLCOW, featureLCOWIntegrity) - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - defaultContainers, err := helpers.PolicyContainersFromConfigs(helpers.DefaultContainerConfigs()) - if err != nil { - t.Fatalf("failed to create policy for default containers: %s", err) - } - - policy := securitypolicy.NewSecurityPolicy(true, defaultContainers) - stringPolicy, err := policy.EncodeToString() - if err != nil { - t.Fatalf("failed to encode policy to base64 string: %s", err) - } - - sandboxRequest := sandboxRequestWithPolicy(t, stringPolicy) - if sandboxRequest.Config.Annotations == nil { - sandboxRequest.Config.Annotations = make(map[string]string) - } - sandboxRequest.Config.Annotations[annotations.SecurityPolicyEnforcer] = "rego" - _, err = client.RunPodSandbox(ctx, sandboxRequest) - if err == nil { - t.Fatal("expected to fail") - } - if !securityPolicyTest.AssertErrorContains(t, err, securitypolicy.ErrInvalidOpenDoorPolicy.Error()) { - t.Fatalf("expected error %s, got %s", securitypolicy.ErrInvalidOpenDoorPolicy, err) - } -} - -func Test_RunContainer_WithPolicy_And_SecurityPolicyEnv_Annotation(t *testing.T) { - requireFeatures(t, featureLCOW, featureLCOWIntegrity) - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - - openDoorPolicy, err := securitypolicy.NewOpenDoorPolicy().EncodeToString() - if err != nil { - t.Fatalf("failed to create open door policy string: %s", err) - } - - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - alpineCmd := []string{"ash", "-c", "sleep 10"} - listDirCmd := []string{"ash", "-c", fmt.Sprintf("ls -l /%s", securitypolicy.SecurityContextDirTemplate)} - opts := []securitypolicy.ContainerConfigOpt{ - securitypolicy.WithCommand(alpineCmd), - securitypolicy.WithAllowStdioAccess(true), - securitypolicy.WithExecProcesses([]securitypolicy.ExecProcessConfig{ - { - Command: listDirCmd, - }, - }), - } - for _, config := range []struct { - name string - policy string - }{ - { - name: "OpenDoorPolicy", - policy: openDoorPolicy, - }, - { - name: "RegoPolicy", - policy: alpineSecurityPolicy(t, "rego", false, false, opts...), - }, - } { - for _, setPolicyEnv := range []bool{true, false} { - testName := fmt.Sprintf("%s_SecurityPolicyEnvSet_%v", config.name, setPolicyEnv) - t.Run(testName, func(t *testing.T) { - sandboxRequest := sandboxRequestWithPolicy(t, config.policy) - - podID := runPodSandbox(t, client, ctx, sandboxRequest) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - - containerRequest := getCreateContainerRequest( - podID, - "alpine-with-policy", - imageLcowAlpine, - alpineCmd, - sandboxRequest.Config, - ) - certValue := "dummy-cert-value" - if setPolicyEnv { - containerRequest.Config.Annotations = map[string]string{ - annotations.UVMSecurityPolicyEnv: "true", - annotations.HostAMDCertificate: certValue, - } - } else { - containerRequest.Config.Annotations = map[string]string{ - annotations.UVMSecurityPolicyEnv: "false", - } - } - - containerID := createContainer(t, client, ctx, containerRequest) - defer removeContainer(t, client, ctx, containerID) - - startContainer(t, client, ctx, containerID) - requireContainerState(ctx, t, client, containerID, runtime.ContainerState_CONTAINER_RUNNING) - - r := execSync(t, client, ctx, &runtime.ExecSyncRequest{ - ContainerId: containerID, - Cmd: listDirCmd, - }) - stdout := string(r.Stdout) - if setPolicyEnv { - if r.ExitCode != 0 { - t.Log(string(r.Stderr)) - t.Fatalf("unexpected exec exit code: %d", r.ExitCode) - } - if !strings.Contains(stdout, securitypolicy.PolicyFilename) { - t.Log(stdout) - t.Fatalf("security policy file not found: %s", securitypolicy.PolicyFilename) - } - if !strings.Contains(stdout, securitypolicy.HostAMDCertFilename) { - t.Log(stdout) - t.Fatalf("AMD certificate file not found: %s", securitypolicy.HostAMDCertFilename) - } - } else { - if r.ExitCode != 1 { - t.Log(stdout) - t.Fatalf("unexpected exec exit code: %d", r.ExitCode) - } - if !strings.Contains(string(r.Stderr), "No such file") { - t.Fatalf("expected different error output, got:\n %s", string(r.Stderr)) - } - } - - // no need to stop the container since it'll exit by itself - requireContainerState(ctx, t, client, containerID, runtime.ContainerState_CONTAINER_EXITED) - }) - } - } -} - -func Test_RunContainer_WithPolicy_And_SecurityPolicyEnv_Dropping(t *testing.T) { - requireFeatures(t, featureLCOW, featureLCOWIntegrity) - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - // The command prints environment variables to stdout, which we can capture - // and validate later - alpineCmd := []string{"ash", "-c", "env && sleep 1"} - - for _, config := range []struct { - name string - policy string - allowed bool - }{ - { - name: "Dropped", - policy: alpineSecurityPolicy(t, "rego", true, false, securitypolicy.WithCommand(alpineCmd)), - allowed: true, - }, - { - name: "NotDropped", - policy: alpineSecurityPolicy(t, "rego", false, false, securitypolicy.WithCommand(alpineCmd)), - allowed: false, - }, - } { - t.Run(config.name, func(t *testing.T) { - sandboxRequest := sandboxRequestWithPolicy(t, config.policy) - - podID := runPodSandbox(t, client, ctx, sandboxRequest) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - - containerRequest := getCreateContainerRequest( - podID, - "alpine-with-policy", - imageLcowAlpine, - alpineCmd, - sandboxRequest.Config, - ) - - // setup logfile to capture stdout - logPath := filepath.Join(t.TempDir(), "log.txt") - containerRequest.Config.LogPath = logPath - - badKV := &runtime.KeyValue{ - Key: "INVALID_ENV", Value: "this/should/cause/an/error/", - } - containerRequest.Config.Envs = append(containerRequest.Config.Envs, badKV) - - response, err := client.CreateContainer(ctx, containerRequest) - if err != nil { - t.Fatalf("error creating container: %v", err) - } - - containerID := response.ContainerId - defer removeContainer(t, client, ctx, containerID) - - _, err = client.StartContainer( - ctx, &runtime.StartContainerRequest{ - ContainerId: containerID, - }, - ) - - if config.allowed { - if err != nil { - t.Fatalf("failed EnforceCreateContainer in sandbox: %s, with: %v", containerRequest.PodSandboxId, err) - } - } else { - if err == nil { - t.Fatalf("expected EnforceCreateContainer to be denied") - } - return - } - - requireContainerState(ctx, t, client, containerID, runtime.ContainerState_CONTAINER_RUNNING) - - // no need to stop the container since it'll exit by itself - requireContainerState(ctx, t, client, containerID, runtime.ContainerState_CONTAINER_EXITED) - - content, err := os.ReadFile(logPath) - if err != nil { - t.Fatalf("error reading log file: %s", err) - } - - badEnv := fmt.Sprintf("%s=%s", badKV.Key, badKV.Value) - if strings.Contains(string(content), badEnv) { - t.Fatalf("INVALID_ENV env var shouldn't be set for init process:\n%s\n", string(content)) - } - }) - } -} - -// The test covers positive test scenarios around scratch encryption: -// - policy allows unencrypted scratch and scratch is encrypted -// - policy allows unencrypted scratch and scratch is not encrypted -// - policy doesn't allow unencrypted scratch and scratch is encrypted -func Test_RunPodSandboxAllowed_WithPolicy_EncryptedScratchPolicy(t *testing.T) { - requireFeatures(t, featureLCOWIntegrity, featureLCOWCrypt) - pullRequiredLCOWImages(t, []string{imageLcowK8sPause}) - - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - for _, tc := range []struct { - allowUnencrypted bool - encryptAnnotation bool - }{ - { - true, - true, - }, - { - true, - false, - }, { - false, - true, - }, - } { - t.Run(fmt.Sprintf("AllowUnencrypted_%t_EncryptionEnabled_%t", tc.allowUnencrypted, tc.encryptAnnotation), func(t *testing.T) { - if tc.encryptAnnotation && !*flagSevSnp { - t.Skip("not running on SNP hardware, dm-crypt not supported") - } - policy := securityPolicyTest.PolicyWithOpts( - t, - "rego", - securitypolicy.WithExternalProcesses(defaultExternalProcesses), - securitypolicy.WithAllowUnencryptedScratch(tc.allowUnencrypted), - securitypolicy.WithAllowEnvVarDropping(true), - securitypolicy.WithAllowCapabilityDropping(true), - ) - sandboxRequest := sandboxRequestWithPolicy(t, policy) - // sandboxRequestWithPolicy sets security policy annotation, so we - // won't get a nil point deref here. - sandboxRequest.Config.Annotations[annotations.EncryptedScratchDisk] = fmt.Sprintf("%t", tc.encryptAnnotation) - podID := runPodSandbox(t, client, ctx, sandboxRequest) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - - if tc.encryptAnnotation { - output := shimDiagExecOutput(ctx, t, podID, []string{"ls", "-l", "/dev/mapper"}) - if !strings.Contains(output, "dm-crypt-scsi-contr") { - t.Log(output) - t.Fatal("expected to find dm-crypt target") - } - } - }) - } -} - -// The test covers negative scenario when policy doesn't allow unencrypted scratch -// and scratch is not encrypted. -func Test_RunPodSandboxNotAllowed_WithPolicy_EncryptedScratchPolicy(t *testing.T) { - requireFeatures(t, featureLCOWIntegrity, featureLCOWCrypt) - pullRequiredLCOWImages(t, []string{imageLcowK8sPause}) - - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - policy := securityPolicyTest.PolicyWithOpts( - t, - "rego", - securitypolicy.WithExternalProcesses(defaultExternalProcesses), - securitypolicy.WithAllowUnencryptedScratch(false), - securitypolicy.WithAllowEnvVarDropping(true), - securitypolicy.WithAllowCapabilityDropping(true), - ) - sandboxRequest := sandboxRequestWithPolicy(t, policy) - sandboxRequest.Config.Annotations[annotations.EncryptedScratchDisk] = "false" - - // we didn't pass encrypt scratch annotation and policy should reject pod creation - response, err := client.RunPodSandbox(ctx, sandboxRequest) - if err == nil { - _, err := client.StopPodSandbox(ctx, &runtime.StopPodSandboxRequest{PodSandboxId: response.PodSandboxId}) - if err != nil { - t.Errorf("failed to stop sandbox: %s", err) - } - _, err = client.RemovePodSandbox(ctx, &runtime.RemovePodSandboxRequest{PodSandboxId: response.PodSandboxId}) - if err != nil { - t.Errorf("failed to remove sandbox: %s", err) - } - t.Fatalf("expected to fail") - } - expectedError := "unencrypted scratch not allowed" - if !securityPolicyTest.AssertErrorContains(t, err, expectedError) { - t.Fatalf("expected '%s' error, got '%s'", expectedError, err) - } -} - -func Test_RunContainer_WithPolicy_And_Binary_Logger_Without_Stdio(t *testing.T) { - requireFeatures(t, featureLCOWIntegrity) - binaryPath := require.Binary(t, "sample-logging-driver.exe") - - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - logPath := "binary:///" + binaryPath - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - - for _, tc := range []struct { - stdioAllowed bool - expectedOutput string - }{ - { - true, - "hello\nworld\n", - }, - { - false, - "", - }, - } { - t.Run(fmt.Sprintf("StdioAllowed_%v", tc.stdioAllowed), func(t *testing.T) { - cmd := []string{"ash", "-c", "echo hello; sleep 1; echo world"} - policy := alpineSecurityPolicy( - t, - "rego", - true, - false, - securitypolicy.WithAllowStdioAccess(tc.stdioAllowed), - securitypolicy.WithCommand(cmd), - ) - podReq := sandboxRequestWithPolicy(t, policy) - podID := runPodSandbox(t, client, ctx, podReq) - defer removePodSandbox(t, client, ctx, podID) - - logFileName := fmt.Sprintf(`%s\stdout.txt`, t.TempDir()) - conReq := getCreateContainerRequest( - podID, - fmt.Sprintf("alpine-stdio-allowed-%v", tc.stdioAllowed), - imageLcowAlpine, - cmd, - podReq.Config, - ) - conReq.Config.LogPath = logPath + fmt.Sprintf("?%s", logFileName) - - containerID := createContainer(t, client, ctx, conReq) - defer removeContainer(t, client, ctx, containerID) - - startContainer(t, client, ctx, containerID) - defer stopContainer(t, client, ctx, containerID) - - requireContainerState(ctx, t, client, containerID, runtime.ContainerState_CONTAINER_RUNNING) - requireContainerState(ctx, t, client, containerID, runtime.ContainerState_CONTAINER_EXITED) - - content, err := os.ReadFile(logFileName) - if err != nil { - t.Fatalf("failed to read log file: %s", err) - } - if tc.expectedOutput != string(content) { - t.Fatalf("expected output %q, got %q", tc.expectedOutput, string(content)) - } - }) - } -} - -func Test_ExecInContainer_WithPolicy(t *testing.T) { - requireFeatures(t, featureLCOWIntegrity) - pullRequiredLCOWImages(t, []string{imageLcowAlpine}) - - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - for _, tc := range []struct { - execProcessConfig securitypolicy.ExecProcessConfig - execProcessRequest []string - shouldFail bool - }{ - { - execProcessConfig: securitypolicy.ExecProcessConfig{ - Command: []string{"ls"}, - }, - execProcessRequest: []string{"ls"}, - shouldFail: false, - }, - { - execProcessConfig: securitypolicy.ExecProcessConfig{ - Command: []string{"ls"}, - }, - execProcessRequest: []string{"ls", "-l"}, - shouldFail: true, - }, - } { - t.Run(fmt.Sprintf("ExecInContainer_ShouldFail_%t", tc.shouldFail), func(t *testing.T) { - cmd := []string{"ash", "-c", "while true; do sleep 1; done"} - policy := alpineSecurityPolicy( - t, - "rego", - true, - false, - securitypolicy.WithExecProcesses([]securitypolicy.ExecProcessConfig{tc.execProcessConfig}), - securitypolicy.WithCommand(cmd), - ) - sandboxRequest := sandboxRequestWithPolicy(t, policy) - podID := runPodSandbox(t, client, ctx, sandboxRequest) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - - conReq := getCreateContainerRequest( - podID, - fmt.Sprintf("alpine-exec-not-allowed-%t", tc.shouldFail), - imageLcowAlpine, - cmd, - sandboxRequest.Config, - ) - - containerID := createContainer(t, client, ctx, conReq) - defer removeContainer(t, client, ctx, containerID) - - startContainer(t, client, ctx, containerID) - defer stopContainer(t, client, ctx, containerID) - - requireContainerState(ctx, t, client, containerID, runtime.ContainerState_CONTAINER_RUNNING) - - execReq := &runtime.ExecSyncRequest{ - ContainerId: containerID, - Cmd: tc.execProcessRequest, - Timeout: 20, - } - _, err := client.ExecSync(ctx, execReq) - if err == nil { - if tc.shouldFail { - t.Fatal("exec should've been denied by policy") - } - } else { - if !tc.shouldFail { - t.Fatalf("unexpected exec failure: %s", err) - } - if !securityPolicyTest.AssertErrorContains(t, err, "invalid command") { - t.Fatalf("expected 'invalid command' error, got '%s' instead", err) - } - } - }) - } -} - -func Test_ExecInContainer_WithPolicy_Privileged(t *testing.T) { - requireFeatures(t, featureLCOWIntegrity) - pullRequiredLCOWImages(t, []string{imageLcowAlpine}) - - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - for _, tc := range []struct { - execProcessConfig securitypolicy.ExecProcessConfig - execProcessRequest []string - shouldFail bool - }{ - { - execProcessConfig: securitypolicy.ExecProcessConfig{ - Command: []string{"ls"}, - }, - execProcessRequest: []string{"ls"}, - shouldFail: false, - }, - { - execProcessConfig: securitypolicy.ExecProcessConfig{ - Command: []string{"ls"}, - }, - execProcessRequest: []string{"ls", "-l"}, - shouldFail: true, - }, - } { - t.Run(fmt.Sprintf("ExecInContainer_ShouldFail_%t", tc.shouldFail), func(t *testing.T) { - cmd := []string{"ash", "-c", "while true; do sleep 1; done"} - policy := alpineSecurityPolicy( - t, - "rego", - true, - false, - securitypolicy.WithExecProcesses([]securitypolicy.ExecProcessConfig{tc.execProcessConfig}), - securitypolicy.WithCommand(cmd), - securitypolicy.WithAllowElevated(true), - ) - sandboxRequest := sandboxRequestWithPolicy(t, policy) - sandboxRequest.Config.Linux = &runtime.LinuxPodSandboxConfig{ - SecurityContext: &runtime.LinuxSandboxSecurityContext{ - Privileged: true, - }, - } - podID := runPodSandbox(t, client, ctx, sandboxRequest) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - - conReq := getCreateContainerRequest( - podID, - fmt.Sprintf("alpine-exec-not-allowed-%t", tc.shouldFail), - imageLcowAlpine, - cmd, - sandboxRequest.Config, - ) - conReq.Config.Linux = &runtime.LinuxContainerConfig{ - SecurityContext: &runtime.LinuxContainerSecurityContext{ - Privileged: true, - }, - } - - containerID := createContainer(t, client, ctx, conReq) - defer removeContainer(t, client, ctx, containerID) - - startContainer(t, client, ctx, containerID) - defer stopContainer(t, client, ctx, containerID) - - requireContainerState(ctx, t, client, containerID, runtime.ContainerState_CONTAINER_RUNNING) - - execReq := &runtime.ExecSyncRequest{ - ContainerId: containerID, - Cmd: tc.execProcessRequest, - Timeout: 20, - } - _, err := client.ExecSync(ctx, execReq) - if err == nil { - if tc.shouldFail { - t.Fatal("exec should've been denied by policy") - } - } else { - if !tc.shouldFail { - t.Fatalf("unexpected exec failure: %s", err) - } - if !securityPolicyTest.AssertErrorContains(t, err, "invalid command") { - t.Fatalf("expected 'invalid command' error, got '%s' instead", err) - } - } - }) - } -} - -func Test_ExecInUVM_WithPolicy(t *testing.T) { - requireFeatures(t, featureLCOWIntegrity) - - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - for _, tc := range []struct { - execInUVMConfig securitypolicy.ExternalProcessConfig - execInUVMRequest []string - shouldFail bool - }{ - { - execInUVMConfig: securitypolicy.ExternalProcessConfig{ - Command: []string{"ls"}, - WorkingDir: "/", - AllowStdioAccess: true, - }, - execInUVMRequest: []string{"ls"}, - shouldFail: false, - }, - { - execInUVMConfig: securitypolicy.ExternalProcessConfig{ - Command: []string{"ls"}, - WorkingDir: "/", - AllowStdioAccess: true, - }, - execInUVMRequest: []string{"ls", "-l"}, - shouldFail: true, - }, - } { - t.Run(fmt.Sprintf("ShouldFail_%t", tc.shouldFail), func(t *testing.T) { - policy := securityPolicyTest.PolicyWithOpts(t, "rego", - securitypolicy.WithExternalProcesses([]securitypolicy.ExternalProcessConfig{tc.execInUVMConfig}), - securitypolicy.WithAllowRuntimeLogging(true), - securitypolicy.WithAllowUnencryptedScratch(true), - ) - sandboxRequest := sandboxRequestWithPolicy(t, policy) - podID := runPodSandbox(t, client, ctx, sandboxRequest) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - - _, err := shimDiagExecOutputWithErr(ctx, t, podID, tc.execInUVMRequest) - if err != nil { - if !tc.shouldFail { - t.Fatalf("external process exec should succeed, got error instead: %s", err) - } - if !securityPolicyTest.AssertErrorContains(t, err, "invalid command") { - t.Fatalf("expected invalid command error, got %s", err) - } - } else { - if tc.shouldFail { - t.Fatal("external process exec should have failed") - } - } - }) - } -} - -func Test_RunPodSandbox_Concurrently(t *testing.T) { - requireFeatures(t, featureLCOWIntegrity) - - for i := 0; i < 20; i++ { - i := i // define a local copy of loop variable that will be captured by `t.Run` closure - t.Run(fmt.Sprintf("ParallelPodRun_%d", i+1), func(t *testing.T) { - t.Parallel() - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - policy := securityPolicyTest.PolicyWithOpts( - t, - "rego", - securitypolicy.WithAllowUnencryptedScratch(true), - ) - runpRequest := &runtime.RunPodSandboxRequest{ - Config: &runtime.PodSandboxConfig{ - Metadata: &runtime.PodSandboxMetadata{ - Name: fmt.Sprintf("%s_%d", t.Name(), i), - Namespace: testNamespace, - }, - Annotations: map[string]string{ - annotations.NoSecurityHardware: strconv.FormatBool(!*flagSevSnp), - annotations.SecurityPolicy: policy, - annotations.SecurityPolicyEnforcer: "rego", - annotations.EncryptedScratchDisk: strconv.FormatBool(*flagSevSnp), - }, - }, - RuntimeHandler: lcowRuntimeHandler, - } - - podID := runPodSandbox(t, client, ctx, runpRequest) - defer func() { - removePodSandbox(t, client, ctx, podID) - }() - defer func() { - stopPodSandbox(t, client, ctx, podID) - }() - time.Sleep(5 * time.Second) - }) - } -} - -func Test_RunContainer_WithPolicy_And_NoNewPrivileges(t *testing.T) { - requireFeatures(t, featureLCOW, featureLCOWIntegrity) - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - policy := alpineSecurityPolicy(t, "rego", false, false, securitypolicy.WithAllowPrivilegeEscalation(false)) - - for _, config := range []struct { - name string - allowPrivilegeEscalation string - allowed bool - }{ - { - name: "AllowPrivilegeEscalation_False_Allow", - allowPrivilegeEscalation: "false", - allowed: true, - }, - { - name: "AllowPrivilegeEscalation_True_Deny", - allowPrivilegeEscalation: "true", - allowed: false, - }, - } { - t.Run(config.name, func(t *testing.T) { - sandboxRequest := sandboxRequestWithPolicy(t, policy) - - podID := runPodSandbox(t, client, ctx, sandboxRequest) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - - containerRequest := getCreateContainerRequest( - podID, - "alpine-with-policy-and-no-new-privs", - imageLcowAlpine, - validPolicyAlpineCommand, - sandboxRequest.Config, - ) - containerRequest.Config.Annotations = map[string]string{ - "io.microsoft.cri.lcow.container.allowprivilegeescalation": config.allowPrivilegeEscalation, - } - - response, err := client.CreateContainer(ctx, containerRequest) - if err != nil { - t.Fatalf("error creating container: %v", err) - } - - containerID := response.ContainerId - defer removeContainer(t, client, ctx, containerID) - - _, err = client.StartContainer( - ctx, &runtime.StartContainerRequest{ - ContainerId: containerID, - }, - ) - - if err == nil { - defer stopContainer(t, client, ctx, containerID) - if !config.allowed { - t.Errorf("expected container to fail to start") - } - } else { - if config.allowed { - t.Errorf("expected container to start successfully: %s", err) - } - } - }) - } -} - -func userConfig(uid, gid int64) securitypolicy.UserConfig { - return securitypolicy.UserConfig{ - UserIDName: securitypolicy.IDNameConfig{ - Strategy: securitypolicy.IDNameStrategyID, - Rule: strconv.FormatInt(uid, 10), - }, - GroupIDNames: []securitypolicy.IDNameConfig{ - { - Strategy: securitypolicy.IDNameStrategyID, - Rule: strconv.FormatInt(gid, 10), - }, - }, - Umask: "0022", - } -} - -func Test_RunContainer_WithPolicy_And_RunAs(t *testing.T) { - requireFeatures(t, featureLCOW, featureLCOWIntegrity) - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowCustomUser}) - - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - cmd := []string{"sh", "-c", "echo 'Hello'"} - policy := securityPolicyTest.PolicyFromImageWithOpts( - t, - imageLcowCustomUser, - "rego", - []securitypolicy.ContainerConfigOpt{ - securitypolicy.WithCommand(cmd), - securitypolicy.WithUser(userConfig(1000, 1000)), - }, - []securitypolicy.PolicyConfigOpt{ - securitypolicy.WithAllowEnvVarDropping(false), - securitypolicy.WithAllowCapabilityDropping(false), - securitypolicy.WithAllowUnencryptedScratch(true), - }, - ) - - for _, config := range []struct { - name string - uid string - gid string - allowed bool - }{ - { - name: "UID_Match_GID_Match_Allow", - uid: "1000", - gid: "1000", - allowed: true, - }, - { - name: "UID_Match_GID_NoMatch_Deny", - uid: "1000", - gid: "0", - allowed: false, - }, - { - name: "UID_NoMatch_GID_Match_Deny", - uid: "0", - gid: "1000", - allowed: false, - }, - { - name: "UID_NoMatch_GID_NoMatch_Deny", - uid: "0", - gid: "0", - allowed: false, - }, - } { - t.Run(config.name, func(t *testing.T) { - sandboxRequest := sandboxRequestWithPolicy(t, policy) - - podID := runPodSandbox(t, client, ctx, sandboxRequest) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - - containerRequest := getCreateContainerRequest( - podID, - "alpine-with-policy-and-custom-user", - imageLcowCustomUser, - cmd, - sandboxRequest.Config, - ) - containerRequest.Config.Annotations = map[string]string{ - "io.microsoft.cri.lcow.container.runasuser": config.uid, - "io.microsoft.cri.lcow.container.runasgroup": config.gid, - } - - response, err := client.CreateContainer(ctx, containerRequest) - if err != nil { - t.Fatalf("error creating container: %v", err) - } - - containerID := response.ContainerId - defer removeContainer(t, client, ctx, containerID) - - _, err = client.StartContainer( - ctx, &runtime.StartContainerRequest{ - ContainerId: containerID, - }, - ) - - if err == nil { - defer stopContainer(t, client, ctx, containerID) - if !config.allowed { - t.Errorf("expected container to fail to start") - } - } else { - if config.allowed { - t.Errorf("expected container to start successfully: %s", err) - } - } - }) - } -} - -func capabilitiesConfig() *securitypolicy.CapabilitiesConfig { - return &securitypolicy.CapabilitiesConfig{ - Bounding: securitypolicy.DefaultUnprivilegedCapabilities(), - Effective: securitypolicy.DefaultUnprivilegedCapabilities(), - Inheritable: securitypolicy.EmptyCapabiltiesSet(), - Permitted: securitypolicy.DefaultUnprivilegedCapabilities(), - Ambient: securitypolicy.EmptyCapabiltiesSet(), - } -} - -func Test_RunContainer_WithPolicy_And_Capabilities(t *testing.T) { - requireFeatures(t, featureLCOW, featureLCOWIntegrity) - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowCustomUser}) - - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - noDropCapsPolicy := alpineSecurityPolicy(t, "rego", false, false, securitypolicy.WithCapabilities(capabilitiesConfig())) - dropCapsPolicy := alpineSecurityPolicy(t, "rego", false, true, securitypolicy.WithCapabilities(capabilitiesConfig())) - - for _, config := range []struct { - name string - policy string - add string - drop string - allowed bool - }{ - { - name: "NoDropping_NoAdd_NoDrop_Allow", - policy: noDropCapsPolicy, - add: "", - drop: "", - allowed: true, - }, - { - name: "NoDropping_Add_NoDrop_Deny", - policy: noDropCapsPolicy, - add: "CAP_SYS_ADMIN", - drop: "", - allowed: false, - }, - { - name: "NoDropping_NoAdd_Drop_Deny", - policy: noDropCapsPolicy, - add: "", - drop: "CAP_CHOWN,CAP_NET_BIND_SERVICE", - allowed: false, - }, - { - name: "Dropping_Add_NoDrop_Allow", - policy: dropCapsPolicy, - add: "CAP_SYS_ADMIN,CAP_NET_BROADCAST", - drop: "", - allowed: true, - }, - { - name: "Dropping_NoAdd_Drop_Deny", - policy: dropCapsPolicy, - add: "", - drop: "CAP_CHOWN", - allowed: false, - }, - } { - t.Run(config.name, func(t *testing.T) { - sandboxRequest := sandboxRequestWithPolicy(t, config.policy) - - podID := runPodSandbox(t, client, ctx, sandboxRequest) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - - containerRequest := getCreateContainerRequest( - podID, - "alpine-with-policy-and-capabilities", - imageLcowAlpine, - validPolicyAlpineCommand, - sandboxRequest.Config, - ) - containerRequest.Config.Annotations = map[string]string{ - "io.microsoft.cri.lcow.container.capabilities.add": config.add, - "io.microsoft.cri.lcow.container.capabilities.drop": config.drop, - } - - response, err := client.CreateContainer(ctx, containerRequest) - if err != nil { - t.Fatalf("error creating container: %v", err) - } - - containerID := response.ContainerId - defer removeContainer(t, client, ctx, containerID) - - _, err = client.StartContainer( - ctx, &runtime.StartContainerRequest{ - ContainerId: containerID, - }, - ) - - if err == nil { - defer stopContainer(t, client, ctx, containerID) - if !config.allowed { - t.Errorf("expected container to fail to start") - } - } else { - if config.allowed { - t.Errorf("expected container to start successfully: %s", err) - } - } - }) - } -} - -//go:embed seccomp_valid.json -var validSeccomp []byte - -//go:embed seccomp_invalid.json -var invalidSeccomp []byte - -func Test_RunContainer_WithPolicy_And_Seccomp(t *testing.T) { - seccompProfilePath := filepath.Join(t.TempDir(), "seccomp_valid.json") - if err := os.WriteFile(seccompProfilePath, validSeccomp, 0755); err != nil { - t.Fatalf("failed to write seccomp profile to temp file: %s", err) - } - defer os.Remove(seccompProfilePath) - - requireFeatures(t, featureLCOW, featureLCOWIntegrity) - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - seccompPolicy := alpineSecurityPolicy(t, "rego", false, false, securitypolicy.WithSeccompProfilePath(seccompProfilePath)) - noSeccompPolicy := alpineSecurityPolicy(t, "rego", false, false) - - validSeccompBase64 := base64.StdEncoding.EncodeToString(validSeccomp) - invalidSeccompBase64 := base64.StdEncoding.EncodeToString(invalidSeccomp) - - for _, config := range []struct { - name string - policy string - seccompBase64 string - allowed bool - }{ - { - name: "Seccomp_Valid_Allow", - policy: seccompPolicy, - seccompBase64: validSeccompBase64, - allowed: true, - }, - { - name: "Seccomp_Invalid_Deny", - policy: seccompPolicy, - seccompBase64: invalidSeccompBase64, - allowed: false, - }, - { - name: "NoSeccomp_Nil_Allow", - policy: noSeccompPolicy, - seccompBase64: "", - allowed: true, - }, - { - name: "NoSeccomp_Valid_Deny", - policy: noSeccompPolicy, - seccompBase64: validSeccompBase64, - allowed: false, - }, - } { - t.Run(config.name, func(t *testing.T) { - sandboxRequest := sandboxRequestWithPolicy(t, config.policy) - - podID := runPodSandbox(t, client, ctx, sandboxRequest) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - - containerRequest := getCreateContainerRequest( - podID, - "alpine-with-policy-and-seccomp", - imageLcowAlpine, - validPolicyAlpineCommand, - sandboxRequest.Config, - ) - - if len(config.seccompBase64) > 0 { - containerRequest.Config.Annotations = map[string]string{ - "io.microsoft.cri.lcow.container.seccompprofile": config.seccompBase64, - } - } - - response, err := client.CreateContainer(ctx, containerRequest) - if err != nil { - t.Fatalf("error creating container: %v", err) - } - - containerID := response.ContainerId - defer removeContainer(t, client, ctx, containerID) - - _, err = client.StartContainer( - ctx, &runtime.StartContainerRequest{ - ContainerId: containerID, - }, - ) - - if err == nil { - defer stopContainer(t, client, ctx, containerID) - if !config.allowed { - t.Errorf("expected container to fail to start") - } - } else { - if config.allowed && !strings.Contains(err.Error(), "seccomp: config provided but seccomp not supported") { - t.Errorf("expected container to start successfully: %s", err) - } - } - }) - } -} - -//go:embed policy-v0.1.0.rego -var oldPolicy []byte - -func Test_RunPrivilegedContainer_WithPolicy_BackwardsCompatible(t *testing.T) { - requireFeatures(t, featureLCOWIntegrity) - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - alpinePolicy := base64.StdEncoding.EncodeToString(oldPolicy) - sandboxRequest := sandboxRequestWithPolicy(t, alpinePolicy) - sandboxRequest.Config.Linux = &runtime.LinuxPodSandboxConfig{ - SecurityContext: &runtime.LinuxSandboxSecurityContext{ - Privileged: true, - }, - } - sandboxRequest.Config.Annotations[annotations.SecurityPolicyEnforcer] = "rego" - - podID := runPodSandbox(t, client, ctx, sandboxRequest) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - - contRequest := getCreateContainerRequest( - podID, - "alpine-privileged", - imageLcowAlpine, - validPolicyAlpineCommand, - sandboxRequest.Config, - ) - contRequest.Config.Linux = &runtime.LinuxContainerConfig{ - SecurityContext: &runtime.LinuxContainerSecurityContext{ - Privileged: true, - }, - } - containerID := createContainer(t, client, ctx, contRequest) - defer removeContainer(t, client, ctx, containerID) - startContainer(t, client, ctx, containerID) - defer stopContainer(t, client, ctx, containerID) -} - -func Test_RunContainer_WithPolicy_BackwardsCompatible(t *testing.T) { - requireFeatures(t, featureLCOWIntegrity) - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - alpinePolicy := base64.StdEncoding.EncodeToString(oldPolicy) - sandboxRequest := sandboxRequestWithPolicy(t, alpinePolicy) - sandboxRequest.Config.Annotations[annotations.SecurityPolicyEnforcer] = "rego" - - podID := runPodSandbox(t, client, ctx, sandboxRequest) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - - contRequest := getCreateContainerRequest( - podID, - "alpine-privileged", - imageLcowAlpine, - validPolicyAlpineCommand, - sandboxRequest.Config, - ) - containerID := createContainer(t, client, ctx, contRequest) - defer removeContainer(t, client, ctx, containerID) - startContainer(t, client, ctx, containerID) - defer stopContainer(t, client, ctx, containerID) -} - -func Test_Plan9Mount_WithPolicy(t *testing.T) { - requireFeatures(t, featureLCOWIntegrity) - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - p9MountConfigs := []securitypolicy.MountConfig{ - { - HostPath: "plan9://", - ContainerPath: "/mounts/p9", - Readonly: false, - }, - } - - for _, allowed := range []bool{true, false} { - t.Run(fmt.Sprintf("Plan9Mount_Allowed_%t", allowed), func(t *testing.T) { - var opts []securitypolicy.ContainerConfigOpt - if allowed { - opts = append(opts, securitypolicy.WithMountConstraints(p9MountConfigs)) - } - alpinePolicy := alpineSecurityPolicy( - t, - "rego", - false, - false, - opts..., - ) - sandboxRequest := sandboxRequestWithPolicy(t, alpinePolicy) - sandboxRequest.Config.Annotations[annotations.SecurityPolicyEnforcer] = "rego" - - podID := runPodSandbox(t, client, ctx, sandboxRequest) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - - containerRequest := getCreateContainerRequest( - podID, - "alpine-plan9", - imageLcowAlpine, - validPolicyAlpineCommand, - sandboxRequest.Config, - ) - containerRequest.Config.Mounts = append(containerRequest.Config.Mounts, &runtime.Mount{ - HostPath: t.TempDir(), - ContainerPath: "/mounts/p9", - Readonly: false, - }) - - containerID := createContainer(t, client, ctx, containerRequest) - defer removeContainer(t, client, ctx, containerID) - - _, err := client.StartContainer(ctx, &runtime.StartContainerRequest{ - ContainerId: containerID, - }) - if err == nil { - defer stopContainer(t, client, ctx, containerID) - if !allowed { - t.Fatal("container start should have failed") - } - } else { - if allowed { - t.Fatalf("container creation should have succeeded: %s", err) - } - expectedErrStr := "invalid mount list: /mounts/p9" - if !securityPolicyTest.AssertErrorContains(t, err, expectedErrStr) { - t.Fatalf("expected '%s' policy error, got: %s", expectedErrStr, err) - } - } - }) - } -} - -func Test_DumpStacks_WithPolicy(t *testing.T) { - requireFeatures(t, featureLCOWIntegrity) - pullRequiredLCOWImages(t, []string{imageLcowK8sPause}) - - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - for _, stacksAllowed := range []bool{true, false} { - t.Run(fmt.Sprintf("Allowed_%t", stacksAllowed), func(t *testing.T) { - policy := securityPolicyTest.PolicyWithOpts( - t, - "rego", - securitypolicy.WithAllowUnencryptedScratch(true), - securitypolicy.WithAllowDumpStacks(stacksAllowed), - ) - sandboxRequest := sandboxRequestWithPolicy(t, policy) - sandboxRequest.Config.Annotations[annotations.EncryptedScratchDisk] = "false" - - podID := runPodSandbox(t, client, ctx, sandboxRequest) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - - shimName := fmt.Sprintf("k8s.io-%s", podID) - shim, err := shimdiag.GetShim(shimName) - if err != nil { - t.Fatal(err) - } - shimClient := shimdiag.NewShimDiagClient(shim) - - // note that this will always return at least the host stacks, but - // we need to explicitly check for the guest stacks to make sure - // that the enforcement works. - stacks, _ := shimClient.DiagStacks(ctx, &shimdiag.StacksRequest{}) - if stacksAllowed { - if len(stacks.GuestStacks) == 0 { - t.Fatal("GCS stacks must be available") - } - } else { - if len(stacks.GuestStacks) > 0 { - t.Fatal("GCS stacks must be empty") - } - } - }) - } -} - -func Test_GetProperties_WithPolicy(t *testing.T) { - requireFeatures(t, featureLCOWIntegrity) - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - for _, allowGetProperties := range []bool{true, false} { - t.Run(fmt.Sprintf("Allowed_%t", allowGetProperties), func(t *testing.T) { - policy := securityPolicyTest.PolicyFromImageWithOpts( - t, - imageLcowAlpine, - "rego", - []securitypolicy.ContainerConfigOpt{ - securitypolicy.WithCommand(validPolicyAlpineCommand), - securitypolicy.WithAllowPrivilegeEscalation(true), - }, - []securitypolicy.PolicyConfigOpt{ - securitypolicy.WithAllowUnencryptedScratch(true), - securitypolicy.WithAllowPropertiesAccess(allowGetProperties), - }, - ) - sandboxRequest := sandboxRequestWithPolicy(t, policy) - sandboxRequest.Config.Annotations[annotations.EncryptedScratchDisk] = "false" - - podID := runPodSandbox(t, client, ctx, sandboxRequest) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - - containerRequest := getCreateContainerRequest( - podID, - "alpine-get-properties", - imageLcowAlpine, - validPolicyAlpineCommand, - sandboxRequest.Config, - ) - containerID := createContainer(t, client, ctx, containerRequest) - defer removeContainer(t, client, ctx, containerID) - startContainer(t, client, ctx, containerID) - defer stopContainer(t, client, ctx, containerID) - - _, err := client.ContainerStats(ctx, &runtime.ContainerStatsRequest{ContainerId: containerID}) - if err != nil { - if allowGetProperties { - t.Fatalf("container stats should be allowed: %s", err) - } - // unfortunately the errors returned during stats collection - // are not bubbled up, so we can only rely on the fact that - // the metrics response is invalid. - if !strings.Contains(err.Error(), " unexpected metrics response: []") { - t.Fatalf("expected different error: %s", err) - } - } else { - if !allowGetProperties { - t.Fatal("container stats should not be allowed") - } - } - }) - } -} diff --git a/test/cri-containerd/pullimage_test.go b/test/cri-containerd/pullimage_test.go index e3e57a1d0d..b9d0ec07ea 100644 --- a/test/cri-containerd/pullimage_test.go +++ b/test/cri-containerd/pullimage_test.go @@ -104,11 +104,3 @@ func Test_PullImageTimestamps(t *testing.T) { t.Fatalf("Timestamps not in order. startTimestamp should be less than testdirTimestamp and fakelinkTimestamp") } } - -func Test_PullImageUnorderedTar(t *testing.T) { - requireFeatures(t, featureLCOW) - - // This is a very minimal hand crafted image so it can't run a container. We just - // want to test if pulling this image succeeds. - pullRequiredLCOWImages(t, []string{imageLinuxUnorderedTar}) -} diff --git a/test/cri-containerd/removepodsandbox_test.go b/test/cri-containerd/removepodsandbox_test.go index 10304b3098..2721df9b88 100644 --- a/test/cri-containerd/removepodsandbox_test.go +++ b/test/cri-containerd/removepodsandbox_test.go @@ -85,7 +85,7 @@ func runContainerInSandboxTest(t *testing.T, tc *TestConfig) { } func Test_RunPodSandbox_Without_Sandbox_Stop(t *testing.T) { - requireAnyFeature(t, featureWCOWProcess, featureWCOWHypervisor, featureLCOW) + requireAnyFeature(t, featureWCOWProcess, featureWCOWHypervisor) tests := []TestConfig{ { @@ -100,23 +100,12 @@ func Test_RunPodSandbox_Without_Sandbox_Stop(t *testing.T) { runtimeHandler: wcowHypervisorRuntimeHandler, sandboxImage: imageWindowsNanoserver, }, - { - name: "LCOW", - requiredFeatures: []string{featureLCOW}, - runtimeHandler: lcowRuntimeHandler, - sandboxImage: imageLcowAlpine, - }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { requireFeatures(t, test.requiredFeatures...) - - if test.runtimeHandler == lcowRuntimeHandler { - pullRequiredLCOWImages(t, []string{test.sandboxImage}) - } else { - pullRequiredImages(t, []string{test.sandboxImage}) - } + pullRequiredImages(t, []string{test.sandboxImage}) request := getRunPodSandboxRequest(t, test.runtimeHandler) runPodSandboxTestWithoutPodStop(t, request) @@ -125,7 +114,7 @@ func Test_RunPodSandbox_Without_Sandbox_Stop(t *testing.T) { } func Test_RunContainer_Without_Sandbox_Stop(t *testing.T) { - requireAnyFeature(t, featureWCOWProcess, featureWCOWHypervisor, featureLCOW) + requireAnyFeature(t, featureWCOWProcess, featureWCOWHypervisor) tests := []TestConfig{ { @@ -146,27 +135,13 @@ func Test_RunContainer_Without_Sandbox_Stop(t *testing.T) { containerImage: imageWindowsNanoserver, cmd: []string{"ping", "-t", "127.0.0.1"}, }, - { - name: "LCOW", - containerName: t.Name() + "-Container-LCOW", - requiredFeatures: []string{featureLCOW}, - runtimeHandler: lcowRuntimeHandler, - sandboxImage: imageLcowK8sPause, - containerImage: imageLcowAlpine, - cmd: []string{"top"}, - }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { requireFeatures(t, test.requiredFeatures...) - if test.runtimeHandler == lcowRuntimeHandler { - pullRequiredLCOWImages(t, []string{test.sandboxImage, test.containerImage}) - } else { - pullRequiredImages(t, []string{test.sandboxImage, test.containerImage}) - } - + pullRequiredImages(t, []string{test.sandboxImage, test.containerImage}) runContainerInSandboxTest(t, &test) }) } diff --git a/test/cri-containerd/runpodsandbox_test.go b/test/cri-containerd/runpodsandbox_test.go index b1ac643e61..920944d03a 100644 --- a/test/cri-containerd/runpodsandbox_test.go +++ b/test/cri-containerd/runpodsandbox_test.go @@ -4,12 +4,10 @@ package cri_containerd import ( - "bufio" - "bytes" "context" "errors" "fmt" - "io" + "os" "path/filepath" "strconv" @@ -17,7 +15,6 @@ import ( "testing" "time" - "github.com/containerd/log" "golang.org/x/sys/windows" runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" @@ -27,11 +24,9 @@ import ( "github.com/Microsoft/hcsshim/test/pkg/definitions/cpugroup" "github.com/Microsoft/hcsshim/test/pkg/definitions/hcs" - "github.com/Microsoft/hcsshim/test/pkg/definitions/lcow" "github.com/Microsoft/hcsshim/test/pkg/definitions/processorinfo" - "github.com/Microsoft/hcsshim/test/pkg/definitions/shimdiag" + "github.com/Microsoft/hcsshim/test/pkg/require" - testuvm "github.com/Microsoft/hcsshim/test/pkg/uvm" ) func runPodSandboxTest(t *testing.T, request *runtime.RunPodSandboxRequest) { @@ -63,66 +58,6 @@ func Test_RunPodSandbox_WCOW_Hypervisor(t *testing.T) { runPodSandboxTest(t, request) } -func Test_RunPodSandbox_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause}) - - request := getRunPodSandboxRequest(t, lcowRuntimeHandler) - runPodSandboxTest(t, request) -} - -func Test_RunPodSandbox_Events_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - client := newTestRuntimeClient(t) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - podctx, podcancel := context.WithCancel(context.Background()) - defer podcancel() - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause}) - - request := getRunPodSandboxRequest(t, lcowRuntimeHandler) - - topicNames, filters := getTargetRunTopics() - targetNamespace := "k8s.io" - - eventService := newTestEventService(t) - stream, errs := eventService.Subscribe(ctx, filters...) - - podID := runPodSandbox(t, client, podctx, request) - stopPodSandbox(t, client, podctx, podID) - removePodSandbox(t, client, podctx, podID) - - for _, topic := range topicNames { - select { - case env := <-stream: - if topic != env.Topic { - t.Fatalf("event topic %v does not match expected topic %v", env.Topic, topic) - } - if targetNamespace != env.Namespace { - t.Fatalf("event namespace %v does not match expected namespace %v", env.Namespace, targetNamespace) - } - t.Logf("event topic seen: %v", env.Topic) - - id, _, err := convertEvent(env.Event) - if err != nil { - t.Fatalf("topic %v event: %v", env.Topic, err) - } - if id != podID { - t.Fatalf("event topic %v belongs to pod %v, not targeted pod %v", env.Topic, id, podID) - } - case err := <-errs: - t.Fatalf("event subscription err %v", err) - case <-ctx.Done(): - if ctx.Err() == context.DeadlineExceeded { //nolint:errorlint - t.Fatalf("event %v deadline exceeded", topic) - } - } - } -} - func Test_RunPodSandbox_VirtualMemory_WCOW_Hypervisor(t *testing.T) { requireFeatures(t, featureWCOWHypervisor) @@ -138,21 +73,6 @@ func Test_RunPodSandbox_VirtualMemory_WCOW_Hypervisor(t *testing.T) { runPodSandboxTest(t, request) } -func Test_RunPodSandbox_VirtualMemory_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause}) - - request := getRunPodSandboxRequest( - t, - lcowRuntimeHandler, - WithSandboxAnnotations(map[string]string{ - annotations.AllowOvercommit: "true", - }), - ) - runPodSandboxTest(t, request) -} - func Test_RunPodSandbox_VirtualMemory_DeferredCommit_WCOW_Hypervisor(t *testing.T) { requireFeatures(t, featureWCOWHypervisor) @@ -169,22 +89,6 @@ func Test_RunPodSandbox_VirtualMemory_DeferredCommit_WCOW_Hypervisor(t *testing. runPodSandboxTest(t, request) } -func Test_RunPodSandbox_VirtualMemory_DeferredCommit_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause}) - - request := getRunPodSandboxRequest( - t, - lcowRuntimeHandler, - WithSandboxAnnotations(map[string]string{ - annotations.AllowOvercommit: "true", - annotations.EnableDeferredCommit: "true", - }), - ) - runPodSandboxTest(t, request) -} - func Test_RunPodSandbox_PhysicalMemory_WCOW_Hypervisor(t *testing.T) { requireFeatures(t, featureWCOWHypervisor) @@ -230,36 +134,6 @@ func Test_RunPodSandbox_VSMBNoDirectMap_WCOW_Hypervisor(t *testing.T) { runPodSandboxTest(t, request) } -func Test_RunPodSandbox_PhysicalMemory_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause}) - - request := getRunPodSandboxRequest( - t, - lcowRuntimeHandler, - WithSandboxAnnotations(map[string]string{ - annotations.AllowOvercommit: "false", - }), - ) - runPodSandboxTest(t, request) -} - -func Test_RunPodSandbox_FullyPhysicallyBacked_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause}) - - request := getRunPodSandboxRequest( - t, - lcowRuntimeHandler, - WithSandboxAnnotations(map[string]string{ - annotations.FullyPhysicallyBacked: "true", - }), - ) - runPodSandboxTest(t, request) -} - func Test_RunPodSandbox_MemorySize_WCOW_Process(t *testing.T) { requireFeatures(t, featureWCOWProcess) @@ -290,21 +164,6 @@ func Test_RunPodSandbox_MemorySize_WCOW_Hypervisor(t *testing.T) { runPodSandboxTest(t, request) } -func Test_RunPodSandbox_MemorySize_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause}) - - request := getRunPodSandboxRequest( - t, - lcowRuntimeHandler, - WithSandboxAnnotations(map[string]string{ - annotations.MemorySizeInMB: "200", - }), - ) - runPodSandboxTest(t, request) -} - func Test_RunPodSandbox_MMIO_WCOW_Process(t *testing.T) { requireFeatures(t, featureWCOWProcess) @@ -345,26 +204,6 @@ func Test_RunPodSandbox_MMIO_WCOW_Hypervisor(t *testing.T) { runPodSandboxTest(t, request) } -func Test_RunPodSandbox_MMIO_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - if osversion.Build() < osversion.V20H1 { - t.Skip("Requires build +20H1") - } - pullRequiredLCOWImages(t, []string{imageLcowK8sPause}) - - request := getRunPodSandboxRequest( - t, - lcowRuntimeHandler, - WithSandboxAnnotations(map[string]string{ - annotations.MemoryLowMMIOGapInMB: "100", - annotations.MemoryHighMMIOBaseInMB: "100", - annotations.MemoryHighMMIOGapInMB: "100", - }), - ) - runPodSandboxTest(t, request) -} - func Test_RunPodSandbox_CPUCount_WCOW_Process(t *testing.T) { requireFeatures(t, featureWCOWProcess) @@ -395,21 +234,6 @@ func Test_RunPodSandbox_CPUCount_WCOW_Hypervisor(t *testing.T) { runPodSandboxTest(t, request) } -func Test_RunPodSandbox_CPUCount_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause}) - - request := getRunPodSandboxRequest( - t, - lcowRuntimeHandler, - WithSandboxAnnotations(map[string]string{ - annotations.ProcessorCount: "1", - }), - ) - runPodSandboxTest(t, request) -} - func Test_RunPodSandbox_CPULimit_WCOW_Process(t *testing.T) { requireFeatures(t, featureWCOWProcess) @@ -440,21 +264,6 @@ func Test_RunPodSandbox_CPULimit_WCOW_Hypervisor(t *testing.T) { runPodSandboxTest(t, request) } -func Test_RunPodSandbox_CPULimit_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause}) - - request := getRunPodSandboxRequest( - t, - lcowRuntimeHandler, - WithSandboxAnnotations(map[string]string{ - annotations.ProcessorLimit: "90000", - }), - ) - runPodSandboxTest(t, request) -} - func Test_RunPodSandbox_CPUWeight_WCOW_Process(t *testing.T) { requireFeatures(t, featureWCOWProcess) @@ -485,21 +294,6 @@ func Test_RunPodSandbox_CPUWeight_WCOW_Hypervisor(t *testing.T) { runPodSandboxTest(t, request) } -func Test_RunPodSandbox_CPUWeight_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause}) - - request := getRunPodSandboxRequest( - t, - lcowRuntimeHandler, - WithSandboxAnnotations(map[string]string{ - annotations.ProcessorWeight: "500", - }), - ) - runPodSandboxTest(t, request) -} - func Test_RunPodSandbox_StorageQoSBandwithMax_WCOW_Process(t *testing.T) { requireFeatures(t, featureWCOWProcess) @@ -530,21 +324,6 @@ func Test_RunPodSandbox_StorageQoSBandwithMax_WCOW_Hypervisor(t *testing.T) { runPodSandboxTest(t, request) } -func Test_RunPodSandbox_StorageQoSBandwithMax_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause}) - - request := getRunPodSandboxRequest( - t, - lcowRuntimeHandler, - WithSandboxAnnotations(map[string]string{ - annotations.StorageQoSBandwidthMaximum: fmt.Sprintf("%d", 1024*1024), // 1MB/s - }), - ) - runPodSandboxTest(t, request) -} - func Test_RunPodSandbox_StorageQoSIopsMax_WCOW_Process(t *testing.T) { requireFeatures(t, featureWCOWProcess) @@ -575,81 +354,6 @@ func Test_RunPodSandbox_StorageQoSIopsMax_WCOW_Hypervisor(t *testing.T) { runPodSandboxTest(t, request) } -func Test_RunPodSandbox_StorageQoSIopsMax_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause}) - - request := getRunPodSandboxRequest( - t, - lcowRuntimeHandler, - WithSandboxAnnotations(map[string]string{ - annotations.StorageQoSIopsMaximum: "300", - }), - ) - runPodSandboxTest(t, request) -} - -func Test_RunPodSandbox_InitrdBoot_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause}) - - request := getRunPodSandboxRequest( - t, - lcowRuntimeHandler, - WithSandboxAnnotations(map[string]string{ - annotations.PreferredRootFSType: "initrd", - }), - ) - runPodSandboxTest(t, request) -} - -func Test_RunPodSandbox_RootfsVhdBoot_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause}) - - request := getRunPodSandboxRequest( - t, - lcowRuntimeHandler, - WithSandboxAnnotations(map[string]string{ - annotations.PreferredRootFSType: "vhd", - }), - ) - runPodSandboxTest(t, request) -} - -func Test_RunPodSandbox_VPCIEnabled_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause}) - - request := getRunPodSandboxRequest( - t, - lcowRuntimeHandler, - WithSandboxAnnotations(map[string]string{ - annotations.VPCIEnabled: "true", - }), - ) - runPodSandboxTest(t, request) -} - -func Test_RunPodSandbox_UEFIBoot_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause}) - - request := getRunPodSandboxRequest( - t, - lcowRuntimeHandler, - WithSandboxAnnotations(map[string]string{ - annotations.KernelDirectBoot: "false", - }), - ) - runPodSandboxTest(t, request) -} - func Test_RunPodSandbox_DnsConfig_WCOW_Process(t *testing.T) { requireFeatures(t, featureWCOWProcess) @@ -680,21 +384,6 @@ func Test_RunPodSandbox_DnsConfig_WCOW_Hypervisor(t *testing.T) { // searches are set. } -func Test_RunPodSandbox_DnsConfig_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause}) - - request := getRunPodSandboxRequest(t, lcowRuntimeHandler) - request.Config.DnsConfig = &runtime.DNSConfig{ - Searches: []string{"8.8.8.8", "8.8.4.4"}, - } - runPodSandboxTest(t, request) - // TODO: JTERRY75 - This is just a boot test at present. We need to create a - // container, cat /etc/resolv.conf and parse the results to verify that the - // searches are set. -} - func Test_RunPodSandbox_PortMappings_WCOW_Process(t *testing.T) { requireFeatures(t, featureWCOWProcess) @@ -727,143 +416,6 @@ func Test_RunPodSandbox_PortMappings_WCOW_Hypervisor(t *testing.T) { runPodSandboxTest(t, request) } -func Test_RunPodSandbox_PortMappings_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause}) - - request := getRunPodSandboxRequest(t, lcowRuntimeHandler) - request.Config.PortMappings = []*runtime.PortMapping{ - { - Protocol: runtime.Protocol_TCP, - ContainerPort: 80, - HostPort: 8080, - }, - } - runPodSandboxTest(t, request) -} - -func Test_RunPodSandbox_CustomizableScratchDefaultSize_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause}) - - annots := map[string]string{ - annotations.AllowOvercommit: "true", - } - - output, errorMsg, exitCode := createSandboxContainerAndExecForCustomScratch(t, annots) - - if exitCode != 0 { - t.Fatalf("Exec into container failed with: %v and exit code: %d, Test_RunPodSandbox_CustomizableScratchDefaultSize_LCOW", errorMsg, exitCode) - } - - // Format of output for df is below - // Filesystem 1K-blocks Used Available Use% Mounted on - // overlay 20642524 36 19577528 0% / - // tmpfs 65536 0 65536 0% /dev - scanner := bufio.NewScanner(strings.NewReader(output)) - found := false - var cols []string - for scanner.Scan() { - outputLine := scanner.Text() - if cols = strings.Fields(outputLine); cols[0] == "overlay" && cols[5] == "/" { - found = true - t.Log(outputLine) - break - } - } - - if !found { - t.Fatalf("could not find the correct output line for overlay mount on / n: error: %v, exitcode: %d", errorMsg, exitCode) - } - - // df command shows size in KB, 20642524 is 20GB - actualMountSize, _ := strconv.ParseInt(cols[1], 10, 64) - expectedMountSize := int64(20642524) - toleranceInKB := int64(10240) - if actualMountSize < (expectedMountSize-toleranceInKB) || actualMountSize > (expectedMountSize+toleranceInKB) { - t.Fatalf("Size of the overlay filesystem mounted at / is not within 10MB of 20642524 (20GB). It is %s", cols[1]) - } -} - -func Test_RunPodSandbox_CustomizableScratchCustomSize_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause}) - - annots := map[string]string{ - annotations.AllowOvercommit: "true", - "containerd.io/snapshot/io.microsoft.container.storage.rootfs.size-gb": "200", - } - - output, errorMsg, exitCode := createSandboxContainerAndExecForCustomScratch(t, annots) - - if exitCode != 0 { - t.Fatalf("Exec into container failed with: %v and exit code: %d, Test_RunPodSandbox_CustomizableScratchDefaultSize_LCOW", errorMsg, exitCode) - } - - // Format of output for df is below - // Filesystem 1K-blocks Used Available Use% Mounted on - // overlay 20642524 36 19577528 0% / - // tmpfs 65536 0 65536 0% /dev - scanner := bufio.NewScanner(strings.NewReader(output)) - found := false - var cols []string - for scanner.Scan() { - outputLine := scanner.Text() - if cols = strings.Fields(outputLine); cols[0] == "overlay" && cols[5] == "/" { - found = true - t.Log(outputLine) - break - } - } - - if !found { - t.Log(output) - t.Fatalf("could not find the correct output line for overlay mount on / n: error: %v, exitcode: %d", errorMsg, exitCode) - } - - // df command shows size in KB, 206425432 is 200GB - actualMountSize, _ := strconv.ParseInt(cols[1], 10, 64) - expectedMountSize := int64(206425432) - toleranceInKB := int64(10240) - if actualMountSize < (expectedMountSize-toleranceInKB) || actualMountSize > (expectedMountSize+toleranceInKB) { - t.Log(output) - t.Fatalf("Size of the overlay filesystem mounted at / is not within 10MB of 206425432 (200GB). It is %s", cols[1]) - } -} - -func Test_RunPodSandbox_Mount_SandboxDir_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - - annots := map[string]string{ - annotations.AllowOvercommit: "true", - } - - mounts := []*runtime.Mount{ - { - HostPath: "sandbox:///boot", - ContainerPath: "/containerUvmDir", - }, - } - cmd := []string{ - "mount", - } - - output, errorMsg, exitCode := createSandboxContainerAndExec(t, annots, mounts, cmd) - - if exitCode != 0 { - t.Fatalf("Exec into container failed with: %v and exit code: %d, %s", errorMsg, exitCode, t.Name()) - } - - t.Log(output) - - //TODO: Parse the output of the exec command to make sure the uvm mount was successful -} - func Test_RunPodSandbox_Mount_SandboxDir_WCOW(t *testing.T) { requireFeatures(t, featureWCOWHypervisor) @@ -989,7 +541,7 @@ func Test_RunPodSandbox_Mount_SandboxDir_NoShare_WCOW(t *testing.T) { } func Test_RunPodSandbox_CPUGroup(t *testing.T) { - requireAnyFeature(t, featureLCOW, featureWCOWHypervisor) + requireAnyFeature(t, featureWCOWHypervisor) require.Build(t, osversion.V21H1) ctx := context.Background() @@ -1004,282 +556,53 @@ func Test_RunPodSandbox_CPUGroup(t *testing.T) { lpIndices[i] = p.LpIndex } - if err := cpugroup.Create(ctx, presentID, lpIndices); err != nil { - t.Fatalf("failed to create test cpugroup with: %v", err) - } - - defer func() { - err := cpugroup.Delete(ctx, presentID) - if err != nil && !errors.Is(err, cpugroup.ErrHVStatusInvalidCPUGroupState) { - t.Fatalf("failed to clean up test cpugroup with: %v", err) - } - }() - - type config struct { - name string - requiredFeatures []string - runtimeHandler string - sandboxImage string - } - - tests := []config{ - { - name: "WCOW_Hypervisor", - requiredFeatures: []string{featureWCOWHypervisor}, - runtimeHandler: wcowHypervisorRuntimeHandler, - sandboxImage: imageWindowsNanoserver, - }, - { - name: "LCOW", - requiredFeatures: []string{featureLCOW}, - runtimeHandler: lcowRuntimeHandler, - sandboxImage: imageLcowK8sPause, - }, - } - - for _, test := range tests { - requireFeatures(t, test.requiredFeatures...) - if test.runtimeHandler == lcowRuntimeHandler { - pullRequiredLCOWImages(t, []string{test.sandboxImage}) - } else { - pullRequiredImages(t, []string{test.sandboxImage}) - } - - request := &runtime.RunPodSandboxRequest{ - Config: &runtime.PodSandboxConfig{ - Metadata: &runtime.PodSandboxMetadata{ - Name: t.Name(), - Uid: "0", - Namespace: testNamespace, - }, - Annotations: map[string]string{ - annotations.CPUGroupID: presentID, - }, - }, - RuntimeHandler: test.runtimeHandler, - } - runPodSandboxTest(t, request) - } -} - -func createExt4VHD(ctx context.Context, t *testing.T, path string) { - t.Helper() - // UVM related functions called below produce a lot debug logs. Set the logger - // output to Discard if verbose flag is not set. This way we can still capture - // these logs in a wpr session. - if !testing.Verbose() { - origLogOut := log.L.Logger.Out - log.L.Logger.SetOutput(io.Discard) - defer log.L.Logger.SetOutput(origLogOut) - } - uvm := testuvm.CreateAndStartLCOW(ctx, t, t.Name()+"-createExt4VHD") - defer uvm.Close() - - if err := lcow.CreateScratch(ctx, uvm, path, 2, ""); err != nil { - t.Fatal(err) - } -} - -func Test_RunPodSandbox_MultipleContainersSameVhd_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - annots := map[string]string{ - annotations.AllowOvercommit: "true", - } - - // Create a temporary ext4 VHD to mount into the container. - vhdHostDir := t.TempDir() - vhdHostPath := filepath.Join(vhdHostDir, "temp.vhdx") - createExt4VHD(ctx, t, vhdHostPath) - - vhdContainerPath := "/containerDir" - - mounts := []*runtime.Mount{ - { - HostPath: "vhd://" + vhdHostPath, - ContainerPath: vhdContainerPath, - }, - } - - sbRequest := getRunPodSandboxRequest(t, lcowRuntimeHandler, WithSandboxAnnotations(annots)) - - podID := runPodSandbox(t, client, ctx, sbRequest) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - - execCommand := []string{ - "ls", - vhdContainerPath, - } - - command := []string{ - "top", - } - - // create 2 containers with vhd mounts and verify both can mount vhd - for i := 1; i < 3; i++ { - containerName := t.Name() + "-Container-" + strconv.Itoa(i) - containerID := createContainerInSandbox(t, client, ctx, podID, containerName, imageLcowAlpine, command, annots, mounts, sbRequest.Config) - defer removeContainer(t, client, ctx, containerID) - - startContainer(t, client, ctx, containerID) - defer stopContainer(t, client, ctx, containerID) - - _, errorMsg, exitCode := execContainer(t, client, ctx, containerID, execCommand) - - // For container 1 and 2 we should find the mounts - if exitCode != 0 { - t.Fatalf("Exec into container failed with: %v and exit code: %d, %s", errorMsg, exitCode, containerID) - } - } - - // For the 3rd container don't add any mounts - // this makes sure you can have containers that share vhd mounts and - // at the same time containers in a pod that don't have any mounts - mounts = []*runtime.Mount{} - containerName := t.Name() + "-Container-3" - containerID := createContainerInSandbox(t, client, ctx, podID, containerName, imageLcowAlpine, command, annots, mounts, sbRequest.Config) - defer removeContainer(t, client, ctx, containerID) - - startContainer(t, client, ctx, containerID) - defer stopContainer(t, client, ctx, containerID) - - output, errorMsg, exitCode := execContainer(t, client, ctx, containerID, execCommand) - - // 3rd container should not have the mount and ls should fail - if exitCode == 0 { - t.Fatalf("Exec into container succeeded but we expected it to fail: %v and exit code: %s, %s", errorMsg, output, containerID) - } -} - -func Test_RunPodSandbox_MultipleContainersSameVhd_RShared_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - sbRequest := getRunPodSandboxRequest(t, lcowRuntimeHandler) - sbRequest.Config.Linux = &runtime.LinuxPodSandboxConfig{ - SecurityContext: &runtime.LinuxSandboxSecurityContext{ - Privileged: true, - }, - } - - podID := runPodSandbox(t, client, ctx, sbRequest) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - - // Create a temporary ext4 VHD to mount into the container. - vhdHostDir := t.TempDir() - vhdHostPath := filepath.Join(vhdHostDir, "temp.vhdx") - createExt4VHD(ctx, t, vhdHostPath) - - vhdContainerPath := "/containerDir" - cRequest := &runtime.CreateContainerRequest{ - Config: &runtime.ContainerConfig{ - Metadata: &runtime.ContainerMetadata{}, - Image: &runtime.ImageSpec{ - Image: imageLcowAlpine, - }, - // Hold this command open until killed - Command: []string{ - "top", - }, - Linux: &runtime.LinuxContainerConfig{ - SecurityContext: &runtime.LinuxContainerSecurityContext{ - Privileged: true, - }, - }, - Mounts: []*runtime.Mount{ - { - HostPath: "vhd://" + vhdHostPath, - ContainerPath: vhdContainerPath, - // set 'rshared' propagation - Propagation: runtime.MountPropagation_PROPAGATION_BIDIRECTIONAL, - }, - }, - }, - PodSandboxId: podID, - SandboxConfig: sbRequest.Config, - } - - containerName := t.Name() + "-Container-0" - cRequest.Config.Metadata.Name = containerName - containerID0 := createContainer(t, client, ctx, cRequest) - defer removeContainer(t, client, ctx, containerID0) - startContainer(t, client, ctx, containerID0) - defer stopContainer(t, client, ctx, containerID0) - - containerName1 := t.Name() + "-Container-1" - cRequest.Config.Metadata.Name = containerName1 - containerID1 := createContainer(t, client, ctx, cRequest) - defer removeContainer(t, client, ctx, containerID1) - startContainer(t, client, ctx, containerID1) - defer stopContainer(t, client, ctx, containerID1) - - // create a test directory that will be the new mountpoint's source - createTestDirCmd := []string{ - "mkdir", - "/tmp/testdir", - } - _, errorMsg, exitCode := execContainer(t, client, ctx, containerID0, createTestDirCmd) - if exitCode != 0 { - t.Fatalf("Exec into container failed with: %v and exit code: %d, %s", errorMsg, exitCode, containerID0) - } - - // create a file in the test directory - createTestDirContentCmd := []string{ - "touch", - "/tmp/testdir/test.txt", - } - _, errorMsg, exitCode = execContainer(t, client, ctx, containerID0, createTestDirContentCmd) - if exitCode != 0 { - t.Fatalf("Exec into container failed with: %v and exit code: %d, %s", errorMsg, exitCode, containerID0) - } - - // create a test directory in the vhd that will be the new mountpoint's destination - createTestDirVhdCmd := []string{ - "mkdir", - fmt.Sprintf("%s/testdir", vhdContainerPath), - } - _, errorMsg, exitCode = execContainer(t, client, ctx, containerID0, createTestDirVhdCmd) - if exitCode != 0 { - t.Fatalf("Exec into container failed with: %v and exit code: %d, %s", errorMsg, exitCode, containerID0) + if err := cpugroup.Create(ctx, presentID, lpIndices); err != nil { + t.Fatalf("failed to create test cpugroup with: %v", err) } - // perform rshared mount of test directory into the vhd - mountTestDirToVhdCmd := []string{ - "mount", - "-o", - "rshared", - "/tmp/testdir", - fmt.Sprintf("%s/testdir", vhdContainerPath), - } - _, errorMsg, exitCode = execContainer(t, client, ctx, containerID0, mountTestDirToVhdCmd) - if exitCode != 0 { - t.Fatalf("Exec into container failed with: %v and exit code: %d, %s", errorMsg, exitCode, containerID0) + defer func() { + err := cpugroup.Delete(ctx, presentID) + if err != nil && !errors.Is(err, cpugroup.ErrHVStatusInvalidCPUGroupState) { + t.Fatalf("failed to clean up test cpugroup with: %v", err) + } + }() + + type config struct { + name string + requiredFeatures []string + runtimeHandler string + sandboxImage string } - // try to list the test file in the second container to verify it was propagated correctly - verifyTestMountCommand := []string{ - "ls", - fmt.Sprintf("%s/testdir/test.txt", vhdContainerPath), + tests := []config{ + { + name: "WCOW_Hypervisor", + requiredFeatures: []string{featureWCOWHypervisor}, + runtimeHandler: wcowHypervisorRuntimeHandler, + sandboxImage: imageWindowsNanoserver, + }, } - _, errorMsg, exitCode = execContainer(t, client, ctx, containerID1, verifyTestMountCommand) - if exitCode != 0 { - t.Fatalf("Exec into container failed with: %v and exit code: %d, %s", errorMsg, exitCode, containerID1) + + for _, test := range tests { + requireFeatures(t, test.requiredFeatures...) + pullRequiredImages(t, []string{test.sandboxImage}) + + request := &runtime.RunPodSandboxRequest{ + Config: &runtime.PodSandboxConfig{ + Metadata: &runtime.PodSandboxMetadata{ + Name: t.Name(), + Uid: "0", + Namespace: testNamespace, + }, + Annotations: map[string]string{ + annotations.CPUGroupID: presentID, + }, + }, + RuntimeHandler: test.runtimeHandler, + } + runPodSandboxTest(t, request) } } - func Test_RunPodSandbox_MultipleContainersSameVhd_WCOW(t *testing.T) { requireFeatures(t, featureWCOWHypervisor) // Prior to 19H1, we aren't able to easily create a formatted VHD, as @@ -1374,128 +697,6 @@ func Test_RunPodSandbox_MultipleContainersSameVhd_WCOW(t *testing.T) { } } -func Test_RunPodSandbox_ProcessDump_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpineCoreDump}) - - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - sbRequest := getRunPodSandboxRequest( - t, - lcowRuntimeHandler, - WithSandboxAnnotations(map[string]string{ - annotations.ContainerProcessDumpLocation: "/coredumps/core", - }), - ) - - podID := runPodSandbox(t, client, ctx, sbRequest) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - - mounts := []*runtime.Mount{ - { - HostPath: "sandbox:///coredump", - ContainerPath: "/coredumps", - }, - } - - annots := map[string]string{ - annotations.RLimitCore: "18446744073709551615;18446744073709551615", - } - - // Setup container 1 that uses an image that stackoverflows shortly after starting. - // This should generate a core dump file in the sandbox mount location - c1Request := &runtime.CreateContainerRequest{ - Config: &runtime.ContainerConfig{ - Metadata: &runtime.ContainerMetadata{ - Name: t.Name() + "-Container1", - }, - Image: &runtime.ImageSpec{ - Image: imageLcowAlpineCoreDump, - }, - Command: []string{ - "./stackoverflow", - }, - Annotations: annots, - Mounts: mounts, - }, - PodSandboxId: podID, - SandboxConfig: sbRequest.Config, - } - - container1ID := createContainer(t, client, ctx, c1Request) - defer removeContainer(t, client, ctx, container1ID) - - startContainer(t, client, ctx, container1ID) - defer stopContainer(t, client, ctx, container1ID) - - // Then setup a secondary container that will mount the same sandbox mount and - // just verify that the core dump file is present. - c2Request := &runtime.CreateContainerRequest{ - Config: &runtime.ContainerConfig{ - Metadata: &runtime.ContainerMetadata{ - Name: t.Name() + "-Container2", - }, - Image: &runtime.ImageSpec{ - Image: imageLcowAlpineCoreDump, - }, - // Hold this command open until killed - Command: []string{ - "top", - }, - Mounts: mounts, - }, - PodSandboxId: podID, - SandboxConfig: sbRequest.Config, - } - - container2ID := createContainer(t, client, ctx, c2Request) - defer removeContainer(t, client, ctx, container2ID) - - startContainer(t, client, ctx, container2ID) - defer stopContainer(t, client, ctx, container2ID) - - checkForDumpFile := func() error { - // Check if the core dump file is present - execCommand := []string{ - "ls", - "/coredumps/core", - } - execRequest := &runtime.ExecSyncRequest{ - ContainerId: container2ID, - Cmd: execCommand, - Timeout: 20, - } - - r := execSync(t, client, ctx, execRequest) - if r.ExitCode != 0 { - return fmt.Errorf("failed with exit code %d running `ls`: %s", r.ExitCode, string(r.Stderr)) - } - return nil - } - - var ( - done bool - timeout = time.After(time.Second * 10) - ) - for !done { - // Keep checking for a core dump until timeout. - select { - case <-timeout: - t.Fatal("failed to find core dump within timeout") - default: - if err := checkForDumpFile(); err == nil { - done = true - } else { - time.Sleep(time.Millisecond * 500) - } - } - } -} - func Test_RunPodSandbox_ProcessDump_WCOW_Hypervisor(t *testing.T) { requireFeatures(t, featureWCOWHypervisor) @@ -1811,14 +1012,6 @@ func Test_RunPodSandbox_AdditionalRegValues_WCOW(t *testing.T) { } } -func createSandboxContainerAndExecForCustomScratch(t *testing.T, annots map[string]string) (string, string, int) { - t.Helper() - cmd := []string{ - "df", - } - return createSandboxContainerAndExec(t, annots, nil, cmd) -} - func createContainerInSandbox( t *testing.T, client runtime.RuntimeServiceClient, @@ -1860,181 +1053,3 @@ func execContainer( return output, errorMsg, exitCode } - -func createSandboxContainerAndExec(t *testing.T, annots map[string]string, mounts []*runtime.Mount, execCommand []string) (output string, errorMsg string, exitCode int) { - t.Helper() - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - sbRequest := getRunPodSandboxRequest(t, lcowRuntimeHandler, WithSandboxAnnotations(annots)) - - podID := runPodSandbox(t, client, ctx, sbRequest) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - - testMounts := []*runtime.Mount{} - - if mounts != nil { - testMounts = mounts - } - - cRequest := &runtime.CreateContainerRequest{ - Config: &runtime.ContainerConfig{ - Metadata: &runtime.ContainerMetadata{ - Name: t.Name() + "-Container", - }, - Image: &runtime.ImageSpec{ - Image: imageLcowAlpine, - }, - // Hold this command open until killed - Command: []string{ - "top", - }, - Annotations: annots, - Mounts: testMounts, - }, - PodSandboxId: podID, - SandboxConfig: sbRequest.Config, - } - - containerID := createContainer(t, client, ctx, cRequest) - defer removeContainer(t, client, ctx, containerID) - - startContainer(t, client, ctx, containerID) - defer stopContainer(t, client, ctx, containerID) - - //exec request - execRequest := &runtime.ExecSyncRequest{ - ContainerId: containerID, - Cmd: execCommand, - Timeout: 20, - } - - r := execSync(t, client, ctx, execRequest) - output = strings.TrimSpace(string(r.Stdout)) - errorMsg = string(r.Stderr) - exitCode = int(r.ExitCode) - - return output, errorMsg, exitCode -} - -func Test_RunPodSandbox_KernelOptions_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - - annots := map[string]string{ - annotations.FullyPhysicallyBacked: "true", - annotations.MemorySizeInMB: "2048", - annotations.KernelBootOptions: "hugepagesz=2M hugepages=10", - } - - hugePagesCmd := []string{"grep", "-i", "HugePages_Total", "/proc/meminfo"} - output, errorMsg, exitCode := createSandboxContainerAndExec(t, annots, nil, hugePagesCmd) - - if exitCode != 0 { - t.Fatalf("Exec into container failed with: %v and exit code: %d, %s", errorMsg, exitCode, t.Name()) - } - - splitOutput := strings.Split(output, ":") - numOfHugePages, err := strconv.Atoi(strings.TrimSpace(splitOutput[1])) - if err != nil { - t.Fatalf("Error happened while extracting number of hugepages: %v from output : %s", err, output) - } - - if numOfHugePages != 10 { - t.Fatalf("Expected number of hugepages to be 10. Got output instead: %d", numOfHugePages) - } -} - -func Test_RunPodSandbox_TimeSyncService(t *testing.T) { - requireFeatures(t, featureLCOW) - - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause}) - - request := getRunPodSandboxRequest( - t, - lcowRuntimeHandler) - - podID := runPodSandbox(t, client, ctx, request) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - - shimName := fmt.Sprintf("k8s.io-%s", podID) - - shim, err := shimdiag.GetShim(shimName) - if err != nil { - t.Fatalf("failed to find shim %s: %s", shimName, err) - } - - psCmd := []string{"ps"} - shimClient := shimdiag.NewShimDiagClient(shim) - outBuf := bytes.Buffer{} - outw := bufio.NewWriter(&outBuf) - errBuf := bytes.Buffer{} - errw := bufio.NewWriter(&errBuf) - exitCode, err := execInHost(ctx, shimClient, psCmd, nil, outw, errw) - if err != nil { - t.Fatalf("failed to exec `%s` in the uvm with %s", psCmd[0], err) - } - if exitCode != 0 { - t.Fatalf("exec `%s` in the uvm failed with exit code: %d, std error: %s", psCmd[0], exitCode, errBuf.String()) - } - if !strings.Contains(outBuf.String(), "chronyd") { - t.Logf("standard output of exec %s is: %s\n", psCmd[0], outBuf.String()) - t.Fatalf("chronyd is not running inside the uvm") - } -} - -func Test_RunPodSandbox_DisableTimeSyncService(t *testing.T) { - requireFeatures(t, featureLCOW) - - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause}) - - request := getRunPodSandboxRequest( - t, - lcowRuntimeHandler, - WithSandboxAnnotations( - map[string]string{ - annotations.DisableLCOWTimeSyncService: "true", - }), - ) - - podID := runPodSandbox(t, client, ctx, request) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - - shimName := fmt.Sprintf("k8s.io-%s", podID) - - shim, err := shimdiag.GetShim(shimName) - if err != nil { - t.Fatalf("failed to find shim %s: %s", shimName, err) - } - - psCmd := []string{"ps"} - shimClient := shimdiag.NewShimDiagClient(shim) - outBuf := bytes.Buffer{} - outw := bufio.NewWriter(&outBuf) - errBuf := bytes.Buffer{} - errw := bufio.NewWriter(&errBuf) - exitCode, err := execInHost(ctx, shimClient, psCmd, nil, outw, errw) - if err != nil { - t.Fatalf("failed to exec `%s` in the uvm with %s", psCmd[0], err) - } - if exitCode != 0 { - t.Fatalf("exec `%s` in the uvm failed with exit code: %d, std error: %s", psCmd[0], exitCode, errBuf.String()) - } - if strings.Contains(outBuf.String(), "chronyd") { - t.Logf("standard output of exec %s is: %s\n", psCmd[0], outBuf.String()) - t.Fatalf("chronyd should not be running inside the uvm") - } -} diff --git a/test/cri-containerd/stats_test.go b/test/cri-containerd/stats_test.go index 15e2040982..738fd759cb 100644 --- a/test/cri-containerd/stats_test.go +++ b/test/cri-containerd/stats_test.go @@ -91,102 +91,8 @@ func verifyPhysicallyBackedWorkingSet(t *testing.T, num uint64, stat *runtime.Co } } -func Test_SandboxStats_Single_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause}) - - request := getRunPodSandboxRequest(t, lcowRuntimeHandler) - - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - podID := runPodSandbox(t, client, ctx, request) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - - statsRequest := &runtime.ContainerStatsRequest{ - ContainerId: podID, - } - - stats, err := client.ContainerStats(ctx, statsRequest) - if err != nil { - t.Fatal(err) - } - - stat := stats.Stats - verifyStatsContent(t, stat) -} - -func Test_SandboxStats_List_ContainerID_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause}) - - request := getRunPodSandboxRequest(t, lcowRuntimeHandler) - - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - podID := runPodSandbox(t, client, ctx, request) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - - statsRequest := &runtime.ListContainerStatsRequest{ - Filter: &runtime.ContainerStatsFilter{ - Id: podID, - }, - } - - stats, err := client.ListContainerStats(ctx, statsRequest) - if err != nil { - t.Fatal(err) - } - - if len(stats.Stats) != 1 { - t.Fatalf("expected 1 stats result but got %d", len(stats.Stats)) - } - stat := stats.Stats[0] - verifyStatsContent(t, stat) -} - -func Test_SandboxStats_List_PodID_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause}) - - request := getRunPodSandboxRequest(t, lcowRuntimeHandler) - - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - podID := runPodSandbox(t, client, ctx, request) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - - statsRequest := &runtime.ListContainerStatsRequest{ - Filter: &runtime.ContainerStatsFilter{ - PodSandboxId: podID, - }, - } - - stats, err := client.ListContainerStats(ctx, statsRequest) - if err != nil { - t.Fatal(err) - } - - if len(stats.Stats) != 1 { - t.Fatalf("expected 1 stats result but got %d", len(stats.Stats)) - } - stat := stats.Stats[0] - verifyStatsContent(t, stat) -} - func Test_ContainerStats_ContainerID(t *testing.T) { - requireAnyFeature(t, featureWCOWProcess, featureWCOWHypervisor, featureLCOW) + requireAnyFeature(t, featureWCOWProcess, featureWCOWHypervisor) type config struct { name string @@ -213,25 +119,12 @@ func Test_ContainerStats_ContainerID(t *testing.T) { containerImage: imageWindowsNanoserver, cmd: []string{"cmd", "/c", "ping", "-t", "127.0.0.1"}, }, - { - name: "LCOW", - requiredFeatures: []string{featureLCOW}, - runtimeHandler: lcowRuntimeHandler, - sandboxImage: imageLcowK8sPause, - containerImage: imageLcowAlpine, - cmd: []string{"top"}, - }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { requireFeatures(t, test.requiredFeatures...) - - if test.runtimeHandler == lcowRuntimeHandler { - pullRequiredLCOWImages(t, []string{test.sandboxImage, test.containerImage}) - } else { - pullRequiredImages(t, []string{test.sandboxImage, test.containerImage}) - } + pullRequiredImages(t, []string{test.sandboxImage, test.containerImage}) podRequest := getRunPodSandboxRequest(t, test.runtimeHandler) @@ -262,7 +155,7 @@ func Test_ContainerStats_ContainerID(t *testing.T) { } func Test_ContainerStats_List_ContainerID(t *testing.T) { - requireAnyFeature(t, featureWCOWProcess, featureWCOWHypervisor, featureLCOW) + requireAnyFeature(t, featureWCOWProcess, featureWCOWHypervisor) type config struct { name string @@ -289,25 +182,12 @@ func Test_ContainerStats_List_ContainerID(t *testing.T) { containerImage: imageWindowsNanoserver, cmd: []string{"cmd", "/c", "ping", "-t", "127.0.0.1"}, }, - { - name: "LCOW", - requiredFeatures: []string{featureLCOW}, - runtimeHandler: lcowRuntimeHandler, - sandboxImage: imageLcowK8sPause, - containerImage: imageLcowAlpine, - cmd: []string{"top"}, - }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { requireFeatures(t, test.requiredFeatures...) - - if test.runtimeHandler == lcowRuntimeHandler { - pullRequiredLCOWImages(t, []string{test.sandboxImage, test.containerImage}) - } else { - pullRequiredImages(t, []string{test.sandboxImage, test.containerImage}) - } + pullRequiredImages(t, []string{test.sandboxImage, test.containerImage}) podRequest := getRunPodSandboxRequest(t, test.runtimeHandler) @@ -338,7 +218,7 @@ func Test_ContainerStats_List_ContainerID(t *testing.T) { } func Test_SandboxStats_WorkingSet_PhysicallyBacked(t *testing.T) { - requireAnyFeature(t, featureLCOW, featureWCOWHypervisor) + requireAnyFeature(t, featureWCOWHypervisor) type config struct { name string @@ -353,23 +233,12 @@ func Test_SandboxStats_WorkingSet_PhysicallyBacked(t *testing.T) { runtimeHandler: wcowHypervisorRuntimeHandler, sandboxImage: imageWindowsNanoserver, }, - { - name: "LCOW", - requiredFeatures: []string{featureLCOW}, - runtimeHandler: lcowRuntimeHandler, - sandboxImage: imageLcowK8sPause, - }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { requireFeatures(t, test.requiredFeatures...) - - if test.runtimeHandler == lcowRuntimeHandler { - pullRequiredLCOWImages(t, []string{test.sandboxImage}) - } else { - pullRequiredImages(t, []string{test.sandboxImage}) - } + pullRequiredImages(t, []string{test.sandboxImage}) // If we go too high we run the risk of getting a not enough memory // for this operation error. Seems like a nice sweetspot thats different than diff --git a/test/cri-containerd/stopcontainer_test.go b/test/cri-containerd/stopcontainer_test.go index d5343cead2..e53f468e4b 100644 --- a/test/cri-containerd/stopcontainer_test.go +++ b/test/cri-containerd/stopcontainer_test.go @@ -14,167 +14,6 @@ import ( runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" ) -func Test_StopContainer_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - sandboxRequest := getRunPodSandboxRequest(t, lcowRuntimeHandler) - - podID := runPodSandbox(t, client, ctx, sandboxRequest) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - - request := &runtime.CreateContainerRequest{ - PodSandboxId: podID, - Config: &runtime.ContainerConfig{ - Metadata: &runtime.ContainerMetadata{ - Name: t.Name() + "-Container", - }, - Image: &runtime.ImageSpec{ - Image: imageLcowAlpine, - }, - Command: []string{ - "top", - }, - }, - SandboxConfig: sandboxRequest.Config, - } - - containerID := createContainer(t, client, ctx, request) - defer removeContainer(t, client, ctx, containerID) - - startContainer(t, client, ctx, containerID) - stopContainer(t, client, ctx, containerID) -} - -func Test_StopContainer_WithTimeout_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - sandboxRequest := getRunPodSandboxRequest(t, lcowRuntimeHandler) - - podID := runPodSandbox(t, client, ctx, sandboxRequest) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - - request := &runtime.CreateContainerRequest{ - PodSandboxId: podID, - Config: &runtime.ContainerConfig{ - Metadata: &runtime.ContainerMetadata{ - Name: t.Name() + "-Container", - }, - Image: &runtime.ImageSpec{ - Image: imageLcowAlpine, - }, - Command: []string{ - "top", - }, - }, - SandboxConfig: sandboxRequest.Config, - } - - containerID := createContainer(t, client, ctx, request) - defer removeContainer(t, client, ctx, containerID) - - startContainer(t, client, ctx, containerID) - stopContainerWithTimeout(t, client, ctx, containerID, 10) -} - -func Test_StopContainer_WithExec_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - sandboxRequest := getRunPodSandboxRequest(t, lcowRuntimeHandler) - - podID := runPodSandbox(t, client, ctx, sandboxRequest) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - - request := &runtime.CreateContainerRequest{ - PodSandboxId: podID, - Config: &runtime.ContainerConfig{ - Metadata: &runtime.ContainerMetadata{ - Name: t.Name() + "-Container", - }, - Image: &runtime.ImageSpec{ - Image: imageLcowAlpine, - }, - Command: []string{ - "top", - }, - }, - SandboxConfig: sandboxRequest.Config, - } - - containerID := createContainer(t, client, ctx, request) - defer removeContainer(t, client, ctx, containerID) - - startContainer(t, client, ctx, containerID) - defer stopContainer(t, client, ctx, containerID) - - execRequest(t, client, ctx, &runtime.ExecRequest{ - ContainerId: containerID, - Cmd: []string{ - "top", - }, - Stdout: true, - }) -} - -func Test_StopContainer_ReusePod_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW) - - pullRequiredLCOWImages(t, []string{alpineAspNet, alpineAspnetUpgrade}) - - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - sandboxRequest := getRunPodSandboxRequest(t, lcowRuntimeHandler) - - podID := runPodSandbox(t, client, ctx, sandboxRequest) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - - request := &runtime.CreateContainerRequest{ - PodSandboxId: podID, - Config: &runtime.ContainerConfig{ - Metadata: &runtime.ContainerMetadata{ - Name: t.Name() + "-Container", - }, - Image: &runtime.ImageSpec{ - Image: alpineAspNet, - }, - Command: []string{ - "top", - }, - }, - SandboxConfig: sandboxRequest.Config, - } - - containerID := createContainer(t, client, ctx, request) - runContainerLifetime(t, client, ctx, containerID) - - request.Config.Image.Image = alpineAspnetUpgrade - containerID = createContainer(t, client, ctx, request) - runContainerLifetime(t, client, ctx, containerID) -} - // This test runs a container with an image that waits for sigterm and then // prints for loop counter down from 60 till the container is stopped with // a timeout of 15 seconds. This is done to mimic graceful termination diff --git a/test/cri-containerd/test-images/imagepull_timestamp/Dockerfile b/test/cri-containerd/test-images/imagepull_timestamp/Dockerfile index 7d76a0c07f..3413cbfdec 100644 --- a/test/cri-containerd/test-images/imagepull_timestamp/Dockerfile +++ b/test/cri-containerd/test-images/imagepull_timestamp/Dockerfile @@ -9,7 +9,9 @@ # `docker push cplatpublic.azurecr.io/timestamp:latest` # Base image + FROM mcr.microsoft.com/windows/nanoserver:ltsc2022@sha256:56a7c823d8aed810849a38f47d995a727bf3de1ec611394025e1de24898a9622 + # Get administrator privileges USER containeradministrator diff --git a/test/cri-containerd/test-images/nanoserver-gracefultermination-repro/latest/delayed-shutdown.go b/test/cri-containerd/test-images/nanoserver-gracefultermination-repro/latest/delayed-shutdown.go index 0c5719c19c..a45ac61675 100644 --- a/test/cri-containerd/test-images/nanoserver-gracefultermination-repro/latest/delayed-shutdown.go +++ b/test/cri-containerd/test-images/nanoserver-gracefultermination-repro/latest/delayed-shutdown.go @@ -42,4 +42,4 @@ func main() { } fmt.Println("Goodbye") -} \ No newline at end of file +} diff --git a/test/cri-containerd/test-images/servercore-gracefultermination-repro/latest/delayed-shutdown.go b/test/cri-containerd/test-images/servercore-gracefultermination-repro/latest/delayed-shutdown.go index 0c5719c19c..a45ac61675 100644 --- a/test/cri-containerd/test-images/servercore-gracefultermination-repro/latest/delayed-shutdown.go +++ b/test/cri-containerd/test-images/servercore-gracefultermination-repro/latest/delayed-shutdown.go @@ -42,4 +42,4 @@ func main() { } fmt.Println("Goodbye") -} \ No newline at end of file +} diff --git a/test/go.mod b/test/go.mod index d1f6e5818e..5af3ec46fd 100644 --- a/test/go.mod +++ b/test/go.mod @@ -8,9 +8,7 @@ require ( github.com/containerd/cgroups/v3 v3.0.2 github.com/containerd/containerd v1.7.0 github.com/containerd/go-runc v1.0.0 - github.com/containerd/log v0.1.0 github.com/containerd/ttrpc v1.2.2 - github.com/containerd/typeurl/v2 v2.1.1 github.com/google/go-containerregistry v0.17.0 github.com/josephspurrier/goversioninfo v1.4.0 github.com/kevpar/cri v1.11.1-0.20220302210600-4c5c347230b2 @@ -20,7 +18,6 @@ require ( github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626 github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.9.3 - github.com/stretchr/testify v1.8.4 go.opencensus.io v0.24.0 golang.org/x/exp v0.0.0-20231006140011-7918f672742d golang.org/x/sync v0.5.0 @@ -44,9 +41,9 @@ require ( github.com/containerd/continuity v0.4.1 // indirect github.com/containerd/fifo v1.1.0 // indirect github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect + github.com/containerd/typeurl/v2 v2.1.1 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cyphar/filepath-securejoin v0.2.3 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect github.com/docker/cli v24.0.0+incompatible // indirect github.com/docker/distribution v2.8.2+incompatible // indirect @@ -86,7 +83,6 @@ require ( github.com/opencontainers/runc v1.1.10 // indirect github.com/opencontainers/selinux v1.11.0 // indirect github.com/pelletier/go-toml v1.9.5 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect github.com/vbatts/tar-split v0.11.3 // indirect @@ -108,7 +104,6 @@ require ( google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) diff --git a/test/go.sum b/test/go.sum index 0264496e3c..8c91259fed 100644 --- a/test/go.sum +++ b/test/go.sum @@ -1006,8 +1006,6 @@ github.com/containerd/imgcrypt v1.1.1/go.mod h1:xpLnwiQmEUJPvQoAapeb2SNCxz7Xr6PJ github.com/containerd/imgcrypt v1.1.3/go.mod h1:/TPA1GIDXMzbj01yd8pIbQiLdQxed5ue1wb8bP7PQu4= github.com/containerd/imgcrypt v1.1.4/go.mod h1:LorQnPtzL/T0IyCeftcsMEO7AqxUDbdO8j/tSUpgxvo= github.com/containerd/imgcrypt v1.1.7/go.mod h1:FD8gqIcX5aTotCtOmjeCsi3A1dHmTZpnMISGKSczt4k= -github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= -github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/containerd/nri v0.0.0-20201007170849-eb1350a75164/go.mod h1:+2wGSDGFYfE5+So4M5syatU0N0f0LbWpuqyMi4/BE8c= github.com/containerd/nri v0.0.0-20210316161719-dbaa18c31c14/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= github.com/containerd/nri v0.1.0/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= From c25f9030b8f42dd13a67b183cab96dc352bda611 Mon Sep 17 00:00:00 2001 From: Yuanyuan Lei Date: Sat, 30 Dec 2023 01:16:24 -0800 Subject: [PATCH 2/5] minor fixes --- test/cri-containerd/container_update_test.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/cri-containerd/container_update_test.go b/test/cri-containerd/container_update_test.go index 7942f692b2..40fb0d1388 100644 --- a/test/cri-containerd/container_update_test.go +++ b/test/cri-containerd/container_update_test.go @@ -157,7 +157,9 @@ func Test_Container_UpdateResources_CPUShare_NotRunning(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { requireFeatures(t, test.requiredFeatures...) - pullRequiredImages(t, []string{test.sandboxImage}) + if test.runtimeHandler == wcowHypervisorRuntimeHandler { + pullRequiredImages(t, []string{test.sandboxImage}) + } podRequest := getRunPodSandboxRequest(t, test.runtimeHandler) @@ -244,7 +246,9 @@ func Test_Container_UpdateResources_Memory(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { requireFeatures(t, test.requiredFeatures...) - pullRequiredImages(t, []string{test.sandboxImage}) + if test.runtimeHandler == wcowHypervisorRuntimeHandler { + pullRequiredImages(t, []string{test.sandboxImage}) + } podRequest := getRunPodSandboxRequest(t, test.runtimeHandler) From 0c34909ccdea3a8b2577100f65591f2a9f07d241 Mon Sep 17 00:00:00 2001 From: Yuanyuan Lei Date: Fri, 2 Feb 2024 16:44:12 -0800 Subject: [PATCH 3/5] Addressed comments --- .github/dependabot.yml | 2 - test/cri-containerd/argsescaped_test.go | 2 +- .../container_downlevel_test.go | 2 +- .../container_fileshare_test.go | 2 +- test/cri-containerd/container_gmsa_test.go | 2 +- test/cri-containerd/container_network_test.go | 8 +- test/cri-containerd/container_update_test.go | 2 +- .../container_virtual_device_test.go | 2 +- test/cri-containerd/containerdrestart_test.go | 420 ------------------ test/cri-containerd/createcontainer_test.go | 2 +- test/cri-containerd/disable_vpmem_test.go | 2 +- test/cri-containerd/helper_container_test.go | 2 +- .../cri-containerd/helper_cri_plugins_test.go | 42 -- test/cri-containerd/helper_exec_test.go | 2 +- test/cri-containerd/helper_sandbox_test.go | 2 +- test/cri-containerd/jobcontainer_test.go | 2 +- test/cri-containerd/logging_binary_test.go | 2 +- test/cri-containerd/main_test.go | 10 +- test/cri-containerd/pod_update_test.go | 2 +- test/cri-containerd/removepodsandbox_test.go | 2 +- test/cri-containerd/runpodsandbox_test.go | 2 +- .../scale_cpu_limits_to_sandbox_test.go | 2 +- test/cri-containerd/stats_test.go | 2 +- test/cri-containerd/stopcontainer_test.go | 2 +- test/go.mod | 7 +- test/go.sum | 16 +- 26 files changed, 37 insertions(+), 506 deletions(-) delete mode 100644 test/cri-containerd/containerdrestart_test.go delete mode 100644 test/cri-containerd/helper_cri_plugins_test.go diff --git a/.github/dependabot.yml b/.github/dependabot.yml index d9ccdc3340..9b8ae6b9ab 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -28,8 +28,6 @@ updates: - go - tests ignore: - # updating CRI would break tests that rely on `v1alpha2` API - - dependency-name: "k8s.io/cri-api" # keep containerd and related dependencies in sync with the shim/the root repo - dependency-name: "github.com/containerd/*" # ignore patch version increment updates in test; they shouldn't be critical diff --git a/test/cri-containerd/argsescaped_test.go b/test/cri-containerd/argsescaped_test.go index 461ca325c0..87f8cb9d5e 100644 --- a/test/cri-containerd/argsescaped_test.go +++ b/test/cri-containerd/argsescaped_test.go @@ -8,7 +8,7 @@ import ( "strings" "testing" - runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" + runtime "k8s.io/cri-api/pkg/apis/runtime/v1" ) func Test_ArgsEscaped_Exec(t *testing.T) { diff --git a/test/cri-containerd/container_downlevel_test.go b/test/cri-containerd/container_downlevel_test.go index fd7a077d65..75a5a121cc 100644 --- a/test/cri-containerd/container_downlevel_test.go +++ b/test/cri-containerd/container_downlevel_test.go @@ -8,7 +8,7 @@ import ( "github.com/Microsoft/hcsshim/osversion" "github.com/Microsoft/hcsshim/test/pkg/require" - runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" + runtime "k8s.io/cri-api/pkg/apis/runtime/v1" ) func Test_CreateContainer_DownLevel_WCOW_Hypervisor(t *testing.T) { diff --git a/test/cri-containerd/container_fileshare_test.go b/test/cri-containerd/container_fileshare_test.go index 4523e4347e..f4fcd2c8ac 100644 --- a/test/cri-containerd/container_fileshare_test.go +++ b/test/cri-containerd/container_fileshare_test.go @@ -13,7 +13,7 @@ import ( "github.com/Microsoft/hcsshim/pkg/annotations" "github.com/Microsoft/hcsshim/test/pkg/definitions/hcs" - runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" + runtime "k8s.io/cri-api/pkg/apis/runtime/v1" ) func Test_Container_File_Share_Writable_WCOW(t *testing.T) { diff --git a/test/cri-containerd/container_gmsa_test.go b/test/cri-containerd/container_gmsa_test.go index 5d34c3426c..ba9c41f1b4 100644 --- a/test/cri-containerd/container_gmsa_test.go +++ b/test/cri-containerd/container_gmsa_test.go @@ -11,7 +11,7 @@ import ( "github.com/Microsoft/hcsshim/pkg/annotations" "github.com/Microsoft/hcsshim/test/pkg/definitions/hcs" - runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" + runtime "k8s.io/cri-api/pkg/apis/runtime/v1" ) func Test_RunContainer_GMSA_WCOW(t *testing.T) { diff --git a/test/cri-containerd/container_network_test.go b/test/cri-containerd/container_network_test.go index 35a7fa0aae..8d11e1bedc 100644 --- a/test/cri-containerd/container_network_test.go +++ b/test/cri-containerd/container_network_test.go @@ -8,7 +8,7 @@ import ( "strings" "testing" - runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" + runtime "k8s.io/cri-api/pkg/apis/runtime/v1" ) func Test_Container_Network_Hostname(t *testing.T) { @@ -45,11 +45,7 @@ func Test_Container_Network_Hostname(t *testing.T) { t.Run(test.name, func(t *testing.T) { requireFeatures(t, test.requiredFeatures...) - if test.runtimeHandler == lcowRuntimeHandler { - pullRequiredLCOWImages(t, []string{test.sandboxImage, test.containerImage}) - } else { - pullRequiredImages(t, []string{test.sandboxImage, test.containerImage}) - } + pullRequiredImages(t, []string{test.sandboxImage, test.containerImage}) sandboxRequest := getRunPodSandboxRequest(t, test.runtimeHandler) sandboxRequest.Config.Hostname = "TestHost" diff --git a/test/cri-containerd/container_update_test.go b/test/cri-containerd/container_update_test.go index 40fb0d1388..4a7f87ced6 100644 --- a/test/cri-containerd/container_update_test.go +++ b/test/cri-containerd/container_update_test.go @@ -12,7 +12,7 @@ import ( "github.com/Microsoft/hcsshim/osversion" "github.com/Microsoft/hcsshim/pkg/annotations" "github.com/Microsoft/hcsshim/test/pkg/require" - runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" + runtime "k8s.io/cri-api/pkg/apis/runtime/v1" ) const processorWeightMax = 10000 diff --git a/test/cri-containerd/container_virtual_device_test.go b/test/cri-containerd/container_virtual_device_test.go index 16fa909a69..7432817777 100644 --- a/test/cri-containerd/container_virtual_device_test.go +++ b/test/cri-containerd/container_virtual_device_test.go @@ -13,7 +13,7 @@ import ( "github.com/Microsoft/hcsshim/osversion" "github.com/Microsoft/hcsshim/pkg/annotations" - runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" + runtime "k8s.io/cri-api/pkg/apis/runtime/v1" ) const containerDeviceUtilPath = "C:\\device-util.exe" diff --git a/test/cri-containerd/containerdrestart_test.go b/test/cri-containerd/containerdrestart_test.go deleted file mode 100644 index c9ba734983..0000000000 --- a/test/cri-containerd/containerdrestart_test.go +++ /dev/null @@ -1,420 +0,0 @@ -//go:build windows && functional -// +build windows,functional - -package cri_containerd - -import ( - "context" - "strings" - "testing" - "time" - - "github.com/Microsoft/hcsshim/pkg/annotations" - criAnnotations "github.com/kevpar/cri/pkg/annotations" - runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" -) - -// CRI will terminate any running containers when it is restarted. -// Run a container, restart containerd, validate the container is terminated. -func Test_ContainerdRestart_LCOW(t *testing.T) { - requireFeatures(t, featureLCOW, featureTerminateOnRestart) - - pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - sandboxRequest := getRunPodSandboxRequest(t, lcowRuntimeHandler) - - podID := runPodSandbox(t, client, ctx, sandboxRequest) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - - request := &runtime.CreateContainerRequest{ - PodSandboxId: podID, - Config: &runtime.ContainerConfig{ - Metadata: &runtime.ContainerMetadata{ - Name: t.Name() + "-Container", - }, - Image: &runtime.ImageSpec{ - Image: imageLcowAlpine, - }, - Command: []string{ - "top", - }, - }, - SandboxConfig: sandboxRequest.Config, - } - - containerID := createContainer(t, client, ctx, request) - defer removeContainer(t, client, ctx, containerID) - startContainer(t, client, ctx, containerID) - defer stopContainer(t, client, ctx, containerID) - - t.Log("Restarting containerd") - stopContainerd(t) - startContainerd(t) - client = newTestRuntimeClient(t) - waitForCRI(ctx, t, client, 15*time.Second) - - containerStatus, err := client.ContainerStatus(ctx, &runtime.ContainerStatusRequest{ContainerId: containerID}) - if err != nil { - t.Fatal(err) - } - if containerStatus.Status.State != runtime.ContainerState_CONTAINER_EXITED { - t.Errorf("Container was not terminated on containerd restart. Status is %d", containerStatus.Status.State) - } - podStatus, err := client.PodSandboxStatus(ctx, &runtime.PodSandboxStatusRequest{PodSandboxId: podID}) - if err != nil { - t.Fatal(err) - } - if podStatus.Status.State != runtime.PodSandboxState_SANDBOX_NOTREADY { - t.Errorf("Pod was not terminated on containerd restart. Status is %d", podStatus.Status.State) - } -} - -// test restarting containers and pods -func Test_Container_CRI_Restart(t *testing.T) { - requireFeatures(t, featureCRIPlugin) - requireAnyFeature(t, featureWCOWProcess, featureWCOWHypervisor, featureLCOW) - - client := newTestRuntimeClient(t) - pluginClient := newTestPluginClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - tests := []struct { - Name string - Feature string - Runtime string - SandboxOpts []SandboxConfigOpt - Image string - Command []string - }{ - { - Name: "LCOW", - Feature: featureLCOW, - Runtime: lcowRuntimeHandler, - SandboxOpts: []SandboxConfigOpt{WithSandboxAnnotations(map[string]string{ - annotations.DisableLCOWTimeSyncService: "true", - })}, - Image: imageLcowAlpine, - Command: []string{ - "ash", - "-c", - "tail -f /dev/null", - }, - }, - { - Name: "WCOW_Hypervisor", - Feature: featureWCOWHypervisor, - Runtime: wcowHypervisorRuntimeHandler, - Image: imageWindowsNanoserver, - Command: []string{ - "cmd", - "/c", - "ping -t 127.0.0.1", - }, - }, - { - Name: "WCOW_Process", - Feature: featureWCOWProcess, - Runtime: wcowProcessRuntimeHandler, - Image: imageWindowsNanoserver, - Command: []string{ - "cmd", - "/c", - "ping -t 127.0.0.1", - }, - }, - } - - for _, tt := range tests { - // Test both implicit and explicit restart - // Implicit restart uses a container annotation to cause pod run and container start - // to automatically restart exited pods and containers. - // Explicit requires an intervening call to the restart pod or container CRI extension - // command between stoping the pod or container, and then calling run or start again. - for _, explicit := range []bool{false, true} { - suffix := "_Implicit" - if explicit { - suffix = "_Explicit" - } - - t.Run(tt.Name+suffix, func(t *testing.T) { - requireFeatures(t, tt.Feature) - - switch tt.Feature { - case featureLCOW: - pullRequiredLCOWImages(t, append([]string{imageLcowK8sPause}, tt.Image)) - case featureWCOWHypervisor, featureWCOWProcess: - pullRequiredImages(t, []string{tt.Image}) - } - - opts := tt.SandboxOpts - if !explicit { - opts = append(tt.SandboxOpts, - WithSandboxAnnotations(map[string]string{ - criAnnotations.EnableReset: "true", - })) - } - sandboxRequest := getRunPodSandboxRequest(t, tt.Runtime, opts...) - podID := runPodSandbox(t, client, ctx, sandboxRequest) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - - request := getCreateContainerRequest(podID, t.Name()+"-Container", tt.Image, tt.Command, sandboxRequest.Config) - request.Config.Annotations = map[string]string{} - - if !explicit { - request.Config.Annotations[criAnnotations.EnableReset] = "true" - } - - containerID := createContainer(t, client, ctx, request) - startContainer(t, client, ctx, containerID) - defer removeContainer(t, client, ctx, containerID) - defer stopContainer(t, client, ctx, containerID) - - /******************************************************************* - * restart container - *******************************************************************/ - stopContainer(t, client, ctx, containerID) - assertContainerState(t, client, ctx, containerID, runtime.ContainerState_CONTAINER_EXITED) - - if explicit { - resetContainer(t, pluginClient, ctx, containerID) - assertContainerState(t, client, ctx, containerID, runtime.ContainerState_CONTAINER_CREATED) - } - - startContainer(t, client, ctx, containerID) - assertContainerState(t, client, ctx, containerID, runtime.ContainerState_CONTAINER_RUNNING) - - /******************************************************************* - * restart pod - *******************************************************************/ - stopContainer(t, client, ctx, containerID) - assertContainerState(t, client, ctx, containerID, runtime.ContainerState_CONTAINER_EXITED) - - stopPodSandbox(t, client, ctx, podID) - assertPodSandboxState(t, client, ctx, podID, runtime.PodSandboxState_SANDBOX_NOTREADY) - - if explicit { - resetPodSandbox(t, pluginClient, ctx, podID) - } else { - newPodID := runPodSandbox(t, client, ctx, sandboxRequest) - if newPodID != podID { - defer removePodSandbox(t, client, ctx, newPodID) - defer stopPodSandbox(t, client, ctx, newPodID) - t.Fatalf("pod restarted with different id (%q) from original (%q)", newPodID, podID) - } - } - - assertPodSandboxState(t, client, ctx, podID, runtime.PodSandboxState_SANDBOX_READY) - assertContainerState(t, client, ctx, containerID, runtime.ContainerState_CONTAINER_CREATED) - - startContainer(t, client, ctx, containerID) - assertContainerState(t, client, ctx, containerID, runtime.ContainerState_CONTAINER_RUNNING) - }) - } - } -} - -// test preserving state after restarting pod -func Test_Container_CRI_Restart_State(t *testing.T) { - testFile := "t.txt" - wcowTestFile := `C:\Users\ContainerUser\t.txt` - - requireFeatures(t, featureCRIPlugin) - requireAnyFeature(t, featureWCOWProcess, featureWCOWHypervisor, featureLCOW) - - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - tests := []struct { - Name string - Feature string - Runtime string - SandboxOpts []SandboxConfigOpt - Image string - Command []string - SetStateCommand []string - GetStateCommand []string - ExpectedResult string - }{ - { - Name: "LCOW", - Feature: featureLCOW, - Runtime: lcowRuntimeHandler, - SandboxOpts: []SandboxConfigOpt{WithSandboxAnnotations(map[string]string{ - annotations.DisableLCOWTimeSyncService: "true", - })}, - Image: imageLcowAlpine, - Command: []string{"ash", "-c", "tail -f /dev/null"}, - SetStateCommand: []string{"ash", "-c", "echo - >> " + testFile}, - GetStateCommand: []string{"ash", "-c", "cat " + testFile}, - ExpectedResult: "-\n", - }, - { - Name: "WCOW_Hypervisor", - Feature: featureWCOWHypervisor, - Runtime: wcowHypervisorRuntimeHandler, - Image: imageWindowsNanoserver, - Command: []string{"cmd", "/c", "ping -t 127.0.0.1"}, - SetStateCommand: []string{"cmd", "/c", "echo - >> " + wcowTestFile}, - GetStateCommand: []string{"cmd", "/c", "type", wcowTestFile}, - ExpectedResult: "- \r\n", - }, - { - Name: "WCOW_Process", - Feature: featureWCOWProcess, - Runtime: wcowProcessRuntimeHandler, - Image: imageWindowsNanoserver, - Command: []string{"cmd", "/c", "ping -t 127.0.0.1"}, - SetStateCommand: []string{"cmd", "/c", "echo - >> " + wcowTestFile}, - GetStateCommand: []string{"cmd", "/c", "type", wcowTestFile}, - ExpectedResult: "- \r\n", - }, - } - - for _, tt := range tests { - for _, restart := range []bool{false, true} { - suffix := "_Restart" - if !restart { - suffix = "_No" + suffix - } - - t.Run(tt.Name+suffix, func(t *testing.T) { - requireFeatures(t, tt.Feature) - if restart { - requireFeatures(t, featureTerminateOnRestart) - } - - switch tt.Feature { - case featureLCOW: - pullRequiredLCOWImages(t, append([]string{imageLcowK8sPause}, tt.Image)) - case featureWCOWHypervisor, featureWCOWProcess: - pullRequiredImages(t, []string{tt.Image}) - } - - sandboxRequest := getRunPodSandboxRequest(t, tt.Runtime, - append(tt.SandboxOpts, - WithSandboxAnnotations(map[string]string{ - criAnnotations.EnableReset: "true", - }))...) - - podID := runPodSandbox(t, client, ctx, sandboxRequest) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - - request := getCreateContainerRequest(podID, t.Name()+"-Container", tt.Image, tt.Command, sandboxRequest.Config) - request.Config.Annotations = map[string]string{ - criAnnotations.EnableReset: "true", - } - - containerID := createContainer(t, client, ctx, request) - startContainer(t, client, ctx, containerID) - defer removeContainer(t, client, ctx, containerID) - defer stopContainer(t, client, ctx, containerID) - - startExecRequest := &runtime.ExecSyncRequest{ - ContainerId: containerID, - Cmd: tt.SetStateCommand, - Timeout: 1, - } - req := execSync(t, client, ctx, startExecRequest) - if req.ExitCode != 0 { - t.Fatalf("exec %v failed with exit code %d: %s", startExecRequest.Cmd, req.ExitCode, string(req.Stderr)) - } - t.Logf("exec: %s", tt.SetStateCommand) - - // check the write worked - startExecRequest = &runtime.ExecSyncRequest{ - ContainerId: containerID, - Cmd: tt.GetStateCommand, - Timeout: 1, - } - - req = execSync(t, client, ctx, startExecRequest) - if req.ExitCode != 0 { - t.Fatalf("exec %v failed with exit code %d: %s %s", startExecRequest.Cmd, req.ExitCode, string(req.Stdout), string(req.Stderr)) - } - - if string(req.Stdout) != tt.ExpectedResult { - t.Fatalf("did not properly set container state; expected %q, got: %q", tt.ExpectedResult, string(req.Stdout)) - } - - /******************************************************************* - * restart pod - *******************************************************************/ - stopContainer(t, client, ctx, containerID) - stopPodSandbox(t, client, ctx, podID) - - if restart { - // allow for any garbage collection and clean up to happen - time.Sleep(time.Second * 1) - stopContainerd(t) - startContainerd(t) - client = newTestRuntimeClient(t) - waitForCRI(ctx, t, client, 15*time.Second) - } - - newPodID := runPodSandbox(t, client, ctx, sandboxRequest) - if newPodID != podID { - defer removePodSandbox(t, client, ctx, newPodID) - defer stopPodSandbox(t, client, ctx, newPodID) - t.Fatalf("pod restarted with different id (%q) from original (%q)", newPodID, podID) - } - - startContainer(t, client, ctx, containerID) - req = execSync(t, client, ctx, startExecRequest) - if req.ExitCode != 0 { - t.Fatalf("exec %v failed with exit code %d: %s %s", startExecRequest.Cmd, req.ExitCode, string(req.Stdout), string(req.Stderr)) - } - - if string(req.Stdout) != tt.ExpectedResult { - t.Fatalf("expected %q, got: %q", tt.ExpectedResult, string(req.Stdout)) - } - }) - } - } -} - -func waitForCRI(ctx context.Context, tb testing.TB, client runtime.RuntimeServiceClient, timeout time.Duration) { - tb.Helper() - - ctx, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - ch := make(chan error) - defer close(ch) - - go func() { - sleep := timeout / 10 - for { - select { - case <-ctx.Done(): - // context timed out or was cancelled - return - default: - } - - _, err := client.Version(ctx, &runtime.VersionRequest{}) - if err == nil || !strings.Contains(err.Error(), "server is not initialized yet") { - ch <- err - return - } - tb.Logf("CRI is not yet initialized, sleeping for %s", sleep.String()) - time.Sleep(sleep) - } - }() - - select { - case err := <-ctx.Done(): - tb.Fatalf("could not wait for CRI plugin to initialize: %v", err) - case err := <-ch: - if err != nil { - tb.Fatalf("error while checking CRI plugin to initialization status: %v", err) - } - } -} diff --git a/test/cri-containerd/createcontainer_test.go b/test/cri-containerd/createcontainer_test.go index 3cb1e847da..17229b91cd 100644 --- a/test/cri-containerd/createcontainer_test.go +++ b/test/cri-containerd/createcontainer_test.go @@ -14,7 +14,7 @@ import ( "github.com/Microsoft/hcsshim/internal/memory" "github.com/Microsoft/hcsshim/pkg/annotations" - runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" + runtime "k8s.io/cri-api/pkg/apis/runtime/v1" ) func runCreateContainerTest(t *testing.T, runtimeHandler string, request *runtime.CreateContainerRequest) { diff --git a/test/cri-containerd/disable_vpmem_test.go b/test/cri-containerd/disable_vpmem_test.go index 963d1fcbc6..4c5104f46c 100644 --- a/test/cri-containerd/disable_vpmem_test.go +++ b/test/cri-containerd/disable_vpmem_test.go @@ -13,7 +13,7 @@ import ( "time" "github.com/Microsoft/hcsshim/pkg/annotations" - runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" + runtime "k8s.io/cri-api/pkg/apis/runtime/v1" ) // Use unique names for pods & containers so that if we run this test multiple times in parallel we don't diff --git a/test/cri-containerd/helper_container_test.go b/test/cri-containerd/helper_container_test.go index 300849252f..1ecf1962bb 100644 --- a/test/cri-containerd/helper_container_test.go +++ b/test/cri-containerd/helper_container_test.go @@ -7,7 +7,7 @@ import ( "context" "testing" - runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" + runtime "k8s.io/cri-api/pkg/apis/runtime/v1" ) func createContainer(tb testing.TB, client runtime.RuntimeServiceClient, ctx context.Context, request *runtime.CreateContainerRequest) string { diff --git a/test/cri-containerd/helper_cri_plugins_test.go b/test/cri-containerd/helper_cri_plugins_test.go deleted file mode 100644 index bf018dcf49..0000000000 --- a/test/cri-containerd/helper_cri_plugins_test.go +++ /dev/null @@ -1,42 +0,0 @@ -//go:build windows && functional -// +build windows,functional - -package cri_containerd - -import ( - "context" - "testing" - - cri "github.com/kevpar/cri/pkg/api/v1" -) - -func newTestPluginClient(tb testing.TB) cri.CRIPluginServiceClient { - tb.Helper() - ctx, cancel := context.WithTimeout(context.Background(), connectTimeout) - defer cancel() - conn, err := createGRPCConn(ctx) - if err != nil { - tb.Fatalf("failed to dial runtime client: %v", err) - } - return cri.NewCRIPluginServiceClient(conn) -} - -func resetContainer(tb testing.TB, client cri.CRIPluginServiceClient, ctx context.Context, containerID string) { - tb.Helper() - _, err := client.ResetContainer(ctx, &cri.ResetContainerRequest{ - ContainerId: containerID, - }) - if err != nil { - tb.Fatalf("failed ResetContainer request for container: %s, with: %v", containerID, err) - } -} - -func resetPodSandbox(tb testing.TB, client cri.CRIPluginServiceClient, ctx context.Context, podID string) { - tb.Helper() - _, err := client.ResetPodSandbox(ctx, &cri.ResetPodSandboxRequest{ - PodSandboxId: podID, - }) - if err != nil { - tb.Fatalf("failed ResetPodSandbox request for container: %s, with: %v", podID, err) - } -} diff --git a/test/cri-containerd/helper_exec_test.go b/test/cri-containerd/helper_exec_test.go index db0c97c56d..5ec3f16689 100644 --- a/test/cri-containerd/helper_exec_test.go +++ b/test/cri-containerd/helper_exec_test.go @@ -14,7 +14,7 @@ import ( "github.com/Microsoft/hcsshim/test/pkg/definitions/cmd" "github.com/Microsoft/hcsshim/test/pkg/definitions/shimdiag" - runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" + runtime "k8s.io/cri-api/pkg/apis/runtime/v1" ) func execSync(tb testing.TB, client runtime.RuntimeServiceClient, ctx context.Context, request *runtime.ExecSyncRequest) *runtime.ExecSyncResponse { diff --git a/test/cri-containerd/helper_sandbox_test.go b/test/cri-containerd/helper_sandbox_test.go index 31d5b7143e..3976b57026 100644 --- a/test/cri-containerd/helper_sandbox_test.go +++ b/test/cri-containerd/helper_sandbox_test.go @@ -7,7 +7,7 @@ import ( "context" "testing" - runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" + runtime "k8s.io/cri-api/pkg/apis/runtime/v1" ) type SandboxConfigOpt func(*runtime.PodSandboxConfig) error diff --git a/test/cri-containerd/jobcontainer_test.go b/test/cri-containerd/jobcontainer_test.go index 48053cf821..e495decfef 100644 --- a/test/cri-containerd/jobcontainer_test.go +++ b/test/cri-containerd/jobcontainer_test.go @@ -15,7 +15,7 @@ import ( "time" "github.com/Microsoft/go-winio/vhd" - runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" + runtime "k8s.io/cri-api/pkg/apis/runtime/v1" "github.com/Microsoft/hcsshim/hcn" "github.com/Microsoft/hcsshim/osversion" diff --git a/test/cri-containerd/logging_binary_test.go b/test/cri-containerd/logging_binary_test.go index aa69e599d6..314cd5631d 100644 --- a/test/cri-containerd/logging_binary_test.go +++ b/test/cri-containerd/logging_binary_test.go @@ -13,7 +13,7 @@ import ( "time" "github.com/Microsoft/hcsshim/test/pkg/require" - runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" + runtime "k8s.io/cri-api/pkg/apis/runtime/v1" ) // This test requires compiling a helper logging binary which can be found diff --git a/test/cri-containerd/main_test.go b/test/cri-containerd/main_test.go index e869e678ce..5ac2b070b3 100644 --- a/test/cri-containerd/main_test.go +++ b/test/cri-containerd/main_test.go @@ -13,7 +13,7 @@ import ( kubeutil "github.com/containerd/containerd/integration/remote/util" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" - runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" + runtime "k8s.io/cri-api/pkg/apis/runtime/v1" "github.com/Microsoft/hcsshim/osversion" testflag "github.com/Microsoft/hcsshim/test/pkg/flag" @@ -27,6 +27,7 @@ const ( connectTimeout = time.Second * 10 testNamespace = "cri-containerd-test" + // TODO: remove lcow when shim only tests are relocated lcowRuntimeHandler = "runhcs-lcow" wcowProcessRuntimeHandler = "runhcs-wcow-process" wcowHypervisorRuntimeHandler = "runhcs-wcow-hypervisor" @@ -42,6 +43,7 @@ const ( testVMServiceAddress = "C:\\ContainerPlat\\vmservice.sock" testVMServiceBinary = "C:\\Containerplat\\vmservice.exe" + // TODO: remove the lcow ones when shim only tests are relocated. imageLcowK8sPause = "mcr.microsoft.com/oss/kubernetes/pause:3.1" imageLcowAlpine = "mcr.microsoft.com/mirror/docker/library/alpine:3.16" imageLcowAlpineCoreDump = "cplatpublic.azurecr.io/stackoverflow-alpine:latest" @@ -104,9 +106,6 @@ const ( featureGPU = "GPU" featureCRIUpdateContainer = "UpdateContainer" featureTerminateOnRestart = "TerminateOnRestart" - featureLCOWIntegrity = "LCOWIntegrity" - featureLCOWCrypt = "LCOWCrypt" - featureCRIPlugin = "CRIPlugin" ) var allFeatures = []string{ @@ -118,9 +117,6 @@ var allFeatures = []string{ featureGPU, featureCRIUpdateContainer, featureTerminateOnRestart, - featureLCOWIntegrity, - featureLCOWCrypt, - featureCRIPlugin, } func TestMain(m *testing.M) { diff --git a/test/cri-containerd/pod_update_test.go b/test/cri-containerd/pod_update_test.go index 05595e2f21..20bf08df4a 100644 --- a/test/cri-containerd/pod_update_test.go +++ b/test/cri-containerd/pod_update_test.go @@ -13,7 +13,7 @@ import ( "github.com/Microsoft/hcsshim/pkg/annotations" "github.com/Microsoft/hcsshim/test/pkg/definitions/cpugroup" "github.com/Microsoft/hcsshim/test/pkg/definitions/processorinfo" - runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" + runtime "k8s.io/cri-api/pkg/apis/runtime/v1" ) func Test_Pod_UpdateResources_Memory(t *testing.T) { diff --git a/test/cri-containerd/removepodsandbox_test.go b/test/cri-containerd/removepodsandbox_test.go index 2721df9b88..6180e5c8e3 100644 --- a/test/cri-containerd/removepodsandbox_test.go +++ b/test/cri-containerd/removepodsandbox_test.go @@ -9,7 +9,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" + runtime "k8s.io/cri-api/pkg/apis/runtime/v1" ) type TestConfig struct { diff --git a/test/cri-containerd/runpodsandbox_test.go b/test/cri-containerd/runpodsandbox_test.go index 920944d03a..ebf6ddca5b 100644 --- a/test/cri-containerd/runpodsandbox_test.go +++ b/test/cri-containerd/runpodsandbox_test.go @@ -16,7 +16,7 @@ import ( "time" "golang.org/x/sys/windows" - runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" + runtime "k8s.io/cri-api/pkg/apis/runtime/v1" iannotations "github.com/Microsoft/hcsshim/internal/annotations" "github.com/Microsoft/hcsshim/osversion" diff --git a/test/cri-containerd/scale_cpu_limits_to_sandbox_test.go b/test/cri-containerd/scale_cpu_limits_to_sandbox_test.go index 609b55efcc..533bdf525e 100644 --- a/test/cri-containerd/scale_cpu_limits_to_sandbox_test.go +++ b/test/cri-containerd/scale_cpu_limits_to_sandbox_test.go @@ -12,7 +12,7 @@ import ( "time" "github.com/Microsoft/hcsshim/pkg/annotations" - criruntime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" + criruntime "k8s.io/cri-api/pkg/apis/runtime/v1" ) const imageWindowsMaxCPUWorkload = "cplatpublic.azurecr.io/golang-1.16.2-nanoserver-1809:max-cpu-workload" diff --git a/test/cri-containerd/stats_test.go b/test/cri-containerd/stats_test.go index 738fd759cb..fc31830458 100644 --- a/test/cri-containerd/stats_test.go +++ b/test/cri-containerd/stats_test.go @@ -11,7 +11,7 @@ import ( "github.com/Microsoft/hcsshim/internal/memory" "github.com/Microsoft/hcsshim/pkg/annotations" - runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" + runtime "k8s.io/cri-api/pkg/apis/runtime/v1" ) func runContainerAndQueryStats(t *testing.T, client runtime.RuntimeServiceClient, ctx context.Context, request *runtime.CreateContainerRequest) { diff --git a/test/cri-containerd/stopcontainer_test.go b/test/cri-containerd/stopcontainer_test.go index e53f468e4b..e08aa7f12d 100644 --- a/test/cri-containerd/stopcontainer_test.go +++ b/test/cri-containerd/stopcontainer_test.go @@ -11,7 +11,7 @@ import ( "github.com/Microsoft/hcsshim/osversion" "github.com/Microsoft/hcsshim/test/pkg/require" - runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" + runtime "k8s.io/cri-api/pkg/apis/runtime/v1" ) // This test runs a container with an image that waits for sigterm and then diff --git a/test/go.mod b/test/go.mod index 5af3ec46fd..be4da33031 100644 --- a/test/go.mod +++ b/test/go.mod @@ -11,7 +11,6 @@ require ( github.com/containerd/ttrpc v1.2.2 github.com/google/go-containerregistry v0.17.0 github.com/josephspurrier/goversioninfo v1.4.0 - github.com/kevpar/cri v1.11.1-0.20220302210600-4c5c347230b2 github.com/opencontainers/go-digest v1.0.0 github.com/opencontainers/image-spec v1.1.0-rc3 github.com/opencontainers/runtime-spec v1.1.0 @@ -107,8 +106,4 @@ require ( sigs.k8s.io/yaml v1.3.0 // indirect ) -replace ( - github.com/Microsoft/hcsshim => ../ - // keep CRI version frozen for cri-containerd tests - k8s.io/cri-api => k8s.io/cri-api v0.25.8 -) +replace github.com/Microsoft/hcsshim => ../ diff --git a/test/go.sum b/test/go.sum index 8c91259fed..677a1f25d3 100644 --- a/test/go.sum +++ b/test/go.sum @@ -1263,6 +1263,7 @@ github.com/gogo/googleapis v1.2.0/go.mod h1:Njal3psf3qN6dwBtQfUmBZh2ybovJ0tlu3o/ github.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -1485,8 +1486,6 @@ github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8 github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= -github.com/kevpar/cri v1.11.1-0.20220302210600-4c5c347230b2 h1:uiHNQd4Qc3o3z1a9n0noEY6unSQ0qGnjEKnH06JKNKg= -github.com/kevpar/cri v1.11.1-0.20220302210600-4c5c347230b2/go.mod h1:aKPTQrCvhUuOmvVdpNmKSQpe36YPEw83fX5Lk3YCo/o= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -2175,6 +2174,7 @@ golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -2390,6 +2390,7 @@ golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210819135213-f52c844e1c1c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210903071746-97244b99971b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -2974,8 +2975,15 @@ k8s.io/component-base v0.20.4/go.mod h1:t4p9EdiagbVCJKrQ1RsA5/V4rFQNDfRlevJajlGw k8s.io/component-base v0.20.6/go.mod h1:6f1MPBAeI+mvuts3sIdtpjljHWBQ2cIy38oBIWMYnrM= k8s.io/component-base v0.22.5/go.mod h1:VK3I+TjuF9eaa+Ln67dKxhGar5ynVbwnGrUiNF4MqCI= k8s.io/component-base v0.26.2/go.mod h1:DxbuIe9M3IZPRxPIzhch2m1eT7uFrSBJUBuVCQEBivs= -k8s.io/cri-api v0.25.8 h1:OwV/xxklUdm3MhvVwXD49cXcywSpICjFKMTgRSHZixc= -k8s.io/cri-api v0.25.8/go.mod h1:ZOI0K9rvD6KaGFndOpvW+zp/fJvudDbkAl26ZBHh2wY= +k8s.io/cri-api v0.17.3/go.mod h1:X1sbHmuXhwaHs9xxYffLqJogVsnI+f6cPRcgPel7ywM= +k8s.io/cri-api v0.20.1/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= +k8s.io/cri-api v0.20.4/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= +k8s.io/cri-api v0.20.6/go.mod h1:ew44AjNXwyn1s0U4xCKGodU7J1HzBeZ1MpGrpa5r8Yc= +k8s.io/cri-api v0.23.1/go.mod h1:REJE3PSU0h/LOV1APBrupxrEJqnoxZC8KWzkBUHwrK4= +k8s.io/cri-api v0.25.0/go.mod h1:J1rAyQkSJ2Q6I+aBMOVgg2/cbbebso6FNa0UagiR0kc= +k8s.io/cri-api v0.25.3/go.mod h1:riC/P0yOGUf2K1735wW+CXs1aY2ctBgePtnnoFLd0dU= +k8s.io/cri-api v0.26.2 h1:Vifw8T4ZFzU5pQ5dj5rdDsPOSzmLAvhVcYEJpbjOYLY= +k8s.io/cri-api v0.26.2/go.mod h1:Oo8O7MKFPNDxfDf2LmrF/3Hf30q1C6iliGuv3la3tIA= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= From 788484094c585bf860986f42ccdb4ad5c1f59aa1 Mon Sep 17 00:00:00 2001 From: Yuanyuan Lei Date: Mon, 5 Feb 2024 10:19:56 -0800 Subject: [PATCH 4/5] removed unused file and code --- test/cri-containerd/helper_sandbox_test.go | 18 ----- test/cri-containerd/helper_service_test.go | 94 ---------------------- test/cri-containerd/main_test.go | 9 +-- 3 files changed, 4 insertions(+), 117 deletions(-) delete mode 100644 test/cri-containerd/helper_service_test.go diff --git a/test/cri-containerd/helper_sandbox_test.go b/test/cri-containerd/helper_sandbox_test.go index 3976b57026..984b53d522 100644 --- a/test/cri-containerd/helper_sandbox_test.go +++ b/test/cri-containerd/helper_sandbox_test.go @@ -66,24 +66,6 @@ func removePodSandbox(tb testing.TB, client runtime.RuntimeServiceClient, ctx co } } -func getPodSandboxStatus(tb testing.TB, client runtime.RuntimeServiceClient, ctx context.Context, podID string) *runtime.PodSandboxStatus { - tb.Helper() - status, err := client.PodSandboxStatus(ctx, &runtime.PodSandboxStatusRequest{ - PodSandboxId: podID, - }) - if err != nil { - tb.Fatalf("failed PodSandboxStatus for sandbox: %s, request with: %v", podID, err) - } - return status.Status -} - -func assertPodSandboxState(tb testing.TB, client runtime.RuntimeServiceClient, ctx context.Context, podID string, state runtime.PodSandboxState) { - tb.Helper() - if st := getPodSandboxStatus(tb, client, ctx, podID).State; st != state { - tb.Fatalf("got pod sandbox %q state %q; wanted %v", podID, st.String(), state.String()) - } -} - func getTestSandboxConfig(tb testing.TB, opts ...SandboxConfigOpt) *runtime.PodSandboxConfig { tb.Helper() c := &runtime.PodSandboxConfig{ diff --git a/test/cri-containerd/helper_service_test.go b/test/cri-containerd/helper_service_test.go deleted file mode 100644 index 5ac7a34acd..0000000000 --- a/test/cri-containerd/helper_service_test.go +++ /dev/null @@ -1,94 +0,0 @@ -//go:build windows && functional - -package cri_containerd - -import ( - "fmt" - "sync" - "testing" - "time" - - "golang.org/x/sys/windows/svc" - "golang.org/x/sys/windows/svc/mgr" -) - -// Implements functionality so tests can start/stop the containerd service. -// Tests assume containerd will be running when they start, since this -// matches with the state a dev box will usually be in. A test that stops containerd should -// therefore ensure it starts it again. - -var ( - svcMgr *mgr.Mgr - svcMgrConnectOnce sync.Once - svcMgrConnectErr error -) - -func getSvcMgr() (*mgr.Mgr, error) { - svcMgrConnectOnce.Do(func() { - s, err := mgr.Connect() - if err != nil { - err = fmt.Errorf("failed to connect to service manager: %w", err) - } - svcMgr, svcMgrConnectErr = s, err - }) - return svcMgr, svcMgrConnectErr -} - -func startService(serviceName string) error { - m, err := getSvcMgr() - if err != nil { - return err - } - s, err := m.OpenService(serviceName) - if err != nil { - return fmt.Errorf("failed to open service %s: %w", serviceName, err) - } - if err := s.Start(); err != nil { - return fmt.Errorf("failed to start service %s: %w", serviceName, err) - } - return nil -} - -func stopService(serviceName string) error { - m, err := getSvcMgr() - if err != nil { - return err - } - s, err := m.OpenService(serviceName) - if err != nil { - return fmt.Errorf("failed to open service %s: %w", serviceName, err) - } - status, err := s.Control(svc.Stop) - if err != nil { - return fmt.Errorf("failed to send stop control to service %s: %w", serviceName, err) - } - tc := time.NewTimer(10 * time.Second) - defer tc.Stop() - for status.State != svc.Stopped { - time.Sleep(1 * time.Second) - select { - case <-tc.C: - return fmt.Errorf("service %s did not stop in time", serviceName) - default: - status, err = s.Query() - if err != nil { - return fmt.Errorf("failed to query service %s status: %w", serviceName, err) - } - } - } - return nil -} - -func startContainerd(tb testing.TB) { - tb.Helper() - if err := startService(*flagContainerdServiceName); err != nil { - tb.Fatal(err) - } -} - -func stopContainerd(tb testing.TB) { - tb.Helper() - if err := stopService(*flagContainerdServiceName); err != nil { - tb.Fatal(err) - } -} diff --git a/test/cri-containerd/main_test.go b/test/cri-containerd/main_test.go index 5ac2b070b3..f19f7d185f 100644 --- a/test/cri-containerd/main_test.go +++ b/test/cri-containerd/main_test.go @@ -88,11 +88,10 @@ var ( // Flags var ( - flagFeatures = testflag.NewFeatureFlag(allFeatures) - flagCRIEndpoint = flag.String("cri-endpoint", "tcp://127.0.0.1:2376", "Address of CRI runtime and image service.") - flagVirtstack = flag.String("virtstack", "", "Virtstack to use for hypervisor isolated containers") - flagVMServiceBinary = flag.String("vmservice-binary", "", "Path to a binary implementing the vmservice ttrpc service") - flagContainerdServiceName = flag.String("containerd-service-name", "containerd", "Name of the containerd Windows service") + flagFeatures = testflag.NewFeatureFlag(allFeatures) + flagCRIEndpoint = flag.String("cri-endpoint", "tcp://127.0.0.1:2376", "Address of CRI runtime and image service.") + flagVirtstack = flag.String("virtstack", "", "Virtstack to use for hypervisor isolated containers") + flagVMServiceBinary = flag.String("vmservice-binary", "", "Path to a binary implementing the vmservice ttrpc service") ) // Features From bbbf09216f9ee50813104d8814d5ad4e64bd2385 Mon Sep 17 00:00:00 2001 From: Yuanyuan Lei Date: Wed, 14 Feb 2024 17:23:01 -0800 Subject: [PATCH 5/5] remove blanks --- test/cri-containerd/test-images/imagepull_timestamp/Dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/cri-containerd/test-images/imagepull_timestamp/Dockerfile b/test/cri-containerd/test-images/imagepull_timestamp/Dockerfile index 3413cbfdec..7d76a0c07f 100644 --- a/test/cri-containerd/test-images/imagepull_timestamp/Dockerfile +++ b/test/cri-containerd/test-images/imagepull_timestamp/Dockerfile @@ -9,9 +9,7 @@ # `docker push cplatpublic.azurecr.io/timestamp:latest` # Base image - FROM mcr.microsoft.com/windows/nanoserver:ltsc2022@sha256:56a7c823d8aed810849a38f47d995a727bf3de1ec611394025e1de24898a9622 - # Get administrator privileges USER containeradministrator