Add tests for k8s and fix cgroup issue

.dockerignore

@@ -0,0 +1,11 @@
+target
+test/k8s/Dockerfile
+test/k8s/_out/*
+.github
+out
+*.md
+LICENSE
+.gitignore
+.dockerignore
+docs
+bin/

.github/workflows/ci.yml

@@ -26,3 +26,16 @@ jobs:
         run: cargo build --all --verbose
       - name: Run tests
         run: cargo test --all --verbose
+
+  e2e:
+    needs: [build]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: setup rust-wasm target
+        run: rustup target add wasm32-wasi
+      - name: run
+        run: make test/k8s/deploy
+      - name: cleanup
+        if: always()
+        run: make test/k8s/clean

.gitignore

@@ -2,3 +2,4 @@ bin/
 test/out/img.tar
 **/target/
 !src/bin/
+test/k8s/_out

Makefile

@@ -9,15 +9,20 @@ ifeq ($(TARGET),release)
 RELEASE_FLAG = --release
 endif
 
+DOCKER_BUILD ?= docker buildx build
+
+KIND_CLUSTER_NAME ?= containerd-wasm
+
 .PHONY: build
 build:
 	cargo build $(RELEASE_FLAG)
 
 .PHONY: install
 install:
-	$(INSTALL) target/$(TARGET)/containerd-shim-wasmtime-v1 $(PREFIX)/bin
-	$(INSTALL) target/$(TARGET)/containerd-shim-wasmtimed-v1 $(PREFIX)/bin
-	$(INSTALL) target/$(TARGET)/containerd-wasmtimed $(PREFIX)/bin
+	mkdir -p $(PREFIX)/bin
+	$(INSTALL) target/$(TARGET)/containerd-shim-wasmtime-v1 $(PREFIX)/bin/
+	$(INSTALL) target/$(TARGET)/containerd-shim-wasmtimed-v1 $(PREFIX)/bin/
+	$(INSTALL) target/$(TARGET)/containerd-wasmtimed $(PREFIX)/bin/
 
 .PHONY: target/wasm32-wasi/$(TARGET)/wasi-demo-app.wasm
 target/wasm32-wasi/$(TARGET)/wasi-demo-app.wasm:
@@ -28,4 +33,25 @@ target/wasm32-wasi/$(TARGET)/img.tar: target/wasm32-wasi/$(TARGET)/wasi-demo-app
 	cd crates/wasi-demo-app && cargo build --features oci-v1-tar
 
 load: target/wasm32-wasi/$(TARGET)/img.tar
-	sudo ctr -n $(CONTAINERD_NAMESPACE) image import --all-platforms $<
+	sudo ctr -n $(CONTAINERD_NAMESPACE) image import --all-platforms $<
+
+bin/kind: test/k8s/Dockerfile
+	$(DOCKER_BUILD) --output=bin/ -f test/k8s/Dockerfile --target=kind .
+
+
+test/k8s/_out/img: test/k8s/Dockerfile Cargo.toml Cargo.lock $(shell find . -type f -name '*.rs')
+	mkdir -p $(@D) && $(DOCKER_BUILD) -f test/k8s/Dockerfile --iidfile=$(@) --load  .
+
+.PHONY: test/k8s/cluster
+test/k8s/cluster: target/wasm32-wasi/$(TARGET)/img.tar bin/kind test/k8s/_out/img bin/kind
+	bin/kind create cluster --name $(KIND_CLUSTER_NAME) --image="$(shell cat test/k8s/_out/img)" && \
+	bin/kind load image-archive --name $(KIND_CLUSTER_NAME) $(<)
+
+.PHONY: test/k8s/deploy
+test/k8s/deploy: test/k8s/cluster
+	kubectl --context=kind-$(KIND_CLUSTER_NAME) apply -f test/k8s/deploy.yaml
+	kubectl --context=kind-$(KIND_CLUSTER_NAME) wait deployment wasi-demo --for condition=Available=True --timeout=90s
+
+.PHONY: test/k8s/clean
+test/k8s/clean:
+	kind delete cluster --name $(KIND_CLUSTER_NAME)

crates/containerd-shim-wasm/src/sandbox/cgroups/cgroupv2.rs

@@ -38,12 +38,6 @@ impl CgroupV2 {
         for d in p.iter() {
             full = safe_join(full.clone(), PathBuf::from(d))?;
 
-            if full.exists() {
-                debug!("skipping creation of existing cgroup: {}", full.display());
-                // This is a pre-existing cgroup, so we won't touch subtree control.
-                continue;
-            }
-
             if full.to_str() == self.full_path()?.to_str() {
                 // This is the last element in the path, so we don't need to create it.
                 // The cgroup will get created later when we try to write to a file in it.

test/k8s/Dockerfile

@@ -0,0 +1,42 @@
+# syntax=docker/dockerfile:1.4
+
+ARG KIND_NODE_VERSION=v1.23.13
+
+FROM kindest/node:${KIND_NODE_VERSION} AS kind-base
+
+# We need the build to link using against same system libs as the kind image otherwise the shim won't work.
+# So use the node image here as a base and install rust on top of it.
+FROM kind-base AS shim-build
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > /tmp/rustup.sh && sh /tmp/rustup.sh -y --profile=minimal
+ENV PATH="/root/.cargo/bin:${PATH}"
+RUN rustup install stable
+WORKDIR /shim
+COPY . .
+RUN apt-get update && apt-get install -y build-essential
+RUN \
+    --mount=type=cache,target=/usr/local/cargo/registry \
+    --mount=type=cache,target=/shim/target \
+    mkdir -p /opt/shim; make build install PREFIX=/opt/shim
+
+FROM scratch AS shim
+COPY --from=shim-build /opt/shim/bin/* /
+
+
+# Reuse kind-base so we don't have to download other images...
+FROM kind-base AS kind-fetch
+ARG TARGETARCH
+ARG KIND_VERSION=v0.17.0
+RUN curl -sSLf https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-${TARGETARCH} > /root/kind && chmod +x /root/kind
+
+FROM scratch AS kind
+COPY --from=kind-fetch /root/kind /kind
+
+FROM kind-base
+RUN <<EOF
+set -e
+echo '[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.wasm]' >> /etc/containerd/config.toml
+echo 'runtime_type = "io.containerd.wasmtime.v1"' >> /etc/containerd/config.toml
+sed -i 's,SystemdCgroup = true,,' /etc/containerd/config.toml
+EOF
+COPY --link --from=shim /* /usr/local/bin/
+

test/k8s/deploy.yaml

@@ -0,0 +1,27 @@
+apiVersion: node.k8s.io/v1
+kind: RuntimeClass
+metadata:
+  name: wasm
+handler: wasm
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: wasi-demo
+  labels:
+    app: wasi-demo
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: wasi-demo
+  template:
+    metadata:
+      labels:
+        app: wasi-demo
+    spec:
+      runtimeClassName: wasm
+      containers:
+      - name: demo
+        image: ghcr.io/containerd/runwasi/wasi-demo-app:latest
+        imagePullPolicy: Never