From 693a69a14b086cc914be543d7132b393d5efd0b3 Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Wed, 4 Jan 2023 01:28:47 +0000 Subject: [PATCH 1/2] Add k8s tests This tests running a deployment against a k8s cluster (created by kind). Signed-off-by: Brian Goff --- .dockerignore | 11 +++++++++++ .github/workflows/ci.yml | 13 +++++++++++++ .gitignore | 1 + Makefile | 35 +++++++++++++++++++++++++++++---- test/k8s/Dockerfile | 42 ++++++++++++++++++++++++++++++++++++++++ test/k8s/deploy.yaml | 27 ++++++++++++++++++++++++++ 6 files changed, 125 insertions(+), 4 deletions(-) create mode 100644 .dockerignore create mode 100644 test/k8s/Dockerfile create mode 100644 test/k8s/deploy.yaml diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..88d966a36 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,11 @@ +target +test/k8s/Dockerfile +test/k8s/_out/* +.github +out +*.md +LICENSE +.gitignore +.dockerignore +docs +bin/ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 823dacd50..0c14036e4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,3 +26,16 @@ jobs: run: cargo build --all --verbose - name: Run tests run: cargo test --all --verbose + + e2e: + needs: [build] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: setup rust-wasm target + run: rustup target add wasm32-wasi + - name: run + run: make test/k8s/deploy + - name: cleanup + if: always() + run: make test/k8s/clean diff --git a/.gitignore b/.gitignore index f6eabc2ac..48da3538f 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ bin/ test/out/img.tar **/target/ !src/bin/ +test/k8s/_out diff --git a/Makefile b/Makefile index 76bb32d51..4aa13808a 100644 --- a/Makefile +++ b/Makefile @@ -9,15 +9,20 @@ ifeq ($(TARGET),release) RELEASE_FLAG = --release endif +DOCKER_BUILD ?= docker buildx build + +KIND_CLUSTER_NAME ?= containerd-wasm + .PHONY: build build: cargo build $(RELEASE_FLAG) .PHONY: install install: - $(INSTALL) target/$(TARGET)/containerd-shim-wasmtime-v1 $(PREFIX)/bin - $(INSTALL) target/$(TARGET)/containerd-shim-wasmtimed-v1 $(PREFIX)/bin - $(INSTALL) target/$(TARGET)/containerd-wasmtimed $(PREFIX)/bin + mkdir -p $(PREFIX)/bin + $(INSTALL) target/$(TARGET)/containerd-shim-wasmtime-v1 $(PREFIX)/bin/ + $(INSTALL) target/$(TARGET)/containerd-shim-wasmtimed-v1 $(PREFIX)/bin/ + $(INSTALL) target/$(TARGET)/containerd-wasmtimed $(PREFIX)/bin/ .PHONY: target/wasm32-wasi/$(TARGET)/wasi-demo-app.wasm target/wasm32-wasi/$(TARGET)/wasi-demo-app.wasm: @@ -28,4 +33,26 @@ target/wasm32-wasi/$(TARGET)/img.tar: target/wasm32-wasi/$(TARGET)/wasi-demo-app cd crates/wasi-demo-app && cargo build --features oci-v1-tar load: target/wasm32-wasi/$(TARGET)/img.tar - sudo ctr -n $(CONTAINERD_NAMESPACE) image import --all-platforms $< \ No newline at end of file + sudo ctr -n $(CONTAINERD_NAMESPACE) image import --all-platforms $< + +bin/kind: test/k8s/Dockerfile + $(DOCKER_BUILD) --output=bin/ -f test/k8s/Dockerfile --target=kind . + + +test/k8s/_out/img: test/k8s/Dockerfile Cargo.toml Cargo.lock $(shell find . -type f -name '*.rs') + mkdir -p $(@D) && $(DOCKER_BUILD) -f test/k8s/Dockerfile --iidfile=$(@) --load . + +.PHONY: test/k8s/cluster +test/k8s/cluster: target/wasm32-wasi/$(TARGET)/img.tar bin/kind test/k8s/_out/img + kind create cluster --name $(KIND_CLUSTER_NAME) --image="$(shell cat test/k8s/_out/img)" && \ + # Load the image into the cluster. We can't use `kind load image-archive` because it doesn't support non-native arch images. \ + docker exec -i $(KIND_CLUSTER_NAME)-control-plane ctr -n k8s.io image import --all-platforms - < $(<) + +.PHONY: test/k8s/deploy +test/k8s/deploy: test/k8s/cluster + kubectl --context=kind-$(KIND_CLUSTER_NAME) apply -f test/k8s/deploy.yaml + kubectl --context=kind-$(KIND_CLUSTER_NAME) wait deployment wasi-demo --for condition=Available=True --timeout=90s + +.PHONY: test/k8s/clean +test/k8s/clean: + kind delete cluster --name $(KIND_CLUSTER_NAME) \ No newline at end of file diff --git a/test/k8s/Dockerfile b/test/k8s/Dockerfile new file mode 100644 index 000000000..94e2d4e6a --- /dev/null +++ b/test/k8s/Dockerfile @@ -0,0 +1,42 @@ +# syntax=docker/dockerfile:1.4 + +ARG KIND_NODE_VERSION=v1.23.13 + +FROM kindest/node:${KIND_NODE_VERSION} AS kind-base + +# We need the build to link using against same system libs as the kind image otherwise the shim won't work. +# So use the node image here as a base and install rust on top of it. +FROM kind-base AS shim-build +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > /tmp/rustup.sh && sh /tmp/rustup.sh -y --profile=minimal +ENV PATH="/root/.cargo/bin:${PATH}" +RUN rustup install stable +WORKDIR /shim +COPY . . +RUN apt-get update && apt-get install -y build-essential +RUN \ + --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=cache,target=/shim/target \ + mkdir -p /opt/shim; make build install PREFIX=/opt/shim + +FROM scratch AS shim +COPY --from=shim-build /opt/shim/bin/* / + + +# Reuse kind-base so we don't have to download other images... +FROM kind-base AS kind-fetch +ARG TARGETARCH +ARG KIND_VERSION=v0.17.0 +RUN curl -sSLf https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-${TARGETARCH} > /root/kind && chmod +x /root/kind + +FROM scratch AS kind +COPY --from=kind-fetch /root/kind /kind + +FROM kind-base +RUN <> /etc/containerd/config.toml +echo 'runtime_type = "io.containerd.wasmtime.v1"' >> /etc/containerd/config.toml +sed -i 's,SystemdCgroup = true,,' /etc/containerd/config.toml +EOF +COPY --link --from=shim /* /usr/local/bin/ + diff --git a/test/k8s/deploy.yaml b/test/k8s/deploy.yaml new file mode 100644 index 000000000..91f59edbf --- /dev/null +++ b/test/k8s/deploy.yaml @@ -0,0 +1,27 @@ +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: wasm +handler: wasm +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: wasi-demo + labels: + app: wasi-demo +spec: + replicas: 3 + selector: + matchLabels: + app: wasi-demo + template: + metadata: + labels: + app: wasi-demo + spec: + runtimeClassName: wasm + containers: + - name: demo + image: ghcr.io/containerd/runwasi/wasi-demo-app:latest + imagePullPolicy: Never \ No newline at end of file From 34c1a641b7e9de62af309e82db53a24393eb91ec Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Wed, 4 Jan 2023 01:29:30 +0000 Subject: [PATCH 2/2] cgroups: v2: always init parent dirs By skipping parents we were hitting an issue with the way k8s cgroups are managed since it is creating those cgroups. This change ensures we apply subtree_control to all parents as needed. Before this change pods would fail to start due to missing `cpu.weight` since we were not delegating cgroup control. Signed-off-by: Brian Goff --- Makefile | 7 +++---- .../containerd-shim-wasm/src/sandbox/cgroups/cgroupv2.rs | 6 ------ 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index 4aa13808a..1b81fecf2 100644 --- a/Makefile +++ b/Makefile @@ -43,10 +43,9 @@ test/k8s/_out/img: test/k8s/Dockerfile Cargo.toml Cargo.lock $(shell find . -typ mkdir -p $(@D) && $(DOCKER_BUILD) -f test/k8s/Dockerfile --iidfile=$(@) --load . .PHONY: test/k8s/cluster -test/k8s/cluster: target/wasm32-wasi/$(TARGET)/img.tar bin/kind test/k8s/_out/img - kind create cluster --name $(KIND_CLUSTER_NAME) --image="$(shell cat test/k8s/_out/img)" && \ - # Load the image into the cluster. We can't use `kind load image-archive` because it doesn't support non-native arch images. \ - docker exec -i $(KIND_CLUSTER_NAME)-control-plane ctr -n k8s.io image import --all-platforms - < $(<) +test/k8s/cluster: target/wasm32-wasi/$(TARGET)/img.tar bin/kind test/k8s/_out/img bin/kind + bin/kind create cluster --name $(KIND_CLUSTER_NAME) --image="$(shell cat test/k8s/_out/img)" && \ + bin/kind load image-archive --name $(KIND_CLUSTER_NAME) $(<) .PHONY: test/k8s/deploy test/k8s/deploy: test/k8s/cluster diff --git a/crates/containerd-shim-wasm/src/sandbox/cgroups/cgroupv2.rs b/crates/containerd-shim-wasm/src/sandbox/cgroups/cgroupv2.rs index fb78ee292..43aff3e1d 100644 --- a/crates/containerd-shim-wasm/src/sandbox/cgroups/cgroupv2.rs +++ b/crates/containerd-shim-wasm/src/sandbox/cgroups/cgroupv2.rs @@ -38,12 +38,6 @@ impl CgroupV2 { for d in p.iter() { full = safe_join(full.clone(), PathBuf::from(d))?; - if full.exists() { - debug!("skipping creation of existing cgroup: {}", full.display()); - // This is a pre-existing cgroup, so we won't touch subtree control. - continue; - } - if full.to_str() == self.full_path()?.to_str() { // This is the last element in the path, so we don't need to create it. // The cgroup will get created later when we try to write to a file in it.