From fe591c1ccac67b6dfef27e99bef7a2326075ad17 Mon Sep 17 00:00:00 2001 From: Moritz Sanft <58110325+msanft@users.noreply.github.com> Date: Fri, 10 Jan 2025 10:47:05 +0100 Subject: [PATCH] Add bare-metal GPU runtime class This adds a runtime class for the local just-based deployments as well as the release artifacts that corresponds to the GPU-enabled runtime for Contrast on bare-metal platforms. --- .github/workflows/release.yml | 4 + internal/kuberesource/parts.go | 6 + justfile | 6 +- .../contrast-node-installer-image/package.nix | 189 ++++++++++-------- .../kata/snp-launch-digest/package.nix | 11 +- packages/containers.nix | 4 + packages/nixos/kata.nix | 8 +- 7 files changed, 135 insertions(+), 93 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5997d011b5..5d9d6031ab 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -247,6 +247,7 @@ jobs: coordinatorImg=$(nix run .#containers.push-coordinator -- "$container_registry/contrast/coordinator") nodeInstallerMsftImg=$(nix run .#containers.push-node-installer-microsoft -- "$container_registry/contrast/node-installer-microsoft") nodeInstallerKataImg=$(nix run .#containers.push-node-installer-kata -- "$container_registry/contrast/node-installer-kata") + nodeInstallerKataGPUImg=$(nix run .#containers.push-node-installer-kata-gpu -- "$container_registry/contrast/node-installer-kata") initializerImg=$(nix run .#containers.push-initializer -- "$container_registry/contrast/initializer") serviceMeshImg=$(nix run .#containers.push-service-mesh-proxy -- "$container_registry/contrast/service-mesh-proxy") tardevSnapshotterImg=$(nix run .#containers.push-tardev-snapshotter -- "$container_registry/contrast/tardev-snapshotter") @@ -256,6 +257,7 @@ jobs: echo "coordinatorImg=$coordinatorImg" | tee -a "$GITHUB_ENV" echo "nodeInstallerMsftImg=$nodeInstallerMsftImg" | tee -a "$GITHUB_ENV" echo "nodeInstallerKataImg=$nodeInstallerKataImg" | tee -a "$GITHUB_ENV" + echo "nodeInstallerKataGPUImg=$nodeInstallerKataGPUImg" | tee -a "$GITHUB_ENV" echo "initializerImg=$initializerImg" | tee -a "$GITHUB_ENV" echo "serviceMeshImg=$serviceMeshImg" | tee -a "$GITHUB_ENV" echo "tardevSnapshotterImg=$tardevSnapshotterImg" | tee -a "$GITHUB_ENV" @@ -272,6 +274,7 @@ jobs: echo "coordinatorImgTagged=$(tag "$coordinatorImg")" | tee -a "$GITHUB_ENV" echo "nodeInstallerMsftImgTagged=$(tag "$nodeInstallerMsftImg")" | tee -a "$GITHUB_ENV" echo "nodeInstallerKataImgTagged=$(tag "$nodeInstallerKataImg")" | tee -a "$GITHUB_ENV" + echo "nodeInstallerKataGPUImgTagged=$(tag "$nodeInstallerKataGPUImg")" | tee -a "$GITHUB_ENV" echo "initializerImgTagged=$(tag "$initializerImg")" | tee -a "$GITHUB_ENV" echo "serviceMeshImgTagged=$(tag "$serviceMeshImg")" | tee -a "$GITHUB_ENV" echo "nydusPullImgTagged=$(tag "$nydusPullImg")" | tee -a "$GITHUB_ENV" @@ -294,6 +297,7 @@ jobs: echo "ghcr.io/edgelesssys/contrast/service-mesh-proxy:latest=$serviceMeshImgTagged" echo "ghcr.io/edgelesssys/contrast/node-installer-microsoft:latest=$nodeInstallerMsftImgTagged" echo "ghcr.io/edgelesssys/contrast/node-installer-kata:latest=$nodeInstallerKataImgTagged" + echo "ghcr.io/edgelesssys/contrast/node-installer-kata-gpu:latest=$nodeInstallerKataGPUImgTagged" echo "ghcr.io/edgelesssys/contrast/tardev-snapshotter:latest=$tardevSnapshotterImgTagged" echo "ghcr.io/edgelesssys/contrast/nydus-snapshotter:latest=$nydusSnapshotterImgTagged" echo "ghcr.io/edgelesssys/contrast/nydus-pull:latest=$nydusPullImgTagged" diff --git a/internal/kuberesource/parts.go b/internal/kuberesource/parts.go index 51a60c92f9..400baf58a4 100644 --- a/internal/kuberesource/parts.go +++ b/internal/kuberesource/parts.go @@ -153,6 +153,9 @@ func NodeInstaller(namespace string, platform platforms.Platform) (*NodeInstalle snapshotterVolumes = tardevSnapshotterVolumes case platforms.MetalQEMUSNP, platforms.MetalQEMUTDX, platforms.MetalQEMUSNPGPU: nodeInstallerImageURL = "ghcr.io/edgelesssys/contrast/node-installer-kata:latest" + if platform == platforms.MetalQEMUSNPGPU { + nodeInstallerImageURL = "ghcr.io/edgelesssys/contrast/node-installer-kata-gpu:latest" + } containers = append(containers, nydusSnapshotter, nydusPull) nydusSnapshotterVolumes = append(nydusSnapshotterVolumes, Volume(). @@ -171,6 +174,9 @@ func NodeInstaller(namespace string, platform platforms.Platform) (*NodeInstalle snapshotterVolumes = nydusSnapshotterVolumes case platforms.K3sQEMUTDX, platforms.K3sQEMUSNP, platforms.K3sQEMUSNPGPU, platforms.RKE2QEMUTDX: nodeInstallerImageURL = "ghcr.io/edgelesssys/contrast/node-installer-kata:latest" + if platform == platforms.K3sQEMUSNPGPU { + nodeInstallerImageURL = "ghcr.io/edgelesssys/contrast/node-installer-kata-gpu:latest" + } containers = append(containers, nydusSnapshotter, nydusPull) nydusSnapshotterVolumes = append(nydusSnapshotterVolumes, Volume(). diff --git a/justfile b/justfile index 8967e67b3e..142c05858d 100644 --- a/justfile +++ b/justfile @@ -50,11 +50,15 @@ node-installer platform=default_platform: just push "tardev-snapshotter" just push "node-installer-microsoft" ;; - "Metal-QEMU-SNP"|"Metal-QEMU-TDX"|"Metal-QEMU-SNP-GPU"|"K3s-QEMU-SNP"|"K3s-QEMU-SNP-GPU"|"K3s-QEMU-TDX"|"RKE2-QEMU-TDX") + "Metal-QEMU-SNP"|"Metal-QEMU-TDX"|"K3s-QEMU-SNP"|"K3s-QEMU-TDX"|"RKE2-QEMU-TDX") just push "nydus-snapshotter" just push "nydus-pull" just push "node-installer-kata" ;; + "Metal-QEMU-SNP-GPU"|"K3s-QEMU-SNP-GPU") + just push "nydus-snapshotter" + just push "node-installer-kata-gpu" + ;; "AKS-PEER-SNP") nix run -L .#scripts.deploy-caa -- \ --kustomization=./infra/azure-peerpods/kustomization.yaml \ diff --git a/packages/by-name/kata/contrast-node-installer-image/package.nix b/packages/by-name/kata/contrast-node-installer-image/package.nix index d0e5c5c22b..fce75bb95e 100644 --- a/packages/by-name/kata/contrast-node-installer-image/package.nix +++ b/packages/by-name/kata/contrast-node-installer-image/package.nix @@ -2,6 +2,7 @@ # SPDX-License-Identifier: AGPL-3.0-only { + lib, ociLayerTar, ociImageManifest, ociImageLayout, @@ -18,9 +19,15 @@ OVMF-TDX, debugRuntime ? false, + withGPU ? false, }: let + os-image = kata.kata-image.override { + inherit withGPU; + withDebug = debugRuntime; + }; + node-installer = ociLayerTar { files = [ { @@ -38,79 +45,82 @@ let files = [ { source = writers.writeJSON "contrast-node-install.json" { - files = [ - { - url = "file:///opt/edgeless/share/kata-containers.img"; - path = "/opt/edgeless/@@runtimeName@@/share/kata-containers.img"; - } - { - url = "file:///opt/edgeless/share/kata-kernel"; - path = "/opt/edgeless/@@runtimeName@@/share/kata-kernel"; - } - { - url = "file:///opt/edgeless/share/kata-initrd.zst"; - path = "/opt/edgeless/@@runtimeName@@/share/kata-initrd.zst"; - } - { - url = "file:///opt/edgeless/snp/bin/qemu-system-x86_64"; - path = "/opt/edgeless/@@runtimeName@@/snp/bin/qemu-system-x86_64"; - executable = true; - } - { - url = "file:///opt/edgeless/tdx/bin/qemu-system-x86_64"; - path = "/opt/edgeless/@@runtimeName@@/tdx/bin/qemu-system-x86_64"; - executable = true; - } - { - url = "file:///opt/edgeless/snp/share/OVMF.fd"; - path = "/opt/edgeless/@@runtimeName@@/snp/share/OVMF.fd"; - } - { - url = "file:///opt/edgeless/tdx/share/OVMF.fd"; - path = "/opt/edgeless/@@runtimeName@@/tdx/share/OVMF.fd"; - } - { - url = "file:///opt/edgeless/bin/containerd-shim-contrast-cc-v2"; - path = "/opt/edgeless/@@runtimeName@@/bin/containerd-shim-contrast-cc-v2"; - executable = true; - } - { - url = "file:///opt/edgeless/bin/kata-runtime"; - path = "/opt/edgeless/@@runtimeName@@/bin/kata-runtime"; - executable = true; - } - { - url = "file:///opt/edgeless/snp/share/qemu/kvmvapic.bin"; - path = "/opt/edgeless/@@runtimeName@@/snp/share/qemu/kvmvapic.bin"; - } - { - url = "file:///opt/edgeless/snp/share/qemu/linuxboot_dma.bin"; - path = "/opt/edgeless/@@runtimeName@@/snp/share/qemu/linuxboot_dma.bin"; - } - { - url = "file:///opt/edgeless/snp/share/qemu/efi-virtio.rom"; - path = "/opt/edgeless/@@runtimeName@@/snp/share/qemu/efi-virtio.rom"; - } - { - url = "file:///opt/edgeless/tdx/share/qemu/kvmvapic.bin"; - path = "/opt/edgeless/@@runtimeName@@/tdx/share/qemu/kvmvapic.bin"; - } - { - url = "file:///opt/edgeless/tdx/share/qemu/linuxboot_dma.bin"; - path = "/opt/edgeless/@@runtimeName@@/tdx/share/qemu/linuxboot_dma.bin"; - } - { - url = "file:///opt/edgeless/tdx/share/qemu/efi-virtio.rom"; - path = "/opt/edgeless/@@runtimeName@@/tdx/share/qemu/efi-virtio.rom"; - } - { - url = "file:///bin/nydus-overlayfs"; - path = "/opt/edgeless/@@runtimeName@@/bin/nydus-overlayfs"; - executable = true; - } - ]; + files = + [ + { + url = "file:///opt/edgeless/share/kata-containers.img"; + path = "/opt/edgeless/@@runtimeName@@/share/kata-containers.img"; + } + { + url = "file:///opt/edgeless/share/kata-kernel"; + path = "/opt/edgeless/@@runtimeName@@/share/kata-kernel"; + } + { + url = "file:///opt/edgeless/share/kata-initrd.zst"; + path = "/opt/edgeless/@@runtimeName@@/share/kata-initrd.zst"; + } + { + url = "file:///opt/edgeless/snp/bin/qemu-system-x86_64"; + path = "/opt/edgeless/@@runtimeName@@/snp/bin/qemu-system-x86_64"; + executable = true; + } + { + url = "file:///opt/edgeless/snp/share/OVMF.fd"; + path = "/opt/edgeless/@@runtimeName@@/snp/share/OVMF.fd"; + } + { + url = "file:///opt/edgeless/bin/containerd-shim-contrast-cc-v2"; + path = "/opt/edgeless/@@runtimeName@@/bin/containerd-shim-contrast-cc-v2"; + executable = true; + } + { + url = "file:///opt/edgeless/bin/kata-runtime"; + path = "/opt/edgeless/@@runtimeName@@/bin/kata-runtime"; + executable = true; + } + { + url = "file:///opt/edgeless/snp/share/qemu/kvmvapic.bin"; + path = "/opt/edgeless/@@runtimeName@@/snp/share/qemu/kvmvapic.bin"; + } + { + url = "file:///opt/edgeless/snp/share/qemu/linuxboot_dma.bin"; + path = "/opt/edgeless/@@runtimeName@@/snp/share/qemu/linuxboot_dma.bin"; + } + { + url = "file:///opt/edgeless/snp/share/qemu/efi-virtio.rom"; + path = "/opt/edgeless/@@runtimeName@@/snp/share/qemu/efi-virtio.rom"; + } + { + url = "file:///bin/nydus-overlayfs"; + path = "/opt/edgeless/@@runtimeName@@/bin/nydus-overlayfs"; + executable = true; + } + ] + ++ lib.optionals (!withGPU) [ + { + url = "file:///opt/edgeless/tdx/share/OVMF.fd"; + path = "/opt/edgeless/@@runtimeName@@/tdx/share/OVMF.fd"; + } + { + url = "file:///opt/edgeless/tdx/bin/qemu-system-x86_64"; + path = "/opt/edgeless/@@runtimeName@@/tdx/bin/qemu-system-x86_64"; + executable = true; + } + { + url = "file:///opt/edgeless/tdx/share/qemu/kvmvapic.bin"; + path = "/opt/edgeless/@@runtimeName@@/tdx/share/qemu/kvmvapic.bin"; + } + { + url = "file:///opt/edgeless/tdx/share/qemu/linuxboot_dma.bin"; + path = "/opt/edgeless/@@runtimeName@@/tdx/share/qemu/linuxboot_dma.bin"; + } + { + url = "file:///opt/edgeless/tdx/share/qemu/efi-virtio.rom"; + path = "/opt/edgeless/@@runtimeName@@/tdx/share/qemu/efi-virtio.rom"; + } + ]; inherit debugRuntime; - qemuExtraKernelParams = kata.kata-image.cmdline; + qemuExtraKernelParams = os-image.cmdline; }; destination = "/config/contrast-node-install.json"; } @@ -120,15 +130,15 @@ let kata-container-img = ociLayerTar { files = [ { - source = "${kata.kata-image.image}/${kata.kata-image.imageFileName}"; + source = "${os-image.image}/${os-image.imageFileName}"; destination = "/opt/edgeless/share/kata-containers.img"; } { - source = "${kata.kata-image.kernel}/bzImage"; + source = "${os-image.kernel}/bzImage"; destination = "/opt/edgeless/share/kata-kernel"; } { - source = "${kata.kata-image.initialRamdisk}/initrd"; + source = "${os-image.initialRamdisk}/initrd"; destination = "/opt/edgeless/share/kata-initrd.zst"; } ]; @@ -216,16 +226,19 @@ let ]; }; - layers = [ - installer-config - kata-container-img - ovmf-snp - ovmf-tdx - qemu-snp - qemu-tdx - kata-runtime - nydus - ]; + layers = + [ + installer-config + kata-container-img + kata-runtime + ovmf-snp + qemu-snp + nydus + ] + ++ lib.optionals (!withGPU) [ + qemu-tdx + ovmf-tdx + ]; manifest = ociImageManifest { layers = layers ++ [ node-installer ]; @@ -251,10 +264,14 @@ in ociImageLayout { manifests = [ manifest ]; passthru = { - inherit debugRuntime; + inherit debugRuntime os-image; runtimeHash = hashDirs { dirs = layers; # Layers without node-installer, or we have a circular dependency! name = "runtime-hash-kata"; }; + gpu = kata.contrast-node-installer-image.override { + inherit debugRuntime; + withGPU = true; + }; }; } diff --git a/packages/by-name/kata/snp-launch-digest/package.nix b/packages/by-name/kata/snp-launch-digest/package.nix index f3759664cc..4cc035acfa 100644 --- a/packages/by-name/kata/snp-launch-digest/package.nix +++ b/packages/by-name/kata/snp-launch-digest/package.nix @@ -4,30 +4,31 @@ { lib, stdenvNoCC, - kata, OVMF-SNP, python3Packages, + kata, debug ? false, + os-image ? kata.kata-image, }: let ovmf-snp = "${OVMF-SNP}/FV/OVMF.fd"; - kernel = "${kata.kata-image}/bzImage"; - initrd = "${kata.kata-image}/initrd"; + kernel = "${os-image}/bzImage"; + initrd = "${os-image}/initrd"; # Kata uses a base command line and then appends the command line from the kata config (i.e. also our node-installer config). # Thus, we need to perform the same steps when calculating the digest. baseCmdline = if debug then kata.kata-runtime.cmdline.debug else kata.kata-runtime.cmdline.default; cmdline = lib.strings.concatStringsSep " " [ baseCmdline - kata.kata-image.cmdline + os-image.cmdline ]; in stdenvNoCC.mkDerivation { name = "snp-launch-digest${lib.optionalString debug "-debug"}"; - inherit (kata.kata-image) version; + inherit (os-image) version; dontUnpack = true; diff --git a/packages/containers.nix b/packages/containers.nix index 533448c943..1196aca850 100644 --- a/packages/containers.nix +++ b/packages/containers.nix @@ -198,5 +198,9 @@ containers push-node-installer-kata = pushOCIDir "push-node-installer-kata" pkgs.kata.contrast-node-installer-image "v${pkgs.contrast.version}"; + push-node-installer-kata-gpu = pushOCIDir "push-node-installer-kata-gpu" ( + pkgs.kata.contrast-node-installer-image.override + { withGPU = true; } + ) "v${pkgs.contrast.version}"; } // (lib.concatMapAttrs (name: container: { "push-${name}" = pushContainer container; }) containers) diff --git a/packages/nixos/kata.nix b/packages/nixos/kata.nix index 0443215e9d..aa56656d98 100644 --- a/packages/nixos/kata.nix +++ b/packages/nixos/kata.nix @@ -78,7 +78,13 @@ in }; # Not used directly, but required for kernel-specific driver builds. - boot.kernelPackages = pkgs.recurseIntoAttrs (pkgs.linuxPackagesFor pkgs.kata-kernel-uvm); + boot.kernelPackages = pkgs.recurseIntoAttrs ( + pkgs.linuxPackagesFor ( + pkgs.kata-kernel-uvm.override { + withGPU = config.contrast.gpu.enable; + } + ) + ); boot.initrd = { # Don't require TPM2 support. (additional modules)