diff --git a/.ci/ci_job_flags.sh b/.ci/ci_job_flags.sh index ee8519994..8d5a52536 100755 --- a/.ci/ci_job_flags.sh +++ b/.ci/ci_job_flags.sh @@ -106,7 +106,7 @@ case "${CI_JOB}" in export CRI_RUNTIME="containerd" export KATA_HYPERVISOR="qemu" ;; -"CRI_CONTAINERD"|"CRI_CONTAINERD_K8S"|"CC_CRI_CONTAINERD"|"CC_CRI_CONTAINERD_K8S"|"CC_SEV_CRI_CONTAINERD_K8S") +"CRI_CONTAINERD"|"CRI_CONTAINERD_K8S"|"CC_CRI_CONTAINERD"|"CC_CRI_CONTAINERD_K8S") # This job only tests containerd + k8s init_ci_flags export CRI_CONTAINERD="yes" @@ -116,7 +116,7 @@ case "${CI_JOB}" in "CRI_CONTAINERD_K8S") export KUBERNETES="yes" ;; - "CC_CRI_CONTAINERD"|"CC_CRI_CONTAINERD_K8S"|"CC_SEV_CRI_CONTAINERD_K8S") + "CC_CRI_CONTAINERD"|"CC_CRI_CONTAINERD_K8S") # Export any CC specific environment variables export KATA_BUILD_CC="yes" export MEASURED_ROOTFS="yes" @@ -124,14 +124,31 @@ case "${CI_JOB}" in if [[ "${CI_JOB}" =~ K8S ]]; then export KUBERNETES=yes fi - if [[ "${CI_JOB}" =~ SEV ]]; then - export TEE_TYPE="sev" - export AA_KBC="online_sev_kbc" - export TEST_INITRD="yes" - fi ;; esac ;; +"CC_SEV_CRI_CONTAINERD_K8S"|"CC_SNP_CRI_CONTAINERD_K8S") + init_ci_flags + export CRI_CONTAINERD="yes" + export CRI_RUNTIME="containerd" + export KATA_HYPERVISOR="qemu" + export KATA_BUILD_CC="yes" + export AA_KBC="offline_fs_kbc" + export TEST_INITRD="yes" + if [[ "${CI_JOB}" =~ K8S ]]; then + export KUBERNETES=yes + fi + if [[ "${CI_JOB}" =~ SEV ]]; then + export TEE_TYPE="sev" + export AA_KBC="online_sev_kbc" + export KATA_BUILD_KERNEL_TYPE="sev" + fi + if [[ "${CI_JOB}" =~ SNP ]]; then + export TEE_TYPE="snp" + export KATA_BUILD_QEMU_TYPE="snp" + export KATA_BUILD_KERNEL_TYPE="sev" + fi + ;; "CC_CRI_CONTAINERD_TDX_QEMU"|"CC_CRI_CONTAINERD_TDX_CLOUD_HYPERVISOR") init_ci_flags export CRI_CONTAINERD="yes" diff --git a/.ci/install_kata.sh b/.ci/install_kata.sh index 38a56b1ba..3af670502 100755 --- a/.ci/install_kata.sh +++ b/.ci/install_kata.sh @@ -33,6 +33,11 @@ if [ "${TEE_TYPE:-}" == "sev" ]; then KATA_BUILD_KERNEL_TYPE=sev fi +if [ "${TEE_TYPE:-}" == "snp" ]; then + KATA_BUILD_KERNEL_TYPE=snp + KATA_BUILD_QEMU_TYPE="${KATA_BUILD_QEMU_TYPE:-snp}" +fi + if [ "${KATA_HYPERVISOR:-}" == "dragonball" ]; then KATA_BUILD_KERNEL_TYPE=dragonball fi @@ -89,6 +94,8 @@ case "${KATA_HYPERVISOR}" in "${cidir}/install_tdvf.sh" elif [ "${TEE_TYPE:-}" == "sev" ]; then "${cidir}/install_ovmf_sev.sh" + elif [ "${TEE_TYPE:-}" == "snp" ]; then + "${cidir}/install_ovmf_x86_64.sh" fi ;; "dragonball") diff --git a/.ci/install_kata_image.sh b/.ci/install_kata_image.sh index 00866dfec..fd7ec521c 100755 --- a/.ci/install_kata_image.sh +++ b/.ci/install_kata_image.sh @@ -21,7 +21,9 @@ TEE_TYPE="${TEE_TYPE:-}" build_image_for_cc () { if [ "${TEST_INITRD}" == "yes" ]; then - [ "${TEE_TYPE}" == "sev" ] || die "SEV is the only TEE type that supports initrd" + if [ "${TEE_TYPE}" != "sev" ] && [ "${TEE_TYPE}" != "snp" ]; then + die "SEV and SNP are the only TEE types that supports initrd" + fi build_static_artifact_and_install "sev-rootfs-initrd" else [ "${osbuilder_distro:-ubuntu}" == "ubuntu" ] || \ diff --git a/.ci/install_kata_kernel.sh b/.ci/install_kata_kernel.sh index 32a275d8b..16939e43a 100755 --- a/.ci/install_kata_kernel.sh +++ b/.ci/install_kata_kernel.sh @@ -34,9 +34,12 @@ build_and_install_kernel_for_cc() { local artifact="kernel" case "$kernel_type" in - tdx|sev) + tdx) artifact="${kernel_type}-${artifact}" ;; + sev|snp) + artifact="sev-${artifact}" + ;; vanilla) ;; *) die_unsupported_kernel_type "$kernel_type" @@ -67,7 +70,7 @@ Usage: Options: -d : Enable bash debug. -h : Display this help. - -t : kernel type, such as vanilla, experimental, dragonball, etc + -t : kernel type, such as vanilla, experimental, dragonball, tdx, sev, snp. EOF exit "$exit_code" } diff --git a/.ci/install_ovmf_sev.sh b/.ci/install_ovmf_sev.sh index 46a9a96b0..51cdc0c9b 100755 --- a/.ci/install_ovmf_sev.sh +++ b/.ci/install_ovmf_sev.sh @@ -21,7 +21,6 @@ main() { pushd $katacontainers_repo_dir sudo -E PATH=$PATH bash ${buildscript} --build=cc-sev-ovmf sudo tar -xvJpf build/kata-static-cc-sev-ovmf.tar.xz -C / - sudo ln -sf /opt/confidential-containers/share/ovmf /usr/share/ovmf popd } diff --git a/.ci/install_ovmf_x86_64.sh b/.ci/install_ovmf_x86_64.sh new file mode 100755 index 000000000..f00cc5c50 --- /dev/null +++ b/.ci/install_ovmf_x86_64.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# Copyright 2022 Advanced Micro Devices, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +set -o errexit +set -o nounset +set -o pipefail +set -o errtrace + +cidir=$(dirname "$0") +source "${cidir}/lib.sh" + +main() { + build_static_artifact_and_install x86_64-ovmf +} + +main "$@" diff --git a/.ci/install_qemu.sh b/.ci/install_qemu.sh index 09e9afde0..7dda7041a 100755 --- a/.ci/install_qemu.sh +++ b/.ci/install_qemu.sh @@ -29,7 +29,7 @@ build_and_install_qemu_for_cc() { local artifact="qemu" case "${qemu_type}" in - tdx) + tdx|snp) artifact="${qemu_type}-${artifact}" ;; vanilla) ;; @@ -109,6 +109,12 @@ main() { export qemu_type case "${qemu_type}" in + snp) + CURRENT_QEMU_VERSION=$(get_version "assets.hypervisor.qemu-snp-experimental.tag") + QEMU_REPO_URL=$(get_version "assets.hypervisor.qemu-snp-experimental.url") + qemu_latest_build_url="${jenkins_url}/job/kata-containers-2.0-qemu-snp-$(uname -m)/${cached_artifacts_path}" + qemu_type="snp-qemu" + ;; vanilla) qemu_type="qemu" ;; diff --git a/.ci/install_runtime.sh b/.ci/install_runtime.sh index 2d0cb7daf..5be2f3933 100755 --- a/.ci/install_runtime.sh +++ b/.ci/install_runtime.sh @@ -134,6 +134,8 @@ case "${KATA_HYPERVISOR}" in enable_hypervisor_config "${PKGDEFAULTSDIR}/configuration-qemu-tdx.toml" elif [ "$TEE_TYPE" == "sev" ]; then enable_hypervisor_config "${PKGDEFAULTSDIR}/configuration-qemu-sev.toml" + elif [ "$TEE_TYPE" == "snp" ]; then + enable_hypervisor_config "${PKGDEFAULTSDIR}/configuration-qemu-snp.toml" elif [ "$TEE_TYPE" == "se" ]; then enable_hypervisor_config "${PKGDEFAULTSDIR}/configuration-qemu-se.toml" else diff --git a/.ci/lib.sh b/.ci/lib.sh index 3b11dbf1e..27238baa4 100755 --- a/.ci/lib.sh +++ b/.ci/lib.sh @@ -56,7 +56,6 @@ if [ "$(uname -m)" == "s390x" ] && grep -Eq "\<(fedora|suse)\>" /etc/os-release export CC=gcc fi -tests_repo="${tests_repo:-github.com/kata-containers/tests}" lib_script="${GOPATH}/src/${tests_repo}/lib/common.bash" source "${lib_script}" diff --git a/.ci/run.sh b/.ci/run.sh index ad21119c2..e14eac16b 100755 --- a/.ci/run.sh +++ b/.ci/run.sh @@ -79,6 +79,10 @@ case "${CI_JOB}" in info "Running Confidential Containers tests for AMD SEV" sudo -E PATH="$PATH" CRI_RUNTIME="containerd" bash -c "make cc-sev-kubernetes" ;; + "CC_SNP_CRI_CONTAINERD_K8S") + info "Running Confidential Containers tests for AMD SEV-SNP" + sudo -E PATH="$PATH" CRI_RUNTIME="containerd" bash -c "make cc-snp-kubernetes" + ;; "CC_CRI_CONTAINERD_K8S"|"CC_CRI_CONTAINERD_K8S_TDX_QEMU"|"CC_CRI_CONTAINERD_K8S_SE_QEMU"|"CC_CRI_CONTAINERD_K8S_TDX_CLOUD_HYPERVISOR") info "Running Confidential Container tests" sudo -E PATH="$PATH" CRI_RUNTIME="containerd" bash -c "make cc-kubernetes" diff --git a/Makefile b/Makefile index 6e48a8283..c32c0977b 100644 --- a/Makefile +++ b/Makefile @@ -113,6 +113,12 @@ cc-sev-kubernetes: K8S_TEST_UNION="confidential/sev.bats" \ bash integration/kubernetes/run_kubernetes_tests.sh +# Run the Confidential Containers AMD SNP specific tests. +cc-snp-kubernetes: + bash -f .ci/install_bats.sh + K8S_TEST_UNION="confidential/snp.bats" \ + bash integration/kubernetes/run_kubernetes_tests.sh + log-parser: make -C cmd/log-parser diff --git a/integration/confidential/lib.sh b/integration/confidential/lib.sh index ba584fa86..9a085c638 100644 --- a/integration/confidential/lib.sh +++ b/integration/confidential/lib.sh @@ -204,7 +204,7 @@ configure_cc_containerd() { sudo systemctl stop containerd sleep 5 [ -n "$saved_containerd_conf_file" ] && \ - cp -f "$containerd_conf_file" "$saved_containerd_conf_file" + sudo cp -f "$containerd_conf_file" "$saved_containerd_conf_file" sudo systemctl start containerd waitForProcess 30 5 "sudo crictl info >/dev/null" @@ -353,3 +353,90 @@ setup_credentials_files() { CREDENTIAL="${auth_json}" envsubst < "${SHARED_FIXTURES_DIR}/offline-fs-kbc/aa-offline_fs_kbc-resources.json.in" > "${dest_file}" cp_to_guest_img "etc" "${dest_file}" } + +############################################################################### + +# simple-kbs + +SIMPLE_KBS_DIR="${SIMPLE_KBS_DIR:-/tmp/simple-kbs}" +KBS_DB_USER="${KBS_DB_USER:-kbsuser}" +KBS_DB_PW="${KBS_DB_PW:-kbspassword}" +KBS_DB="${KBS_DB:-simple_kbs}" +#KBS_DB_TYPE="{KBS_DB_TYPE:-mysql}" + +# Run the simple-kbs +simple_kbs_run() { + # Retrieve simple-kbs repo and tag from versions.yaml + local simple_kbs_url=$(get_test_version "externals.simple-kbs.url") + local simple_kbs_tag=$(get_test_version "externals.simple-kbs.tag") + + # Cleanup and create installation directory + esudo rm -rf "${SIMPLE_KBS_DIR}" + mkdir -p "${SIMPLE_KBS_DIR}" + pushd "${SIMPLE_KBS_DIR}" + + # Clone and run + git clone "${simple_kbs_url}" --branch main + pushd simple-kbs + + # Checkout, build and start + git checkout -b "branch_${simple_kbs_tag}" "${simple_kbs_tag}" + esudo docker-compose build + esudo docker-compose up -d + + # Wait for simple-kbs to start + waitForProcess 15 1 "esudo docker-compose top | grep -q simple-kbs" + popd + + # Get simple-kbs database container ip + local kbs_db_host=$(simple_kbs_get_db_ip) + + # Confirm connection to the database is possible + waitForProcess 5 1 "mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} -e '\q'" + popd +} + +# Stop simple-kbs and database containers +simple_kbs_stop() { + (cd ${SIMPLE_KBS_DIR}/simple-kbs && esudo docker-compose down 2>/dev/null) +} + +# Delete all test inserted data in the simple-kbs +simple_kbs_delete_data() { + # Get simple-kbs database container ip + local kbs_db_host=$(simple_kbs_get_db_ip) + + # Delete all data with 'id = 10' + mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} <&2 echo "Cannot find ${versions_file}"; return 1) - - # Install yq - #"${TESTS_REPO_DIR}/.ci/install_yq.sh" >&2 - - # Parse versions file with yq for dependency - result=$("${GOPATH}/bin/yq" r -X "$versions_file" "$dependency") - [ "$result" = "null" ] && result="" - echo "$result" -} - -generate_service_yaml() { - local name="${1}" - local image="${2}" - - # Default policy is 3: - # - NODBG (1): Debugging of the guest is disallowed when set - # - NOKS (2): Sharing keys with other guests is disallowed when set - local policy="${3:-3}" - - local kbs_ip="$(ip -o route get to 8.8.8.8 | sed -n 's/.*src \([0-9.]\+\).*/\1/p')" - local service_yaml_template="${FIXTURES_DIR}/service.yaml.in" - - local service_yaml="${TEST_DIR}/${name}.yaml" - rm -f "${service_yaml}" - - NAME="${name}" IMAGE="${image}" RUNTIMECLASS="${RUNTIMECLASS}" \ - KBS_URI="${kbs_ip}:44444" \ - POLICY="$policy" \ - envsubst < "${service_yaml_template}" > "${service_yaml}" -} - -# Wait until the pod is 'Ready'. Fail if it hits the timeout. -kubernetes_wait_for_pod_ready_state() { - local pod_name="${1}" - local wait_time="${2:-60}" - - esudo kubectl wait --for=condition=ready pod/$pod_name --timeout=${wait_time}s -} - -# Wait until the pod is 'Deleted'. Fail if it hits the timeout. -kubernetes_wait_for_pod_delete_state() { - local pod_name="${1}" - local wait_time="${2:-60}" - - esudo kubectl wait --for=delete pod/$pod_name --timeout=${wait_time}s -} - -# Find container id -get_container_id() { - local pod_name="${1}" - - # Get container id from pod info - local container_id=$(esudo kubectl get pod "${pod_name}" \ - -o jsonpath='{.status.containerStatuses..containerID}' \ - | sed "s|containerd://||g") - - echo "${container_id}" -} - -# Find sandbox id using container ID -get_sandbox_id() { - local container_id="${1}" - local sandbox_dir="/run/kata-containers/shared/sandboxes" - - # Find container directory inside sandbox directory - local container_dir=$(esudo find "${sandbox_dir}" -name "${container_id}" | head -1) - - # Ensure directory path pattern is correct - [[ ${container_dir} =~ ^${sandbox_dir}/.*/.*/${container_id} ]] \ - || (>&2 echo "Incorrect container folder path: ${container_dir}"; return 1) - - # Two levels up, and trim the sandbox dir off the front - local sandbox_id=$(dirname $(dirname "${container_dir}") | sed "s|${sandbox_dir}/||g") - - echo "${sandbox_id}" -} - -# Get guest kernel append from qemu command line -get_guest_kernel_append() { - local pod_name="${1}" - local duration=$((SECONDS+20)) - local kernel_append - - # Attempt to get qemu command line from qemu process - while [ $SECONDS -lt $duration ]; do - container_id=$(get_container_id "${pod_name}") - sandbox_id=$(get_sandbox_id "${container_id}") - qemu_process=$(ps aux | grep qemu | grep ${sandbox_id} | grep append || true) - if [ -n "${qemu_process}" ]; then - kernel_append=$(echo ${qemu_process} \ - | sed "s|.*-append \(.*$\)|\1|g" \ - | sed "s| -.*$||") - break - fi - sleep 1 +load "${TESTS_REPO_DIR}/integration/kubernetes/lib.sh" +load "${TESTS_REPO_DIR}/integration/kubernetes/confidential/lib.sh" + +# Delete all test services +k8s_delete_all() { + for file in $(ls "${TEST_DIR}/*.yaml") ; do + # Removing extension to get the pod name + local pod_name="${file%.*}" + kubernetes_delete_by_yaml "${pod_name}" "${TEST_DIR}/${file}" done - - [ -n "${kernel_append}" ] \ - || (>&2 echo "Could not retrieve guest kernel append parameters"; return 1) - - echo "${kernel_append}" -} - -# Delete pods -delete_pods() { - # Retrieve pod names - local encrypted_pod_name=$(esudo kubectl get pod -o wide | grep encrypted-image-tests | awk '{print $1;}' || true) - local unencrypted_pod_name=$(esudo kubectl get pod -o wide | grep unencrypted-image-tests | awk '{print $1;}' || true) - local encrypted_pod_name_es=$(esudo kubectl get pod -o wide | grep encrypted-image-tests-es | awk '{print $1;}' || true) - - # Delete both encrypted and unencrypted pods - esudo kubectl delete -f \ - "${TEST_DIR}/unencrypted-image-tests.yaml" 2>/dev/null || true - esudo kubectl delete -f \ - "${TEST_DIR}/encrypted-image-tests.yaml" 2>/dev/null || true - esudo kubectl delete -f \ - "${TEST_DIR}/encrypted-image-tests-es.yaml" 2>/dev/null || true - - [ -z "${encrypted_pod_name}" ] || (kubernetes_wait_for_pod_delete_state "${encrypted_pod_name}" || true) - [ -z "${unencrypted_pod_name}" ] || (kubernetes_wait_for_pod_delete_state "${unencrypted_pod_name}" || true) - [ -z "${encrypted_pod_name_es}" ] || (kubernetes_wait_for_pod_delete_state "${encrypted_pod_name_es}" || true) } -run_kbs() { - KBS_DIR="$(mktemp -d /tmp/kbs.XXXXXXXX)" - pushd "${KBS_DIR}" - - # Retrieve simple-kbs repo and tag from versions.yaml - local simple_kbs_url=$(get_version "externals.simple-kbs.url") - local simple_kbs_tag=$(get_version "externals.simple-kbs.tag") - - # Clone and run - git clone "${simple_kbs_url}" --branch main - - pushd simple-kbs - git checkout -b "branch_${simple_kbs_tag}" "${simple_kbs_tag}" - esudo docker-compose build - - esudo docker-compose up -d - until docker-compose top | grep -q "simple-kbs" - do - echo "waiting for simple-kbs to start" - sleep 5 - done - popd - - # Set KBS_DB_HOST to kbs db container IP - KBS_DB_HOST=$(esudo docker network inspect simple-kbs_default \ - | jq -r '.[].Containers[] | select(.Name | test("simple-kbs[_-]db.*")).IPv4Address' \ - | sed "s|/.*$||g") - - waitForProcess 15 1 "mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${KBS_DB_HOST} -D ${KBS_DB} -e '\q'" - popd -} - -pull_unencrypted_image_and_set_keys() { - # Pull unencrypted test image to get labels - local unencrypted_image_url="${IMAGE_REPO}:unencrypted" - esudo docker pull "${unencrypted_image_url}" - - # Get encryption key from docker image label - ENCRYPTION_KEY=$(esudo docker inspect ${unencrypted_image_url} \ - | jq -r '.[0].Config.Labels.enc_key') - - # Get ssh key from docker image label and save to file - esudo docker inspect ${unencrypted_image_url} \ - | jq -r '.[0].Config.Labels.ssh_key' \ - | sed "s|\(-----BEGIN OPENSSH PRIVATE KEY-----\)|\1\n|g" \ - | sed "s|\(-----END OPENSSH PRIVATE KEY-----\)|\n\1|g" \ - > "${SSH_KEY_FILE}" - - # Set permissions on private key file - chmod 600 "${SSH_KEY_FILE}" -} - -generate_firmware_measurement_with_append() { - - # Gather firmware locations and kernel append for measurement - local append="${1}" - local mode="${2:-sev}" - local vcpu_sig=$(cpuid -1 --leaf 0x1 --raw | cut -s -f2 -d= | cut -f1 -d" ") - local ovmf_path=$(grep "firmware = " $SEV_CONFIG | cut -d'"' -f2) - local kernel_path="$(esudo /opt/confidential-containers/bin/kata-runtime \ - --config ${SEV_CONFIG} kata-env --json | jq -r .Kernel.Path)" - local initrd_path="$(esudo /opt/confidential-containers/bin/kata-runtime \ - --config ${SEV_CONFIG} kata-env --json | jq -r .Initrd.Path)" - - # Return error if files don't exist - [ -f "${ovmf_path}" ] || return 1 - [ -f "${kernel_path}" ] || return 1 - [ -f "${initrd_path}" ] || return 1 - - # Generate digest from sev-snp-measure output - this also inserts measurement values inside OVMF image - measurement=$(PATH="${PATH}:${HOME}/.local/bin" sev-snp-measure \ - --mode="${mode}" \ - --vcpus=1 \ - --vcpu-sig="${vcpu_sig}" \ - --output-format=base64 \ - --ovmf="${ovmf_path}" \ - --kernel="${kernel_path}" \ - --initrd="${initrd_path}" \ - --append="${append}" \ - ) - if [[ -z "${measurement}" ]]; then return 1; fi - echo ${measurement} -} - -add_key_to_kbs_db() { - measurement=${1} - - # Add key and keyset to DB; If set, add policy with measurement to DB - if [ -n "${measurement}" ]; then - mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${KBS_DB_HOST} -D ${KBS_DB} <&2 - SAVED_CONTAINERD_CONF_FILE="/etc/containerd/config.toml.$$" - configure_cc_containerd "$SAVED_CONTAINERD_CONF_FILE" + # Configure CoCo settings in containerd config + local saved_containerd_conf_file="/etc/containerd/config.toml.$$" + configure_cc_containerd "${saved_containerd_conf_file}" # KBS setup and run echo "Setting up simple-kbs..." - run_kbs + simple_kbs_run # Pull unencrypted image and retrieve encryption and ssh keys - echo "Pulling unencrypted image and setting keys..." - pull_unencrypted_image_and_set_keys + echo "Pulling unencrypted image and retrieving keys..." + ENCRYPTION_KEY=$(docker_image_label_get_encryption_key "${UNENCRYPTED_IMAGE_URL}") + docker_image_label_save_ssh_key "${UNENCRYPTED_IMAGE_URL}" "${SSH_KEY_FILE}" - generate_service_yaml "unencrypted-image-tests" "${IMAGE_REPO}:unencrypted" - generate_service_yaml "encrypted-image-tests" "${IMAGE_REPO}:multi-arch-encrypted" + # Get host ip and set as simple-kbs ip; uri uses default port 44444 + # These values will be set as k8s annotations in the service yamls + local kbs_ip="$(ip -o route get to 8.8.8.8 | sed -n 's/.*src \([0-9.]\+\).*/\1/p')" + local kbs_uri="${kbs_ip}:44444" + # SEV unencrypted service yaml generation + kubernetes_generate_service_yaml "${TEST_DIR}/sev-unencrypted.yaml" "${IMAGE_REPO}:unencrypted" + kubernetes_yaml_set_annotation "${TEST_DIR}/sev-unencrypted.yaml" "io.katacontainers.config.guest_pre_attestation.enabled" "false" + + # SEV encrypted service yaml generation + # SEV policy is 3 (default): + # - NODBG (1): Debugging of the guest is disallowed when set + # - NOKS (2): Sharing keys with other guests is disallowed when set + kubernetes_generate_service_yaml "${TEST_DIR}/sev-encrypted.yaml" "${IMAGE_REPO}:multi-arch-encrypted" + kubernetes_yaml_set_annotation "${TEST_DIR}/sev-encrypted.yaml" "io.katacontainers.config.pre_attestation.uri" "${kbs_uri}" + kubernetes_yaml_set_annotation "${TEST_DIR}/sev-encrypted.yaml" "io.katacontainers.config.sev.policy" "3" + # SEV-ES policy is 7: # - NODBG (1): Debugging of the guest is disallowed when set # - NOKS (2): Sharing keys with other guests is disallowed when set # - ES (4): SEV-ES is required when set - generate_service_yaml "encrypted-image-tests-es" "${IMAGE_REPO}:multi-arch-encrypted" "7" + kubernetes_generate_service_yaml "${TEST_DIR}/sev-es-encrypted.yaml" "${IMAGE_REPO}:multi-arch-encrypted" + kubernetes_yaml_set_annotation "${TEST_DIR}/sev-es-encrypted.yaml" "io.katacontainers.config.pre_attestation.uri" "${kbs_uri}" + kubernetes_yaml_set_annotation "${TEST_DIR}/sev-es-encrypted.yaml" "io.katacontainers.config.sev.policy" "7" +} + +teardown_file() { + # Allow to not destroy the environment if you are developing/debugging tests + if [[ "${CI:-false}" == "false" && "${DEBUG:-}" == true ]]; then + echo "Leaving changes and created resources untouched" + return + fi + + # Remove all k8s test services + k8s_delete_all + + # Stop the simple-kbs + simple_kbs_stop - echo "SETUP FILE - COMPLETE" - echo "###############################################################################" + # Cleanup directories + esudo rm -rf "${SIMPLE_KBS_DIR}" + esudo rm -rf "${TEST_DIR}" } setup() { - # Remove the service/deployment/pod if it exists - echo "Deleting previous test pods..." - delete_pods - - # Delete any previous data in the DB - mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${KBS_DB_HOST} -D ${KBS_DB} <&2 echo -e "${RED}KATA CC TEST - FAIL: SEV is NOT Enabled${NC}" + local sev_enabled=$(ssh_dmesg_grep \ + "${SSH_KEY_FILE}" \ + "${pod_ip}" \ + "${SEV_DMESG_GREP_TEXT}") + + if [ -z "${sev_enabled}" ]; then + >&2 echo -e "KATA SEV TEST - FAIL: SEV is NOT Enabled" return 1 else - echo "DMESG REPORT: $sev_enabled" - echo -e "${GREEN}KATA CC TEST - PASS: SEV is Enabled${NC}" + echo "DMESG REPORT: ${sev_enabled}" + echo -e "KATA SEV TEST - PASS: SEV is Enabled" fi - } -@test "$test_tag Test SEV encrypted container launch failure with INVALID measurement" { - # Make sure pre-attestation is enabled. - esudo sed -i 's/guest_pre_attestation = false/guest_pre_attestation = true/g' ${SEV_CONFIG} - +@test "${TEST_TAG} Test SEV encrypted container launch failure with INVALID measurement" { # Generate firmware measurement local append="INVALID-INPUT" - measurement=$(generate_firmware_measurement_with_append ${append}) + local measurement=$(generate_firmware_measurement_with_append "${SEV_CONFIG_FILE}" "${append}") echo "Firmware Measurement: ${measurement}" # Add key to KBS with policy measurement - add_key_to_kbs_db ${measurement} + simple_kbs_add_key_to_db "${ENCRYPTION_KEY}" "${measurement}" # Start the service/deployment/pod - esudo kubectl apply -f "${TEST_DIR}/encrypted-image-tests.yaml" + esudo kubectl apply -f "${TEST_DIR}/sev-encrypted.yaml" # Retrieve pod name, wait for it to fail - pod_name=$(esudo kubectl get pod -o wide | grep encrypted-image-tests | awk '{print $1;}') + local pod_name=$(esudo kubectl get pod -o wide | grep sev-encrypted | awk '{print $1;}') kubernetes_wait_for_pod_ready_state "$pod_name" 20 || true - print_service_info + kubernetes_print_info "sev-encrypted" # Save guest qemu kernel append to file - kernel_append=$(get_guest_kernel_append "${pod_name}") + local kernel_append=$(kata_get_guest_kernel_append "${pod_name}") echo "${kernel_append}" > "${TEST_DIR}/guest-kernel-append" echo "Kernel Append Retrieved from QEMU Process: ${kernel_append}" # Get pod info - pod_info=$(esudo kubectl describe pod ${pod_name}) + local pod_info=$(esudo kubectl describe pod ${pod_name}) # Check failure condition if [[ ! ${pod_info} =~ "Failed to pull image" ]]; then - >&2 echo -e "${RED}TEST - FAIL${NC}" + >&2 echo -e "TEST - FAIL" return 1 else echo "Pod message contains: Failed to pull image" - echo -e "${GREEN}TEST - PASS${NC}" + echo -e "TEST - PASS" fi } -@test "$test_tag Test SEV encrypted container launch success with NO measurement" { - +@test "${TEST_TAG} Test SEV encrypted container launch success with NO measurement" { # Add key to KBS without a policy measurement - add_key_to_kbs_db + simple_kbs_add_key_to_db "${ENCRYPTION_KEY}" # Start the service/deployment/pod - esudo kubectl apply -f "${TEST_DIR}/encrypted-image-tests.yaml" + esudo kubectl apply -f "${TEST_DIR}/sev-encrypted.yaml" # Retrieve pod name, wait for it to come up, retrieve pod ip - pod_name=$(esudo kubectl get pod -o wide | grep encrypted-image-tests | awk '{print $1;}') + local pod_name=$(esudo kubectl get pod -o wide | grep sev-encrypted | awk '{print $1;}') kubernetes_wait_for_pod_ready_state "$pod_name" 20 - pod_ip=$(esudo kubectl get pod -o wide | grep encrypted-image-tests | awk '{print $6;}') + local pod_ip=$(esudo kubectl get pod -o wide | grep sev-encrypted | awk '{print $6;}') - print_service_info + kubernetes_print_info "sev-encrypted" # Look for SEV enabled in container dmesg output - sev_enabled=$(ssh -i ${SSH_KEY_FILE} \ - -o "StrictHostKeyChecking no" \ - -o "PasswordAuthentication=no" \ - -t root@${pod_ip} \ - 'dmesg | grep SEV' || true) - - if [ -z "$sev_enabled" ]; then - >&2 echo -e "${RED}KATA CC TEST - FAIL: SEV is NOT Enabled${NC}" + local sev_enabled=$(ssh_dmesg_grep \ + "${SSH_KEY_FILE}" \ + "${pod_ip}" \ + "${SEV_DMESG_GREP_TEXT}") + + if [ -z "${sev_enabled}" ]; then + >&2 echo -e "KATA SEV TEST - FAIL: SEV is NOT Enabled" return 1 else - echo "DMESG REPORT: $sev_enabled" - echo -e "${GREEN}KATA CC TEST - PASS: SEV is Enabled${NC}" + echo "DMESG REPORT: ${sev_enabled}" + echo -e "KATA SEV TEST - PASS: SEV is Enabled" fi } -@test "$test_tag Test SEV encrypted container launch success with VALID measurement" { - +@test "${TEST_TAG} Test SEV encrypted container launch success with VALID measurement" { # Generate firmware measurement local append=$(cat ${TEST_DIR}/guest-kernel-append) echo "Kernel Append: ${append}" - measurement=$(generate_firmware_measurement_with_append "${append}") + local measurement=$(generate_firmware_measurement_with_append "${SEV_CONFIG_FILE}" "${append}") echo "Firmware Measurement: ${measurement}" # Add key to KBS with policy measurement - add_key_to_kbs_db ${measurement} + simple_kbs_add_key_to_db "${ENCRYPTION_KEY}" "${measurement}" # Start the service/deployment/pod - esudo kubectl apply -f "${TEST_DIR}/encrypted-image-tests.yaml" + esudo kubectl apply -f "${TEST_DIR}/sev-encrypted.yaml" # Retrieve pod name, wait for it to come up, retrieve pod ip - pod_name=$(esudo kubectl get pod -o wide | grep encrypted-image-tests | awk '{print $1;}') + local pod_name=$(esudo kubectl get pod -o wide | grep sev-encrypted | awk '{print $1;}') kubernetes_wait_for_pod_ready_state "$pod_name" 20 - pod_ip=$(esudo kubectl get pod -o wide | grep encrypted-image-tests | awk '{print $6;}') + local pod_ip=$(esudo kubectl get pod -o wide | grep sev-encrypted | awk '{print $6;}') - print_service_info + kubernetes_print_info "sev-encrypted" # Look for SEV enabled in container dmesg output - sev_enabled=$(ssh -i ${SSH_KEY_FILE} \ - -o "StrictHostKeyChecking no" \ - -o "PasswordAuthentication=no" \ - -t root@${pod_ip} \ - 'dmesg | grep SEV' || true) - - if [ -z "$sev_enabled" ]; then - >&2 echo -e "${RED}KATA CC TEST - FAIL: SEV is NOT Enabled${NC}" + local sev_enabled=$(ssh_dmesg_grep \ + "${SSH_KEY_FILE}" \ + "${pod_ip}" \ + "${SEV_DMESG_GREP_TEXT}") + + if [ -z "${sev_enabled}" ]; then + >&2 echo -e "KATA SEV TEST - FAIL: SEV is NOT Enabled" return 1 else - echo "DMESG REPORT: $sev_enabled" - echo -e "${GREEN}KATA CC TEST - PASS: SEV is Enabled${NC}" + echo "DMESG REPORT: ${sev_enabled}" + echo -e "KATA SEV TEST - PASS: SEV is Enabled" fi } -@test "$test_tag Test SEV-ES encrypted container launch success with VALID measurement" { - +@test "${TEST_TAG} Test SEV-ES encrypted container launch success with VALID measurement" { # Generate firmware measurement local append=$(cat ${TEST_DIR}/guest-kernel-append) echo "Kernel Append: ${append}" - measurement=$(generate_firmware_measurement_with_append "${append}" "seves") + local measurement=$(generate_firmware_measurement_with_append "${SEV_CONFIG_FILE}" "${append}" "seves") echo "Firmware Measurement: ${measurement}" # Add key to KBS with policy measurement - add_key_to_kbs_db ${measurement} + simple_kbs_add_key_to_db "${ENCRYPTION_KEY}" "${measurement}" # Start the service/deployment/pod - esudo kubectl apply -f "${TEST_DIR}/encrypted-image-tests-es.yaml" + esudo kubectl apply -f "${TEST_DIR}/sev-es-encrypted.yaml" # Retrieve pod name, wait for it to come up, retrieve pod ip - pod_name=$(esudo kubectl get pod -o wide | grep encrypted-image-tests-es | awk '{print $1;}') + local pod_name=$(esudo kubectl get pod -o wide | grep sev-es-encrypted | awk '{print $1;}') kubernetes_wait_for_pod_ready_state "$pod_name" 20 - pod_ip=$(esudo kubectl get pod -o wide | grep encrypted-image-tests-es | awk '{print $6;}') + local pod_ip=$(esudo kubectl get pod -o wide | grep sev-es-encrypted | awk '{print $6;}') - print_service_info + kubernetes_print_info "sev-es-encrypted" # Look for SEV-ES enabled in container dmesg output - seves_enabled=$(ssh -i ${SSH_KEY_FILE} \ - -o "StrictHostKeyChecking no" \ - -o "PasswordAuthentication=no" \ - -t root@${pod_ip} \ - 'dmesg | grep SEV-ES' || true) - - if [ -z "$seves_enabled" ]; then - >&2 echo -e "${RED}KATA CC TEST - FAIL: SEV-ES is NOT Enabled${NC}" + local sev_es_enabled=$(ssh_dmesg_grep \ + "${SSH_KEY_FILE}" \ + "${pod_ip}" \ + "${SEV_ES_DMESG_GREP_TEXT}") + + if [ -z "${sev_es_enabled}" ]; then + >&2 echo -e "KATA SEV-ES TEST - FAIL: SEV-ES is NOT Enabled" return 1 else - echo "DMESG REPORT: $seves_enabled" - echo -e "${GREEN}KATA CC TEST - PASS: SEV-ES is Enabled${NC}" - fi -} - - - -teardown_file() { - echo "###############################################################################" - echo -e "TEARDOWN - STARTED\n" - - # Allow to not destroy the environment if you are developing/debugging tests - if [[ "${CI:-false}" == "false" && "${DEBUG:-}" == true ]]; then - echo "Leaving changes and created resources untouched" - return + echo "DMESG REPORT: ${sev_es_enabled}" + echo -e "KATA SEV-ES TEST - PASS: SEV-ES is Enabled" fi - - # Remove the service/deployment/pod - delete_pods - - # Stop KBS and KBS DB containers - (cd ${KBS_DIR}/simple-kbs && esudo docker-compose down 2>/dev/null) - - # Cleanup directories - esudo rm -rf "${KBS_DIR}" - esudo rm -rf "${TEST_DIR}" - - echo "TEARDOWN - COMPLETE" - echo "###############################################################################" } diff --git a/integration/kubernetes/confidential/snp.bats b/integration/kubernetes/confidential/snp.bats new file mode 100644 index 000000000..3f03ebda2 --- /dev/null +++ b/integration/kubernetes/confidential/snp.bats @@ -0,0 +1,95 @@ +#!/usr/bin/env bats +# Copyright 2023 Advanced Micro Devices, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# Environment variables +TEST_TAG="[cc][kubernetes][containerd][snp]" +TESTS_REPO_DIR=$(realpath "${BATS_TEST_DIRNAME}/../../..") +RUNTIMECLASS="${RUNTIMECLASS:-"kata"}" +IMAGE_REPO="ghcr.io/confidential-containers/test-container" +UNENCRYPTED_IMAGE_URL="${IMAGE_REPO}:unencrypted" + +# Text to grep for active feature in guest dmesg output +SNP_DMESG_GREP_TEXT="Memory Encryption Features active:.*SEV-SNP" + +export TEST_DIR +export ENCRYPTION_KEY +export SSH_KEY_FILE + +load "${BATS_TEST_DIRNAME}/../../confidential/lib.sh" +load "${TESTS_REPO_DIR}/lib/common.bash" +load "${TESTS_REPO_DIR}/integration/kubernetes/lib.sh" + +# Delete all test services +k8s_delete_all() { + kubernetes_delete_by_yaml "snp-unencrypted" "${TEST_DIR}/snp-unencrypted.yaml" +} + +setup_file() { + TEST_DIR="$(mktemp -d /tmp/test-kata-snp.XXXXXXXX)" + SSH_KEY_FILE="${TEST_DIR}/container-ssh-key" + + # Install package dependencies + echo "Installing required packages..." + esudo apt install -y jq + + # Configure CoCo settings in containerd config + local saved_containerd_conf_file="/etc/containerd/config.toml.$$" + configure_cc_containerd "${saved_containerd_conf_file}" + + # Pull unencrypted image and retrieve ssh keys + echo "Pulling unencrypted image and retrieve ssh key..." + docker_image_label_save_ssh_key "${UNENCRYPTED_IMAGE_URL}" "${SSH_KEY_FILE}" + + # SEV service yaml generation + kubernetes_generate_service_yaml "${TEST_DIR}/snp-unencrypted.yaml" "${IMAGE_REPO}:unencrypted" +} + +teardown_file() { + # Allow to not destroy the environment if you are developing/debugging tests + if [[ "${CI:-false}" == "false" && "${DEBUG:-}" == true ]]; then + echo "Leaving changes and created resources untouched" + return + fi + + # Remove all k8s test services + k8s_delete_all + + # Cleanup directories + esudo rm -rf "${TEST_DIR}" +} + +setup() { + # Remove any previous k8s test services + echo "Deleting previous test services..." + k8s_delete_all +} + + +@test "${TEST_TAG} Test SNP unencrypted container launch success" { + # Start the service/deployment/pod + esudo kubectl apply -f "${TEST_DIR}/snp-unencrypted.yaml" + + # Retrieve pod name, wait for it to come up, retrieve pod ip + local pod_name=$(esudo kubectl get pod -o wide | grep snp-unencrypted | awk '{print $1;}') + kubernetes_wait_for_pod_ready_state "$pod_name" 20 + local pod_ip=$(esudo kubectl get pod -o wide | grep snp-unencrypted | awk '{print $6;}') + + kubernetes_print_info "snp-unencrypted" + + # Look for SEV enabled in container dmesg output + local snp_enabled=$(ssh_dmesg_grep \ + "${SSH_KEY_FILE}" \ + "${pod_ip}" \ + "${SNP_DMESG_GREP_TEXT}") + + if [ -z "${snp_enabled}" ]; then + >&2 echo -e "KATA SNP TEST - FAIL: SNP is NOT Enabled" + return 1 + else + echo "DMESG REPORT: ${snp_enabled}" + echo -e "KATA SNP TEST - PASS: SNP is Enabled" + fi +} diff --git a/integration/kubernetes/lib.sh b/integration/kubernetes/lib.sh new file mode 100755 index 000000000..72b561a4c --- /dev/null +++ b/integration/kubernetes/lib.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +# Copyright 2023 Advanced Micro Devices, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +TESTS_REPO_DIR=$(realpath $(dirname "${BASH_SOURCE[0]}")/../..) +FIXTURES_DIR="${TESTS_REPO_DIR}/integration/kubernetes/confidential/fixtures" + +# Generate kubernetes service yaml from template +kubernetes_generate_service_yaml() { + local service_yaml="${1}" + local image="${2}" + + # Extract name from the file name + local name=$(basename "${service_yaml%.*}") + + local service_yaml_template="${FIXTURES_DIR}/service.yaml.in" + + NAME="${name}" IMAGE="${image}" RUNTIMECLASS="${RUNTIMECLASS}" \ + envsubst < "${service_yaml_template}" > "${service_yaml}" +} + +# Set annotation for yaml +kubernetes_yaml_set_annotation() { + local yaml="${1}" + local key="${2}" + local value="${3}" + + # yaml annotation key name + local annotation_key="spec.template.metadata.annotations.\"${key}\"" + + # yq set annotations in yaml + "${GOPATH}/bin/yq" w -i --style=double -d1 "${yaml}" "${annotation_key}" "${value}" +} + +# Wait until the pod is 'Ready'. Fail if it hits the timeout. +kubernetes_wait_for_pod_ready_state() { + local pod_name="${1}" + local wait_time="${2:-10}" + + kubectl wait --for=condition=ready "pod/${pod_name}" --timeout=${wait_time}s +} + +# Wait until the pod is 'Deleted'. Fail if it hits the timeout. +kubernetes_wait_for_pod_delete_state() { + local pod_name="${1}" + local wait_time="${2:-10}" + + kubectl wait --for=delete "pod/${pod_name}" --timeout=${wait_time}s +} + +# Find container id +kubernetes_get_container_id() { + local pod_name="${1}" + + # Get container id from pod info + local container_id=$(kubectl get pod "${pod_name}" \ + -o jsonpath='{.status.containerStatuses..containerID}' \ + | sed "s|containerd://||g") + + echo "${container_id}" +} + +# Delete k8s entity by yaml +kubernetes_delete_by_yaml() { + local partial_pod_name="${1}" + local yaml="${2}" + + # Retrieve pod name + local pod_name=$(kubectl get pod -o wide | grep ${partial_pod_name} | awk '{print $1;}' || true) + + # Delete by yaml + kubectl delete -f "${yaml}" 2>/dev/null || true + + # Verify pod deleted + [ -z "${pod_name}" ] || (kubernetes_wait_for_pod_delete_state "${pod_name}" || true) +} + +# Retrieve pod name and log kubernetes environment information: +# nodes, services, deployments, pods +kubernetes_print_info() { + local partial_pod_name="${1}" + + echo "-------------------------------------------------------------------------------" + kubectl get nodes -o wide + echo "-------------------------------------------------------------------------------" + kubectl get services -o wide + echo "-------------------------------------------------------------------------------" + kubectl get deployments -o wide + echo "-------------------------------------------------------------------------------" + kubectl get pods -o wide + echo "-------------------------------------------------------------------------------" + local pod_name=$(kubectl get pod -o wide | grep "${partial_pod_name}" | awk '{print $1;}') + kubectl describe pod "${pod_name}" + echo "-------------------------------------------------------------------------------" +} diff --git a/lib/common.bash b/lib/common.bash index 90ee83d05..afb5171a2 100755 --- a/lib/common.bash +++ b/lib/common.bash @@ -532,3 +532,117 @@ check_dockerfiles_images() build_dockerfile_image "$image" "$dockerfile_path" fi } + +############################################################################### + +# Misc functions. + +TESTS_REPO_DIR=$(realpath $(dirname "${BASH_SOURCE[0]}")/..) + +# sudo shortcut function with environment +esudo() { + sudo -E PATH=$PATH "${@}" +} + + +# SSH grep for a string in the target dmesg output +ssh_dmesg_grep() { + local private_key_file="${1}" + local pod_ip="${2}" + local grep_text="${3}" + + ssh -i ${private_key_file} \ + -o "StrictHostKeyChecking no" \ + -o "PasswordAuthentication=no" \ + -o ConnectTimeout=1 \ + -t root@${pod_ip} \ + "dmesg 2>&1 | grep \"${grep_text}\"" || true +} + + +############################################################################### + +# docker Image Handling + +# Retrieve encryption key from docker image label +# 'enc_key' image label must be set when image created or uploaded +docker_image_label_get_encryption_key() { + local image_url="${1}" + + # Pull image + esudo docker pull "${image_url}" &>/dev/null + + # Get encryption key from docker image label + esudo docker inspect ${image_url} \ + | jq -r '.[0].Config.Labels.enc_key' +} + +# Retrieve ssh key from docker image label +# 'ssh_key' image label must be set when image created or uploaded +docker_image_label_save_ssh_key() { + local image_url="${1}" + local ssh_key_file="${2}" + + # Pull image + esudo docker pull "${image_url}" + + # Get ssh key from docker image label and save to file + esudo docker inspect ${image_url} \ + | jq -r '.[0].Config.Labels.ssh_key' \ + | sed "s|\(-----BEGIN OPENSSH PRIVATE KEY-----\)|\1\n|g" \ + | sed "s|\(-----END OPENSSH PRIVATE KEY-----\)|\n\1|g" \ + > "${ssh_key_file}" + + # Set permissions on private key file + chmod 600 "${ssh_key_file}" +} + + + +############################################################################### + +# kata + +# Find sandbox id using container ID +kata_get_sandbox_id() { + local container_id="${1}" + local sandbox_dir="/run/kata-containers/shared/sandboxes" + + # Find container directory inside sandbox directory + local container_dir=$(esudo find "${sandbox_dir}" -name "${container_id}" | head -1) + + # Ensure directory path pattern is correct + [[ ${container_dir} =~ ^${sandbox_dir}/.*/.*/${container_id} ]] \ + || (>&2 echo "Incorrect container folder path: ${container_dir}"; return 1) + + # Two levels up, and trim the sandbox dir off the front + local sandbox_id=$(dirname $(dirname "${container_dir}") | sed "s|${sandbox_dir}/||g") + + echo "${sandbox_id}" +} + +# Get guest kernel append from qemu command line +kata_get_guest_kernel_append() { + local pod_name="${1}" + local duration=$((SECONDS+20)) + local kernel_append + + # Attempt to get qemu command line from qemu process + while [ $SECONDS -lt $duration ]; do + container_id=$(kubernetes_get_container_id "${pod_name}") + sandbox_id=$(kata_get_sandbox_id "${container_id}") + qemu_process=$(ps aux | grep qemu | grep ${sandbox_id} | grep append || true) + if [ -n "${qemu_process}" ]; then + kernel_append=$(echo ${qemu_process} \ + | sed "s|.*-append \(.*$\)|\1|g" \ + | sed "s| -.*$||") + break + fi + sleep 1 + done + + [ -n "${kernel_append}" ] \ + || (>&2 echo "Could not retrieve guest kernel append parameters"; return 1) + + echo "${kernel_append}" +}