diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml new file mode 100644 index 0000000..329cde4 --- /dev/null +++ b/.github/workflows/integration.yml @@ -0,0 +1,71 @@ +--- +name: integration +on: + push: + branches: + - main + pull_request: +jobs: + single-trust-zone: + name: single trust zone + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install just + uses: taiki-e/install-action@just + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: Build and run tests + run: just build + + - name: Install kind + run: just install-kind + + - name: Install ko + uses: ko-build/setup-ko@v0.6 + env: + KO_DOCKER_REPO: kind.local + + - name: Create a kind cluster + run: just create-kind-cluster + + - name: Test + run: just integration-test single-trust-zone + + federation: + name: federation + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install just + uses: taiki-e/install-action@just + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: Build and run tests + run: just build + + - name: Install kind + run: just install-kind + + - name: Install ko + uses: ko-build/setup-ko@v0.6 + env: + KO_DOCKER_REPO: kind.local + + - name: Create kind clusters + run: just create-kind-clusters 2 + + - name: Test + run: just integration-test federation diff --git a/Justfile b/Justfile index fcea971..74f0b17 100644 --- a/Justfile +++ b/Justfile @@ -6,3 +6,15 @@ test: lint *args: golangci-lint run --show-stats {{args}} + +install-kind: + tests/integration/install-kind.sh + +create-kind-cluster: + tests/integration/create-kind-cluster.sh + +create-kind-clusters num_clusters: + tests/integration/create-kind-clusters.sh {{num_clusters}} + +integration-test test: + tests/integration/{{test}}/test.sh diff --git a/demos/Justfile b/demos/Justfile index 8acbed0..24ff815 100644 --- a/demos/Justfile +++ b/demos/Justfile @@ -1,12 +1,11 @@ set export set shell := ["bash", "-euo", "pipefail", "-c"] -# assume a local kind cluster for the demos unless otherwise configured -export KO_DOCKER_REPO := env_var_or_default("KO_DOCKER_REPO", "kind.local") -export KIND_CLUSTER_NAME := env_var_or_default("KIND_CLUSTER_NAME", "kind") - namespace := "demo" +# Set prompt_namespace=no to avoid prompting before namespace creation. +prompt_namespace := 'yes' + # Check for demo script dependencies check-deps: for cmd in ko kubectl; do \ @@ -21,28 +20,23 @@ ensure-namespace context: if [[ ! -z "{{context}}" ]]; then \ if ! kubectl --context {{context}} get namespace "{{namespace}}" &> /dev/null; then \ echo "Namespace {{namespace}} does not exist"; \ - read -p "Create namespace? (y/n) " -r; \ - if [[ $REPLY =~ ^[Yy]$ ]]; then \ - kubectl --context {{context}} create namespace "{{namespace}}"; \ - else \ - echo "Aborting..."; \ - exit 1; \ - fi \ + if [[ "{{prompt_namespace}}" != "no" ]]; then \ + read -p "Create namespace? (y/n) " -r; \ + if [[ ! $REPLY =~ ^[Yy]$ ]]; then \ + echo "Aborting..."; \ + exit 1; \ + fi \ + fi; \ + kubectl --context {{context}} create namespace "{{namespace}}"; \ fi \ fi +# Clone the cofide-demos git repo. clone-demos-repo: if [ ! -d cofide-demos ]; then \ git clone https://github.com/cofide/cofide-demos; \ fi -# Build all demo ping-pong applications -build-demos: build-ping-pong - -# Build the ping-pong application -build-ping-pong: clone-demos-repo - just -f cofide-demos/Justfile build-ping-pong - # Deploy ping-pong server and client -deploy-ping-pong client_context server_context="": build-ping-pong (ensure-namespace client_context) (ensure-namespace server_context) +deploy-ping-pong client_context server_context="": clone-demos-repo (ensure-namespace client_context) (ensure-namespace server_context) ping-pong/deploy.sh {{namespace}} {{client_context}} {{server_context}} diff --git a/demos/ping-pong/deploy.sh b/demos/ping-pong/deploy.sh index f95bc1d..3c4a5b9 100755 --- a/demos/ping-pong/deploy.sh +++ b/demos/ping-pong/deploy.sh @@ -6,6 +6,9 @@ NAMESPACE="$1" CLIENT_CTX="$2" SERVER_CTX="${3:-$CLIENT_CTX}" +# assume a local kind cluster for the demos unless otherwise configured +export KO_DOCKER_REPO=${KO_DOCKER_REPO:-kind.local} + pushd cofide-demos echo "Deploying pong server to: $SERVER_CTX" @@ -17,12 +20,12 @@ if ! ko resolve -f workloads/ping-pong/server/deploy.yaml | kubectl apply -n "$N exit 1 fi echo "Server deployment complete" -if [ "$CLIENT_CTX" == kind-* ]; then +if [[ "$CLIENT_CTX" == kind-* ]]; then export KIND_CLUSTER_NAME="${CLIENT_CTX#kind-}" fi echo "Deploying ping client to: $CLIENT_CTX" -if [ "$SERVER_CTX" != "$CLIENT_CTX" ]; then +if [[ "$SERVER_CTX" != "$CLIENT_CTX" ]]; then echo "Discovering server IP..." export PING_PONG_SERVER_SERVICE_HOST=$(kubectl --context "$SERVER_CTX" wait --for=jsonpath="{.status.loadBalancer.ingress[0].ip}" service/ping-pong-server -n $NAMESPACE --timeout=60s > /dev/null 2>&1 \ && kubectl --context "$SERVER_CTX" get service ping-pong-server -n $NAMESPACE -o "jsonpath={.status.loadBalancer.ingress[0].ip}") diff --git a/tests/integration/create-kind-cluster.sh b/tests/integration/create-kind-cluster.sh new file mode 100755 index 0000000..8b1e60a --- /dev/null +++ b/tests/integration/create-kind-cluster.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# This script deploys a local Kubernetes cluster using Kind (https://kind.sigs.k8s.io). + +set -euxo pipefail + +DELETE_EXISTING_KIND_CLUSTER=${DELETE_EXISTING_KIND_CLUSTER:-true} + +K8S_CLUSTER_NAME=${K8S_CLUSTER_NAME:-local1} + +CLOUD_PROVIDER_KIND_CONTAINER_NAME=cloud-provider-kind-cloud-provider-1 + +function delete_kind_cluster() { + cluster=$(kind get clusters | egrep "\b${K8S_CLUSTER_NAME}\b" || true) + if [[ -n $cluster ]]; then + kind delete cluster -n $K8S_CLUSTER_NAME + fi +} + +function create_kind_cluster() { + kind create cluster -n $K8S_CLUSTER_NAME +} + +function restart_cloud_provider_kind() { + # Workaround: cloud-provider-kind often stops working when a kind cluster is created. Restart it. + if [[ $(docker ps -q --filter "name=$CLOUD_PROVIDER_KIND_CONTAINER_NAME") != "" ]]; then + docker restart $CLOUD_PROVIDER_KIND_CONTAINER_NAME + fi +} + +function main() { + if $DELETE_EXISTING_KIND_CLUSTER; then + delete_kind_cluster + fi + create_kind_cluster + restart_cloud_provider_kind + echo "Success!" +} + +main diff --git a/tests/integration/create-kind-clusters.sh b/tests/integration/create-kind-clusters.sh new file mode 100755 index 0000000..bab776b --- /dev/null +++ b/tests/integration/create-kind-clusters.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# This script deploys multiple local Kubernetes clusters using Kind (https://kind.sigs.k8s.io). + +set -euxo pipefail + +NUM_K8S_CLUSTERS=${1:?Number of kind clusters to create} +K8S_CLUSTER_NAME_PREFIX=${K8S_CLUSTER_NAME_PREFIX:-local} + +function create_kind_cluster() { + suffix=$1 + parent_dir=$(dirname $BASH_SOURCE) + export K8S_CLUSTER_NAME=${K8S_CLUSTER_NAME_PREFIX}${suffix} + $parent_dir/create-kind-cluster.sh +} + +function main() { + for i in $(seq $NUM_K8S_CLUSTERS); do + create_kind_cluster $i + done +} + +main diff --git a/tests/integration/federation/test.sh b/tests/integration/federation/test.sh new file mode 100755 index 0000000..2a657ed --- /dev/null +++ b/tests/integration/federation/test.sh @@ -0,0 +1,96 @@ +#!/bin/bash + +# This script deploys two federated trust zones, runs some basic tests against them, then tears them down. +# The trust zones have one attestation policy matching workloads in namespace ns1, and another matching pods with a label foo=bar. + +set -euxo pipefail + +K8S_CLUSTER_1_NAME=${K8S_CLUSTER_1_NAME:-local1} +K8S_CLUSTER_1_CONTEXT=${K8S_CLUSTER_1_CONTEXT:-kind-$K8S_CLUSTER_1_NAME} + +K8S_CLUSTER_2_NAME=${K8S_CLUSTER_2_NAME:-local2} +K8S_CLUSTER_2_CONTEXT=${K8S_CLUSTER_2_CONTEXT:-kind-$K8S_CLUSTER_2_NAME} + +TRUST_ZONE_1=${TRUST_ZONE_1:-tz1} +TRUST_DOMAIN_1=${TRUST_DOMAIN_1:-td1} + +TRUST_ZONE_2=${TRUST_ZONE_2:-tz2} +TRUST_DOMAIN_2=${TRUST_DOMAIN_2:-td2} + +NAMESPACE_POLICY_NAMESPACE=${NAMESPACE_POLICY_NAMESPACE:-demo} +POD_POLICY_POD_LABEL=${POD_POLICY_POD_LABEL:-"foo=bar"} + +function configure() { + rm -f cofide.yaml + ./cofidectl init + ./cofidectl trust-zone add $TRUST_ZONE_1 --trust-domain $TRUST_DOMAIN_1 --kubernetes-context $K8S_CLUSTER_1_CONTEXT --kubernetes-cluster $K8S_CLUSTER_1_NAME --profile kubernetes + ./cofidectl trust-zone add $TRUST_ZONE_2 --trust-domain $TRUST_DOMAIN_2 --kubernetes-context $K8S_CLUSTER_2_CONTEXT --kubernetes-cluster $K8S_CLUSTER_2_NAME --profile kubernetes + ./cofidectl federation add --from $TRUST_ZONE_1 --to $TRUST_ZONE_2 + ./cofidectl federation add --from $TRUST_ZONE_2 --to $TRUST_ZONE_1 + ./cofidectl attestation-policy add kubernetes --name namespace --namespace $NAMESPACE_POLICY_NAMESPACE + ./cofidectl attestation-policy add kubernetes --name pod-label --pod-label $POD_POLICY_POD_LABEL + ./cofidectl attestation-policy-binding add --trust-zone $TRUST_ZONE_1 --attestation-policy namespace --federates-with $TRUST_ZONE_2 + ./cofidectl attestation-policy-binding add --trust-zone $TRUST_ZONE_1 --attestation-policy pod-label --federates-with $TRUST_ZONE_2 + ./cofidectl attestation-policy-binding add --trust-zone $TRUST_ZONE_2 --attestation-policy namespace --federates-with $TRUST_ZONE_1 + ./cofidectl attestation-policy-binding add --trust-zone $TRUST_ZONE_2 --attestation-policy pod-label --federates-with $TRUST_ZONE_1 +} + +function up() { + ./cofidectl up +} + +function list_resources() { + ./cofidectl trust-zone list + ./cofidectl attestation-policy list + ./cofidectl attestation-policy-binding list +} + +function show_config() { + cat cofide.yaml +} + +function show_status() { + ./cofidectl workload discover + ./cofidectl workload list + ./cofidectl trust-zone status $TRUST_ZONE_1 + ./cofidectl trust-zone status $TRUST_ZONE_2 +} + +function run_tests() { + just -f demos/Justfile prompt_namespace=no deploy-ping-pong $K8S_CLUSTER_1_CONTEXT $K8S_CLUSTER_2_CONTEXT + if ! wait_for_pong; then + echo "Timed out waiting for pong from server" + echo "Client logs:" + kubectl --context $K8S_CLUSTER_1_CONTEXT logs -n demo deployments/ping-pong-client + echo "Server logs:" + kubectl --context $K8S_CLUSTER_2_CONTEXT logs -n demo deployments/ping-pong-server + exit 1 + fi +} + +function wait_for_pong() { + kubectl --context $K8S_CLUSTER_1_CONTEXT wait -n demo --for=condition=Available --timeout 60s deployments/ping-pong-client + for i in $(seq 30); do + if kubectl --context $K8S_CLUSTER_1_CONTEXT logs -n demo deployments/ping-pong-client | grep pong; then + return + fi + sleep 2 + done +} + +function down() { + ./cofidectl down +} + +function main() { + configure + up + list_resources + show_config + show_status + run_tests + down + echo "Success!" +} + +main diff --git a/tests/integration/install-kind.sh b/tests/integration/install-kind.sh new file mode 100755 index 0000000..0c2d7ac --- /dev/null +++ b/tests/integration/install-kind.sh @@ -0,0 +1,90 @@ +#!/bin/bash + +# This script installs Kind (https://kind.sigs.k8s.io) for test and development of cofidectl with local Kubernetes clusters. + +set -euxo pipefail + +function prechecks() { + if ! type apt; then + echo "Only Ubuntu is supported" + fi +} + +function install_package_deps() { + sudo apt update + sudo apt install -y git +} + +function install_docker() { + # https://docs.docker.com/engine/install/ubuntu/ + # Add Docker's official GPG key: + sudo apt-get update + sudo apt-get install -y ca-certificates curl + sudo install -m 0755 -d /etc/apt/keyrings + sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc + sudo chmod a+r /etc/apt/keyrings/docker.asc + + # Add the repository to Apt sources: + echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ + $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + sudo apt-get update + sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin +} + +function install_kind () { + # https://kind.sigs.k8s.io/docs/user/quick-start#installing-from-release-binaries + # For AMD64 / x86_64 + [ "$(uname -m)" = x86_64 ] && curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.24.0/kind-linux-amd64 + # For ARM64 + [ "$(uname -m)" = aarch64 ] && curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.24.0/kind-linux-arm64 + chmod +x ./kind + sudo mv ./kind /usr/local/bin/kind +} + +function set_inotify_limits_for_kind() { + # https://kind.sigs.k8s.io/docs/user/known-issues/#pod-errors-due-to-too-many-open-files + cat << EOF | sudo tee /etc/sysctl.d/10-kind.conf +fs.inotify.max_user_watches = 524288 +fs.inotify.max_user_instances = 512 +EOF +} + +function ensure_kind_network() { + # cloud-provider-kind requires the kind network to exist, which gets created the first time a kind cluster is created. + if ! sudo docker network inspect kind &>/dev/null; then + sudo kind create cluster + sudo kind delete cluster + fi +} + +function install_cloud_provider_kind() { + # https://github.com/kubernetes-sigs/cloud-provider-kind + if [[ ! -d cloud-provider-kind ]]; then + git clone --depth 1 https://github.com/kubernetes-sigs/cloud-provider-kind + fi + pushd cloud-provider-kind + sudo bash -c 'NET_MODE=kind docker compose up -d' + popd +} + +function check_non_root_docker_access() { + if ! docker ps &>/dev/null; then + echo "You may need to log out and in again to obtain non-root access to Docker" + fi +} + +function main() { + prechecks + install_package_deps + install_docker + install_kind + set_inotify_limits_for_kind + ensure_kind_network + install_cloud_provider_kind + echo "Success!" + check_non_root_docker_access +} + +main diff --git a/tests/integration/single-trust-zone/test.sh b/tests/integration/single-trust-zone/test.sh new file mode 100755 index 0000000..9425b97 --- /dev/null +++ b/tests/integration/single-trust-zone/test.sh @@ -0,0 +1,84 @@ +#!/bin/bash + +# This script deploys a single trust zone, runs some basic tests against it, then tears it down. +# The trust zone has one attestation policy matching workloads in namespace ns1, and another matching pods with a label foo=bar. + +set -euxo pipefail + +K8S_CLUSTER_NAME=${K8S_CLUSTER_NAME:-local1} +K8S_CLUSTER_CONTEXT=${K8S_CLUSTER_CONTEXT:-kind-$K8S_CLUSTER_NAME} + +TRUST_ZONE=${TRUST_ZONE:-tz1} +TRUST_DOMAIN=${TRUST_DOMAIN:-td1} + +NAMESPACE_POLICY_NAMESPACE=${NAMESPACE_POLICY_NAMESPACE:-demo} +POD_POLICY_POD_LABEL=${POD_POLICY_POD_LABEL:-"foo=bar"} + +function configure() { + rm -f cofide.yaml + ./cofidectl init + ./cofidectl trust-zone add $TRUST_ZONE --trust-domain $TRUST_DOMAIN --kubernetes-context $K8S_CLUSTER_CONTEXT --kubernetes-cluster $K8S_CLUSTER_NAME --profile kubernetes + ./cofidectl attestation-policy add kubernetes --name namespace --namespace $NAMESPACE_POLICY_NAMESPACE + ./cofidectl attestation-policy add kubernetes --name pod-label --pod-label $POD_POLICY_POD_LABEL + ./cofidectl attestation-policy-binding add --trust-zone $TRUST_ZONE --attestation-policy namespace + ./cofidectl attestation-policy-binding add --trust-zone $TRUST_ZONE --attestation-policy pod-label +} + +function up() { + ./cofidectl up +} + +function list_resources() { + ./cofidectl trust-zone list + ./cofidectl attestation-policy list + ./cofidectl attestation-policy-binding list +} + +function show_config() { + cat cofide.yaml +} + +function show_status() { + ./cofidectl workload discover + ./cofidectl workload list + ./cofidectl trust-zone status $TRUST_ZONE +} + +function run_tests() { + just -f demos/Justfile prompt_namespace=no deploy-ping-pong $K8S_CLUSTER_CONTEXT + if ! wait_for_pong; then + echo "Timed out waiting for pong from server" + echo "Client logs:" + kubectl --context $K8S_CLUSTER_CONTEXT logs -n demo deployments/ping-pong-client + echo "Server logs:" + kubectl --context $K8S_CLUSTER_CONTEXT logs -n demo deployments/ping-pong-server + exit 1 + fi +} + +function wait_for_pong() { + kubectl --context $K8S_CLUSTER_CONTEXT wait -n demo --for=condition=Available --timeout 60s deployments/ping-pong-client + for i in $(seq 30); do + if kubectl --context $K8S_CLUSTER_CONTEXT logs -n demo deployments/ping-pong-client | grep pong; then + return + fi + sleep 2 + done +} + +function down() { + ./cofidectl down +} + +function main() { + configure + up + list_resources + show_config + show_status + run_tests + down + echo "Success!" +} + +main